<commit_before>// +build e2e

/*
Copyright 2019 The Knative Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"path/filepath"
	"runtime"
	"testing"

	mf "github.com/jcrossley3/manifestival"
	apierrs "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	"knative.dev/pkg/test/logstream"
	"knative.dev/serving-operator/test"
	"knative.dev/serving-operator/test/resources"
)

// TestKnativeServingDeployment verifies the KnativeServing creation, deployment recreation, and KnativeServing deletion.
func TestKnativeServingDeployment(t *testing.T) {
	cancel := logstream.Start(t)
	defer cancel()
	clients := Setup(t)

	names := test.ResourceNames{
		KnativeServing: test.ServingOperatorName,
		Namespace:      test.ServingOperatorNamespace,
	}

	test.CleanupOnInterrupt(func() { test.TearDown(clients, names) })
	defer test.TearDown(clients, names)

	// Create a KnativeServing
	if _, err := resources.CreateKnativeServing(clients.KnativeServingAlphaClient, names); err != nil {
		t.Fatalf("KnativeService %q failed to create: %v", names.KnativeServing, err)
	}

	// Test if KnativeServing can reach the READY status
	t.Run("create", func(t *testing.T) {
		knativeServingVerify(t, clients, names)
	})

	// Delete the deployments one by one to see if they will be recreated.
	t.Run("restore", func(t *testing.T) {
		knativeServingVerify(t, clients, names)
		deploymentRecreation(t, clients, names)
	})

	// Delete the KnativeServing to see if all the deployments will be removed as well
	t.Run("delete", func(t *testing.T) {
		knativeServingVerify(t, clients, names)
		knativeServingDeletion(t, clients, names)
		verifyClusterResourceDeletion(t, clients)
	})
}

// knativeServingVerify verifies if the KnativeServing can reach the READY status.
func knativeServingVerify(t *testing.T, clients *test.Clients, names test.ResourceNames) {
	if _, err := resources.WaitForKnativeServingState(clients.KnativeServingAlphaClient, names.KnativeServing,
		resources.IsKnativeServingReady); err != nil {
		t.Fatalf("KnativeService %q failed to get to the READY status: %v", names.KnativeServing, err)
	}

}

// deploymentRecreation verify whether all the deployments for knative serving are able to recreate, when they are deleted.
func deploymentRecreation(t *testing.T, clients *test.Clients, names test.ResourceNames) {
	dpList, err := clients.KubeClient.Kube.AppsV1().Deployments(names.Namespace).List(metav1.ListOptions{})
	if err != nil {
		t.Fatalf("Failed to get any deployment under the namespace %q: %v",
			test.ServingOperatorNamespace, err)
	}
	if len(dpList.Items) == 0 {
		t.Fatalf("No deployment under the namespace %q was found",
			test.ServingOperatorNamespace)
	}
	// Delete the first deployment and verify the operator recreates it
	deployment := dpList.Items[0]
	if err := clients.KubeClient.Kube.AppsV1().Deployments(deployment.Namespace).Delete(deployment.Name,
		&metav1.DeleteOptions{}); err != nil {
		t.Fatalf("Failed to delete deployment %s/%s: %v", deployment.Namespace, deployment.Name, err)
	}

	waitErr := wait.PollImmediate(resources.Interval, resources.Timeout, func() (bool, error) {
		dep, err := clients.KubeClient.Kube.AppsV1().Deployments(deployment.Namespace).Get(deployment.Name, metav1.GetOptions{})
		if err != nil {
			// If the deployment is not found, we continue to wait for the availability.
			if apierrs.IsNotFound(err) {
				return false, nil
			}
			return false, err
		}
		return resources.IsDeploymentAvailable(dep)
	})

	if waitErr != nil {
		t.Fatalf("The deployment %s/%s failed to reach the desired state: %v", deployment.Namespace, deployment.Name, err)
	}

	if _, err := resources.WaitForKnativeServingState(clients.KnativeServingAlphaClient, test.ServingOperatorName,
		resources.IsKnativeServingReady); err != nil {
		t.Fatalf("KnativeService %q failed to reach the desired state: %v", test.ServingOperatorName, err)
	}
	t.Logf("The deployment %s/%s reached the desired state.", deployment.Namespace, deployment.Name)
}

// knativeServingDeletion deletes tha KnativeServing to see if all the deployments will be removed.
func knativeServingDeletion(t *testing.T, clients *test.Clients, names test.ResourceNames) {
	if err := clients.KnativeServingAlphaClient.Delete(names.KnativeServing, &metav1.DeleteOptions{}); err != nil {
		t.Fatalf("KnativeService %q failed to delete: %v", names.KnativeServing, err)
	}

	dpList, err := clients.KubeClient.Kube.AppsV1().Deployments(names.Namespace).List(metav1.ListOptions{})
	if err != nil {
		t.Fatalf("Error getting any deployment under the namespace %q: %v", names.Namespace, err)
	}

	for _, deployment := range dpList.Items {
		waitErr := wait.PollImmediate(resources.Interval, resources.Timeout, func() (bool, error) {
			if _, err := clients.KubeClient.Kube.AppsV1().Deployments(deployment.Namespace).Get(deployment.Name, metav1.GetOptions{}); err != nil {
				if apierrs.IsNotFound(err) {
					return true, nil
				}
				return false, err
			}
			return false, nil
		})

		if waitErr != nil {
			t.Fatalf("The deployment %s/%s failed to be deleted: %v", deployment.Namespace, deployment.Name, waitErr)
		}
		t.Logf("The deployment %s/%s has been deleted.", deployment.Namespace, deployment.Name)
	}
}

func verifyClusterResourceDeletion(t *testing.T, clients *test.Clients) {
	_, b, _, _ := runtime.Caller(0)
	m, err := mf.NewManifest(filepath.Join((filepath.Dir(b) + "/.."), "config/"), false, clients.Config)
	if err != nil {
		t.Fatal("Failed to load manifest", err)
	}
	for _, u := range m.Resources {
		if u.GetNamespace() == "" && u.GetKind() != "Namespace" {
			waitErr := wait.PollImmediate(resources.Interval, resources.Timeout, func() (bool, error) {
				gvrs, _ := meta.UnsafeGuessKindToResource(u.GroupVersionKind())
				if _, err := clients.Dynamic.Resource(gvrs).Get(u.GetName(), metav1.GetOptions{}); apierrs.IsNotFound(err) {
					return true, nil
				} else {
					return false, err
				}
			})

			if waitErr != nil {
				t.Fatalf("The %s %s failed to be deleted: %v", u.GetKind(), u.GetName(), waitErr)
			}
			t.Logf("The %s %s has been deleted.", u.GetKind(), u.GetName())
		}
	}
}
<commit_msg>golang format tools (#155)<commit_after>// +build e2e

/*
Copyright 2019 The Knative Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"path/filepath"
	"runtime"
	"testing"

	mf "github.com/jcrossley3/manifestival"
	apierrs "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	"knative.dev/pkg/test/logstream"
	"knative.dev/serving-operator/test"
	"knative.dev/serving-operator/test/resources"
)

// TestKnativeServingDeployment verifies the KnativeServing creation, deployment recreation, and KnativeServing deletion.
func TestKnativeServingDeployment(t *testing.T) {
	cancel := logstream.Start(t)
	defer cancel()
	clients := Setup(t)

	names := test.ResourceNames{
		KnativeServing: test.ServingOperatorName,
		Namespace:      test.ServingOperatorNamespace,
	}

	test.CleanupOnInterrupt(func() { test.TearDown(clients, names) })
	defer test.TearDown(clients, names)

	// Create a KnativeServing
	if _, err := resources.CreateKnativeServing(clients.KnativeServingAlphaClient, names); err != nil {
		t.Fatalf("KnativeService %q failed to create: %v", names.KnativeServing, err)
	}

	// Test if KnativeServing can reach the READY status
	t.Run("create", func(t *testing.T) {
		knativeServingVerify(t, clients, names)
	})

	// Delete the deployments one by one to see if they will be recreated.
	t.Run("restore", func(t *testing.T) {
		knativeServingVerify(t, clients, names)
		deploymentRecreation(t, clients, names)
	})

	// Delete the KnativeServing to see if all the deployments will be removed as well
	t.Run("delete", func(t *testing.T) {
		knativeServingVerify(t, clients, names)
		knativeServingDeletion(t, clients, names)
		verifyClusterResourceDeletion(t, clients)
	})
}

// knativeServingVerify verifies if the KnativeServing can reach the READY status.
func knativeServingVerify(t *testing.T, clients *test.Clients, names test.ResourceNames) {
	if _, err := resources.WaitForKnativeServingState(clients.KnativeServingAlphaClient, names.KnativeServing,
		resources.IsKnativeServingReady); err != nil {
		t.Fatalf("KnativeService %q failed to get to the READY status: %v", names.KnativeServing, err)
	}

}

// deploymentRecreation verify whether all the deployments for knative serving are able to recreate, when they are deleted.
func deploymentRecreation(t *testing.T, clients *test.Clients, names test.ResourceNames) {
	dpList, err := clients.KubeClient.Kube.AppsV1().Deployments(names.Namespace).List(metav1.ListOptions{})
	if err != nil {
		t.Fatalf("Failed to get any deployment under the namespace %q: %v",
			test.ServingOperatorNamespace, err)
	}
	if len(dpList.Items) == 0 {
		t.Fatalf("No deployment under the namespace %q was found",
			test.ServingOperatorNamespace)
	}
	// Delete the first deployment and verify the operator recreates it
	deployment := dpList.Items[0]
	if err := clients.KubeClient.Kube.AppsV1().Deployments(deployment.Namespace).Delete(deployment.Name,
		&metav1.DeleteOptions{}); err != nil {
		t.Fatalf("Failed to delete deployment %s/%s: %v", deployment.Namespace, deployment.Name, err)
	}

	waitErr := wait.PollImmediate(resources.Interval, resources.Timeout, func() (bool, error) {
		dep, err := clients.KubeClient.Kube.AppsV1().Deployments(deployment.Namespace).Get(deployment.Name, metav1.GetOptions{})
		if err != nil {
			// If the deployment is not found, we continue to wait for the availability.
			if apierrs.IsNotFound(err) {
				return false, nil
			}
			return false, err
		}
		return resources.IsDeploymentAvailable(dep)
	})

	if waitErr != nil {
		t.Fatalf("The deployment %s/%s failed to reach the desired state: %v", deployment.Namespace, deployment.Name, err)
	}

	if _, err := resources.WaitForKnativeServingState(clients.KnativeServingAlphaClient, test.ServingOperatorName,
		resources.IsKnativeServingReady); err != nil {
		t.Fatalf("KnativeService %q failed to reach the desired state: %v", test.ServingOperatorName, err)
	}
	t.Logf("The deployment %s/%s reached the desired state.", deployment.Namespace, deployment.Name)
}

// knativeServingDeletion deletes tha KnativeServing to see if all the deployments will be removed.
func knativeServingDeletion(t *testing.T, clients *test.Clients, names test.ResourceNames) {
	if err := clients.KnativeServingAlphaClient.Delete(names.KnativeServing, &metav1.DeleteOptions{}); err != nil {
		t.Fatalf("KnativeService %q failed to delete: %v", names.KnativeServing, err)
	}

	dpList, err := clients.KubeClient.Kube.AppsV1().Deployments(names.Namespace).List(metav1.ListOptions{})
	if err != nil {
		t.Fatalf("Error getting any deployment under the namespace %q: %v", names.Namespace, err)
	}

	for _, deployment := range dpList.Items {
		waitErr := wait.PollImmediate(resources.Interval, resources.Timeout, func() (bool, error) {
			if _, err := clients.KubeClient.Kube.AppsV1().Deployments(deployment.Namespace).Get(deployment.Name, metav1.GetOptions{}); err != nil {
				if apierrs.IsNotFound(err) {
					return true, nil
				}
				return false, err
			}
			return false, nil
		})

		if waitErr != nil {
			t.Fatalf("The deployment %s/%s failed to be deleted: %v", deployment.Namespace, deployment.Name, waitErr)
		}
		t.Logf("The deployment %s/%s has been deleted.", deployment.Namespace, deployment.Name)
	}
}

func verifyClusterResourceDeletion(t *testing.T, clients *test.Clients) {
	_, b, _, _ := runtime.Caller(0)
	m, err := mf.NewManifest(filepath.Join((filepath.Dir(b)+"/.."), "config/"), false, clients.Config)
	if err != nil {
		t.Fatal("Failed to load manifest", err)
	}
	for _, u := range m.Resources {
		if u.GetNamespace() == "" && u.GetKind() != "Namespace" {
			waitErr := wait.PollImmediate(resources.Interval, resources.Timeout, func() (bool, error) {
				gvrs, _ := meta.UnsafeGuessKindToResource(u.GroupVersionKind())
				if _, err := clients.Dynamic.Resource(gvrs).Get(u.GetName(), metav1.GetOptions{}); apierrs.IsNotFound(err) {
					return true, nil
				} else {
					return false, err
				}
			})

			if waitErr != nil {
				t.Fatalf("The %s %s failed to be deleted: %v", u.GetKind(), u.GetName(), waitErr)
			}
			t.Logf("The %s %s has been deleted.", u.GetKind(), u.GetName())
		}
	}
}
<|endoftext|>


<commit_before>package utils

import "os/exec"

// GitRepo type manages a VCS repository with Git
type GitRepo struct {
	// Local path to the repository
	Path string

	// Upstream URL of the Git repository
	Upstream string

	// Path to the Git tool
	git string
}

// NewGitRepo creates a new Git repository
func NewGitRepo(path, upstream string) (*GitRepo, error) {
	path, err := exec.LookPath("git")
	if err != nil {
		return nil, err
	}

	repo := &GitRepo{
		Path:     path,
		Upstream: upstream,
		cmd:      path,
	}

	return repo, nil
}

// Fetch fetches from the given remote
func (gr *GitRepo) Fetch(remote string) ([]byte, error) {
	return exec.Command(gr.git, "--git-dir", gr.Path, "fetch", remote).CombinedOutput()
}

// Pull pulls from the given remote and merges changes into the
// local branch
func (gr *GitRepo) Pull(remote, branch string) ([]byte, error) {
	out, err := gr.CheckoutBranch(branch)
	if err != nil {
		return out, err
	}

	return exec.Command(gr.git, "--git-dir", gr.Path, "pull", remote).CombinedOutput()
}

// CheckoutBranch checks out a given local branch
func (gr *GitRepo) CheckoutBranch(branch string) ([]byte, error) {
	return exec.Command(gr.git, "--git-dir", gr.Path, "checkout", branch).CombinedOutput()
}

// Clone clones the upstream repository
func (gr *GitRepo) Clone() ([]byte, error) {
	return exec.Command(gr.git, "clone", gr.Upstream, gr.Path).CombinedOutput()
}
<commit_msg>utils: Rename CheckoutBranch to Checkout method for GitRepo type<commit_after>package utils

import "os/exec"

// GitRepo type manages a VCS repository with Git
type GitRepo struct {
	// Local path to the repository
	Path string

	// Upstream URL of the Git repository
	Upstream string

	// Path to the Git tool
	git string
}

// NewGitRepo creates a new Git repository
func NewGitRepo(path, upstream string) (*GitRepo, error) {
	path, err := exec.LookPath("git")
	if err != nil {
		return nil, err
	}

	repo := &GitRepo{
		Path:     path,
		Upstream: upstream,
		cmd:      path,
	}

	return repo, nil
}

// Fetch fetches from the given remote
func (gr *GitRepo) Fetch(remote string) ([]byte, error) {
	return exec.Command(gr.git, "--git-dir", gr.Path, "fetch", remote).CombinedOutput()
}

// Pull pulls from the given remote and merges changes into the
// local branch
func (gr *GitRepo) Pull(remote, branch string) ([]byte, error) {
	out, err := gr.Checkout(branch)
	if err != nil {
		return out, err
	}

	return exec.Command(gr.git, "--git-dir", gr.Path, "pull", remote).CombinedOutput()
}

// CheckoutBranch checks out a given local branch
func (gr *GitRepo) Checkout(branch string) ([]byte, error) {
	return exec.Command(gr.git, "--git-dir", gr.Path, "checkout", branch).CombinedOutput()
}

// Clone clones the upstream repository
func (gr *GitRepo) Clone() ([]byte, error) {
	return exec.Command(gr.git, "clone", gr.Upstream, gr.Path).CombinedOutput()
}
<|endoftext|>


<commit_before>package router

import (
	"bytes"
	"crypto/rand"
	"encoding/gob"
	"fmt"
	"hash/fnv"
	"log"
	"net"
)

type Interaction struct {
	code       int
	resultChan chan<- interface{}
}

func checkFatal(e error) {
	if e != nil {
		log.Fatal(e)
	}
}

func checkWarn(e error) {
	if e != nil {
		log.Println(e)
	}
}

func PosixError(err error) error {
	if err == nil {
		return nil
	}
	operr, ok := err.(*net.OpError)
	if !ok {
		return nil
	}
	return operr.Err
}

func (mtbe MsgTooBigError) Error() string {
	return fmt.Sprint("Msg too big error. PMTU is ", mtbe.PMTU)
}

func (ftbe FrameTooBigError) Error() string {
	return fmt.Sprint("Frame too big error. Effective PMTU is ", ftbe.EPMTU)
}

func (upe UnknownPeerError) Error() string {
	return fmt.Sprint("Reference to unknown peer ", upe.Name)
}

func (nce NameCollisionError) Error() string {
	return fmt.Sprint("Multiple peers found with same name: ", nce.Name)
}

func (pde PacketDecodingError) Error() string {
	return fmt.Sprint("Failed to decode packet: ", pde.Desc)
}

func Concat(elems ...[]byte) []byte {
	res := []byte{}
	for _, e := range elems {
		res = append(res, e...)
	}
	return res
}

func randUint64() (r uint64) {
	buf := make([]byte, 8)
	_, err := rand.Read(buf)
	checkFatal(err)
	for _, v := range buf {
		r <<= 8
		r |= uint64(v)
	}
	return
}

func hash(s string) uint32 {
	h := fnv.New32a()
	h.Write([]byte(s))
	return h.Sum32()
}

func GobEncode(items ...interface{}) []byte {
	buf := new(bytes.Buffer)
	enc := gob.NewEncoder(buf)
	for _, i := range items {
		checkFatal(enc.Encode(i))
	}
	return buf.Bytes()
}

func macint(mac net.HardwareAddr) (r uint64) {
	for _, b := range mac {
		r <<= 8
		r |= uint64(b)
	}
	return
}

func intmac(key uint64) (r net.HardwareAddr) {
	r = make([]byte, 6)
	for i := 5; i >= 0; i-- {
		r[i] = byte(key)
		key >>= 8
	}
	return
}

type ListOfPeers []*Peer

func (lop ListOfPeers) Len() int {
	return len(lop)
}
func (lop ListOfPeers) Swap(i, j int) {
	lop[i], lop[j] = lop[j], lop[i]
}
func (lop ListOfPeers) Less(i, j int) bool {
	return lop[i].Name < lop[j].Name
}

// given an address like '1.2.3.4:567', return the address if it has a port,
// otherwise return the address with weave's standard port number
func NormalisePeerAddr(peerAddr string) string {
	_, _, err := net.SplitHostPort(peerAddr)
	if err == nil {
		return peerAddr
	} else {
		return fmt.Sprintf("%s:%d", peerAddr, Port)
	}
}
<commit_msg>get rid of Interaction it's no longer needed in the closure-based actor method scheme<commit_after>package router

import (
	"bytes"
	"crypto/rand"
	"encoding/gob"
	"fmt"
	"hash/fnv"
	"log"
	"net"
)

func checkFatal(e error) {
	if e != nil {
		log.Fatal(e)
	}
}

func checkWarn(e error) {
	if e != nil {
		log.Println(e)
	}
}

func PosixError(err error) error {
	if err == nil {
		return nil
	}
	operr, ok := err.(*net.OpError)
	if !ok {
		return nil
	}
	return operr.Err
}

func (mtbe MsgTooBigError) Error() string {
	return fmt.Sprint("Msg too big error. PMTU is ", mtbe.PMTU)
}

func (ftbe FrameTooBigError) Error() string {
	return fmt.Sprint("Frame too big error. Effective PMTU is ", ftbe.EPMTU)
}

func (upe UnknownPeerError) Error() string {
	return fmt.Sprint("Reference to unknown peer ", upe.Name)
}

func (nce NameCollisionError) Error() string {
	return fmt.Sprint("Multiple peers found with same name: ", nce.Name)
}

func (pde PacketDecodingError) Error() string {
	return fmt.Sprint("Failed to decode packet: ", pde.Desc)
}

func Concat(elems ...[]byte) []byte {
	res := []byte{}
	for _, e := range elems {
		res = append(res, e...)
	}
	return res
}

func randUint64() (r uint64) {
	buf := make([]byte, 8)
	_, err := rand.Read(buf)
	checkFatal(err)
	for _, v := range buf {
		r <<= 8
		r |= uint64(v)
	}
	return
}

func hash(s string) uint32 {
	h := fnv.New32a()
	h.Write([]byte(s))
	return h.Sum32()
}

func GobEncode(items ...interface{}) []byte {
	buf := new(bytes.Buffer)
	enc := gob.NewEncoder(buf)
	for _, i := range items {
		checkFatal(enc.Encode(i))
	}
	return buf.Bytes()
}

func macint(mac net.HardwareAddr) (r uint64) {
	for _, b := range mac {
		r <<= 8
		r |= uint64(b)
	}
	return
}

func intmac(key uint64) (r net.HardwareAddr) {
	r = make([]byte, 6)
	for i := 5; i >= 0; i-- {
		r[i] = byte(key)
		key >>= 8
	}
	return
}

type ListOfPeers []*Peer

func (lop ListOfPeers) Len() int {
	return len(lop)
}
func (lop ListOfPeers) Swap(i, j int) {
	lop[i], lop[j] = lop[j], lop[i]
}
func (lop ListOfPeers) Less(i, j int) bool {
	return lop[i].Name < lop[j].Name
}

// given an address like '1.2.3.4:567', return the address if it has a port,
// otherwise return the address with weave's standard port number
func NormalisePeerAddr(peerAddr string) string {
	_, _, err := net.SplitHostPort(peerAddr)
	if err == nil {
		return peerAddr
	} else {
		return fmt.Sprintf("%s:%d", peerAddr, Port)
	}
}
<|endoftext|>


<commit_before>package neko

import (
	"github.com/julienschmidt/httprouter"
	"mime/multipart"
	"net/http"
)

type routerParams struct {
	req    *http.Request
	params httprouter.Params
}

func (c *routerParams) ByGet(name string) string {
	val := c.params.ByName(name)
	if val == "" {
		val = c.req.URL.Query().Get(name)
	}
	return val
}

func (c *routerParams) ByPost(name string) string {
	return c.req.FormValue(name)
}

func (c *routerParams) File(name string) (multipart.File, *multipart.FileHeader, error) {
	return c.req.FormFile(name)
}
<commit_msg>Add parameter to get json data<commit_after>package neko

import (
	"encoding/json"
	"github.com/julienschmidt/httprouter"
	"mime/multipart"
	"net/http"
	"io/ioutil"
)

type routerParams struct {
	req    *http.Request
	params httprouter.Params
}

func (c *routerParams) ByGet(name string) string {
	val := c.params.ByName(name)
	if val == "" {
		val = c.req.URL.Query().Get(name)
	}
	return val
}

func (c *routerParams) ByPost(name string) string {
	return c.req.FormValue(name)
}

func (c *routerParams) File(name string) (multipart.File, *multipart.FileHeader, error) {
	return c.req.FormFile(name)
}

func (c *routerParams) Json() *jsonParams {
	defer c.req.Body.Close()

	data, _ := ioutil.ReadAll(c.req.Body)
	objJson := &jsonParams{ data: map[string]string{}}
	objJson.source = string(data)
	json.Unmarshal(data, &objJson.data);

	return objJson
}

type jsonParams struct {
	source string
	data map[string]string
}

func (c *jsonParams) Get(name string) string {
	if len(c.data) == 0 {
		return ""
	}
	return c.data[name]
}

func (c *jsonParams) String() string {
	return c.source
}
<|endoftext|>


<commit_before>package ui

import (
	".."
	"../../element"
	"bytes"
	"fmt"
	"html"
	"html/template"
	"io/ioutil"
	"log"
	"mime/multipart"
	"net/http"
)

type Page struct {
	Head    string
	header  string
	Content string
	W       http.ResponseWriter
	R       *http.Request
	P       webapp.HandlerParams
}

type uiParams struct {
	Head    template.HTML
	Header  template.HTML
	Content template.HTML
	Menu    template.HTML
}

func New(w http.ResponseWriter, r *http.Request, p webapp.HandlerParams, title, icon string) *Page {
	head := "<title>" + html.EscapeString(title) + "</title>"
	if icon != "" {
		head += webapp.Icon(icon)
	}
	head += webapp.Stylesheet("/res/css/ui.css")
	head += webapp.Script("/res/js/ui.js")
	pg := Page{Head: head, W: w, R: r, P: p}
	return &pg
}

func (pg *Page) AddHeaderGlyph(src string) {
	pg.header += "

	"
}

func (pg *Page) AddHeaderIcon(src string, url string) {
	if url == "" {
		pg.header += "

	"
	} else {
		pg.header += "

	"
	}
}

func (pg *Page) AddHeaderLabel(label string, url string) {
	if url == "" {
		pg.header += "" + html.EscapeString(label) + "	"
	} else {
		pg.header += "" + html.EscapeString(label) + "	"
	}
}

func (pg *Page) AddScript(url string) (err error) {
	elements, err := element.Parse(pg.Head)
	if err == nil {
		for _, e := range elements {
			if e.Tag == "script" && e.Attributes["src"] == url {
				return
			}
		}
		pg.Head += webapp.Script(url)
	}
	return
}

func (pg *Page) AddStylesheet(url string) (err error) {
	elements, err := element.Parse(pg.Head)
	if err == nil {
		for _, e := range elements {
			if e.Tag == "link" && e.Attributes["rel"] == "stylesheet" && e.Attributes["href"] == url {
				return
			}
		}
		pg.Head += webapp.Stylesheet(url)
	}
	return
}

func (pg *Page) FormFile(key string) (multipart.File, *multipart.FileHeader, error) {
	return pg.R.FormFile(key)
}

func (pg *Page) HasPermission(permission string) bool {
	return webapp.HasPermission(pg.P.Username, permission)
}

func (pg *Page) Method() string {
	return pg.R.Method
}

func (pg *Page) Param(name string) string {
	return pg.R.URL.Query().Get(name)
}

func (pg *Page) ParseMultipartForm(size int64) {
	pg.R.ParseMultipartForm(32 << 20)
}

func (pg *Page) PostParam(name string) string {
	return pg.R.Form.Get(name)
}

func (pg *Page) Render() {
	header := "

"
	var menu string
	if pg.P.Username != "" {
		header += "<button class='ui_menu_button' onClick='UIMenuShow(\"ui_menu_content\")'> </button>	"
		if webapp.HasPermission(pg.P.Username, "switch_user") {
			menu += "Switch User
"
		}
		menu += "Log Out"
	}
	header += pg.header
	if pg.P.Username != "" {
		user := webapp.User(pg.P.Username)
		display_name := user["first_name"] + " " + user["last_name"]
		if display_name == " " {
			display_name = pg.P.Username
		}
		header += "<button class='ui_menu_button' onClick='UIMenuShow(\"ui_usermenu_content\")'>" + display_name + "</button>	"
	}
	header += "

"
	webapp.Render(pg.W, "ui.html", uiParams{Head: template.HTML(pg.Head),
		Header: template.HTML(header), Content: template.HTML(pg.Content),
		Menu: template.HTML(menu)})
}

func (pg *Page) RenderError(message string, url string) {
	log.Printf(" ** ERROR: %s\n", message)
	pg.RenderInfo(message, url)
}

func (pg *Page) RenderInfo(message string, url string) {
	pg.Content = "

"
	if url != "" {
		pg.Content += " "
	}
	pg.Content += "
" + message + "

"
	pg.Render()
}

func (pg *Page) RenderFile(filename string) {
	mime_type := webapp.ContentType(filename)
	fb, err := ioutil.ReadFile(filename)
	if (err != nil) || (mime_type == "") {
		pg.W.WriteHeader(http.StatusNotFound)
		fmt.Fprint(pg.W, "Not Found")
		return
	}
	b := bytes.NewBuffer(fb)
	pg.W.Header().Set("Content-type", mime_type)
	b.WriteTo(pg.W)
}

func (pg *Page) Redirect(url string) {
	webapp.Redirect(pg.W, pg.R, url)
}

func (pg *Page) SessionValuesRead(keys ...string) string {
	if len(keys) > 0 {
		if len(keys) > 1 {
			return webapp.SessionValuesRead(pg.P, keys[0], keys[1])
		}
		return webapp.SessionValuesRead(pg.P, keys[0])
	}
	return ""
}

func (pg *Page) SessionValuesWrite(key, value string) error {
	return webapp.SessionValuesWrite(pg.P, key, value)
}

func (pg *Page) SubPath(index int) string {
	return webapp.UrlPath(pg.R, index)
}

func (pg *Page) Username() string {
	return pg.P.Username
}
<commit_msg>Added RenderQuestion & Output<commit_after>package ui

import (
	".."
	"../../element"
	"bytes"
	"fmt"
	"html"
	"html/template"
	"io/ioutil"
	"log"
	"mime/multipart"
	"net/http"
)

type Page struct {
	Head    string
	header  string
	Content string
	W       http.ResponseWriter
	R       *http.Request
	P       webapp.HandlerParams
}

type uiParams struct {
	Head    template.HTML
	Header  template.HTML
	Content template.HTML
	Menu    template.HTML
}

func New(w http.ResponseWriter, r *http.Request, p webapp.HandlerParams, title, icon string) *Page {
	head := "<title>" + html.EscapeString(title) + "</title>"
	if icon != "" {
		head += webapp.Icon(icon)
	}
	head += webapp.Stylesheet("/res/css/ui.css")
	head += webapp.Script("/res/js/ui.js")
	pg := Page{Head: head, W: w, R: r, P: p}
	return &pg
}

func (pg *Page) AddHeaderGlyph(src string) {
	pg.header += "

	"
}

func (pg *Page) AddHeaderIcon(src string, url string) {
	if url == "" {
		pg.header += "

	"
	} else {
		pg.header += "

	"
	}
}

func (pg *Page) AddHeaderLabel(label string, url string) {
	if url == "" {
		pg.header += "" + html.EscapeString(label) + "	"
	} else {
		pg.header += "" + html.EscapeString(label) + "	"
	}
}

func (pg *Page) AddScript(url string) (err error) {
	elements, err := element.Parse(pg.Head)
	if err == nil {
		for _, e := range elements {
			if e.Tag == "script" && e.Attributes["src"] == url {
				return
			}
		}
		pg.Head += webapp.Script(url)
	}
	return
}

func (pg *Page) AddStylesheet(url string) (err error) {
	elements, err := element.Parse(pg.Head)
	if err == nil {
		for _, e := range elements {
			if e.Tag == "link" && e.Attributes["rel"] == "stylesheet" && e.Attributes["href"] == url {
				return
			}
		}
		pg.Head += webapp.Stylesheet(url)
	}
	return
}

func (pg *Page) FormFile(key string) (multipart.File, *multipart.FileHeader, error) {
	return pg.R.FormFile(key)
}

func (pg *Page) HasPermission(permission string) bool {
	return webapp.HasPermission(pg.P.Username, permission)
}

func (pg *Page) Method() string {
	return pg.R.Method
}

func (pg *Page) Output(content string) {
    fmt.Fprintf(pg.W, "%s", content)
}

func (pg *Page) Param(name string) string {
	return pg.R.URL.Query().Get(name)
}

func (pg *Page) ParseMultipartForm(size int64) {
	pg.R.ParseMultipartForm(32 << 20)
}

func (pg *Page) PostParam(name string) string {
	return pg.R.Form.Get(name)
}

func (pg *Page) Redirect(url string) {
	webapp.Redirect(pg.W, pg.R, url)
}

func (pg *Page) Render() {
	header := "

"
	var menu string
	if pg.P.Username != "" {
		header += "<button class='ui_menu_button' onClick='UIMenuShow(\"ui_menu_content\")'> </button>	"
		if webapp.HasPermission(pg.P.Username, "switch_user") {
			menu += "Switch User
"
		}
		menu += "Log Out"
	}
	header += pg.header
	if pg.P.Username != "" {
		user := webapp.User(pg.P.Username)
		display_name := user["first_name"] + " " + user["last_name"]
		if display_name == " " {
			display_name = pg.P.Username
		}
		header += "<button class='ui_menu_button' onClick='UIMenuShow(\"ui_usermenu_content\")'>" + display_name + "</button>	"
	}
	header += "

"
	webapp.Render(pg.W, "ui.html", uiParams{Head: template.HTML(pg.Head),
		Header: template.HTML(header), Content: template.HTML(pg.Content),
		Menu: template.HTML(menu)})
}

func (pg *Page) RenderError(message string, url string) {
	log.Printf(" ** ERROR: %s\n", message)
	pg.RenderInfo(message, url)
}

func (pg *Page) RenderFile(filename string) {
	mime_type := webapp.ContentType(filename)
	fb, err := ioutil.ReadFile(filename)
	if (err != nil) || (mime_type == "") {
		pg.W.WriteHeader(http.StatusNotFound)
		fmt.Fprint(pg.W, "Not Found")
		return
	}
	b := bytes.NewBuffer(fb)
	pg.W.Header().Set("Content-type", mime_type)
	b.WriteTo(pg.W)
}

func (pg *Page) RenderInfo(message string, url string) {
	pg.Content = "

"
	if url != "" {
		pg.Content += " "
	}
	pg.Content += "
" + message + "

"
	pg.Render()
}

func (pg *Page) RenderQuestion(question string, url_yes string, url_no string) {
	pg.RenderInfo(question+"
Yes No", "")
}

func (pg *Page) SessionValuesRead(keys ...string) string {
	if len(keys) > 0 {
		if len(keys) > 1 {
			return webapp.SessionValuesRead(pg.P, keys[0], keys[1])
		}
		return webapp.SessionValuesRead(pg.P, keys[0])
	}
	return ""
}

func (pg *Page) SessionValuesWrite(key, value string) error {
	return webapp.SessionValuesWrite(pg.P, key, value)
}

func (pg *Page) SubPath(index int) string {
	return webapp.UrlPath(pg.R, index)
}

func (pg *Page) Username() string {
	return pg.P.Username
}
<|endoftext|>


<commit_before>package stageCmd

import (
	// Stdlib
	"errors"
	"fmt"
	"os"

	// Internal
	"github.com/salsaflow/salsaflow/action"
	"github.com/salsaflow/salsaflow/app"
	"github.com/salsaflow/salsaflow/changes"
	"github.com/salsaflow/salsaflow/errs"
	"github.com/salsaflow/salsaflow/git"
	"github.com/salsaflow/salsaflow/log"
	"github.com/salsaflow/salsaflow/modules"
	"github.com/salsaflow/salsaflow/modules/common"
	"github.com/salsaflow/salsaflow/prompt"
	"github.com/salsaflow/salsaflow/releases"
	"github.com/salsaflow/salsaflow/version"

	// Other
	"gopkg.in/tchap/gocli.v2"
)

var Command = &gocli.Command{
	UsageLine: `
  stage`,
	Short: "stage and close the current release",
	Long: `
  Stage and close the release that is currently running, i.e.

    1) Make sure the release can be staged (the stories are reviewed and tested)
    2) Tag the release branch and delete it.
    3) Move the staging branch to point to the tag.
    4) Push the changes.
	`,
	Action: run,
}

func run(cmd *gocli.Command, args []string) {
	if len(args) != 0 {
		cmd.Usage()
		os.Exit(2)
	}

	app.InitOrDie()

	defer prompt.RecoverCancel()

	if err := runMain(); err != nil {
		errs.Fatal(err)
	}
}

func runMain() (err error) {
	// Load git config.
	gitConfig, err := git.LoadConfig()
	if err != nil {
		return err
	}
	var (
		remoteName    = gitConfig.RemoteName()
		releaseBranch = gitConfig.ReleaseBranchName()
		stagingBranch = gitConfig.StagingBranchName()
	)

	// Instantiate the issue tracker.
	tracker, err := modules.GetIssueTracker()
	if err != nil {
		return err
	}

	// Get the current branch.
	task := "Get the current branch"
	currentBranch, err := git.CurrentBranch()
	if err != nil {
		return err
	}

	// Cannot be on the release branch, it will be deleted.
	task = "Make sure that the release branch is not checked out"
	if currentBranch == releaseBranch {
		return errs.NewError(
			task, errors.New("cannot stage the release while on the release branch"), nil)
	}

	// Fetch the remote repository.
	task = "Fetch the remote repository"
	log.Run(task)
	if err := git.UpdateRemotes(remoteName); err != nil {
		return errs.NewError(task, err, nil)
	}

	// Make sure the remote release branch exists.
	task = "Make sure that the remote release branch exists"
	exists, err := git.RemoteBranchExists(releaseBranch, remoteName)
	if err != nil {
		return errs.NewError(task, err, nil)
	}
	if !exists {
		return errs.NewError(task,
			fmt.Errorf("branch '%v' not found in remote '%v'", releaseBranch, remoteName), nil)
	}

	// Make sure that the local release branch exists.
	task = "Make sure that the local release branch exists"
	if err := git.CreateTrackingBranchUnlessExists(releaseBranch, remoteName); err != nil {
		return errs.NewError(task, err, nil)
	}

	// Make sure that the release branch is up to date.
	task = fmt.Sprintf("Make sure branch '%v' is up to date", releaseBranch)
	log.Run(task)
	if err := git.EnsureBranchSynchronized(releaseBranch, remoteName); err != nil {
		return errs.NewError(task, err, nil)
	}

	// Read the current release version.
	task = "Read the current release version"
	releaseVersion, err := version.GetByBranch(releaseBranch)
	if err != nil {
		return errs.NewError(task, err, nil)
	}

	// Make sure the release is stageable.
	release, err := tracker.RunningRelease(releaseVersion)
	if err != nil {
		return err
	}
	if err := release.EnsureStageable(); err != nil {
		return err
	}

	// Make sure there are no commits being left behind,
	// e.g. make sure no commits are forgotten on the trunk branch,
	// i.e. make sure that everything necessary was cherry-picked.
	if err := checkCommits(tracker, release, releaseBranch); err != nil {
		return err
	}

	// Tag the release branch with the associated version string.
	tag := releaseVersion.ReleaseTagString()
	task = fmt.Sprintf("Tag branch '%v' (tag = %v)", releaseBranch, tag)
	log.Run(task)
	tag = releaseVersion.ReleaseTagString()
	if err := git.Tag(tag, releaseBranch); err != nil {
		return errs.NewError(task, err, nil)
	}
	defer action.RollbackTaskOnError(&err, task, action.ActionFunc(func() error {
		// On error, delete the release tag.
		task := "Delete the release tag"
		if err := git.DeleteTag(tag); err != nil {
			return errs.NewError(task, err, nil)
		}
		return nil
	}))

	// Reset the staging branch to point to the newly created tag.
	task = fmt.Sprintf("Reset branch '%v' to point to tag '%v'", stagingBranch, tag)
	log.Run(task)
	act, err := git.CreateOrResetBranch(stagingBranch, tag)
	if err != nil {
		return errs.NewError(task, err, nil)
	}
	defer action.RollbackTaskOnError(&err, task, act)

	// Delete the local release branch.
	task = fmt.Sprintf("Delete branch '%v'", releaseBranch)
	log.Run(task)
	if err := git.Branch("-d", releaseBranch); err != nil {
		return errs.NewError(task, err, nil)
	}
	defer action.RollbackTaskOnError(&err, task, action.ActionFunc(func() error {
		task := fmt.Sprintf("Recreate branch '%v'", releaseBranch)

		// In case the release branch exists locally, do nothing.
		// This might look like an extra and useless check, but it looks like
		// the final git push at the end of the command function actually creates
		// the release branch locally when it is aborted from the pre-push hook.
		// Not sure why and how that is happening.
		exists, err := git.LocalBranchExists(releaseBranch)
		if err != nil {
			return errs.NewError(task, err, nil)
		}
		if exists {
			return nil
		}

		// In case the branch indeed does not exist, create it.
		if err := git.Branch(releaseBranch, remoteName+"/"+releaseBranch); err != nil {
			return errs.NewError(task, err, nil)
		}
		return nil
	}))

	// Finalise the release in the code review tool.
	codeReviewTool, err := modules.GetCodeReviewTool()
	if err != nil {
		return err
	}
	act, err = codeReviewTool.FinaliseRelease(releaseVersion)
	if err != nil {
		return err
	}
	// No need to pass any task string, the module rollback functions
	// are expected to take care of printing messages on their own.
	defer action.RollbackOnError(&err, act)

	// Stage the release in the issue tracker.
	act, err = release.Stage()
	if err != nil {
		return err
	}
	defer action.RollbackOnError(&err, act)

	// Push to create the tag, reset client and delete release in the remote repository.
	task = "Push changes to the remote repository"
	log.Run(task)
	return git.Push(remoteName,
		"-f", "--tags", // Force push, push the release tag.
		":"+releaseBranch,               // Delete the release branch.
		stagingBranch+":"+stagingBranch) // Push the staging branch.
}

func checkCommits(
	tracker common.IssueTracker,
	release common.RunningRelease,
	releaseBranch string,
) error {

	var task = "Make sure no changes are being left behind"
	log.Run(task)

	stories, err := release.Stories()
	if err != nil {
		return errs.NewError(task, err, nil)
	}
	if len(stories) == 0 {
		return nil
	}

	groups, err := changes.StoryChanges(stories)
	if err != nil {
		return errs.NewError(task, err, nil)
	}

	toCherryPick, err := releases.StoryChangesToCherryPick(groups)
	if err != nil {
		return errs.NewError(task, err, nil)
	}

	// In case there are some changes being left behind,
	// ask the user to confirm whether to proceed or not.
	if len(toCherryPick) == 0 {
		return nil
	}

	fmt.Println(`
Some changes are being left behind!

In other words, some changes that are assigned to the current release
have not been cherry-picked onto the release branch yet.
	`)
	if err := changes.DumpStoryChanges(os.Stdout, toCherryPick, tracker, false); err != nil {
		panic(err)
	}
	fmt.Println()

	confirmed, err := prompt.Confirm("Are you sure you really want to stage the release?")
	if err != nil {
		return errs.NewError(task, err, nil)
	}
	if !confirmed {
		prompt.PanicCancel()
	}
	fmt.Println()

	return nil
}
<commit_msg>release stage: Update the release workflow<commit_after>package stageCmd

import (
	// Stdlib
	"errors"
	"fmt"
	"os"

	// Internal
	"github.com/salsaflow/salsaflow/action"
	"github.com/salsaflow/salsaflow/app"
	"github.com/salsaflow/salsaflow/changes"
	"github.com/salsaflow/salsaflow/errs"
	"github.com/salsaflow/salsaflow/git"
	"github.com/salsaflow/salsaflow/log"
	"github.com/salsaflow/salsaflow/modules"
	"github.com/salsaflow/salsaflow/modules/common"
	"github.com/salsaflow/salsaflow/prompt"
	"github.com/salsaflow/salsaflow/releases"
	"github.com/salsaflow/salsaflow/version"

	// Other
	"gopkg.in/tchap/gocli.v2"
)

var Command = &gocli.Command{
	UsageLine: `
  stage`,
	Short: "stage current release",
	Long: `
  Stage the release that is currently running, i.e.

    1) Make sure the release can be staged (the stories are reviewed and tested)
    2) Reset the staging branch to point to the release branch.
    3) Delete the release branch.
    4) Bump the version for the staging branch.
    5) Push the changes.
	`,
	Action: run,
}

func run(cmd *gocli.Command, args []string) {
	if len(args) != 0 {
		cmd.Usage()
		os.Exit(2)
	}

	app.InitOrDie()

	defer prompt.RecoverCancel()

	if err := runMain(); err != nil {
		errs.Fatal(err)
	}
}

func runMain() (err error) {
	// Load git config.
	gitConfig, err := git.LoadConfig()
	if err != nil {
		return err
	}
	var (
		remoteName    = gitConfig.RemoteName()
		releaseBranch = gitConfig.ReleaseBranchName()
		stagingBranch = gitConfig.StagingBranchName()
	)

	// Instantiate the issue tracker.
	tracker, err := modules.GetIssueTracker()
	if err != nil {
		return err
	}

	// Get the current branch.
	task := "Get the current branch"
	currentBranch, err := git.CurrentBranch()
	if err != nil {
		return err
	}

	// Cannot be on the release branch, it will be deleted.
	task = "Make sure that the release branch is not checked out"
	if currentBranch == releaseBranch {
		return errs.NewError(
			task, errors.New("cannot stage the release while on the release branch"), nil)
	}

	// Fetch the remote repository.
	task = "Fetch the remote repository"
	log.Run(task)
	if err := git.UpdateRemotes(remoteName); err != nil {
		return errs.NewError(task, err, nil)
	}

	// Make sure the remote release branch exists.
	task = "Make sure that the remote release branch exists"
	exists, err := git.RemoteBranchExists(releaseBranch, remoteName)
	if err != nil {
		return errs.NewError(task, err, nil)
	}
	if !exists {
		return errs.NewError(task,
			fmt.Errorf("branch '%v' not found in remote '%v'", releaseBranch, remoteName), nil)
	}

	// Make sure that the local release branch exists.
	task = "Make sure that the local release branch exists"
	if err := git.CreateTrackingBranchUnlessExists(releaseBranch, remoteName); err != nil {
		return errs.NewError(task, err, nil)
	}

	// Make sure that the release branch is up to date.
	task = fmt.Sprintf("Make sure branch '%v' is up to date", releaseBranch)
	log.Run(task)
	if err := git.EnsureBranchSynchronized(releaseBranch, remoteName); err != nil {
		return errs.NewError(task, err, nil)
	}

	// Read the current release version.
	task = "Read the current release version"
	releaseVersion, err := version.GetByBranch(releaseBranch)
	if err != nil {
		return errs.NewError(task, err, nil)
	}

	// Make sure the release is stageable.
	release, err := tracker.RunningRelease(releaseVersion)
	if err != nil {
		return err
	}
	if err := release.EnsureStageable(); err != nil {
		return err
	}

	// Make sure there are no commits being left behind,
	// e.g. make sure no commits are forgotten on the trunk branch,
	// i.e. make sure that everything necessary was cherry-picked.
	if err := checkCommits(tracker, release, releaseBranch); err != nil {
		return err
	}

	// Reset the staging branch to point to the newly created tag.
	task = fmt.Sprintf("Reset branch '%v' to point to branch '%v'", stagingBranch, releaseBranch)
	log.Run(task)
	act, err := git.CreateOrResetBranch(stagingBranch, releaseBranch)
	if err != nil {
		return errs.NewError(task, err, nil)
	}
	defer action.RollbackTaskOnError(&err, task, act)

	// Delete the local release branch.
	task = fmt.Sprintf("Delete branch '%v'", releaseBranch)
	log.Run(task)
	if err := git.Branch("-d", releaseBranch); err != nil {
		return errs.NewError(task, err, nil)
	}
	defer action.RollbackTaskOnError(&err, task, action.ActionFunc(func() error {
		task := fmt.Sprintf("Recreate branch '%v'", releaseBranch)

		// In case the release branch exists locally, do nothing.
		// This might look like an extra and useless check, but it looks like
		// the final git push at the end of the command function actually creates
		// the release branch locally when it is aborted from the pre-push hook.
		// Not sure why and how that is happening.
		exists, err := git.LocalBranchExists(releaseBranch)
		if err != nil {
			return errs.NewError(task, err, nil)
		}
		if exists {
			return nil
		}

		// In case the branch indeed does not exist, create it.
		if err := git.Branch(releaseBranch, remoteName+"/"+releaseBranch); err != nil {
			return errs.NewError(task, err, nil)
		}
		return nil
	}))

	// Update the version string on the staging branch.
	stagingVersion, err := releaseVersion.ToStageVersion()
	if err != nil {
		return err
	}

	task = fmt.Sprintf("Bump version (branch '%v' -> %v)", stagingBranch, stagingVersion)
	log.Run(task)
	act, err = version.SetForBranch(stagingVersion, stagingBranch)
	if err != nil {
		return errs.NewError(task, err, nil)
	}
	defer action.RollbackTaskOnError(&err, task, act)

	// Finalise the release in the code review tool.
	codeReviewTool, err := modules.GetCodeReviewTool()
	if err != nil {
		return err
	}
	act, err = codeReviewTool.FinaliseRelease(releaseVersion)
	if err != nil {
		return err
	}
	// No need to pass any task string, the module rollback functions
	// are expected to take care of printing messages on their own.
	defer action.RollbackOnError(&err, act)

	// Stage the release in the issue tracker.
	act, err = release.Stage()
	if err != nil {
		return err
	}
	defer action.RollbackOnError(&err, act)

	// Push to create the tag, reset client and delete release in the remote repository.
	task = "Push changes to the remote repository"
	log.Run(task)
	return git.Push(remoteName,
		"-f",                            // Use the Force, Luke.
		":"+releaseBranch,               // Delete the release branch.
		stagingBranch+":"+stagingBranch) // Push the staging branch.
}

func checkCommits(
	tracker common.IssueTracker,
	release common.RunningRelease,
	releaseBranch string,
) error {

	var task = "Make sure no changes are being left behind"
	log.Run(task)

	stories, err := release.Stories()
	if err != nil {
		return errs.NewError(task, err, nil)
	}
	if len(stories) == 0 {
		return nil
	}

	groups, err := changes.StoryChanges(stories)
	if err != nil {
		return errs.NewError(task, err, nil)
	}

	toCherryPick, err := releases.StoryChangesToCherryPick(groups)
	if err != nil {
		return errs.NewError(task, err, nil)
	}

	// In case there are some changes being left behind,
	// ask the user to confirm whether to proceed or not.
	if len(toCherryPick) == 0 {
		return nil
	}

	fmt.Println(`
Some changes are being left behind!

In other words, some changes that are assigned to the current release
have not been cherry-picked onto the release branch yet.
	`)
	if err := changes.DumpStoryChanges(os.Stdout, toCherryPick, tracker, false); err != nil {
		panic(err)
	}
	fmt.Println()

	confirmed, err := prompt.Confirm("Are you sure you really want to stage the release?")
	if err != nil {
		return errs.NewError(task, err, nil)
	}
	if !confirmed {
		prompt.PanicCancel()
	}
	fmt.Println()

	return nil
}
<|endoftext|>


<commit_before>// Package nl has low level primitives for making Netlink calls.
package nl

import (
	"bytes"
	"encoding/binary"
	"net"
	"fmt"
	"sync/atomic"
	"syscall"
	"unsafe"
)

const (
	// Family type definitions
	FAMILY_ALL = syscall.AF_UNSPEC
	FAMILY_V4  = syscall.AF_INET
	FAMILY_V6  = syscall.AF_INET6
)

var nextSeqNr uint32

// GetIPFamily returns the family type of a net.IP.
func GetIPFamily(ip net.IP) int {
	if len(ip) <= net.IPv4len {
		return FAMILY_V4
	}
	if ip.To4() != nil {
		return FAMILY_V4
	}
	return FAMILY_V6
}
// Get native endianness for the system
func NativeEndian() binary.ByteOrder {
	var x uint32 = 0x01020304
	if *(*byte)(unsafe.Pointer(&x)) == 0x01 {
		return binary.BigEndian
	}
	return binary.LittleEndian
}

// Byte swap a 16 bit value in place
func Swap16(i uint16) uint16 {
	return (i&0xff00)>>8 | (i&0xff)<<8
}

// Byte swap a 32 bit value in place
func Swap32(i uint32) uint32 {
	return (i&0xff000000)>>24 | (i&0xff0000)>>8 | (i&0xff00)<<8 | (i&0xff)<<24
}

type NetlinkRequestData interface {
	Len() int
	Serialize() []byte
}

// IfInfomsg is related to links, but it is used for list requests as well
type IfInfomsg struct {
	syscall.IfInfomsg
}

// Create an IfInfomsg with family specified
func NewIfInfomsg(family int) *IfInfomsg {
	return &IfInfomsg{
		IfInfomsg: syscall.IfInfomsg{
			Family: uint8(family),
		},
	}
}

func DeserializeIfInfomsg(b []byte) *IfInfomsg {
	return (*IfInfomsg)(unsafe.Pointer(&b[0:syscall.SizeofIfInfomsg][0]))
}

func (msg *IfInfomsg) Serialize() []byte {
	return (*(*[syscall.SizeofIfInfomsg]byte)(unsafe.Pointer(msg)))[:]
}

func (msg *IfInfomsg) Len() int {
	return syscall.SizeofIfInfomsg
}

func rtaAlignOf(attrlen int) int {
	return (attrlen + syscall.RTA_ALIGNTO - 1) & ^(syscall.RTA_ALIGNTO - 1)
}

func NewIfInfomsgChild(parent *RtAttr, family int) *IfInfomsg {
	msg := NewIfInfomsg(family)
	parent.children = append(parent.children, msg)
	return msg
}

// Extend RtAttr to handle data and children
type RtAttr struct {
	syscall.RtAttr
	Data     []byte
	children []NetlinkRequestData
}

// Create a new Extended RtAttr object
func NewRtAttr(attrType int, data []byte) *RtAttr {
	return &RtAttr{
		RtAttr: syscall.RtAttr{
			Type: uint16(attrType),
		},
		children: []NetlinkRequestData{},
		Data:     data,
	}
}

// Create a new RtAttr obj anc add it as a child of an existing object
func NewRtAttrChild(parent *RtAttr, attrType int, data []byte) *RtAttr {
	attr := NewRtAttr(attrType, data)
	parent.children = append(parent.children, attr)
	return attr
}

func (a *RtAttr) Len() int {
	if len(a.children) == 0 {
		return (syscall.SizeofRtAttr + len(a.Data))
	}

	l := 0
	for _, child := range a.children {
		l += child.Len()
	}
	l += syscall.SizeofRtAttr
	return rtaAlignOf(l + len(a.Data))
}

// Serialize the RtAttr into a byte array
// This can't ust unsafe.cast because it must iterate through children.
func (a *RtAttr) Serialize() []byte {
	native := NativeEndian()

	length := a.Len()
	buf := make([]byte, rtaAlignOf(length))

	if a.Data != nil {
		copy(buf[4:], a.Data)
	} else {
		next := 4
		for _, child := range a.children {
			childBuf := child.Serialize()
			copy(buf[next:], childBuf)
			next += rtaAlignOf(len(childBuf))
		}
	}

	if l := uint16(length); l != 0 {
		native.PutUint16(buf[0:2], l)
	}
	native.PutUint16(buf[2:4], a.Type)
	return buf
}

type NetlinkRequest struct {
	syscall.NlMsghdr
	Data []NetlinkRequestData
}

// Serialize the Netlink Request into a byte array
func (msg *NetlinkRequest) Serialize() []byte {
	length := syscall.SizeofNlMsghdr
	dataBytes := make([][]byte, len(msg.Data))
	for i, data := range msg.Data {
		dataBytes[i] = data.Serialize()
		length = length + len(dataBytes[i])
	}
	msg.Len = uint32(length)
	b := make([]byte, length)
	hdr := (*(*[syscall.SizeofNlMsghdr]byte)(unsafe.Pointer(msg)))[:]
	next := syscall.SizeofNlMsghdr
	copy(b[0:next], hdr)
	for _, data := range dataBytes {
		for _, dataByte := range data {
			b[next] = dataByte
			next = next + 1
		}
	}
	return b
}

func (msg *NetlinkRequest) AddData(data NetlinkRequestData) {
	if data != nil {
		msg.Data = append(msg.Data, data)
	}
}

// Execute the request against a the given sockType.
// Returns a list of netlink messages in seriaized format, optionally filtered
// by resType.
func (req *NetlinkRequest) Execute(sockType int, resType uint16) ([][]byte, error) {
	s, err := getNetlinkSocket(sockType)
	if err != nil {
		return nil, err
	}
	defer s.Close()

	if err := s.Send(req); err != nil {
		return nil, err
	}

	pid, err := s.GetPid()
	if err != nil {
		return nil, err
	}

	res := make([][]byte, 0)

done:
	for {
		msgs, err := s.Recieve()
		if err != nil {
			return nil, err
		}
		for _, m := range msgs {
			if m.Header.Seq != req.Seq {
				return nil, fmt.Errorf("Wrong Seq nr %d, expected 1", m.Header.Seq)
			}
			if m.Header.Pid != pid {
				return nil, fmt.Errorf("Wrong pid %d, expected %d", m.Header.Pid, pid)
			}
			if m.Header.Type == syscall.NLMSG_DONE {
				break done
			}
			if m.Header.Type == syscall.NLMSG_ERROR {
				native := NativeEndian()
				error := int32(native.Uint32(m.Data[0:4]))
				if error == 0 {
					break done
				}
				return nil, syscall.Errno(-error)
			}
			if resType != 0 && m.Header.Type != resType {
				continue
			}
			res = append(res, m.Data)
		}
	}
	return res, nil
}

// Create a new netlink request from proto and flags
// Note the Len value will be inaccurate once data is added until
// the message is serialized
func NewNetlinkRequest(proto, flags int) *NetlinkRequest {
	return &NetlinkRequest{
		NlMsghdr: syscall.NlMsghdr{
			Len:   uint32(syscall.SizeofNlMsghdr),
			Type:  uint16(proto),
			Flags: syscall.NLM_F_REQUEST | uint16(flags),
			Seq:   atomic.AddUint32(&nextSeqNr, 1),
		},
	}
}

type NetlinkSocket struct {
	fd  int
	lsa syscall.SockaddrNetlink
}

func getNetlinkSocket(protocol int) (*NetlinkSocket, error) {
	fd, err := syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, protocol)
	if err != nil {
		return nil, err
	}
	s := &NetlinkSocket{
		fd: fd,
	}
	s.lsa.Family = syscall.AF_NETLINK
	if err := syscall.Bind(fd, &s.lsa); err != nil {
		syscall.Close(fd)
		return nil, err
	}

	return s, nil
}

func (s *NetlinkSocket) Close() {
	syscall.Close(s.fd)
}

func (s *NetlinkSocket) Send(request *NetlinkRequest) error {
	if err := syscall.Sendto(s.fd, request.Serialize(), 0, &s.lsa); err != nil {
		return err
	}
	return nil
}

func (s *NetlinkSocket) Recieve() ([]syscall.NetlinkMessage, error) {
	rb := make([]byte, syscall.Getpagesize())
	nr, _, err := syscall.Recvfrom(s.fd, rb, 0)
	if err != nil {
		return nil, err
	}
	if nr < syscall.NLMSG_HDRLEN {
		return nil, fmt.Errorf("Got short response from netlink")
	}
	rb = rb[:nr]
	return syscall.ParseNetlinkMessage(rb)
}

func (s *NetlinkSocket) GetPid() (uint32, error) {
	lsa, err := syscall.Getsockname(s.fd)
	if err != nil {
		return 0, err
	}
	switch v := lsa.(type) {
	case *syscall.SockaddrNetlink:
		return v.Pid, nil
	}
	return 0, fmt.Errorf("Wrong socket type")
}

func ZeroTerminated(s string) []byte {
	bytes := make([]byte, len(s)+1)
	for i := 0; i < len(s); i++ {
		bytes[i] = s[i]
	}
	bytes[len(s)] = 0
	return bytes
}

func NonZeroTerminated(s string) []byte {
	bytes := make([]byte, len(s))
	for i := 0; i < len(s); i++ {
		bytes[i] = s[i]
	}
	return bytes
}

func BytesToString(b []byte) string {
	n := bytes.Index(b, []byte{0})
	return string(b[:n])
}

func ParseRouteAttr(b []byte) ([]syscall.NetlinkRouteAttr, error) {
	var attrs []syscall.NetlinkRouteAttr
	for len(b) >= syscall.SizeofRtAttr {
		a, vbuf, alen, err := netlinkRouteAttrAndValue(b)
		if err != nil {
			return nil, err
		}
		ra := syscall.NetlinkRouteAttr{Attr: *a, Value: vbuf[:int(a.Len)-syscall.SizeofRtAttr]}
		attrs = append(attrs, ra)
		b = b[alen:]
	}
	return attrs, nil
}

func netlinkRouteAttrAndValue(b []byte) (*syscall.RtAttr, []byte, int, error) {
	a := (*syscall.RtAttr)(unsafe.Pointer(&b[0]))
	if int(a.Len) < syscall.SizeofRtAttr || int(a.Len) > len(b) {
		return nil, nil, 0, syscall.EINVAL
	}
	return a, b[syscall.SizeofRtAttr:], rtaAlignOf(int(a.Len)), nil
}
<commit_msg>Optimize byte order handling<commit_after>// Package nl has low level primitives for making Netlink calls.
package nl

import (
	"bytes"
	"encoding/binary"
	"net"
	"fmt"
	"sync/atomic"
	"syscall"
	"unsafe"
)

const (
	// Family type definitions
	FAMILY_ALL = syscall.AF_UNSPEC
	FAMILY_V4  = syscall.AF_INET
	FAMILY_V6  = syscall.AF_INET6
)

var nextSeqNr uint32

// GetIPFamily returns the family type of a net.IP.
func GetIPFamily(ip net.IP) int {
	if len(ip) <= net.IPv4len {
		return FAMILY_V4
	}
	if ip.To4() != nil {
		return FAMILY_V4
	}
	return FAMILY_V6
}

var nativeEndian binary.ByteOrder

// Get native endianness for the system
func NativeEndian() binary.ByteOrder {
	if nativeEndian == nil {
		var x uint32 = 0x01020304
		if *(*byte)(unsafe.Pointer(&x)) == 0x01 {
			nativeEndian = binary.BigEndian
		}
		nativeEndian = binary.LittleEndian
	}
	return nativeEndian
}

// Byte swap a 16 bit value if we aren't big endian
func Swap16(i uint16) uint16 {
	if NativeEndian() == binary.BigEndian {
		return i
	}
	return (i&0xff00)>>8 | (i&0xff)<<8
}

// Byte swap a 32 bit value if aren't big endian
func Swap32(i uint32) uint32 {
	if NativeEndian() == binary.BigEndian {
		return i
	}
	return (i&0xff000000)>>24 | (i&0xff0000)>>8 | (i&0xff00)<<8 | (i&0xff)<<24
}

type NetlinkRequestData interface {
	Len() int
	Serialize() []byte
}

// IfInfomsg is related to links, but it is used for list requests as well
type IfInfomsg struct {
	syscall.IfInfomsg
}

// Create an IfInfomsg with family specified
func NewIfInfomsg(family int) *IfInfomsg {
	return &IfInfomsg{
		IfInfomsg: syscall.IfInfomsg{
			Family: uint8(family),
		},
	}
}

func DeserializeIfInfomsg(b []byte) *IfInfomsg {
	return (*IfInfomsg)(unsafe.Pointer(&b[0:syscall.SizeofIfInfomsg][0]))
}

func (msg *IfInfomsg) Serialize() []byte {
	return (*(*[syscall.SizeofIfInfomsg]byte)(unsafe.Pointer(msg)))[:]
}

func (msg *IfInfomsg) Len() int {
	return syscall.SizeofIfInfomsg
}

func rtaAlignOf(attrlen int) int {
	return (attrlen + syscall.RTA_ALIGNTO - 1) & ^(syscall.RTA_ALIGNTO - 1)
}

func NewIfInfomsgChild(parent *RtAttr, family int) *IfInfomsg {
	msg := NewIfInfomsg(family)
	parent.children = append(parent.children, msg)
	return msg
}

// Extend RtAttr to handle data and children
type RtAttr struct {
	syscall.RtAttr
	Data     []byte
	children []NetlinkRequestData
}

// Create a new Extended RtAttr object
func NewRtAttr(attrType int, data []byte) *RtAttr {
	return &RtAttr{
		RtAttr: syscall.RtAttr{
			Type: uint16(attrType),
		},
		children: []NetlinkRequestData{},
		Data:     data,
	}
}

// Create a new RtAttr obj anc add it as a child of an existing object
func NewRtAttrChild(parent *RtAttr, attrType int, data []byte) *RtAttr {
	attr := NewRtAttr(attrType, data)
	parent.children = append(parent.children, attr)
	return attr
}

func (a *RtAttr) Len() int {
	if len(a.children) == 0 {
		return (syscall.SizeofRtAttr + len(a.Data))
	}

	l := 0
	for _, child := range a.children {
		l += child.Len()
	}
	l += syscall.SizeofRtAttr
	return rtaAlignOf(l + len(a.Data))
}

// Serialize the RtAttr into a byte array
// This can't ust unsafe.cast because it must iterate through children.
func (a *RtAttr) Serialize() []byte {
	native := NativeEndian()

	length := a.Len()
	buf := make([]byte, rtaAlignOf(length))

	if a.Data != nil {
		copy(buf[4:], a.Data)
	} else {
		next := 4
		for _, child := range a.children {
			childBuf := child.Serialize()
			copy(buf[next:], childBuf)
			next += rtaAlignOf(len(childBuf))
		}
	}

	if l := uint16(length); l != 0 {
		native.PutUint16(buf[0:2], l)
	}
	native.PutUint16(buf[2:4], a.Type)
	return buf
}

type NetlinkRequest struct {
	syscall.NlMsghdr
	Data []NetlinkRequestData
}

// Serialize the Netlink Request into a byte array
func (msg *NetlinkRequest) Serialize() []byte {
	length := syscall.SizeofNlMsghdr
	dataBytes := make([][]byte, len(msg.Data))
	for i, data := range msg.Data {
		dataBytes[i] = data.Serialize()
		length = length + len(dataBytes[i])
	}
	msg.Len = uint32(length)
	b := make([]byte, length)
	hdr := (*(*[syscall.SizeofNlMsghdr]byte)(unsafe.Pointer(msg)))[:]
	next := syscall.SizeofNlMsghdr
	copy(b[0:next], hdr)
	for _, data := range dataBytes {
		for _, dataByte := range data {
			b[next] = dataByte
			next = next + 1
		}
	}
	return b
}

func (msg *NetlinkRequest) AddData(data NetlinkRequestData) {
	if data != nil {
		msg.Data = append(msg.Data, data)
	}
}

// Execute the request against a the given sockType.
// Returns a list of netlink messages in seriaized format, optionally filtered
// by resType.
func (req *NetlinkRequest) Execute(sockType int, resType uint16) ([][]byte, error) {
	s, err := getNetlinkSocket(sockType)
	if err != nil {
		return nil, err
	}
	defer s.Close()

	if err := s.Send(req); err != nil {
		return nil, err
	}

	pid, err := s.GetPid()
	if err != nil {
		return nil, err
	}

	res := make([][]byte, 0)

done:
	for {
		msgs, err := s.Recieve()
		if err != nil {
			return nil, err
		}
		for _, m := range msgs {
			if m.Header.Seq != req.Seq {
				return nil, fmt.Errorf("Wrong Seq nr %d, expected 1", m.Header.Seq)
			}
			if m.Header.Pid != pid {
				return nil, fmt.Errorf("Wrong pid %d, expected %d", m.Header.Pid, pid)
			}
			if m.Header.Type == syscall.NLMSG_DONE {
				break done
			}
			if m.Header.Type == syscall.NLMSG_ERROR {
				native := NativeEndian()
				error := int32(native.Uint32(m.Data[0:4]))
				if error == 0 {
					break done
				}
				return nil, syscall.Errno(-error)
			}
			if resType != 0 && m.Header.Type != resType {
				continue
			}
			res = append(res, m.Data)
		}
	}
	return res, nil
}

// Create a new netlink request from proto and flags
// Note the Len value will be inaccurate once data is added until
// the message is serialized
func NewNetlinkRequest(proto, flags int) *NetlinkRequest {
	return &NetlinkRequest{
		NlMsghdr: syscall.NlMsghdr{
			Len:   uint32(syscall.SizeofNlMsghdr),
			Type:  uint16(proto),
			Flags: syscall.NLM_F_REQUEST | uint16(flags),
			Seq:   atomic.AddUint32(&nextSeqNr, 1),
		},
	}
}

type NetlinkSocket struct {
	fd  int
	lsa syscall.SockaddrNetlink
}

func getNetlinkSocket(protocol int) (*NetlinkSocket, error) {
	fd, err := syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, protocol)
	if err != nil {
		return nil, err
	}
	s := &NetlinkSocket{
		fd: fd,
	}
	s.lsa.Family = syscall.AF_NETLINK
	if err := syscall.Bind(fd, &s.lsa); err != nil {
		syscall.Close(fd)
		return nil, err
	}

	return s, nil
}

func (s *NetlinkSocket) Close() {
	syscall.Close(s.fd)
}

func (s *NetlinkSocket) Send(request *NetlinkRequest) error {
	if err := syscall.Sendto(s.fd, request.Serialize(), 0, &s.lsa); err != nil {
		return err
	}
	return nil
}

func (s *NetlinkSocket) Recieve() ([]syscall.NetlinkMessage, error) {
	rb := make([]byte, syscall.Getpagesize())
	nr, _, err := syscall.Recvfrom(s.fd, rb, 0)
	if err != nil {
		return nil, err
	}
	if nr < syscall.NLMSG_HDRLEN {
		return nil, fmt.Errorf("Got short response from netlink")
	}
	rb = rb[:nr]
	return syscall.ParseNetlinkMessage(rb)
}

func (s *NetlinkSocket) GetPid() (uint32, error) {
	lsa, err := syscall.Getsockname(s.fd)
	if err != nil {
		return 0, err
	}
	switch v := lsa.(type) {
	case *syscall.SockaddrNetlink:
		return v.Pid, nil
	}
	return 0, fmt.Errorf("Wrong socket type")
}

func ZeroTerminated(s string) []byte {
	bytes := make([]byte, len(s)+1)
	for i := 0; i < len(s); i++ {
		bytes[i] = s[i]
	}
	bytes[len(s)] = 0
	return bytes
}

func NonZeroTerminated(s string) []byte {
	bytes := make([]byte, len(s))
	for i := 0; i < len(s); i++ {
		bytes[i] = s[i]
	}
	return bytes
}

func BytesToString(b []byte) string {
	n := bytes.Index(b, []byte{0})
	return string(b[:n])
}

func ParseRouteAttr(b []byte) ([]syscall.NetlinkRouteAttr, error) {
	var attrs []syscall.NetlinkRouteAttr
	for len(b) >= syscall.SizeofRtAttr {
		a, vbuf, alen, err := netlinkRouteAttrAndValue(b)
		if err != nil {
			return nil, err
		}
		ra := syscall.NetlinkRouteAttr{Attr: *a, Value: vbuf[:int(a.Len)-syscall.SizeofRtAttr]}
		attrs = append(attrs, ra)
		b = b[alen:]
	}
	return attrs, nil
}

func netlinkRouteAttrAndValue(b []byte) (*syscall.RtAttr, []byte, int, error) {
	a := (*syscall.RtAttr)(unsafe.Pointer(&b[0]))
	if int(a.Len) < syscall.SizeofRtAttr || int(a.Len) > len(b) {
		return nil, nil, 0, syscall.EINVAL
	}
	return a, b[syscall.SizeofRtAttr:], rtaAlignOf(int(a.Len)), nil
}
<|endoftext|>


<commit_before>package main

import (
	"compress/gzip"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
)

var logger = log.New(os.Stderr, "", 0)

func main() {
	if err := run(); err != nil {
		logger.Printf(err.Error())
		flag.Usage()
		os.Exit(1)
	}
}

var (
	address   = flag.String("a", "http://127.0.0.1:9200", "Address")
	index     = flag.String("i", "", "Index Name")
	out       = flag.String("o", "", "Output file")
	batchSize = flag.Int("b", 1000, "Batch Size")
	scroll    = flag.String("s", "1m", "Scroll duration")
)

func run() error {
	flag.Parse()
	if *address == "" {
		return errors.New("Address must be set")
	}
	if *index == "" {
		return errors.New("Index Name must be set")
	}

	if *out == "" {
		return errors.New("Output file must be set")
	}

	f, err := os.Create(*out)
	if err != nil {
		return err
	}
	defer f.Close()
	gz := gzip.NewWriter(f)
	defer gz.Close()

	docs, err := iterateIndex(*address, *index, *batchSize, *scroll)
	if err != nil {
		return err
	}

	c, wg := progress()
	for d := range docs {
		if _, err = io.WriteString(gz, string(d)+"\n"); err != nil {
			return err
		}
		c <- struct{}{}
	}
	close(c)
	wg.Wait()
	return nil
}

func progress() (chan struct{}, *sync.WaitGroup) {
	wg := &sync.WaitGroup{}
	wg.Add(1)
	c := make(chan struct{})
	go func() {
		defer wg.Done()
		t := time.Tick(1 * time.Second)
		started := time.Now()
		cnt := 0

		printStatus := func() {
			diff := time.Since(started).Seconds()
			perSecond := float64(cnt) / diff
			logger.Printf("cnt=%d time=%.01f per_second=%.1f/second", cnt, diff, perSecond)
		}
		for {
			select {
			case _, ok := <-c:
				if !ok {
					printStatus()
					return
				}
				cnt++
			case <-t:
				printStatus()
			}

		}
	}()
	return c, wg
}

func iterateIndex(addr, name string, size int, scroll string) (chan json.RawMessage, error) {
	scrollID, err := openIndex(addr, name, size, scroll)
	if err != nil {
		return nil, err
	}
	c := make(chan json.RawMessage)
	var docs []json.RawMessage
	go func() {
		defer close(c)
		for {
			scrollID, docs, err = loadDocumentsWithScroll(addr, scrollID)
			if err != nil {
				logger.Printf("err=%q", err)
				return
			} else if len(docs) == 0 {
				return
			}
			for _, d := range docs {
				c <- d
			}
		}
	}()
	return c, nil
}

type scrollResponse struct {
	ScrollID string `json:"_scroll_id"`
	Hits     struct {
		Hits []json.RawMessage `json:"hits"`
	} `json:"hits"`
}

func loadDocumentsWithScroll(addr string, id string) (string, []json.RawMessage, error) {
	req, err := http.NewRequest("GET", addr+"/_search/scroll?scroll=1m", strings.NewReader(id))
	if err != nil {
		return "", nil, err
	}
	rsp, err := http.DefaultClient.Do(req)
	if err != nil {
		return "", nil, err
	}
	defer rsp.Body.Close()
	switch rsp.StatusCode {
	case 404:
		b, _ := ioutil.ReadAll(rsp.Body)
		logger.Printf("%s", string(b))
		return "", nil, nil
	case 200:
		var rr *scrollResponse
		err = json.NewDecoder(rsp.Body).Decode(&rr)
		if err != nil {
			return "", nil, err
		}
		return rr.ScrollID, rr.Hits.Hits, nil
	default:
		b, _ := ioutil.ReadAll(rsp.Body)
		return "", nil, fmt.Errorf("got status %s but expected 2x. body=%s", rsp.Status, string(b))
	}
}

func openIndex(addr, name string, size int, scroll string) (scrollID string, err error) {
	rsp, err := http.Post(addr+"/"+name+"/_search?search_type=scan&scroll="+scroll,
		"application/json",
		strings.NewReader(`{"query": {"match_all": {} }, "size": `+strconv.Itoa(size)+`}`),
	)
	if err != nil {
		return "", err
	}
	defer rsp.Body.Close()
	if rsp.Status[0] != '2' {
		b, _ := ioutil.ReadAll(rsp.Body)
		return "", fmt.Errorf("expected status 2xx, got %s: %s", rsp.Status, string(b))
	}
	var s *scrollResponse
	err = json.NewDecoder(rsp.Body).Decode(&s)
	if err != nil {
		return "", err
	}
	return s.ScrollID, nil
}
<commit_msg>treat 404s as errors when scrolling<commit_after>package main

import (
	"compress/gzip"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
)

var logger = log.New(os.Stderr, "", 0)

func main() {
	if err := run(); err != nil {
		logger.Printf(err.Error())
		flag.Usage()
		os.Exit(1)
	}
}

var (
	address   = flag.String("a", "http://127.0.0.1:9200", "Address")
	index     = flag.String("i", "", "Index Name")
	out       = flag.String("o", "", "Output file")
	batchSize = flag.Int("b", 1000, "Batch Size")
	scroll    = flag.String("s", "1m", "Scroll duration")
)

func run() error {
	flag.Parse()
	if *address == "" {
		return errors.New("Address must be set")
	}
	if *index == "" {
		return errors.New("Index Name must be set")
	}

	if *out == "" {
		return errors.New("Output file must be set")
	}

	f, err := os.Create(*out)
	if err != nil {
		return err
	}
	defer f.Close()
	gz := gzip.NewWriter(f)
	defer gz.Close()

	docs, err := iterateIndex(*address, *index, *batchSize, *scroll)
	if err != nil {
		return err
	}

	c, wg := progress()
	for d := range docs {
		if _, err = io.WriteString(gz, string(d)+"\n"); err != nil {
			return err
		}
		c <- struct{}{}
	}
	close(c)
	wg.Wait()
	return nil
}

func progress() (chan struct{}, *sync.WaitGroup) {
	wg := &sync.WaitGroup{}
	wg.Add(1)
	c := make(chan struct{})
	go func() {
		defer wg.Done()
		t := time.Tick(1 * time.Second)
		started := time.Now()
		cnt := 0

		printStatus := func() {
			diff := time.Since(started).Seconds()
			perSecond := float64(cnt) / diff
			logger.Printf("cnt=%d time=%.01f per_second=%.1f/second", cnt, diff, perSecond)
		}
		for {
			select {
			case _, ok := <-c:
				if !ok {
					printStatus()
					return
				}
				cnt++
			case <-t:
				printStatus()
			}

		}
	}()
	return c, wg
}

func iterateIndex(addr, name string, size int, scroll string) (chan json.RawMessage, error) {
	scrollID, err := openIndex(addr, name, size, scroll)
	if err != nil {
		return nil, err
	}
	c := make(chan json.RawMessage)
	var docs []json.RawMessage
	go func() {
		defer close(c)
		for {
			scrollID, docs, err = loadDocumentsWithScroll(addr, scrollID)
			if err != nil {
				logger.Printf("err=%q", err)
				return
			} else if len(docs) == 0 {
				return
			}
			for _, d := range docs {
				c <- d
			}
		}
	}()
	return c, nil
}

type scrollResponse struct {
	ScrollID string `json:"_scroll_id"`
	Hits     struct {
		Hits []json.RawMessage `json:"hits"`
	} `json:"hits"`
}

func loadDocumentsWithScroll(addr string, id string) (string, []json.RawMessage, error) {
	req, err := http.NewRequest("GET", addr+"/_search/scroll?scroll=1m", strings.NewReader(id))
	if err != nil {
		return "", nil, err
	}
	rsp, err := http.DefaultClient.Do(req)
	if err != nil {
		return "", nil, err
	}
	defer rsp.Body.Close()
	switch rsp.StatusCode {
	case 200:
		var rr *scrollResponse
		err = json.NewDecoder(rsp.Body).Decode(&rr)
		if err != nil {
			return "", nil, err
		}
		return rr.ScrollID, rr.Hits.Hits, nil
	default:
		b, _ := ioutil.ReadAll(rsp.Body)
		return "", nil, fmt.Errorf("got status %s but expected 2x. body=%s", rsp.Status, string(b))
	}
}

func openIndex(addr, name string, size int, scroll string) (scrollID string, err error) {
	rsp, err := http.Post(addr+"/"+name+"/_search?search_type=scan&scroll="+scroll,
		"application/json",
		strings.NewReader(`{"query": {"match_all": {} }, "size": `+strconv.Itoa(size)+`}`),
	)
	if err != nil {
		return "", err
	}
	defer rsp.Body.Close()
	if rsp.Status[0] != '2' {
		b, _ := ioutil.ReadAll(rsp.Body)
		return "", fmt.Errorf("expected status 2xx, got %s: %s", rsp.Status, string(b))
	}
	var s *scrollResponse
	err = json.NewDecoder(rsp.Body).Decode(&s)
	if err != nil {
		return "", err
	}
	return s.ScrollID, nil
}
<|endoftext|>


<commit_before>package app_workspace

import (
	"errors"
	"fmt"
	"github.com/watermint/toolbox/essentials/file/es_filepath"
	"github.com/watermint/toolbox/essentials/log/esl"
	"github.com/watermint/toolbox/infra/app"
	"go.uber.org/atomic"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"time"
)

type Application interface {
	// Toolbox home path
	Home() string
}

type Job interface {
	// Path for job
	Job() string

	// Job ID
	JobId() string

	// Log path for job
	Log() string

	// Test
	Test() string

	// Report path for job
	Report() string

	// Path for KVS storage
	KVS() string

	// Create or get child folder of job folder
	Descendant(name string) (path string, err error)
}

type User interface {
	// Secrets path
	Secrets() string
}

type MultiUser interface {
	User

	// User home path
	UserHome() string
}

type Workspace interface {
	Application
	User
	Job
}

const (
	NameSecrets = "secrets"
	NameUser    = "user"
	NameJobs    = "jobs"
	NameLogs    = "logs"
	NameReport  = "report"
	NameTest    = "test"
	NameKvs     = "kvs"
	JobIdFormat = "20060102-150405"
)

var (
	jobSequence = atomic.Int64{}
)

func NewJobId() string {
	return fmt.Sprintf("%s.%03d", time.Now().Format(JobIdFormat), jobSequence.Add(1))
}

func DefaultAppPath() (path string, err error) {
	if eh := os.Getenv(app.EnvNameToolboxHome); eh != "" {
		return "", nil
	}

	u, err := user.Current()
	if err != nil {
		return "", err
	}
	return filepath.Join(u.HomeDir, ".toolbox"), nil
}

func NewWorkspace(home string, transient bool) (Workspace, error) {
	if transient {
		if path, err := ioutil.TempDir("", "transient"); err != nil {
			return nil, err
		} else {
			return newWorkspaceWithJobIdNoSetup(path, NewJobId()), nil
		}
	}

	if home == "" {
		if path, err := DefaultAppPath(); err != nil {
			return nil, err
		} else {
			return newWorkspace(path)
		}
	} else {
		if path, err := es_filepath.FormatPathWithPredefinedVariables(home); err != nil {
			return nil, err
		} else {
			return newWorkspace(path)
		}
	}
}

// Create workspace instance by job path.
// This will not create directories. Just matches to existing folder structure.
// Returns error if a
func NewWorkspaceByJobPath(home Application, jobId string) (ws Workspace, err error) {
	ws = newWorkspaceWithJobIdNoSetup(home.Home(), jobId)
	p, err := os.Lstat(ws.Log())
	if err != nil {
		return nil, err
	}
	if p.IsDir() {
		return ws, nil
	}
	return nil, errors.New("given path look like not a workspace path")
}

// create or get fully qualified path
func getOrCreate(fqp string) (path string, err error) {
	l := esl.Default().With(esl.String("path", fqp))
	st, err := os.Stat(fqp)
	switch {
	case err != nil && os.IsNotExist(err):
		err = os.MkdirAll(fqp, 0701)
		if err != nil {
			l.Error("Unable to create workspace path", esl.Error(err))
			return "", err
		}
	case err != nil:
		l.Error("Unable to setup path", esl.Error(err))
		return "", err

	case !st.IsDir():
		l.Error("Workspace path is not a directory")
		return "", errors.New("workspace path is not a directory")

	case st.Mode()&0700 == 0:
		l.Error("No permission to read and write at workspace path", esl.Any("mode", st.Mode()))
		return "", errors.New("no permission")
	}
	return fqp, nil
}
<commit_msg>fix #386 : TOOLBOX_HOME is not working<commit_after>package app_workspace

import (
	"errors"
	"fmt"
	"github.com/watermint/toolbox/essentials/file/es_filepath"
	"github.com/watermint/toolbox/essentials/log/esl"
	"github.com/watermint/toolbox/infra/app"
	"go.uber.org/atomic"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"time"
)

type Application interface {
	// Toolbox home path
	Home() string
}

type Job interface {
	// Path for job
	Job() string

	// Job ID
	JobId() string

	// Log path for job
	Log() string

	// Test
	Test() string

	// Report path for job
	Report() string

	// Path for KVS storage
	KVS() string

	// Create or get child folder of job folder
	Descendant(name string) (path string, err error)
}

type User interface {
	// Secrets path
	Secrets() string
}

type MultiUser interface {
	User

	// User home path
	UserHome() string
}

type Workspace interface {
	Application
	User
	Job
}

const (
	NameSecrets = "secrets"
	NameUser    = "user"
	NameJobs    = "jobs"
	NameLogs    = "logs"
	NameReport  = "report"
	NameTest    = "test"
	NameKvs     = "kvs"
	JobIdFormat = "20060102-150405"
)

var (
	jobSequence = atomic.Int64{}
)

func NewJobId() string {
	return fmt.Sprintf("%s.%03d", time.Now().Format(JobIdFormat), jobSequence.Add(1))
}

func DefaultAppPath() (path string, err error) {
	if eh := os.Getenv(app.EnvNameToolboxHome); eh != "" {
		return eh, nil
	}

	u, err := user.Current()
	if err != nil {
		return "", err
	}
	return filepath.Join(u.HomeDir, ".toolbox"), nil
}

func NewWorkspace(home string, transient bool) (Workspace, error) {
	if transient {
		if path, err := ioutil.TempDir("", "transient"); err != nil {
			return nil, err
		} else {
			return newWorkspaceWithJobIdNoSetup(path, NewJobId()), nil
		}
	}

	if home == "" {
		if path, err := DefaultAppPath(); err != nil {
			return nil, err
		} else {
			return newWorkspace(path)
		}
	} else {
		if path, err := es_filepath.FormatPathWithPredefinedVariables(home); err != nil {
			return nil, err
		} else {
			return newWorkspace(path)
		}
	}
}

// Create workspace instance by job path.
// This will not create directories. Just matches to existing folder structure.
// Returns error if a
func NewWorkspaceByJobPath(home Application, jobId string) (ws Workspace, err error) {
	ws = newWorkspaceWithJobIdNoSetup(home.Home(), jobId)
	p, err := os.Lstat(ws.Log())
	if err != nil {
		return nil, err
	}
	if p.IsDir() {
		return ws, nil
	}
	return nil, errors.New("given path look like not a workspace path")
}

// create or get fully qualified path
func getOrCreate(fqp string) (path string, err error) {
	l := esl.Default().With(esl.String("path", fqp))
	st, err := os.Stat(fqp)
	switch {
	case err != nil && os.IsNotExist(err):
		err = os.MkdirAll(fqp, 0701)
		if err != nil {
			l.Error("Unable to create workspace path", esl.Error(err))
			return "", err
		}
	case err != nil:
		l.Error("Unable to setup path", esl.Error(err))
		return "", err

	case !st.IsDir():
		l.Error("Workspace path is not a directory")
		return "", errors.New("workspace path is not a directory")

	case st.Mode()&0700 == 0:
		l.Error("No permission to read and write at workspace path", esl.Any("mode", st.Mode()))
		return "", errors.New("no permission")
	}
	return fqp, nil
}
<|endoftext|>


<commit_before>package kafka

import (
	"bufio"
	"encoding/binary"
	"fmt"
	"hash/crc32"
	"io"
	"reflect"
)

type apiKey int16

const (
	produceRequest          apiKey = 0
	fetchRequest            apiKey = 1
	offsetRequest           apiKey = 2
	metadataRequest         apiKey = 3
	offsetCommitRequest     apiKey = 8
	offsetFetchRequest      apiKey = 9
	groupCoordinatorRequest apiKey = 10
	joinGroupRequest        apiKey = 11
	heartbeatRequest        apiKey = 12
	leaveGroupRequest       apiKey = 13
	syncGroupRequest        apiKey = 14
	describeGroupsRequest   apiKey = 15
	listGroupsRequest       apiKey = 16
)

type apiVersion int16

const (
	v0 apiVersion = 0
	v1 apiVersion = 1
	v2 apiVersion = 2
)

type requestHeader struct {
	Size          int32
	ApiKey        int16
	ApiVersion    int16
	CorrelationID int32
	ClientID      string
}

// Message types
type messageSet []messageSetItem

type messageSetItem struct {
	Offset      int64
	MessageSize int32
	Message     message
}

type message struct {
	CRC        int32
	MagicByte  int8
	Attributes int8
	Timestamp  int64
	Key        []byte
	Value      []byte
}

func makeMessage(timestamp int64, key []byte, value []byte) message {
	m := message{
		MagicByte: 1,
		Timestamp: timestamp,
		Key:       key,
		Value:     value,
	}
	m.CRC = m.crc32()
	return m
}

func (m message) crc32() int32 {
	sum := uint32(0)
	buf := [10]byte{}

	buf[0] = byte(m.MagicByte)
	buf[1] = byte(m.Attributes)
	binary.BigEndian.PutUint64(buf[2:], uint64(m.Timestamp))
	sum = crc32.Update(sum, crc32.IEEETable, buf[:])

	if m.Key == nil {
		binary.BigEndian.PutUint32(buf[:4], 0xFFFFFFFF) // -1
	} else {
		binary.BigEndian.PutUint32(buf[:4], uint32(len(m.Key)))
	}
	sum = crc32.Update(sum, crc32.IEEETable, buf[:4])
	sum = crc32.Update(sum, crc32.IEEETable, m.Key)

	binary.BigEndian.PutUint32(buf[:4], uint32(len(m.Value)))
	sum = crc32.Update(sum, crc32.IEEETable, buf[:4])
	sum = crc32.Update(sum, crc32.IEEETable, m.Value)

	return int32(sum)
}

// Metadata API
type topicMetadataRequest []string

type metadataResponse struct {
	Brokers []broker
	Topics  []topicMetadata
}

type broker struct {
	NodeID int32
	Host   string
	Port   int32
}

type topicMetadata struct {
	TopicErrorCode int16
	TopicName      string
	Partitions     []partitionMetadata
}

type partitionMetadata struct {
	PartitionErrorCode int16
	PartitionID        int32
	Leader             int32
	Replicas           []int32
	Isr                []int32
}

// Produce API
type produceRequestType struct {
	RequiredAcks int16
	Timeout      int32
	Topics       []produceRequestTopic
}

type produceRequestTopic struct {
	TopicName  string
	Partitions []produceRequestPartition
}

type produceRequestPartition struct {
	Partition      int32
	MessageSetSize int32
	MessageSet     messageSet
}

type produceResponse struct {
	Topics       []produceResponseTopic
	ThrottleTime int32
}

type produceResponseTopic struct {
	TopicName  string
	Partitions []produceResponsePartition
}

type produceResponsePartition struct {
	Partition int32
	ErrorCode int16
	Offset    int64
	Timestamp int64
}

// Offset API
type listOffsetRequest struct {
	ReplicaID int32
	Topics    []listOffsetRequestTopic
}

type listOffsetRequestTopic struct {
	TopicName  string
	Partitions []listOffsetRequestPartition
}

type listOffsetRequestPartition struct {
	Partition int32
	Time      int64
}

type listOffsetResponse struct {
	TopicName        string
	PartitionOffsets []partitionOffset
}

type partitionOffset struct {
	Partition int32
	ErrorCode int16
	Timestamp int64
	Offsets   []int64
}

func makeInt8(b []byte) int8 {
	return int8(b[0])
}

func makeInt16(b []byte) int16 {
	return int16(binary.BigEndian.Uint16(b))
}

func makeInt32(b []byte) int32 {
	return int32(binary.BigEndian.Uint32(b))
}

func makeInt64(b []byte) int64 {
	return int64(binary.BigEndian.Uint64(b))
}

func peekRead(r *bufio.Reader, n int, f func([]byte)) error {
	b, err := r.Peek(n)
	if err != nil {
		return err
	}
	f(b)
	_, err = r.Discard(n)
	return err
}

func readInt8(r *bufio.Reader, v *int8) error {
	return peekRead(r, 1, func(b []byte) { *v = makeInt8(b) })
}

func readInt16(r *bufio.Reader, v *int16) error {
	return peekRead(r, 2, func(b []byte) { *v = makeInt16(b) })
}

func readInt32(r *bufio.Reader, v *int32) error {
	return peekRead(r, 4, func(b []byte) { *v = makeInt32(b) })
}

func readInt64(r *bufio.Reader, v *int64) error {
	return peekRead(r, 8, func(b []byte) { *v = makeInt64(b) })
}

func readString(r *bufio.Reader, v *string) error {
	var b []byte
	var n int16

	if err := readInt16(r, &n); err != nil {
		return err
	}

	if n >= 0 {
		b = make([]byte, int(n))
		if _, err := io.ReadFull(r, b); err != nil {
			return err
		}
	}

	*v = string(b)
	return nil
}

func readBytes(r *bufio.Reader, v *[]byte) error {
	var b []byte
	var n int32

	if err := readInt32(r, &n); err != nil {
		return err
	}

	if n >= 0 {
		b = make([]byte, int(n))
		if _, err := io.ReadFull(r, b); err != nil {
			return err
		}
	}

	*v = b
	return nil
}

func readResponse(r *bufio.Reader, size int32, res interface{}) error {
	// TODO: check size?
	return read(r, res)
}

func read(r *bufio.Reader, a interface{}) error {
	switch v := a.(type) {
	case *int8:
		return readInt8(r, v)
	case *int16:
		return readInt16(r, v)
	case *int32:
		return readInt32(r, v)
	case *int64:
		return readInt64(r, v)
	case *string:
		return readString(r, v)
	case *[]byte:
		return readBytes(r, v)
	}

	v := reflect.ValueOf(a)
	if v.IsNil() {
		v.Set(reflect.New(v.Type().Elem()))
	}

	switch v = v.Elem(); v.Kind() {
	case reflect.Struct:
		return readStruct(r, v)
	case reflect.Slice:
		return readSlice(r, v)
	default:
		panic(fmt.Sprintf("unsupported type: %T", a))
	}
}

func readStruct(r *bufio.Reader, v reflect.Value) error {
	for i, n := 0, v.NumField(); i != n; i++ {
		if err := read(r, v.Field(i).Addr().Interface()); err != nil {
			return err
		}
	}
	return nil
}

func readSlice(r *bufio.Reader, v reflect.Value) error {
	size := int32(0)

	if err := readInt32(r, &size); err != nil {
		return err
	}

	n := int(size)
	v.Set(reflect.MakeSlice(v.Type(), n, n))

	for i := 0; i != n; i++ {
		if err := read(r, v.Index(i).Addr().Interface()); err != nil {
			return err
		}
	}

	return nil
}

func writeInt8(w *bufio.Writer, i int8) error {
	return w.WriteByte(byte(i))
}

func writeInt16(w *bufio.Writer, i int16) error {
	var b [2]byte
	binary.BigEndian.PutUint16(b[:], uint16(i))
	return writeSmallBuffer(w, b[:])
}

func writeInt32(w *bufio.Writer, i int32) error {
	var b [4]byte
	binary.BigEndian.PutUint32(b[:], uint32(i))
	return writeSmallBuffer(w, b[:])
}

func writeInt64(w *bufio.Writer, i int64) error {
	var b [8]byte
	binary.BigEndian.PutUint64(b[:], uint64(i))
	return writeSmallBuffer(w, b[:])
}

func writeString(w *bufio.Writer, s string) error {
	if err := writeInt16(w, int16(len(s))); err != nil {
		return err
	}
	_, err := w.WriteString(s)
	return err
}

func writeBytes(w *bufio.Writer, b []byte) error {
	n := int32(len(b))
	if b == nil {
		n = -1
	}
	if err := writeInt32(w, n); err != nil {
		return err
	}
	_, err := w.Write(b)
	return err
}

func writeMessageSet(w *bufio.Writer, mset messageSet) error {
	// message sets are not preceded by their size for some reason
	for _, m := range mset {
		if err := write(w, m); err != nil {
			return err
		}
	}
	return nil
}

func writeSmallBuffer(w *bufio.Writer, b []byte) error {
	// faster for short byte slices because it doesn't cause a memory allocation
	// because the byte slice may be passed to the underlying io.Writer.
	for _, c := range b {
		if err := w.WriteByte(c); err != nil {
			return err
		}
	}
	return nil
}

func writeRequest(w *bufio.Writer, hdr requestHeader, req interface{}) error {
	hdr.Size = (sizeof(hdr) + sizeof(req)) - 4
	write(w, hdr)
	write(w, req)
	return w.Flush()
}

func write(w *bufio.Writer, a interface{}) error {
	switch v := a.(type) {
	case int8:
		return writeInt8(w, v)
	case int16:
		return writeInt16(w, v)
	case int32:
		return writeInt32(w, v)
	case int64:
		return writeInt64(w, v)
	case string:
		return writeString(w, v)
	case []byte:
		return writeBytes(w, v)
	case messageSet:
		return writeMessageSet(w, v)
	}
	switch v := reflect.ValueOf(a); v.Kind() {
	case reflect.Struct:
		return writeStruct(w, v)
	case reflect.Slice:
		return writeSlice(w, v)
	default:
		panic(fmt.Sprintf("unsupported type: %T", a))
	}
}

func writeStruct(w *bufio.Writer, v reflect.Value) error {
	for i, n := 0, v.NumField(); i != n; i++ {
		if err := write(w, v.Field(i).Interface()); err != nil {
			return err
		}
	}
	return nil
}

func writeSlice(w *bufio.Writer, v reflect.Value) error {
	n := v.Len()
	if err := writeInt32(w, int32(n)); err != nil {
		return err
	}
	for i := 0; i != n; i++ {
		if err := write(w, v.Index(i).Interface()); err != nil {
			return err
		}
	}
	return nil
}

func sizeof(a interface{}) int32 {
	switch v := a.(type) {
	case int8:
		return 1
	case int16:
		return 2
	case int32:
		return 4
	case int64:
		return 8
	case string:
		return 2 + int32(len(v))
	case []byte:
		return 4 + int32(len(v))
	case messageSet:
		return sizeofMessageSet(v)
	}
	switch v := reflect.ValueOf(a); v.Kind() {
	case reflect.Struct:
		return sizeofStruct(v)
	case reflect.Slice:
		return sizeofSlice(v)
	default:
		panic(fmt.Sprintf("unsupported type: %T", a))
	}
}

func sizeofMessageSet(set messageSet) (size int32) {
	for _, msg := range set {
		size += sizeof(msg)
	}
	return
}

func sizeofStruct(v reflect.Value) (size int32) {
	for i, n := 0, v.NumField(); i != n; i++ {
		size += sizeof(v.Field(i).Interface())
	}
	return
}

func sizeofSlice(v reflect.Value) (size int32) {
	size = 4
	for i, n := 0, v.Len(); i != n; i++ {
		size += sizeof(v.Index(i).Interface())
	}
	return
}
<commit_msg>fix crc32 for null message values<commit_after>package kafka

import (
	"bufio"
	"encoding/binary"
	"fmt"
	"hash/crc32"
	"io"
	"reflect"
)

type apiKey int16

const (
	produceRequest          apiKey = 0
	fetchRequest            apiKey = 1
	offsetRequest           apiKey = 2
	metadataRequest         apiKey = 3
	offsetCommitRequest     apiKey = 8
	offsetFetchRequest      apiKey = 9
	groupCoordinatorRequest apiKey = 10
	joinGroupRequest        apiKey = 11
	heartbeatRequest        apiKey = 12
	leaveGroupRequest       apiKey = 13
	syncGroupRequest        apiKey = 14
	describeGroupsRequest   apiKey = 15
	listGroupsRequest       apiKey = 16
)

type apiVersion int16

const (
	v0 apiVersion = 0
	v1 apiVersion = 1
	v2 apiVersion = 2
)

type requestHeader struct {
	Size          int32
	ApiKey        int16
	ApiVersion    int16
	CorrelationID int32
	ClientID      string
}

// Message types
type messageSet []messageSetItem

type messageSetItem struct {
	Offset      int64
	MessageSize int32
	Message     message
}

type message struct {
	CRC        int32
	MagicByte  int8
	Attributes int8
	Timestamp  int64
	Key        []byte
	Value      []byte
}

func makeMessage(timestamp int64, key []byte, value []byte) message {
	m := message{
		MagicByte: 1,
		Timestamp: timestamp,
		Key:       key,
		Value:     value,
	}
	m.CRC = m.crc32()
	return m
}

func (m message) crc32() int32 {
	sum := uint32(0)

	buf := [10]byte{}
	buf[0] = byte(m.MagicByte)
	buf[1] = byte(m.Attributes)
	binary.BigEndian.PutUint64(buf[2:], uint64(m.Timestamp))

	sum = crc32.Update(sum, crc32.IEEETable, buf[:])
	sum = crc32Bytes(sum, m.Key)
	sum = crc32Bytes(sum, m.Value)

	return int32(sum)
}

func crc32Bytes(sum uint32, b []byte) uint32 {
	buf := [4]byte{}
	if b == nil {
		binary.BigEndian.PutUint32(buf[:4], 0xFFFFFFFF) // -1
	} else {
		binary.BigEndian.PutUint32(buf[:4], uint32(len(b)))
	}
	sum = crc32.Update(sum, crc32.IEEETable, buf[:4])
	sum = crc32.Update(sum, crc32.IEEETable, b)
	return sum
}

// Metadata API
type topicMetadataRequest []string

type metadataResponse struct {
	Brokers []broker
	Topics  []topicMetadata
}

type broker struct {
	NodeID int32
	Host   string
	Port   int32
}

type topicMetadata struct {
	TopicErrorCode int16
	TopicName      string
	Partitions     []partitionMetadata
}

type partitionMetadata struct {
	PartitionErrorCode int16
	PartitionID        int32
	Leader             int32
	Replicas           []int32
	Isr                []int32
}

// Produce API
type produceRequestType struct {
	RequiredAcks int16
	Timeout      int32
	Topics       []produceRequestTopic
}

type produceRequestTopic struct {
	TopicName  string
	Partitions []produceRequestPartition
}

type produceRequestPartition struct {
	Partition      int32
	MessageSetSize int32
	MessageSet     messageSet
}

type produceResponse struct {
	Topics       []produceResponseTopic
	ThrottleTime int32
}

type produceResponseTopic struct {
	TopicName  string
	Partitions []produceResponsePartition
}

type produceResponsePartition struct {
	Partition int32
	ErrorCode int16
	Offset    int64
	Timestamp int64
}

// Offset API
type listOffsetRequest struct {
	ReplicaID int32
	Topics    []listOffsetRequestTopic
}

type listOffsetRequestTopic struct {
	TopicName  string
	Partitions []listOffsetRequestPartition
}

type listOffsetRequestPartition struct {
	Partition int32
	Time      int64
}

type listOffsetResponse struct {
	TopicName        string
	PartitionOffsets []partitionOffset
}

type partitionOffset struct {
	Partition int32
	ErrorCode int16
	Timestamp int64
	Offsets   []int64
}

func makeInt8(b []byte) int8 {
	return int8(b[0])
}

func makeInt16(b []byte) int16 {
	return int16(binary.BigEndian.Uint16(b))
}

func makeInt32(b []byte) int32 {
	return int32(binary.BigEndian.Uint32(b))
}

func makeInt64(b []byte) int64 {
	return int64(binary.BigEndian.Uint64(b))
}

func peekRead(r *bufio.Reader, n int, f func([]byte)) error {
	b, err := r.Peek(n)
	if err != nil {
		return err
	}
	f(b)
	_, err = r.Discard(n)
	return err
}

func readInt8(r *bufio.Reader, v *int8) error {
	return peekRead(r, 1, func(b []byte) { *v = makeInt8(b) })
}

func readInt16(r *bufio.Reader, v *int16) error {
	return peekRead(r, 2, func(b []byte) { *v = makeInt16(b) })
}

func readInt32(r *bufio.Reader, v *int32) error {
	return peekRead(r, 4, func(b []byte) { *v = makeInt32(b) })
}

func readInt64(r *bufio.Reader, v *int64) error {
	return peekRead(r, 8, func(b []byte) { *v = makeInt64(b) })
}

func readString(r *bufio.Reader, v *string) error {
	var b []byte
	var n int16

	if err := readInt16(r, &n); err != nil {
		return err
	}

	if n >= 0 {
		b = make([]byte, int(n))
		if _, err := io.ReadFull(r, b); err != nil {
			return err
		}
	}

	*v = string(b)
	return nil
}

func readBytes(r *bufio.Reader, v *[]byte) error {
	var b []byte
	var n int32

	if err := readInt32(r, &n); err != nil {
		return err
	}

	if n >= 0 {
		b = make([]byte, int(n))
		if _, err := io.ReadFull(r, b); err != nil {
			return err
		}
	}

	*v = b
	return nil
}

func readResponse(r *bufio.Reader, size int32, res interface{}) error {
	// TODO: check size?
	return read(r, res)
}

func read(r *bufio.Reader, a interface{}) error {
	switch v := a.(type) {
	case *int8:
		return readInt8(r, v)
	case *int16:
		return readInt16(r, v)
	case *int32:
		return readInt32(r, v)
	case *int64:
		return readInt64(r, v)
	case *string:
		return readString(r, v)
	case *[]byte:
		return readBytes(r, v)
	}

	v := reflect.ValueOf(a)
	if v.IsNil() {
		v.Set(reflect.New(v.Type().Elem()))
	}

	switch v = v.Elem(); v.Kind() {
	case reflect.Struct:
		return readStruct(r, v)
	case reflect.Slice:
		return readSlice(r, v)
	default:
		panic(fmt.Sprintf("unsupported type: %T", a))
	}
}

func readStruct(r *bufio.Reader, v reflect.Value) error {
	for i, n := 0, v.NumField(); i != n; i++ {
		if err := read(r, v.Field(i).Addr().Interface()); err != nil {
			return err
		}
	}
	return nil
}

func readSlice(r *bufio.Reader, v reflect.Value) error {
	size := int32(0)

	if err := readInt32(r, &size); err != nil {
		return err
	}

	n := int(size)
	v.Set(reflect.MakeSlice(v.Type(), n, n))

	for i := 0; i != n; i++ {
		if err := read(r, v.Index(i).Addr().Interface()); err != nil {
			return err
		}
	}

	return nil
}

func writeInt8(w *bufio.Writer, i int8) error {
	return w.WriteByte(byte(i))
}

func writeInt16(w *bufio.Writer, i int16) error {
	var b [2]byte
	binary.BigEndian.PutUint16(b[:], uint16(i))
	return writeSmallBuffer(w, b[:])
}

func writeInt32(w *bufio.Writer, i int32) error {
	var b [4]byte
	binary.BigEndian.PutUint32(b[:], uint32(i))
	return writeSmallBuffer(w, b[:])
}

func writeInt64(w *bufio.Writer, i int64) error {
	var b [8]byte
	binary.BigEndian.PutUint64(b[:], uint64(i))
	return writeSmallBuffer(w, b[:])
}

func writeString(w *bufio.Writer, s string) error {
	if err := writeInt16(w, int16(len(s))); err != nil {
		return err
	}
	_, err := w.WriteString(s)
	return err
}

func writeBytes(w *bufio.Writer, b []byte) error {
	n := int32(len(b))
	if b == nil {
		n = -1
	}
	if err := writeInt32(w, n); err != nil {
		return err
	}
	_, err := w.Write(b)
	return err
}

func writeMessageSet(w *bufio.Writer, mset messageSet) error {
	// message sets are not preceded by their size for some reason
	for _, m := range mset {
		if err := write(w, m); err != nil {
			return err
		}
	}
	return nil
}

func writeSmallBuffer(w *bufio.Writer, b []byte) error {
	// faster for short byte slices because it doesn't cause a memory allocation
	// because the byte slice may be passed to the underlying io.Writer.
	for _, c := range b {
		if err := w.WriteByte(c); err != nil {
			return err
		}
	}
	return nil
}

func writeRequest(w *bufio.Writer, hdr requestHeader, req interface{}) error {
	hdr.Size = (sizeof(hdr) + sizeof(req)) - 4
	write(w, hdr)
	write(w, req)
	return w.Flush()
}

func write(w *bufio.Writer, a interface{}) error {
	switch v := a.(type) {
	case int8:
		return writeInt8(w, v)
	case int16:
		return writeInt16(w, v)
	case int32:
		return writeInt32(w, v)
	case int64:
		return writeInt64(w, v)
	case string:
		return writeString(w, v)
	case []byte:
		return writeBytes(w, v)
	case messageSet:
		return writeMessageSet(w, v)
	}
	switch v := reflect.ValueOf(a); v.Kind() {
	case reflect.Struct:
		return writeStruct(w, v)
	case reflect.Slice:
		return writeSlice(w, v)
	default:
		panic(fmt.Sprintf("unsupported type: %T", a))
	}
}

func writeStruct(w *bufio.Writer, v reflect.Value) error {
	for i, n := 0, v.NumField(); i != n; i++ {
		if err := write(w, v.Field(i).Interface()); err != nil {
			return err
		}
	}
	return nil
}

func writeSlice(w *bufio.Writer, v reflect.Value) error {
	n := v.Len()
	if err := writeInt32(w, int32(n)); err != nil {
		return err
	}
	for i := 0; i != n; i++ {
		if err := write(w, v.Index(i).Interface()); err != nil {
			return err
		}
	}
	return nil
}

func sizeof(a interface{}) int32 {
	switch v := a.(type) {
	case int8:
		return 1
	case int16:
		return 2
	case int32:
		return 4
	case int64:
		return 8
	case string:
		return 2 + int32(len(v))
	case []byte:
		return 4 + int32(len(v))
	case messageSet:
		return sizeofMessageSet(v)
	}
	switch v := reflect.ValueOf(a); v.Kind() {
	case reflect.Struct:
		return sizeofStruct(v)
	case reflect.Slice:
		return sizeofSlice(v)
	default:
		panic(fmt.Sprintf("unsupported type: %T", a))
	}
}

func sizeofMessageSet(set messageSet) (size int32) {
	for _, msg := range set {
		size += sizeof(msg)
	}
	return
}

func sizeofStruct(v reflect.Value) (size int32) {
	for i, n := 0, v.NumField(); i != n; i++ {
		size += sizeof(v.Field(i).Interface())
	}
	return
}

func sizeofSlice(v reflect.Value) (size int32) {
	size = 4
	for i, n := 0, v.Len(); i != n; i++ {
		size += sizeof(v.Index(i).Interface())
	}
	return
}
<|endoftext|>