<commit_before>/*
Copyright 2020 The cert-manager Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package certificates

import (
	"context"
	"encoding/pem"
	"strconv"
	"testing"
	"time"

	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/wait"

	apiutil "github.com/jetstack/cert-manager/pkg/api/util"
	cmapi "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1"
	cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1"
	controllerpkg "github.com/jetstack/cert-manager/pkg/controller"
	"github.com/jetstack/cert-manager/pkg/controller/certificates/revisionmanager"
	logf "github.com/jetstack/cert-manager/pkg/logs"
	"github.com/jetstack/cert-manager/pkg/metrics"
	utilpki "github.com/jetstack/cert-manager/pkg/util/pki"
	"github.com/jetstack/cert-manager/test/integration/framework"
	"github.com/jetstack/cert-manager/test/unit/gen"
)

// TestRevisionManagerController will ensure that the revision manager
// controller will delete old CertificateRequests occording to the
// spec.revisionHistoryLimit value
func TestRevisionManagerController(t *testing.T) {
	config, stopFn := framework.RunControlPlane(t)
	defer stopFn()

	// Build, instantiate and run the revision manager controller.
	kubeClient, factory, cmCl, cmFactory := framework.NewClients(t, config)

	ctrl, queue, mustSync := revisionmanager.NewController(logf.Log, cmCl, cmFactory)

	c := controllerpkg.NewController(
		context.Background(),
		"issuing_test",
		metrics.New(logf.Log),
		ctrl.ProcessItem,
		mustSync,
		nil,
		queue,
	)
	stopController := framework.StartInformersAndController(t, factory, cmFactory, c)
	defer stopController()

	ctx, cancel := context.WithTimeout(context.TODO(), time.Second*20)
	defer cancel()

	var (
		crtName    = "testcrt"
		namespace  = "testns"
		secretName = "test-crt-tls"
	)

	// Create Namespace
	ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}
	_, err := kubeClient.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{})
	if err != nil {
		t.Fatal(err)
	}

	// Create Certificate
	crt := gen.Certificate(crtName,
		gen.SetCertificateNamespace(namespace),
		gen.SetCertificateCommonName("my-common-name"),
		gen.SetCertificateSecretName(secretName),
		gen.SetCertificateRevisionHistoryLimit(3),
		gen.SetCertificateIssuer(cmmeta.ObjectReference{Name: "testissuer", Group: "foo.io", Kind: "Issuer"}),
	)

	crt, err = cmCl.CertmanagerV1().Certificates(namespace).Create(ctx, crt, metav1.CreateOptions{})
	if err != nil {
		t.Fatal(err)
	}

	// Set Certificate to Ready
	apiutil.SetCertificateCondition(crt, crt.Generation,
		cmapi.CertificateConditionReady, cmmeta.ConditionTrue, "Issued", "integration test")
	crt, err = cmCl.CertmanagerV1().Certificates(namespace).UpdateStatus(ctx, crt, metav1.UpdateOptions{})
	if err != nil {
		t.Fatal(err)
	}

	// Create a new private key
	sk, err := utilpki.GenerateRSAPrivateKey(2048)
	if err != nil {
		t.Fatal(err)
	}

	csr, err := utilpki.GenerateCSR(crt)
	if err != nil {
		t.Fatal(err)
	}

	// Encode CSR
	csrDER, err := utilpki.EncodeCSR(csr, sk)
	if err != nil {
		t.Fatal(err)
	}

	csrPEM := pem.EncodeToMemory(&pem.Block{
		Type: "CERTIFICATE REQUEST", Bytes: csrDER,
	})

	// Create 6 CertificateRequests which are owned by this Certificate
	for i := 0; i < 6; i++ {
		_, err = cmCl.CertmanagerV1().CertificateRequests(namespace).Create(ctx, &cmapi.CertificateRequest{
			ObjectMeta: metav1.ObjectMeta{
				GenerateName: crtName + "-",
				Namespace:    namespace,
				Annotations: map[string]string{
					cmapi.CertificateRequestRevisionAnnotationKey: strconv.Itoa(i),
				},
				OwnerReferences: []metav1.OwnerReference{
					*metav1.NewControllerRef(crt, cmapi.SchemeGroupVersion.WithKind("Certificate")),
				},
			},
			Spec: cmapi.CertificateRequestSpec{
				Request:   csrPEM,
				IssuerRef: cmmeta.ObjectReference{Name: "testissuer", Group: "foo.io", Kind: "Issuer"},
			},
		}, metav1.CreateOptions{})
		if err != nil {
			t.Fatal(err)
		}
	}

	var crs []cmapi.CertificateRequest

	// Wait for 3 CertificateRequests to be deleted, and that they have the correct revisions
	err = wait.Poll(time.Millisecond*100, time.Second*5, func() (done bool, err error) {
		requests, err := cmCl.CertmanagerV1().CertificateRequests(namespace).List(context.TODO(), metav1.ListOptions{})
		if err != nil {
			return false, err
		}

		if len(requests.Items) != 3 {
			t.Logf("waiting for 3 requests to be deleted, got=%d", len(requests.Items))
			return false, nil
		}

		crs = requests.Items

		return true, nil
	})
	if err != nil {
		t.Fatal(err)
	}

	// Expect that the remaining requests have the largest revisions (3, 4, 5)
	for _, revision := range []string{"3", "4", "5"} {
		var found bool
		for _, cr := range crs {
			if cr.Annotations[cmapi.CertificateRequestRevisionAnnotationKey] == revision {
				found = true
				break
			}
		}

		if !found {
			t.Errorf("failed to find a CertificateRequest with a revision=%s", revision)
		}
	}
}
<commit_msg>Fix test controller name of revision manager in integration test<commit_after>/*
Copyright 2020 The cert-manager Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package certificates

import (
	"context"
	"encoding/pem"
	"strconv"
	"testing"
	"time"

	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/wait"

	apiutil "github.com/jetstack/cert-manager/pkg/api/util"
	cmapi "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1"
	cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1"
	controllerpkg "github.com/jetstack/cert-manager/pkg/controller"
	"github.com/jetstack/cert-manager/pkg/controller/certificates/revisionmanager"
	logf "github.com/jetstack/cert-manager/pkg/logs"
	"github.com/jetstack/cert-manager/pkg/metrics"
	utilpki "github.com/jetstack/cert-manager/pkg/util/pki"
	"github.com/jetstack/cert-manager/test/integration/framework"
	"github.com/jetstack/cert-manager/test/unit/gen"
)

// TestRevisionManagerController will ensure that the revision manager
// controller will delete old CertificateRequests occording to the
// spec.revisionHistoryLimit value
func TestRevisionManagerController(t *testing.T) {
	config, stopFn := framework.RunControlPlane(t)
	defer stopFn()

	// Build, instantiate and run the revision manager controller.
	kubeClient, factory, cmCl, cmFactory := framework.NewClients(t, config)

	ctrl, queue, mustSync := revisionmanager.NewController(logf.Log, cmCl, cmFactory)

	c := controllerpkg.NewController(
		context.Background(),
		"revisionmanager_controller_test",
		metrics.New(logf.Log),
		ctrl.ProcessItem,
		mustSync,
		nil,
		queue,
	)
	stopController := framework.StartInformersAndController(t, factory, cmFactory, c)
	defer stopController()

	ctx, cancel := context.WithTimeout(context.TODO(), time.Second*20)
	defer cancel()

	var (
		crtName    = "testcrt"
		namespace  = "testns"
		secretName = "test-crt-tls"
	)

	// Create Namespace
	ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}
	_, err := kubeClient.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{})
	if err != nil {
		t.Fatal(err)
	}

	// Create Certificate
	crt := gen.Certificate(crtName,
		gen.SetCertificateNamespace(namespace),
		gen.SetCertificateCommonName("my-common-name"),
		gen.SetCertificateSecretName(secretName),
		gen.SetCertificateRevisionHistoryLimit(3),
		gen.SetCertificateIssuer(cmmeta.ObjectReference{Name: "testissuer", Group: "foo.io", Kind: "Issuer"}),
	)

	crt, err = cmCl.CertmanagerV1().Certificates(namespace).Create(ctx, crt, metav1.CreateOptions{})
	if err != nil {
		t.Fatal(err)
	}

	// Set Certificate to Ready
	apiutil.SetCertificateCondition(crt, crt.Generation,
		cmapi.CertificateConditionReady, cmmeta.ConditionTrue, "Issued", "integration test")
	crt, err = cmCl.CertmanagerV1().Certificates(namespace).UpdateStatus(ctx, crt, metav1.UpdateOptions{})
	if err != nil {
		t.Fatal(err)
	}

	// Create a new private key
	sk, err := utilpki.GenerateRSAPrivateKey(2048)
	if err != nil {
		t.Fatal(err)
	}

	csr, err := utilpki.GenerateCSR(crt)
	if err != nil {
		t.Fatal(err)
	}

	// Encode CSR
	csrDER, err := utilpki.EncodeCSR(csr, sk)
	if err != nil {
		t.Fatal(err)
	}

	csrPEM := pem.EncodeToMemory(&pem.Block{
		Type: "CERTIFICATE REQUEST", Bytes: csrDER,
	})

	// Create 6 CertificateRequests which are owned by this Certificate
	for i := 0; i < 6; i++ {
		_, err = cmCl.CertmanagerV1().CertificateRequests(namespace).Create(ctx, &cmapi.CertificateRequest{
			ObjectMeta: metav1.ObjectMeta{
				GenerateName: crtName + "-",
				Namespace:    namespace,
				Annotations: map[string]string{
					cmapi.CertificateRequestRevisionAnnotationKey: strconv.Itoa(i),
				},
				OwnerReferences: []metav1.OwnerReference{
					*metav1.NewControllerRef(crt, cmapi.SchemeGroupVersion.WithKind("Certificate")),
				},
			},
			Spec: cmapi.CertificateRequestSpec{
				Request:   csrPEM,
				IssuerRef: cmmeta.ObjectReference{Name: "testissuer", Group: "foo.io", Kind: "Issuer"},
			},
		}, metav1.CreateOptions{})
		if err != nil {
			t.Fatal(err)
		}
	}

	var crs []cmapi.CertificateRequest

	// Wait for 3 CertificateRequests to be deleted, and that they have the correct revisions
	err = wait.Poll(time.Millisecond*100, time.Second*5, func() (done bool, err error) {
		requests, err := cmCl.CertmanagerV1().CertificateRequests(namespace).List(context.TODO(), metav1.ListOptions{})
		if err != nil {
			return false, err
		}

		if len(requests.Items) != 3 {
			t.Logf("waiting for 3 requests to be deleted, got=%d", len(requests.Items))
			return false, nil
		}

		crs = requests.Items

		return true, nil
	})
	if err != nil {
		t.Fatal(err)
	}

	// Expect that the remaining requests have the largest revisions (3, 4, 5)
	for _, revision := range []string{"3", "4", "5"} {
		var found bool
		for _, cr := range crs {
			if cr.Annotations[cmapi.CertificateRequestRevisionAnnotationKey] == revision {
				found = true
				break
			}
		}

		if !found {
			t.Errorf("failed to find a CertificateRequest with a revision=%s", revision)
		}
	}
}
<|endoftext|>


<commit_before>package core

import (
	"fmt"
	"github.com/APTrust/bagit/util"
	"github.com/APTrust/bagit/util/fileutil"
	"os"
	"path/filepath"
	"strings"
)

type Bagger struct {
	Bag            *Bag
	Profile        *BagItProfile
	PayloadDir     string
	TagValues      map[string]string
	ForceOverwrite bool
	tarWriter      *fileutil.TarWriter
	errors         []string
}

// NewBagger creates a new Bagger.
//
// Param bagPath is the path to the location where the bag should
// be built. This should be a directory name. The bagger will
// create the directory, if it does not already exist. If the directory
// exists, the bagger will not overwrite it, unless the forceOverwrite
// (see below) is true. If bagPath ends with ".tar", the bag will be
// created as a tar file instead of as a directory.
//
// Param payloadDir is the name of the directory containing the files to be
// bagged.
//
// Param profile is the BagIt profile that describes the requirements
// for the bag.
//
// Param tagValues is a map of tag values that will be used
// to create tag files.
//
// If param forceOverwrite is true, the bagger will delete and then
// overwrite the directory at bagPath.
func NewBagger(bagPath, payloadDir string, profile *BagItProfile, tagValues map[string]string, forceOverwrite bool) *Bagger {
	bagger := &Bagger{
		Bag:            NewBag(bagPath),
		Profile:        profile,
		PayloadDir:     payloadDir,
		TagValues:      tagValues,
		ForceOverwrite: forceOverwrite,
		errors:         make([]string, 0),
	}
	if strings.HasSuffix(bagPath, ".tar") {
		bagger.tarWriter = fileutil.NewTarWriter(bagPath)
	}
	return bagger
}

/*

- Validate Profile
- Validate that all required tags are present
- Include progress callback or io.Writer for writing progress messages
- Add all payload files to bag in working dir
- Build tag files
- Create manifests

- Tar bag if serialization is required and serlialization format is tar?
- Provide a way of specifying tag files and where to put them?
- Parse existing tag files, and skip defaults if they're already defined in existing tag files?

*/

func (bagger *Bagger) BuildBag() bool {
	ok := bagger.initFileOrDir()
	if !ok {
		return false
	}
	ok = bagger.hasRequiredTags()
	if !ok {
		return false
	}
	bagger.copyPayload()

	// write tag files
	// create manifests

	return true
}

func (bagger *Bagger) hasRequiredTags() bool {
	ok := true
	// Avoid nil pointer errors
	tagValues := bagger.TagValues
	if tagValues == nil {
		tagValues = make(map[string]string)
	}
	for filename, tagmap := range bagger.Profile.TagFilesRequired {
		for tagname, tag := range tagmap {
			value, keyExists := tagValues[tagname]
			if tag.Required && !keyExists {
				bagger.addError("Required tag %s for file %s is missing", tagname, filename)
				ok = false
			}
			if !tag.EmptyOk && keyExists && value == "" {
				bagger.addError("Tag %s for file %s cannot be empty", tagname, filename)
				ok = false
			}
			if len(tag.Values) > 0 && keyExists && !util.StringListContains(tag.Values, value) {
				bagger.addError("Value '%s' is not allowed for tag %s. Valid values: %s",
					value, tagname, strings.Join(tag.Values, ", "))
				ok = false
			}
		}
	}
	return ok
}

// initFileOrDir creates the directory in which we'll assemble the bag,
// performing some safety checks along the way. Returns true on
// success, false otherwise.
func (bagger *Bagger) initFileOrDir() bool {
	if fileutil.FileExists(bagger.Bag.Path) {
		if bagger.ForceOverwrite {
			if fileutil.LooksSafeToDelete(bagger.Bag.Path, 12, 3) {
				err := os.RemoveAll(bagger.Bag.Path)
				if err != nil {
					bagger.addError("Error removing existing %s", bagger.Bag.Path, err.Error())
					return false
				}
			} else {
				bagger.addError("%s already exists and does not look safe to delete. "+
					"Choose a different path for this bag.",
					bagger.Bag.Path)
				return false
			}
		} else {
			bagger.addError("%s already exists. Use forceOverwrite=true to replace it.",
				bagger.Bag.Path)
			return false
		}
	}
	err := os.MkdirAll(bagger.Bag.Path, 0755)
	if err != nil {
		bagger.addError(err.Error())
		return false
	}
	return true
}

// copyPayload copies files from PayloadDir into the bag's data
// directory.
func (bagger *Bagger) copyPayload() bool {
	files, err := fileutil.RecursiveFileList(bagger.PayloadDir)
	if err != nil {
		bagger.addError("Error listing %s: %v", bagger.PayloadDir, err)
		return false
	}
	for _, filePath := range files {
		if !bagger.addPayloadFile(filePath) {
			return false
		}
	}
	return true
}

// addPayloadFile adds one file from the source directory into the
// bag's payload (data) directory.
func (bagger *Bagger) addPayloadFile(filePath string) bool {
	var checksums map[string]string
	var err error
	// Use forward slash, even on Windows, for tar file paths and manifest entries.
	relativePath := fmt.Sprintf("data/%s", bagger.getBasePath(filePath))
	if bagger.tarWriter != nil {
		// Copy the new file into the tar archive, and keep track of its checksums
		checksums, err = bagger.tarWriter.AddToArchive(filePath, relativePath, bagger.Profile.ManifestsRequired)
	} else {
		// targetPath is the path we will copy the file to
		targetPath := filepath.Join(bagger.Bag.Path, "data", bagger.getBasePath(filePath))
		if fileutil.FileExists(targetPath) {
			bagger.addError(fmt.Sprintf("Cannot copy to %s: file already exists", targetPath))
			return false
		}

		// srcFile is the reader we'll copy from
		srcFile, err := os.Open(filePath)
		defer srcFile.Close()
		if err != nil {
			bagger.addError(err.Error())
			return false
		}

		// fileInfo tells us the mode the new file should have
		fileInfo, err := os.Stat(filePath)
		if err != nil {
			bagger.addError(err.Error())
			return false
		}

		targetDir := filepath.Dir(targetPath)
		if !fileutil.IsDir(targetDir) {
			err := os.MkdirAll(targetDir, 0755)
			if err != nil {
				bagger.addError(err.Error())
				return false
			}
		}

		// destFile is the writer we'll copy to
		destFile, err := os.OpenFile(targetPath, os.O_RDWR|os.O_CREATE, fileInfo.Mode())
		defer destFile.Close()
		if err != nil {
			bagger.addError(err.Error())
			return false
		}

		// Copy src to dest, and keep track of the checksums
		checksums, err = fileutil.WriteWithChecksums(srcFile, destFile, bagger.Profile.ManifestsRequired)
	}
	if err != nil {
		bagger.addError(err.Error())
		return false
	}
	bagger.addChecksums(relativePath, checksums)
	return true
}

func (bagger *Bagger) getBasePath(filePath string) string {
	basePath := strings.Replace(filePath, bagger.PayloadDir+string(os.PathSeparator), "", 1)
	if strings.HasSuffix(basePath, string(os.PathSeparator)) {
		basePath = strings.TrimRight(basePath, string(os.PathSeparator))
	}
	return basePath
}

// addChecksums adds a file's checksums to the appropriate manifests.
func (bagger *Bagger) addChecksums(relativePath string, checksums map[string]string) {
	for algorithm, digest := range checksums {
		manifestFileName := fmt.Sprintf("manifest-%s.txt", algorithm)
		manifestFile := bagger.Bag.Manifests[manifestFileName]
		if manifestFile == nil {
			manifestFile = NewFile(int64(0))
			bagger.Bag.Manifests[manifestFileName] = manifestFile
		}
		manifestFile.Checksums[relativePath] = digest
	}
}

// addError adds a message to the list of bagging errors.
func (bagger *Bagger) addError(format string, a ...interface{}) {
	bagger.errors = append(bagger.errors, fmt.Sprintf(format, a...))
}

// Returns a list of error messages describing what went wrong with
// the bagging process.
func (bagger *Bagger) Errors() []string {
	return bagger.errors
}
<commit_msg>Going down the wrong road...<commit_after>package core

import (
	"fmt"
	"github.com/APTrust/bagit/util"
	"github.com/APTrust/bagit/util/fileutil"
	"os"
	"path/filepath"
	"strings"
)

type Bagger struct {
	Bag            *Bag
	Profile        *BagItProfile
	PayloadDir     string
	TagValues      map[string]string
	ForceOverwrite bool
	tarWriter      *fileutil.TarWriter
	errors         []string
}

// NewBagger creates a new Bagger.
//
// Param bagPath is the path to the location where the bag should
// be built. This should be a directory name. The bagger will
// create the directory, if it does not already exist. If the directory
// exists, the bagger will not overwrite it, unless the forceOverwrite
// (see below) is true. If bagPath ends with ".tar", the bag will be
// created as a tar file instead of as a directory.
//
// Param payloadDir is the name of the directory containing the files to be
// bagged.
//
// Param profile is the BagIt profile that describes the requirements
// for the bag.
//
// Param tagValues is a map of tag values that will be used
// to create tag files.
//
// If param forceOverwrite is true, the bagger will delete and then
// overwrite the directory at bagPath.
func NewBagger(bagPath, payloadDir string, profile *BagItProfile, tagValues map[string]string, forceOverwrite bool) *Bagger {
	bagger := &Bagger{
		Bag:            NewBag(bagPath),
		Profile:        profile,
		PayloadDir:     payloadDir,
		TagValues:      tagValues,
		ForceOverwrite: forceOverwrite,
		errors:         make([]string, 0),
	}
	if strings.HasSuffix(bagPath, ".tar") {
		bagger.tarWriter = fileutil.NewTarWriter(bagPath)
	}
	return bagger
}

/*

- Validate Profile
- Validate that all required tags are present
- Include progress callback or io.Writer for writing progress messages
- Add all payload files to bag in working dir
- Build tag files
- Create manifests

- Tar bag if serialization is required and serlialization format is tar?
- Provide a way of specifying tag files and where to put them?
- Parse existing tag files, and skip defaults if they're already defined in existing tag files?

*/

func (bagger *Bagger) BuildBag() bool {
	ok := bagger.initFileOrDir()
	if !ok {
		return false
	}
	ok = bagger.hasRequiredTags()
	if !ok {
		return false
	}
	bagger.copyPayload()
	//bagger.writeTagFiles()

	// write tag files
	// create manifests

	return true
}

func (bagger *Bagger) writeTagFiles() bool {
	for filename, tagmap := range bagger.Profile.TagFilesRequired {
		if !bagger.writeTagFile(filename, tagmap) {
			return false
		}
	}
	return true
}

func (bagger *Bagger) writeTagFile(filename string, tagmap map[string]*Tag) bool {
	//var checksums map[string]string
	var err error
	if bagger.tarWriter != nil {
		// Write the tag file into the tar archive, and keep track of its checksums
		// checksums, err = bagger.tarWriter.AddToArchive(filename, tagmap, bagger.Profile.ManifestsRequired)
	} else {
		targetPath := filepath.Join(bagger.Bag.Path, filename)
		if fileutil.FileExists(targetPath) {
			bagger.addError(fmt.Sprintf("Cannot write tag file %s: file already exists", targetPath))
			return false
		}
		targetDir := filepath.Dir(targetPath)
		if !fileutil.IsDir(targetDir) {
			err := os.MkdirAll(targetDir, 0755)
			if err != nil {
				bagger.addError(err.Error())
				return false
			}
		}
		tagFile, err := os.Create(targetPath)
		defer tagFile.Close()
		if err != nil {
			bagger.addError(err.Error())
			return false
		}

		// Copy src to dest, and keep track of the checksums
		// checksums, err = fileutil.WriteWithChecksums(srcFile, destFile, bagger.Profile.ManifestsRequired)
	}
	if err != nil {
		bagger.addError(err.Error())
		return false
	}
	//bagger.addChecksums(relativePath, checksums)
	return true
}

func (bagger *Bagger) hasRequiredTags() bool {
	ok := true
	// Avoid nil pointer errors
	tagValues := bagger.TagValues
	if tagValues == nil {
		tagValues = make(map[string]string)
	}
	for filename, tagmap := range bagger.Profile.TagFilesRequired {
		for tagname, tag := range tagmap {
			value, keyExists := tagValues[tagname]
			if tag.Required && !keyExists {
				bagger.addError("Required tag %s for file %s is missing", tagname, filename)
				ok = false
			}
			if !tag.EmptyOk && keyExists && value == "" {
				bagger.addError("Tag %s for file %s cannot be empty", tagname, filename)
				ok = false
			}
			if len(tag.Values) > 0 && keyExists && !util.StringListContains(tag.Values, value) {
				bagger.addError("Value '%s' is not allowed for tag %s. Valid values: %s",
					value, tagname, strings.Join(tag.Values, ", "))
				ok = false
			}
		}
	}
	return ok
}

// initFileOrDir creates the directory in which we'll assemble the bag,
// performing some safety checks along the way. Returns true on
// success, false otherwise.
func (bagger *Bagger) initFileOrDir() bool {
	if fileutil.FileExists(bagger.Bag.Path) {
		if bagger.ForceOverwrite {
			if fileutil.LooksSafeToDelete(bagger.Bag.Path, 12, 3) {
				err := os.RemoveAll(bagger.Bag.Path)
				if err != nil {
					bagger.addError("Error removing existing %s", bagger.Bag.Path, err.Error())
					return false
				}
			} else {
				bagger.addError("%s already exists and does not look safe to delete. "+
					"Choose a different path for this bag.",
					bagger.Bag.Path)
				return false
			}
		} else {
			bagger.addError("%s already exists. Use forceOverwrite=true to replace it.",
				bagger.Bag.Path)
			return false
		}
	}
	err := os.MkdirAll(bagger.Bag.Path, 0755)
	if err != nil {
		bagger.addError(err.Error())
		return false
	}
	return true
}

// copyPayload copies files from PayloadDir into the bag's data
// directory.
func (bagger *Bagger) copyPayload() bool {
	files, err := fileutil.RecursiveFileList(bagger.PayloadDir)
	if err != nil {
		bagger.addError("Error listing %s: %v", bagger.PayloadDir, err)
		return false
	}
	for _, filePath := range files {
		if !bagger.addPayloadFile(filePath) {
			return false
		}
	}
	return true
}

// addPayloadFile adds one file from the source directory into the
// bag's payload (data) directory.
func (bagger *Bagger) addPayloadFile(filePath string) bool {
	var checksums map[string]string
	var err error
	// Use forward slash, even on Windows, for tar file paths and manifest entries.
	relativePath := fmt.Sprintf("data/%s", bagger.getBasePath(filePath))
	if bagger.tarWriter != nil {
		// Copy the new file into the tar archive, and keep track of its checksums
		checksums, err = bagger.tarWriter.AddToArchive(filePath, relativePath, bagger.Profile.ManifestsRequired)
	} else {
		// targetPath is the path we will copy the file to
		targetPath := filepath.Join(bagger.Bag.Path, "data", bagger.getBasePath(filePath))
		if fileutil.FileExists(targetPath) {
			bagger.addError(fmt.Sprintf("Cannot copy to %s: file already exists", targetPath))
			return false
		}

		// srcFile is the reader we'll copy from
		srcFile, err := os.Open(filePath)
		defer srcFile.Close()
		if err != nil {
			bagger.addError(err.Error())
			return false
		}

		// fileInfo tells us the mode the new file should have
		fileInfo, err := os.Stat(filePath)
		if err != nil {
			bagger.addError(err.Error())
			return false
		}

		targetDir := filepath.Dir(targetPath)
		if !fileutil.IsDir(targetDir) {
			err := os.MkdirAll(targetDir, 0755)
			if err != nil {
				bagger.addError(err.Error())
				return false
			}
		}

		// destFile is the writer we'll copy to
		destFile, err := os.OpenFile(targetPath, os.O_RDWR|os.O_CREATE, fileInfo.Mode())
		defer destFile.Close()
		if err != nil {
			bagger.addError(err.Error())
			return false
		}

		// Copy src to dest, and keep track of the checksums
		checksums, err = fileutil.WriteWithChecksums(srcFile, destFile, bagger.Profile.ManifestsRequired)
	}
	if err != nil {
		bagger.addError(err.Error())
		return false
	}
	bagger.addChecksums(relativePath, checksums)
	return true
}

func (bagger *Bagger) getBasePath(filePath string) string {
	basePath := strings.Replace(filePath, bagger.PayloadDir+string(os.PathSeparator), "", 1)
	if strings.HasSuffix(basePath, string(os.PathSeparator)) {
		basePath = strings.TrimRight(basePath, string(os.PathSeparator))
	}
	return basePath
}

// addChecksums adds a file's checksums to the appropriate manifests.
func (bagger *Bagger) addChecksums(relativePath string, checksums map[string]string) {
	for algorithm, digest := range checksums {
		manifestFileName := fmt.Sprintf("manifest-%s.txt", algorithm)
		fileMap := bagger.Bag.Manifests
		if !strings.HasPrefix(relativePath, "data/") {
			manifestFileName = fmt.Sprintf("tagmanifest-%s.txt", algorithm)
			fileMap = bagger.Bag.TagManifests
		}
		manifestFile := fileMap[manifestFileName]
		if manifestFile == nil {
			manifestFile = NewFile(int64(0))
			bagger.Bag.Manifests[manifestFileName] = manifestFile
		}
		manifestFile.Checksums[relativePath] = digest
	}
}

// addError adds a message to the list of bagging errors.
func (bagger *Bagger) addError(format string, a ...interface{}) {
	bagger.errors = append(bagger.errors, fmt.Sprintf(format, a...))
}

// Returns a list of error messages describing what went wrong with
// the bagging process.
func (bagger *Bagger) Errors() []string {
	return bagger.errors
}
<|endoftext|>


<commit_before>package sched

import (
	"reflect"
	"testing"
)

func TestNewParseError(t *testing.T) {
	err := NewParseError("expression", "description")
	if err.Expression != "expression" || err.Description != "description" {
		t.Fail()
	}
}

func TestParseError_Error(t *testing.T) {
	err := NewParseError("expression", "description")
	const want = `sched: could not parse "expression": description`
	if result := err.Error(); result != want {
		t.Errorf("err.Error() = %v WANT %v", result, want)
	}
}

func TestFieldsFunc(t *testing.T) {
	tests := []struct {
		value  rune
		result bool
	}{
		{' ', true},
		{'\t', true},
		{'e', false},
		{'0', false},
	}
	for _, test := range tests {
		if result := FieldsFunc(test.value); result != test.result {
			t.Errorf("FieldsFunc(%v) = %v WANT %v", test.value, result, test.result)
		}
	}
}

func TestFields(t *testing.T) {
	tests := []struct {
		exp    string
		result []string
	}{
		{"", []string{}},
		{"expression", []string{"expression"}},
		{"a b", []string{"a", "b"}},
		{"a  b", []string{"a", "b"}},
		{"a\tb", []string{"a", "b"}},
		{"a\t\tb", []string{"a", "b"}},
		{"a\t \tb", []string{"a", "b"}},
		{" a\t \tb\t", []string{"a", "b"}},
		{" a\t \tb\t\n    ", []string{"a", "b"}},
	}
	for _, test := range tests {
		if result := Fields(test.exp); !reflect.DeepEqual(result, test.result) {
			t.Errorf("Fields(%v) = %q WANT %q", test.exp, result, test.result)
		}
	}
}

func TestFieldParts(t *testing.T) {
	tests := []struct {
		field  string
		result []string
	}{
		{"", []string{}},
		{"field", []string{"field"}},
		{"a,b", []string{"a", "b"}},
		{"a,,b", []string{"a", "", "b"}},
		{"a,b,c", []string{"a", "b", "c"}},
		{"this is a string a,this is a string b", []string{"this is a string a", "this is a string b"}},
	}
	for _, test := range tests {
		if result := FieldParts(test.field); !reflect.DeepEqual(result, test.result) {
			t.Errorf("FieldParts(%v) = %q WANT %q", test.field, result, test.result)
		}
	}
}

func TestGetNormalizedDirectiveFields(t *testing.T) {
	tests := []struct {
		directive string
		result    []string
		err       string
	}{
		{Yearly, Fields(YearlyFormat), ""},
		{Annually, Fields(AnnuallyFormat), ""},
		{Monthly, Fields(MonthlyFormat), ""},
		{Weekly, Fields(WeeklyFormat), ""},
		{Daily, Fields(DailyFormat), ""},
		{Hourly, Fields(HourlyFormat), ""},
		{Minutely, Fields(MinutelyFormat), ""},
		{Secondly, Fields(SecondlyFormat), ""},
		{"@reboot", nil, `the directive "@reboot" is not recognized`},
	}
	for _, test := range tests {
		result, err := getNormalizedDirectiveFields(test.directive)
		if !reflect.DeepEqual(result, test.result) || (err != nil && err.Error() != test.err) {
			t.Errorf(
				"getNormalizedDirectiveFields(%v) = %v, %v WANT %v, %v",
				test.directive,
				result,
				err,
				test.result,
				test.err,
			)
		}
	}
}
<commit_msg>Adds test cases for validateNumberOfFields().<commit_after>package sched

import (
	"reflect"
	"testing"
)

func TestNewParseError(t *testing.T) {
	err := NewParseError("expression", "description")
	if err.Expression != "expression" || err.Description != "description" {
		t.Fail()
	}
}

func TestParseError_Error(t *testing.T) {
	err := NewParseError("expression", "description")
	const want = `sched: could not parse "expression": description`
	if result := err.Error(); result != want {
		t.Errorf("err.Error() = %v WANT %v", result, want)
	}
}

func TestFieldsFunc(t *testing.T) {
	tests := []struct {
		value  rune
		result bool
	}{
		{' ', true},
		{'\t', true},
		{'e', false},
		{'0', false},
	}
	for _, test := range tests {
		if result := FieldsFunc(test.value); result != test.result {
			t.Errorf("FieldsFunc(%v) = %v WANT %v", test.value, result, test.result)
		}
	}
}

func TestFields(t *testing.T) {
	tests := []struct {
		exp    string
		result []string
	}{
		{"", []string{}},
		{"expression", []string{"expression"}},
		{"a b", []string{"a", "b"}},
		{"a  b", []string{"a", "b"}},
		{"a\tb", []string{"a", "b"}},
		{"a\t\tb", []string{"a", "b"}},
		{"a\t \tb", []string{"a", "b"}},
		{" a\t \tb\t", []string{"a", "b"}},
		{" a\t \tb\t\n    ", []string{"a", "b"}},
	}
	for _, test := range tests {
		if result := Fields(test.exp); !reflect.DeepEqual(result, test.result) {
			t.Errorf("Fields(%v) = %q WANT %q", test.exp, result, test.result)
		}
	}
}

func TestFieldParts(t *testing.T) {
	tests := []struct {
		field  string
		result []string
	}{
		{"", []string{}},
		{"field", []string{"field"}},
		{"a,b", []string{"a", "b"}},
		{"a,,b", []string{"a", "", "b"}},
		{"a,b,c", []string{"a", "b", "c"}},
		{"this is a string a,this is a string b", []string{"this is a string a", "this is a string b"}},
	}
	for _, test := range tests {
		if result := FieldParts(test.field); !reflect.DeepEqual(result, test.result) {
			t.Errorf("FieldParts(%v) = %q WANT %q", test.field, result, test.result)
		}
	}
}

func TestValidateNumberOfFields(t *testing.T) {
	errString := "number of fields must be 1, 5, 6, or 7"
	tests := []struct {
		fields []string
		count  int
		err    string
	}{
		{nil, 0, errString},
		{[]string{}, 0, errString},
		{make([]string, 1), 1, ""},
		{make([]string, 2), 0, errString},
		{make([]string, 3), 0, errString},
		{make([]string, 4), 0, errString},
		{make([]string, 5), 5, ""},
		{make([]string, 6), 6, ""},
		{make([]string, 7), 7, ""},
		{make([]string, 8), 0, errString},
	}
	for _, test := range tests {
		count, err := validateNumberOfFields(test.fields)
		if (err != nil || test.err != "") && err.Error() != test.err {
			t.Errorf("validateNumberOfFields(%v) error = %v WANT %v", test.fields, err, test.err)
		}
		if count != test.count {
			t.Errorf("validateNumberOfFields(%v) count = %v WANT %v", test.fields, count, test.count)
		}
	}
}

func TestGetNormalizedDirectiveFields(t *testing.T) {
	tests := []struct {
		directive string
		result    []string
		err       string
	}{
		{Yearly, Fields(YearlyFormat), ""},
		{Annually, Fields(AnnuallyFormat), ""},
		{Monthly, Fields(MonthlyFormat), ""},
		{Weekly, Fields(WeeklyFormat), ""},
		{Daily, Fields(DailyFormat), ""},
		{Hourly, Fields(HourlyFormat), ""},
		{Minutely, Fields(MinutelyFormat), ""},
		{Secondly, Fields(SecondlyFormat), ""},
		{"@reboot", nil, `the directive "@reboot" is not recognized`},
	}
	for _, test := range tests {
		result, err := getNormalizedDirectiveFields(test.directive)
		if (err != nil || test.err != "") && err.Error() != test.err {
			t.Errorf("getNormalizedDirectiveFields(%v) error = %v WANT %v", test.directive, err, test.err)
		}
		if !reflect.DeepEqual(result, test.result) {
			t.Errorf("getNormalizedDirectiveFields(%v) result = %v WANT %v", test.directive, result, test.result)
		}
	}
}
<|endoftext|>


<commit_before>package core

import (
	"encoding/base64"
	"fmt"
	"github.com/OpenBazaar/openbazaar-go/ipfs"
	"github.com/OpenBazaar/openbazaar-go/pb"
	"github.com/nfnt/resize"
	"image"
	"image/jpeg"
	"os"
	"path"
	"strings"
)

type Images struct {
	Tiny     string `json:"tiny"`
	Small    string `json:"small"`
	Medium   string `json:"medium"`
	Large    string `json:"large"`
	Original string `json:"original"`
}

func (n *OpenBazaarNode) SetAvatarImages(base64ImageData string) (*Images, error) {
	reader := base64.NewDecoder(base64.StdEncoding, strings.NewReader(base64ImageData))
	img, _, err := image.Decode(reader)
	if err != nil {
		return nil, err
	}
	reader = base64.NewDecoder(base64.StdEncoding, strings.NewReader(base64ImageData))
	imgCfg, _, err := image.DecodeConfig(reader)
	if err != nil {
		return nil, err
	}
	w := uint(imgCfg.Width)
	h := uint(imgCfg.Height)
	if w > h {
		w = w / h
		h = uint(1)
	} else if h > w {
		h = h / w
		w = uint(1)
	} else {
		w = uint(1)
		h = uint(1)
	}

	ty := resize.Resize(w*50, h*50, img, resize.Lanczos3)
	sm := resize.Resize(w*100, h*100, img, resize.Lanczos3)
	md := resize.Resize(w*140, h*140, img, resize.Lanczos3)
	lg := resize.Resize(w*280, h*280, img, resize.Lanczos3)

	imgPath := path.Join(n.RepoPath, "root", "images")

	out, err := os.Create(path.Join(imgPath, "tiny", "avatar"))
	if err != nil {
		return nil, err
	}
	defer out.Close()
	jpeg.Encode(out, ty, nil)

	out, err = os.Create(path.Join(imgPath, "small", "avatar"))
	if err != nil {
		return nil, err
	}
	jpeg.Encode(out, sm, nil)

	out, err = os.Create(path.Join(imgPath, "medium", "avatar"))
	if err != nil {
		return nil, err
	}
	jpeg.Encode(out, md, nil)

	out, err = os.Create(path.Join(imgPath, "large", "avatar"))
	if err != nil {
		return nil, err
	}
	jpeg.Encode(out, lg, nil)

	out, err = os.Create(path.Join(imgPath, "original", "avatar"))
	if err != nil {
		return nil, err
	}
	jpeg.Encode(out, img, nil)

	// Add hash to profile
	t, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "tiny", "avatar"))
	if aerr != nil {
		return nil, err
	}
	s, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "small", "avatar"))
	if aerr != nil {
		return nil, err
	}
	m, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "medium", "avatar"))
	if aerr != nil {
		return nil, err
	}
	l, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "large", "avatar"))
	if aerr != nil {
		return nil, err
	}
	o, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "original", "avatar"))
	if aerr != nil {
		return nil, err
	}
	profile, err := n.GetProfile()
	if err != nil {
		return nil, err
	}
	i := new(pb.Profile_Image)
	i.Tiny = t
	i.Small = s
	i.Medium = m
	i.Large = l
	i.Original = o
	profile.AvatarHashes = i
	err = n.UpdateProfile(&profile)
	if aerr != nil {
		return nil, err
	}
	return &Images{t, s, m, l, o}, nil
}

func (n *OpenBazaarNode) SetHeaderImages(base64ImageData string) (*Images, error) {
	reader := base64.NewDecoder(base64.StdEncoding, strings.NewReader(base64ImageData))
	img, _, err := image.Decode(reader)
	if err != nil {
		return nil, err
	}
	reader = base64.NewDecoder(base64.StdEncoding, strings.NewReader(base64ImageData))
	imgCfg, _, err := image.DecodeConfig(reader)
	if err != nil {
		return nil, err
	}
	imgWidth := uint(imgCfg.Width)
	imgHeight := uint(imgCfg.Height)

	w, h := getImageAttributes(304, 87, imgWidth, imgHeight)
	ty := resize.Resize(w, h, img, resize.Lanczos3)
	w, h = getImageAttributes(608, 174, imgWidth, imgHeight)
	sm := resize.Resize(w, h, img, resize.Lanczos3)
	w, h = getImageAttributes(1225, 350, imgWidth, imgHeight)
	md := resize.Resize(w, h, img, resize.Lanczos3)
	w, h = getImageAttributes(2450, 700, imgWidth, imgHeight)
	lg := resize.Resize(w, h, img, resize.Lanczos3)

	imgPath := path.Join(n.RepoPath, "root", "images")

	out, err := os.Create(path.Join(imgPath, "tiny", "header"))
	if err != nil {
		return nil, err
	}
	defer out.Close()
	jpeg.Encode(out, ty, nil)

	out, err = os.Create(path.Join(imgPath, "small", "header"))
	if err != nil {
		return nil, err
	}
	jpeg.Encode(out, sm, nil)

	out, err = os.Create(path.Join(imgPath, "medium", "header"))
	if err != nil {
		return nil, err
	}
	jpeg.Encode(out, md, nil)

	out, err = os.Create(path.Join(imgPath, "large", "header"))
	if err != nil {
		return nil, err
	}
	jpeg.Encode(out, lg, nil)

	out, err = os.Create(path.Join(imgPath, "original", "header"))
	if err != nil {
		return nil, err
	}
	jpeg.Encode(out, img, nil)

	// Add hash to profile
	t, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "tiny", "header"))
	if aerr != nil {
		return nil, err
	}
	s, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "small", "header"))
	if aerr != nil {
		return nil, err
	}
	m, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "medium", "header"))
	if aerr != nil {
		return nil, err
	}
	l, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "large", "header"))
	if aerr != nil {
		return nil, err
	}
	o, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "original", "header"))
	if aerr != nil {
		return nil, err
	}
	profile, err := n.GetProfile()
	if err != nil {
		return nil, err
	}
	i := new(pb.Profile_Image)
	i.Tiny = t
	i.Small = s
	i.Medium = m
	i.Large = l
	i.Original = o
	profile.HeaderHashes = i
	err = n.UpdateProfile(&profile)
	if aerr != nil {
		return nil, err
	}
	return &Images{t, s, m, l, o}, nil
}

func (n *OpenBazaarNode) SetProductImages(base64ImageData, filename string) (*Images, error) {
	// Decode base64 image data
	reader := base64.NewDecoder(base64.StdEncoding, strings.NewReader(base64ImageData))
	img, _, err := image.Decode(reader)
	if err != nil {
		fmt.Println(1)
		return nil, err
	}

	// Get the image config
	reader = base64.NewDecoder(base64.StdEncoding, strings.NewReader(base64ImageData))
	imgCfg, _, err := image.DecodeConfig(reader)
	if err != nil {
		return nil, err
	}
	imgWidth := uint(imgCfg.Width)
	imgHeight := uint(imgCfg.Height)

	w, h := getImageAttributes(60, 60, imgWidth, imgHeight)
	ty := resize.Resize(w, h, img, resize.Lanczos3)
	w, h = getImageAttributes(228, 228, imgWidth, imgHeight)
	sm := resize.Resize(w, h, img, resize.Lanczos3)
	w, h = getImageAttributes(500, 500, imgWidth, imgHeight)
	md := resize.Resize(w, h, img, resize.Lanczos3)
	w, h = getImageAttributes(1000, 1000, imgWidth, imgHeight)
	lg := resize.Resize(w, h, img, resize.Lanczos3)

	imgPath := path.Join(n.RepoPath, "root", "images")

	out, err := os.Create(path.Join(imgPath, "tiny", filename))
	if err != nil {
		return nil, err
	}
	defer out.Close()
	jpeg.Encode(out, ty, nil)

	out, err = os.Create(path.Join(imgPath, "small", filename))
	if err != nil {
		return nil, err
	}
	jpeg.Encode(out, sm, nil)

	out, err = os.Create(path.Join(imgPath, "medium", filename))
	if err != nil {
		return nil, err
	}
	jpeg.Encode(out, md, nil)

	out, err = os.Create(path.Join(imgPath, "large", filename))
	if err != nil {
		return nil, err
	}
	jpeg.Encode(out, lg, nil)

	out, err = os.Create(path.Join(imgPath, "original", filename))
	if err != nil {
		return nil, err
	}
	jpeg.Encode(out, img, nil)

	// Get the image hashes
	t, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "tiny", filename))
	if aerr != nil {
		return nil, err
	}
	s, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "small", filename))
	if aerr != nil {
		return nil, err
	}
	m, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "medium", filename))
	if aerr != nil {
		return nil, err
	}
	l, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "large", filename))
	if aerr != nil {
		return nil, err
	}
	o, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "original", filename))
	if aerr != nil {
		return nil, err
	}
	ret := &Images{t, s, m, l, o}
	return ret, nil
}

func getImageAttributes(targetWidth, targetHeight, imgWidth, imgHeight uint) (width, height uint) {
	targetRatio := targetWidth / targetHeight
	imageRatio := imgWidth / imgHeight
	if imageRatio > targetRatio {
		height = targetHeight
		width = targetHeight * imageRatio
	} else {
		width = targetWidth
		height = targetWidth * (imgHeight / imgWidth)
	}
	return height, width
}

<commit_msg>Remove extra spaces.<commit_after>package core

import (
	"encoding/base64"
	"fmt"
	"github.com/OpenBazaar/openbazaar-go/ipfs"
	"github.com/OpenBazaar/openbazaar-go/pb"
	"github.com/nfnt/resize"
	"image"
	"image/jpeg"
	"os"
	"path"
	"strings"
)

type Images struct {
	Tiny     string `json:"tiny"`
	Small    string `json:"small"`
	Medium   string `json:"medium"`
	Large    string `json:"large"`
	Original string `json:"original"`
}

func (n *OpenBazaarNode) SetAvatarImages(base64ImageData string) (*Images, error) {
	reader := base64.NewDecoder(base64.StdEncoding, strings.NewReader(base64ImageData))
	img, _, err := image.Decode(reader)
	if err != nil {
		return nil, err
	}
	reader = base64.NewDecoder(base64.StdEncoding, strings.NewReader(base64ImageData))
	imgCfg, _, err := image.DecodeConfig(reader)
	if err != nil {
		return nil, err
	}
	w := uint(imgCfg.Width)
	h := uint(imgCfg.Height)
	if w > h {
		w = w / h
		h = uint(1)
	} else if h > w {
		h = h / w
		w = uint(1)
	} else {
		w = uint(1)
		h = uint(1)
	}

	ty := resize.Resize(w*50, h*50, img, resize.Lanczos3)
	sm := resize.Resize(w*100, h*100, img, resize.Lanczos3)
	md := resize.Resize(w*140, h*140, img, resize.Lanczos3)
	lg := resize.Resize(w*280, h*280, img, resize.Lanczos3)

	imgPath := path.Join(n.RepoPath, "root", "images")

	out, err := os.Create(path.Join(imgPath, "tiny", "avatar"))
	if err != nil {
		return nil, err
	}
	defer out.Close()
	jpeg.Encode(out, ty, nil)

	out, err = os.Create(path.Join(imgPath, "small", "avatar"))
	if err != nil {
		return nil, err
	}
	jpeg.Encode(out, sm, nil)

	out, err = os.Create(path.Join(imgPath, "medium", "avatar"))
	if err != nil {
		return nil, err
	}
	jpeg.Encode(out, md, nil)

	out, err = os.Create(path.Join(imgPath, "large", "avatar"))
	if err != nil {
		return nil, err
	}
	jpeg.Encode(out, lg, nil)

	out, err = os.Create(path.Join(imgPath, "original", "avatar"))
	if err != nil {
		return nil, err
	}
	jpeg.Encode(out, img, nil)

	// Add hash to profile
	t, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "tiny", "avatar"))
	if aerr != nil {
		return nil, err
	}
	s, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "small", "avatar"))
	if aerr != nil {
		return nil, err
	}
	m, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "medium", "avatar"))
	if aerr != nil {
		return nil, err
	}
	l, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "large", "avatar"))
	if aerr != nil {
		return nil, err
	}
	o, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "original", "avatar"))
	if aerr != nil {
		return nil, err
	}
	profile, err := n.GetProfile()
	if err != nil {
		return nil, err
	}
	i := new(pb.Profile_Image)
	i.Tiny = t
	i.Small = s
	i.Medium = m
	i.Large = l
	i.Original = o
	profile.AvatarHashes = i
	err = n.UpdateProfile(&profile)
	if aerr != nil {
		return nil, err
	}
	return &Images{t, s, m, l, o}, nil
}

func (n *OpenBazaarNode) SetHeaderImages(base64ImageData string) (*Images, error) {
	reader := base64.NewDecoder(base64.StdEncoding, strings.NewReader(base64ImageData))
	img, _, err := image.Decode(reader)
	if err != nil {
		return nil, err
	}
	reader = base64.NewDecoder(base64.StdEncoding, strings.NewReader(base64ImageData))
	imgCfg, _, err := image.DecodeConfig(reader)
	if err != nil {
		return nil, err
	}
	imgWidth := uint(imgCfg.Width)
	imgHeight := uint(imgCfg.Height)

	w, h := getImageAttributes(304, 87, imgWidth, imgHeight)
	ty := resize.Resize(w, h, img, resize.Lanczos3)
	w, h = getImageAttributes(608, 174, imgWidth, imgHeight)
	sm := resize.Resize(w, h, img, resize.Lanczos3)
	w, h = getImageAttributes(1225, 350, imgWidth, imgHeight)
	md := resize.Resize(w, h, img, resize.Lanczos3)
	w, h = getImageAttributes(2450, 700, imgWidth, imgHeight)
	lg := resize.Resize(w, h, img, resize.Lanczos3)

	imgPath := path.Join(n.RepoPath, "root", "images")

	out, err := os.Create(path.Join(imgPath, "tiny", "header"))
	if err != nil {
		return nil, err
	}
	defer out.Close()
	jpeg.Encode(out, ty, nil)

	out, err = os.Create(path.Join(imgPath, "small", "header"))
	if err != nil {
		return nil, err
	}
	jpeg.Encode(out, sm, nil)

	out, err = os.Create(path.Join(imgPath, "medium", "header"))
	if err != nil {
		return nil, err
	}
	jpeg.Encode(out, md, nil)

	out, err = os.Create(path.Join(imgPath, "large", "header"))
	if err != nil {
		return nil, err
	}
	jpeg.Encode(out, lg, nil)

	out, err = os.Create(path.Join(imgPath, "original", "header"))
	if err != nil {
		return nil, err
	}
	jpeg.Encode(out, img, nil)

	// Add hash to profile
	t, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "tiny", "header"))
	if aerr != nil {
		return nil, err
	}
	s, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "small", "header"))
	if aerr != nil {
		return nil, err
	}
	m, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "medium", "header"))
	if aerr != nil {
		return nil, err
	}
	l, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "large", "header"))
	if aerr != nil {
		return nil, err
	}
	o, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "original", "header"))
	if aerr != nil {
		return nil, err
	}
	profile, err := n.GetProfile()
	if err != nil {
		return nil, err
	}
	i := new(pb.Profile_Image)
	i.Tiny = t
	i.Small = s
	i.Medium = m
	i.Large = l
	i.Original = o
	profile.HeaderHashes = i
	err = n.UpdateProfile(&profile)
	if aerr != nil {
		return nil, err
	}
	return &Images{t, s, m, l, o}, nil
}

func (n *OpenBazaarNode) SetProductImages(base64ImageData, filename string) (*Images, error) {
	// Decode base64 image data
	reader := base64.NewDecoder(base64.StdEncoding, strings.NewReader(base64ImageData))
	img, _, err := image.Decode(reader)
	if err != nil {
		fmt.Println(1)
		return nil, err
	}

	// Get the image config
	reader = base64.NewDecoder(base64.StdEncoding, strings.NewReader(base64ImageData))
	imgCfg, _, err := image.DecodeConfig(reader)
	if err != nil {
		return nil, err
	}
	imgWidth := uint(imgCfg.Width)
	imgHeight := uint(imgCfg.Height)

	w, h := getImageAttributes(60, 60, imgWidth, imgHeight)
	ty := resize.Resize(w, h, img, resize.Lanczos3)
	w, h = getImageAttributes(228, 228, imgWidth, imgHeight)
	sm := resize.Resize(w, h, img, resize.Lanczos3)
	w, h = getImageAttributes(500, 500, imgWidth, imgHeight)
	md := resize.Resize(w, h, img, resize.Lanczos3)
	w, h = getImageAttributes(1000, 1000, imgWidth, imgHeight)
	lg := resize.Resize(w, h, img, resize.Lanczos3)

	imgPath := path.Join(n.RepoPath, "root", "images")

	out, err := os.Create(path.Join(imgPath, "tiny", filename))
	if err != nil {
		return nil, err
	}
	defer out.Close()
	jpeg.Encode(out, ty, nil)

	out, err = os.Create(path.Join(imgPath, "small", filename))
	if err != nil {
		return nil, err
	}
	jpeg.Encode(out, sm, nil)

	out, err = os.Create(path.Join(imgPath, "medium", filename))
	if err != nil {
		return nil, err
	}
	jpeg.Encode(out, md, nil)

	out, err = os.Create(path.Join(imgPath, "large", filename))
	if err != nil {
		return nil, err
	}
	jpeg.Encode(out, lg, nil)

	out, err = os.Create(path.Join(imgPath, "original", filename))
	if err != nil {
		return nil, err
	}
	jpeg.Encode(out, img, nil)

	// Get the image hashes
	t, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "tiny", filename))
	if aerr != nil {
		return nil, err
	}
	s, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "small", filename))
	if aerr != nil {
		return nil, err
	}
	m, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "medium", filename))
	if aerr != nil {
		return nil, err
	}
	l, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "large", filename))
	if aerr != nil {
		return nil, err
	}
	o, aerr := ipfs.AddFile(n.Context, path.Join(imgPath, "original", filename))
	if aerr != nil {
		return nil, err
	}
	ret := &Images{t, s, m, l, o}
	return ret, nil
}

func getImageAttributes(targetWidth, targetHeight, imgWidth, imgHeight uint) (width, height uint) {
	targetRatio := targetWidth / targetHeight
	imageRatio := imgWidth / imgHeight
	if imageRatio > targetRatio {
		height = targetHeight
		width = targetHeight * imageRatio
	} else {
		width = targetWidth
		height = targetWidth * (imgHeight / imgWidth)
	}
	return height, width
}
<|endoftext|>


<commit_before>package tinkerforge

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
)

// ErrorCode represents the error value returned by the brick(let)s
type ErrorCode uint8

const (
	// ECOkay says no errors occured
	ECOkay ErrorCode = 0
	// ECInvalidParam says you sent an invalid parameter in the packet
	ECInvalidParam = 1
	// ECFuncNotSupported says the function number you sent is not available
	ECFuncNotSupported = 2
)

var (
	// ErrInvalidParam represents ECInvalidParam in Go
	ErrInvalidParam = errors.New("Invalid Parameter")
	// ErrFuncNotSupported represents ECFuncNotSupported in Go
	ErrFuncNotSupported = errors.New("Function is not supported")
)

// Packet holds all information about a sent or received packet
type Packet struct {
	uid       uint32
	funcID    uint8
	seqNum    uint8
	respExp   bool
	errorCode ErrorCode
	callback  bool

	payload []byte
}

// NewPacket creates a new packet to be sent to the TinkerForge daemon
func NewPacket(uid uint32, funcID uint8, respExp bool, params ...interface{}) (*Packet, error) {

	payload, err := parseParams(params)
	if err != nil {
		return nil, err
	}

	return &Packet{
		uid:       uid,
		funcID:    funcID,
		seqNum:    0,
		respExp:   respExp,
		errorCode: 0,
		callback:  false,
		payload:   payload,
	}, nil

}

func readPacket(data []byte) (*Packet, error) {

	re := bytes.NewReader(data)

	header := struct {
		UID   uint32
		Len   uint8
		Func  uint8
		Seq   uint8
		Flags uint8
	}{}

	if err := binary.Read(re, binary.LittleEndian, &header); err != nil && err.Error() != "EOF" {
		return nil, err
	}

	respExp := header.Seq&0x08 != 0
	seqNum := header.Seq >> 4
	callback := seqNum == 0
	errCode := header.Flags >> 6

	payload := make([]byte, header.Len-8)
	re.Read(payload)
	if _, err := re.Read(payload); err != nil && err.Error() != "EOF" {
		return nil, err
	}

	p := &Packet{
		uid:       header.UID,
		funcID:    header.Func,
		seqNum:    seqNum,
		respExp:   respExp,
		errorCode: ErrorCode(errCode),
		callback:  callback,

		payload: payload,
	}

	return p, nil
}

// Decode decodes the payload of a packet into a number of variables
func (p *Packet) Decode(vars ...interface{}) error {

	re := bytes.NewReader(p.payload)

	for _, v := range vars {

		if err := binary.Read(re, binary.LittleEndian, v); err != nil {
			if err.Error() == "EOF" {
				return nil
			}

			return err
		}

	}

	return nil
}

// UID returns the UID of the packet source / destination
func (p *Packet) UID() uint32 {
	return p.uid
}

// Length returns the overall length (header + payload) of the packet
func (p *Packet) Length() uint8 {
	if p.payload == nil {
		return 8
	}

	return uint8(8 + len(p.payload))
}

// FunctionID returns the function ID of the packet
func (p *Packet) FunctionID() uint8 {
	return p.funcID
}

// SequenceNum returns the 4-bit sequence number of the packet.
func (p *Packet) SequenceNum() uint8 {
	return p.seqNum
}

// ResponseExpected returns wether the caller expects an answer
func (p *Packet) ResponseExpected() bool {
	return p.respExp
}

// ErrorID returns the ID of the error (or ECOkay)
func (p *Packet) ErrorID() ErrorCode {
	return p.errorCode
}

// Error returns the corresponding error for the error ID
func (p *Packet) Error() error {
	switch p.errorCode {
	case ECInvalidParam:
		return ErrInvalidParam

	case ECFuncNotSupported:
		return ErrFuncNotSupported
	}

	return nil
}

// Callback indicates if this packet is a callback
func (p *Packet) Callback() bool {
	return p.callback
}

// Payload returns the payload of the packet
func (p *Packet) Payload() []byte {
	return p.payload
}

// Serialize converts the packet into a byte slice for sending
func (p *Packet) Serialize(wr io.Writer, seqNum byte) error {
	// Send header and payload
	if err := p.writeHeader(wr, seqNum); err != nil {
		return err
	}

	return p.writePayload(wr)
}

func parseParams(params []interface{}) ([]byte, error) {

	wr := bytes.NewBuffer(make([]byte, 0))

	for _, p := range params {

		if err := binary.Write(wr, binary.LittleEndian, p); err != nil {
			return nil, err
		}

	}

	return wr.Bytes(), nil

}

func (p *Packet) writeHeader(wr io.Writer, seqNum byte) error {

	// Header structure
	header := &struct {
		UID    uint32
		Len    uint8
		FuncID uint8
		Seq    uint8
		Flags  uint8
	}{}

	// Fill header
	header.UID = p.uid
	header.Len = p.Length()
	header.FuncID = p.funcID
	seqNum = seqNum << 4
	if p.respExp {
		seqNum |= 0x08
	}
	header.Seq = uint8(seqNum)
	header.Flags = uint8(p.errorCode << 6)

	// Send header
	return binary.Write(wr, binary.LittleEndian, header)
}

func (p *Packet) writePayload(wr io.Writer) error {
	_, err := wr.Write(p.payload)
	return err
}

// Scans a byte stream for packets and returns them as byte arrays
func scanPacket(data []byte, atEOF bool) (advance int, token []byte, err error) {
	// We are unable to read the length of the packet.
	if len(data) < 5 {
		if atEOF {
			return 0, nil, errors.New("EOF")
		}

		return 0, nil, nil
	}

	// Check the length of the data and the packet
	if len(data) >= int(data[4]) {
		fmt.Println(data[:data[4]])
		return int(data[4]), data[:data[4]], nil
	}

	// The packet is incomplete but we are at EOF
	if atEOF {
		return 0, nil, errors.New("EOF")
	}

	return 0, nil, nil
}
<commit_msg>Fixed typo in packet.go<commit_after>package tinkerforge

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
)

// ErrorCode represents the error value returned by the brick(let)s
type ErrorCode uint8

const (
	// ECOkay says no errors occurred
	ECOkay ErrorCode = 0
	// ECInvalidParam says you sent an invalid parameter in the packet
	ECInvalidParam = 1
	// ECFuncNotSupported says the function number you sent is not available
	ECFuncNotSupported = 2
)

var (
	// ErrInvalidParam represents ECInvalidParam in Go
	ErrInvalidParam = errors.New("Invalid Parameter")
	// ErrFuncNotSupported represents ECFuncNotSupported in Go
	ErrFuncNotSupported = errors.New("Function is not supported")
)

// Packet holds all information about a sent or received packet
type Packet struct {
	uid       uint32
	funcID    uint8
	seqNum    uint8
	respExp   bool
	errorCode ErrorCode
	callback  bool

	payload []byte
}

// NewPacket creates a new packet to be sent to the TinkerForge daemon
func NewPacket(uid uint32, funcID uint8, respExp bool, params ...interface{}) (*Packet, error) {

	payload, err := parseParams(params)
	if err != nil {
		return nil, err
	}

	return &Packet{
		uid:       uid,
		funcID:    funcID,
		seqNum:    0,
		respExp:   respExp,
		errorCode: 0,
		callback:  false,
		payload:   payload,
	}, nil

}

func readPacket(data []byte) (*Packet, error) {

	re := bytes.NewReader(data)

	header := struct {
		UID   uint32
		Len   uint8
		Func  uint8
		Seq   uint8
		Flags uint8
	}{}

	if err := binary.Read(re, binary.LittleEndian, &header); err != nil && err.Error() != "EOF" {
		return nil, err
	}

	respExp := header.Seq&0x08 != 0
	seqNum := header.Seq >> 4
	callback := seqNum == 0
	errCode := header.Flags >> 6

	payload := make([]byte, header.Len-8)
	re.Read(payload)
	if _, err := re.Read(payload); err != nil && err.Error() != "EOF" {
		return nil, err
	}

	p := &Packet{
		uid:       header.UID,
		funcID:    header.Func,
		seqNum:    seqNum,
		respExp:   respExp,
		errorCode: ErrorCode(errCode),
		callback:  callback,

		payload: payload,
	}

	return p, nil
}

// Decode decodes the payload of a packet into a number of variables
func (p *Packet) Decode(vars ...interface{}) error {

	re := bytes.NewReader(p.payload)

	for _, v := range vars {

		if err := binary.Read(re, binary.LittleEndian, v); err != nil {
			if err.Error() == "EOF" {
				return nil
			}

			return err
		}

	}

	return nil
}

// UID returns the UID of the packet source / destination
func (p *Packet) UID() uint32 {
	return p.uid
}

// Length returns the overall length (header + payload) of the packet
func (p *Packet) Length() uint8 {
	if p.payload == nil {
		return 8
	}

	return uint8(8 + len(p.payload))
}

// FunctionID returns the function ID of the packet
func (p *Packet) FunctionID() uint8 {
	return p.funcID
}

// SequenceNum returns the 4-bit sequence number of the packet.
func (p *Packet) SequenceNum() uint8 {
	return p.seqNum
}

// ResponseExpected returns wether the caller expects an answer
func (p *Packet) ResponseExpected() bool {
	return p.respExp
}

// ErrorID returns the ID of the error (or ECOkay)
func (p *Packet) ErrorID() ErrorCode {
	return p.errorCode
}

// Error returns the corresponding error for the error ID
func (p *Packet) Error() error {
	switch p.errorCode {
	case ECInvalidParam:
		return ErrInvalidParam

	case ECFuncNotSupported:
		return ErrFuncNotSupported
	}

	return nil
}

// Callback indicates if this packet is a callback
func (p *Packet) Callback() bool {
	return p.callback
}

// Payload returns the payload of the packet
func (p *Packet) Payload() []byte {
	return p.payload
}

// Serialize converts the packet into a byte slice for sending
func (p *Packet) Serialize(wr io.Writer, seqNum byte) error {
	// Send header and payload
	if err := p.writeHeader(wr, seqNum); err != nil {
		return err
	}

	return p.writePayload(wr)
}

func parseParams(params []interface{}) ([]byte, error) {

	wr := bytes.NewBuffer(make([]byte, 0))

	for _, p := range params {

		if err := binary.Write(wr, binary.LittleEndian, p); err != nil {
			return nil, err
		}

	}

	return wr.Bytes(), nil

}

func (p *Packet) writeHeader(wr io.Writer, seqNum byte) error {

	// Header structure
	header := &struct {
		UID    uint32
		Len    uint8
		FuncID uint8
		Seq    uint8
		Flags  uint8
	}{}

	// Fill header
	header.UID = p.uid
	header.Len = p.Length()
	header.FuncID = p.funcID
	seqNum = seqNum << 4
	if p.respExp {
		seqNum |= 0x08
	}
	header.Seq = uint8(seqNum)
	header.Flags = uint8(p.errorCode << 6)

	// Send header
	return binary.Write(wr, binary.LittleEndian, header)
}

func (p *Packet) writePayload(wr io.Writer) error {
	_, err := wr.Write(p.payload)
	return err
}

// Scans a byte stream for packets and returns them as byte arrays
func scanPacket(data []byte, atEOF bool) (advance int, token []byte, err error) {
	// We are unable to read the length of the packet.
	if len(data) < 5 {
		if atEOF {
			return 0, nil, errors.New("EOF")
		}

		return 0, nil, nil
	}

	// Check the length of the data and the packet
	if len(data) >= int(data[4]) {
		fmt.Println(data[:data[4]])
		return int(data[4]), data[:data[4]], nil
	}

	// The packet is incomplete but we are at EOF
	if atEOF {
		return 0, nil, errors.New("EOF")
	}

	return 0, nil, nil
}
<|endoftext|>


<commit_before>// Package thrift provides a Thrift client for the add service.
package thrift

import (
	"time"

	jujuratelimit "github.com/juju/ratelimit"
	"github.com/sony/gobreaker"

	"github.com/go-kit/kit/circuitbreaker"
	"github.com/go-kit/kit/endpoint"
	"github.com/go-kit/kit/examples/addsvc"
	thriftadd "github.com/go-kit/kit/examples/addsvc/thrift/gen-go/addsvc"
	"github.com/go-kit/kit/ratelimit"
)

// New returns an AddService backed by a Thrift server described by the provided
// client. The caller is responsible for constructing the client, and eventually
// closing the underlying transport.
func New(client *thriftadd.AddServiceClient) addsvc.Service {
	// We construct a single ratelimiter middleware, to limit the total outgoing
	// QPS from this client to all methods on the remote instance. We also
	// construct per-endpoint circuitbreaker middlewares to demonstrate how
	// that's done, although they could easily be combined into a single breaker
	// for the entire remote instance, too.

	limiter := ratelimit.NewTokenBucketLimiter(jujuratelimit.NewBucketWithRate(100, 100))

	// Thrift does not currently have tracer bindings, so we skip tracing.

	var sumEndpoint endpoint.Endpoint
	{
		sumEndpoint = addsvc.MakeThriftSumEndpoint(client)
		sumEndpoint = limiter(sumEndpoint)
		sumEndpoint = circuitbreaker.Gobreaker(gobreaker.NewCircuitBreaker(gobreaker.Settings{
			Name:    "Sum",
			Timeout: 30 * time.Second,
		}))(sumEndpoint)
	}

	var concatEndpoint endpoint.Endpoint
	{
		concatEndpoint = addsvc.MakeThriftConcatEndpoint(client)
		concatEndpoint = limiter(concatEndpoint)
		sumEndpoint = circuitbreaker.Gobreaker(gobreaker.NewCircuitBreaker(gobreaker.Settings{
			Name:    "Concat",
			Timeout: 30 * time.Second,
		}))(sumEndpoint)
	}

	return addsvc.Endpoints{
		SumEndpoint:    addsvc.MakeThriftSumEndpoint(client),
		ConcatEndpoint: addsvc.MakeThriftConcatEndpoint(client),
	}
}
<commit_msg>Fix small bug in addsvc Thrift client<commit_after>// Package thrift provides a Thrift client for the add service.
package thrift

import (
	"time"

	jujuratelimit "github.com/juju/ratelimit"
	"github.com/sony/gobreaker"

	"github.com/go-kit/kit/circuitbreaker"
	"github.com/go-kit/kit/endpoint"
	"github.com/go-kit/kit/examples/addsvc"
	thriftadd "github.com/go-kit/kit/examples/addsvc/thrift/gen-go/addsvc"
	"github.com/go-kit/kit/ratelimit"
)

// New returns an AddService backed by a Thrift server described by the provided
// client. The caller is responsible for constructing the client, and eventually
// closing the underlying transport.
func New(client *thriftadd.AddServiceClient) addsvc.Service {
	// We construct a single ratelimiter middleware, to limit the total outgoing
	// QPS from this client to all methods on the remote instance. We also
	// construct per-endpoint circuitbreaker middlewares to demonstrate how
	// that's done, although they could easily be combined into a single breaker
	// for the entire remote instance, too.

	limiter := ratelimit.NewTokenBucketLimiter(jujuratelimit.NewBucketWithRate(100, 100))

	// Thrift does not currently have tracer bindings, so we skip tracing.

	var sumEndpoint endpoint.Endpoint
	{
		sumEndpoint = addsvc.MakeThriftSumEndpoint(client)
		sumEndpoint = limiter(sumEndpoint)
		sumEndpoint = circuitbreaker.Gobreaker(gobreaker.NewCircuitBreaker(gobreaker.Settings{
			Name:    "Sum",
			Timeout: 30 * time.Second,
		}))(sumEndpoint)
	}

	var concatEndpoint endpoint.Endpoint
	{
		concatEndpoint = addsvc.MakeThriftConcatEndpoint(client)
		concatEndpoint = limiter(concatEndpoint)
		sumEndpoint = circuitbreaker.Gobreaker(gobreaker.NewCircuitBreaker(gobreaker.Settings{
			Name:    "Concat",
			Timeout: 30 * time.Second,
		}))(sumEndpoint)
	}

	return addsvc.Endpoints{
		SumEndpoint:    sumEndpoint,
		ConcatEndpoint: concatEndpoint,
	}
}
<|endoftext|>


<commit_before>package kubo_deployment_tests_test

import (
	"fmt"

	. "github.com/jhvhs/gob-mock"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"path"
)

var _ = Describe("DeployUtils", func() {
	Describe("set_cloud_config", func() {
		It("Should generate cloud config", func() {
			bash.Source(pathToScript("lib/deploy_utils"), nil)

			bash.Export("BOSH_ENV", "kubo-env")
			bash.Source("", func(string) ([]byte, error) {
				return []byte(fmt.Sprintf("export PATH=%s:$PATH", pathFromRoot("bin"))), nil
			})

			generateCloudConfigMock := Mock("generate_cloud_config", `echo -n "cc"`)
			boshCliMock := Mock("bosh-cli", `echo -n "$@"`)
			ApplyMocks(bash, []Gob{generateCloudConfigMock, boshCliMock})

			code, err := bash.Run("set_cloud_config", []string{})
			Expect(err).NotTo(HaveOccurred())
			Expect(code).To(Equal(0))
			Expect(stderr).To(gbytes.Say("generate_cloud_config kubo-env"))
		})

		It("Should update cloud config", func() {
			bash.Source(pathToScript("lib/deploy_utils"), nil)

			bash.Export("BOSH_ENV", "kubo-env")
			bash.Export("BOSH_NAME", "env-name")
			bash.Source("", func(string) ([]byte, error) {
				return []byte(fmt.Sprintf("export PATH=%s:$PATH", pathFromRoot("bin"))), nil
			})

			generateCloudConfigMock := Mock("generate_cloud_config", `echo -n "cc"`)
			boshCliMock := Mock("bosh-cli", `echo -n "$@"`)
			ApplyMocks(bash, []Gob{generateCloudConfigMock, boshCliMock})

			code, err := bash.Run("set_cloud_config", []string{})
			Expect(err).NotTo(HaveOccurred())
			Expect(code).To(Equal(0))
			Expect(stderr).To(gbytes.Say("bosh-cli -n -e env-name update-cloud-config -"))
		})
	})

	Describe("export_bosh_environment", func() {
		It("should set BOSH_ENV and BOSH_NAME", func() {
			bash.Source(pathToScript("lib/deploy_utils"), nil)
			code, err := bash.Run("export_bosh_environment /envs/foo && echo $BOSH_ENV && echo $BOSH_NAME", []string{})
			Expect(err).NotTo(HaveOccurred())
			Expect(code).To(Equal(0))
			Expect(stdout).To(gbytes.Say("/envs/foo"))
			Expect(stdout).To(gbytes.Say("foo"))
		})
	})

	Describe("deploy_to_bosh", func() {
		It("should deploy to bosh", func() {
			bash.Source(pathToScript("lib/deploy_utils"), nil)

			getBoshSecretMock := Mock("get_bosh_secret", `echo "the-secret"`)
			boshCliMock := Mock("bosh-cli", `echo -n "$@"`)
			ApplyMocks(bash, []Gob{ getBoshSecretMock, boshCliMock })

			bash.Export("BOSH_ENV", "kubo-env")
			bash.Export("BOSH_NAME", "env-name")

			code, err := bash.Run("deploy_to_bosh", []string{"manifest", "deployment-name"})
			Expect(err).NotTo(HaveOccurred())
			Expect(code).To(Equal(0))
			Expect(stderr).To(gbytes.Say("bosh-cli -d deployment-name -n deploy --no-redact -"))
		})
	})

	Describe("get_bosh_secret", func() {
		It("should get bosh_admin_client_secret setting", func() {
			bash.Source(pathToScript("lib/deploy_utils"), nil)

			getSettingMock := Mock("get_setting", `echo "the-secret"`)
			ApplyMocks(bash, []Gob{ getSettingMock })

			code, err := bash.Run("get_bosh_secret", []string{})
			Expect(err).NotTo(HaveOccurred())
			Expect(code).To(Equal(0))
			Expect(stdout).To(gbytes.Say("the-secret"))
		})
	})

	Describe("get_setting", func() {
		It("should call bosh interpolate", func() {
			bash.Source(pathToScript("lib/deploy_utils"), nil)

			boshCliMock := Mock("bosh-cli", `echo -n "$@"`)
			ApplyMocks(bash, []Gob{ boshCliMock })

			bash.Export("BOSH_ENV", "kubo-env")

			code, err := bash.Run("get_setting", []string{"fileToQuery.yml", "path/subpath"})
			Expect(err).NotTo(HaveOccurred())
			Expect(code).To(Equal(0))
			Expect(stderr).To(gbytes.Say("bosh-cli int kubo-env/fileToQuery.yml --path path/subpath"))
		})

		It("should return the setting value", func() {
			bash.Source(pathToScript("lib/deploy_utils"), nil)

			boshCliMock := Mock("bosh-cli", `echo "value-at-path"`)
			ApplyMocks(bash, []Gob{ boshCliMock })

			code, err := bash.Run("get_setting", []string{"fileToQuery.yml", "path/subpath"})
			Expect(err).NotTo(HaveOccurred())
			Expect(code).To(Equal(0))
			Expect(stdout).To(gbytes.Say("value-at-path"))
		})
	})

	Describe("create_and_upload_release", func(){
		Context("is called with a valid directory path", func(){
			It("should create a release and upload it", func(){
				bash.Source(pathToScript("lib/deploy_utils"), nil)

				getBoshSecretMock := Mock("get_bosh_secret", `echo "the-secret"`)
				boshCliMock := Mock("bosh-cli", `echo -n "$@"`)
				uploadReleaseMock := Mock("upload_release", `echo`)
				ApplyMocks(bash, []Gob{ getBoshSecretMock, boshCliMock, uploadReleaseMock})

				releasePath := path.Join(resourcesPath, "releases/mock-release")

				code, err := bash.Run("create_and_upload_release", []string{releasePath})
				Expect(err).NotTo(HaveOccurred())
				Expect(code).To(Equal(0))
				Expect(stderr).To(gbytes.Say("bosh-cli create-release --force --name mock"))
				Expect(stderr).To(gbytes.Say("upload_release --name=mock"))
			})
		})

		Context("is called with an invalid argument", func(){
			It("should exit", func(){
				bash.Source(pathToScript("lib/deploy_utils"), nil)
				code, err := bash.Run("create_and_upload_release", []string{"path_does_not_exist"})
				Expect(err).NotTo(HaveOccurred())
				Expect(code).To(Equal(1))
			})
		})
	})

	Describe("upload_release", func(){
		It("should upload release", func() {
			bash.Source(pathToScript("lib/deploy_utils"), nil)

			getBoshSecretMock := Mock("get_bosh_secret", `echo "the-secret"`)
			boshCliMock := Mock("bosh-cli", `echo -n "$@"`)
			ApplyMocks(bash, []Gob{getBoshSecretMock, boshCliMock})

			bash.Export("BOSH_ENV", "kubo-env")
			bash.Export("BOSH_NAME", "env-name")

			code, err := bash.Run("upload_release", []string{"release-name"})
			Expect(err).NotTo(HaveOccurred())
			Expect(code).To(Equal(0))
			Expect(stderr).To(gbytes.Say("bosh-cli upload-release release-name"))
		})
	})
})
<commit_msg>Test for generate_manifest with default/minimum options<commit_after>package kubo_deployment_tests_test

import (
	"fmt"

	. "github.com/jhvhs/gob-mock"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"path"
)

var _ = Describe("DeployUtils", func() {
	Describe("set_cloud_config", func() {
		It("Should generate cloud config", func() {
			bash.Source(pathToScript("lib/deploy_utils"), nil)

			bash.Export("BOSH_ENV", "kubo-env")
			bash.Source("", func(string) ([]byte, error) {
				return []byte(fmt.Sprintf("export PATH=%s:$PATH", pathFromRoot("bin"))), nil
			})

			generateCloudConfigMock := Mock("generate_cloud_config", `echo -n "cc"`)
			boshCliMock := Mock("bosh-cli", `echo -n "$@"`)
			ApplyMocks(bash, []Gob{generateCloudConfigMock, boshCliMock})

			code, err := bash.Run("set_cloud_config", []string{})
			Expect(err).NotTo(HaveOccurred())
			Expect(code).To(Equal(0))
			Expect(stderr).To(gbytes.Say("generate_cloud_config kubo-env"))
		})

		It("Should update cloud config", func() {
			bash.Source(pathToScript("lib/deploy_utils"), nil)

			bash.Export("BOSH_ENV", "kubo-env")
			bash.Export("BOSH_NAME", "env-name")
			bash.Source("", func(string) ([]byte, error) {
				return []byte(fmt.Sprintf("export PATH=%s:$PATH", pathFromRoot("bin"))), nil
			})

			generateCloudConfigMock := Mock("generate_cloud_config", `echo -n "cc"`)
			boshCliMock := Mock("bosh-cli", `echo -n "$@"`)
			ApplyMocks(bash, []Gob{generateCloudConfigMock, boshCliMock})

			code, err := bash.Run("set_cloud_config", []string{})
			Expect(err).NotTo(HaveOccurred())
			Expect(code).To(Equal(0))
			Expect(stderr).To(gbytes.Say("bosh-cli -n -e env-name update-cloud-config -"))
		})
	})

	Describe("export_bosh_environment", func() {
		It("should set BOSH_ENV and BOSH_NAME", func() {
			bash.Source(pathToScript("lib/deploy_utils"), nil)
			code, err := bash.Run("export_bosh_environment /envs/foo && echo $BOSH_ENV && echo $BOSH_NAME", []string{})
			Expect(err).NotTo(HaveOccurred())
			Expect(code).To(Equal(0))
			Expect(stdout).To(gbytes.Say("/envs/foo"))
			Expect(stdout).To(gbytes.Say("foo"))
		})
	})

	Describe("deploy_to_bosh", func() {
		It("should deploy to bosh", func() {
			bash.Source(pathToScript("lib/deploy_utils"), nil)

			getBoshSecretMock := Mock("get_bosh_secret", `echo "the-secret"`)
			boshCliMock := Mock("bosh-cli", `echo -n "$@"`)
			ApplyMocks(bash, []Gob{ getBoshSecretMock, boshCliMock })

			bash.Export("BOSH_ENV", "kubo-env")
			bash.Export("BOSH_NAME", "env-name")

			code, err := bash.Run("deploy_to_bosh", []string{"manifest", "deployment-name"})
			Expect(err).NotTo(HaveOccurred())
			Expect(code).To(Equal(0))
			Expect(stderr).To(gbytes.Say("bosh-cli -d deployment-name -n deploy --no-redact -"))
		})
	})

	Describe("get_bosh_secret", func() {
		It("should get bosh_admin_client_secret setting", func() {
			bash.Source(pathToScript("lib/deploy_utils"), nil)

			getSettingMock := Mock("get_setting", `echo "the-secret"`)
			ApplyMocks(bash, []Gob{ getSettingMock })

			code, err := bash.Run("get_bosh_secret", []string{})
			Expect(err).NotTo(HaveOccurred())
			Expect(code).To(Equal(0))
			Expect(stdout).To(gbytes.Say("the-secret"))
		})
	})

	Describe("get_setting", func() {
		It("should call bosh interpolate", func() {
			bash.Source(pathToScript("lib/deploy_utils"), nil)

			boshCliMock := Mock("bosh-cli", `echo -n "$@"`)
			ApplyMocks(bash, []Gob{ boshCliMock })

			bash.Export("BOSH_ENV", "kubo-env")

			code, err := bash.Run("get_setting", []string{"fileToQuery.yml", "path/subpath"})
			Expect(err).NotTo(HaveOccurred())
			Expect(code).To(Equal(0))
			Expect(stderr).To(gbytes.Say("bosh-cli int kubo-env/fileToQuery.yml --path path/subpath"))
		})

		It("should return the setting value", func() {
			bash.Source(pathToScript("lib/deploy_utils"), nil)

			boshCliMock := Mock("bosh-cli", `echo "value-at-path"`)
			ApplyMocks(bash, []Gob{ boshCliMock })

			code, err := bash.Run("get_setting", []string{"fileToQuery.yml", "path/subpath"})
			Expect(err).NotTo(HaveOccurred())
			Expect(code).To(Equal(0))
			Expect(stdout).To(gbytes.Say("value-at-path"))
		})
	})

	Describe("create_and_upload_release", func(){
		Context("is called with a valid directory path", func(){
			It("should create a release and upload it", func(){
				bash.Source(pathToScript("lib/deploy_utils"), nil)

				getBoshSecretMock := Mock("get_bosh_secret", `echo "the-secret"`)
				boshCliMock := Mock("bosh-cli", `echo -n "$@"`)
				uploadReleaseMock := Mock("upload_release", `echo`)
				ApplyMocks(bash, []Gob{ getBoshSecretMock, boshCliMock, uploadReleaseMock})

				releasePath := path.Join(resourcesPath, "releases/mock-release")

				code, err := bash.Run("create_and_upload_release", []string{releasePath})
				Expect(err).NotTo(HaveOccurred())
				Expect(code).To(Equal(0))
				Expect(stderr).To(gbytes.Say("bosh-cli create-release --force --name mock"))
				Expect(stderr).To(gbytes.Say("upload_release --name=mock"))
			})
		})

		Context("is called with an invalid argument", func(){
			It("should exit", func(){
				bash.Source(pathToScript("lib/deploy_utils"), nil)
				code, err := bash.Run("create_and_upload_release", []string{"path_does_not_exist"})
				Expect(err).NotTo(HaveOccurred())
				Expect(code).To(Equal(1))
			})
		})
	})

	Describe("upload_release", func(){
		It("should upload release", func() {
			bash.Source(pathToScript("lib/deploy_utils"), nil)

			getBoshSecretMock := Mock("get_bosh_secret", `echo "the-secret"`)
			boshCliMock := Mock("bosh-cli", `echo -n "$@"`)
			ApplyMocks(bash, []Gob{getBoshSecretMock, boshCliMock})

			bash.Export("BOSH_ENV", "kubo-env")
			bash.Export("BOSH_NAME", "env-name")

			code, err := bash.Run("upload_release", []string{"release-name"})
			Expect(err).NotTo(HaveOccurred())
			Expect(code).To(Equal(0))
			Expect(stderr).To(gbytes.Say("bosh-cli upload-release release-name"))
		})
	})

	Describe("generate_manifest", func(){
		Context("default manifest", func() {
			Context("without creds.yml or director-secrets.yml", func() {
				It("applies dev, bootstrap and use-runtime-config-bosh-dns", func() {
					bash.Source(pathToScript("lib/deploy_utils"), nil)

					directorMock := Mock("bosh-cli", `
					if [[ "$3" =~ "addons_spec_path" \
						|| "$3" =~ "http_proxy" \
						|| "$3" =~ "https_proxy" \
						|| "$3" =~ "no_proxy" ]]; then
						return 1
					elif [[ "$3" =~ "routing_mode" ]]; then
						echo "the-routing-mode"
					elif [[ "$3" =~ "iaas" ]]; then
						echo "the-iaas"
					else
						echo
					fi`)
					ApplyMocks(bash, []Gob{directorMock})

					code, err := bash.Run("generate_manifest", []string{"environment-path", "deployment-name", "non-existent-manifest-path", "director-uuid"})
					Expect(err).NotTo(HaveOccurred())
					Expect(code).To(Equal(0))
					Expect(stderr).To(gbytes.Say("routing_mode"))
					Expect(stderr).To(gbytes.Say("iaas"))
					Expect(stderr).To(gbytes.Say("misc/dev.yml"))
					Expect(stderr).To(gbytes.Say("misc/bootstrap.yml"))
					Expect(stderr).To(gbytes.Say("use-runtime-config-bosh-dns.yml"))
					Expect(stderr).To(gbytes.Say("--var=deployment_name=\"deployment-name\""))
				})
			})
		})
	})
})
<|endoftext|>


<commit_before>var juju_config = {
    "baseUrl": "{{.base}}",
    "staticURL": "{{.staticURL}}",
    "jujuCoreVersion": "{{.version}}",
    "jujuEnvUUID": "{{.uuid}}",
    "apiAddress": "wss://{{.host}}",
    "controllerSocketTemplate": "{{.controllerSocket}}",
    "socketTemplate": "{{.socket}}",
    "socket_protocol": "wss",
    "charmstoreAPIPath": "v4",
    "charmstoreURL": "https://api.jujucharms.com/charmstore/",
    "bundleServiceURL": "https://api.jujucharms.com/bundleservice/",
    "plansURL": "https://api.jujucharms.com/omnibus/",
    "registerURL": "https://api.jujucharms.com/payment/",
    "termsURL": "https://api.jujucharms.com/terms/",
    "interactiveLogin": true,
    "html5": true,
    "container": "#main",
    "viewContainer": "#main",
    "consoleEnabled": true,
    "serverRouting": false,
};
<commit_msg>Missed rename.<commit_after>var juju_config = {
    "baseUrl": "{{.base}}",
    "staticURL": "{{.staticURL}}",
    "jujuCoreVersion": "{{.version}}",
    "jujuEnvUUID": "{{.uuid}}",
    "apiAddress": "wss://{{.host}}",
    "controllerSocketTemplate": "{{.controllerSocket}}",
    "socketTemplate": "{{.socket}}",
    "socket_protocol": "wss",
    "charmstoreAPIPath": "v4",
    "charmstoreURL": "https://api.jujucharms.com/charmstore/",
    "bundleServiceURL": "https://api.jujucharms.com/bundleservice/",
    "plansURL": "https://api.jujucharms.com/omnibus/",
    "paymentURL": "https://api.jujucharms.com/payment/",
    "termsURL": "https://api.jujucharms.com/terms/",
    "interactiveLogin": true,
    "html5": true,
    "container": "#main",
    "viewContainer": "#main",
    "consoleEnabled": true,
    "serverRouting": false,
};
<|endoftext|>


<commit_before>// Copyright 2014-2015 The Dename Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
// 	http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.

package raftlog

import (
	"encoding/binary"
	"fmt"
	"log"
	"math"
	"sync"
	"time"

	"github.com/andres-erbsen/clock"
	"github.com/coreos/etcd/raft"
	"github.com/coreos/etcd/raft/raftpb"

	"github.com/yahoo/coname/server/kv"
	"github.com/yahoo/coname/server/replication"
	"golang.org/x/net/context"
)

const (
	HARDSTATE_KEY    = "HS"
	CONFSTATE_KEY    = "CS"
	ENTRY_KEY_PREFIX = "E"
	COMMITTED_BUFFER = 10 // It's fine to let commit run asynchronously ahead of apply
)

type raftLog struct {
	config       raft.Config
	initialNodes []raft.Peer
	storage      *raftStorage
	node         raft.Node

	clk          clock.Clock
	tickInterval time.Duration

	waitCommitted chan replication.LogEntry

	leaderHintSet chan bool
	leaderHint    bool

	stopOnce sync.Once
	stop     chan struct{}
	stopped  chan struct{}
}

var _ replication.LogReplicator = (*raftLog)(nil)

// New initializes a replication.LogReplicator using an already open kv.DB.
// TODO: config.Applied and config.Storage are useless for the caller, and
// initialNodes and tickInterval are included; may want our own config struct
func New(
	db kv.DB, prefix []byte, config *raft.Config, initialNodes []raft.Peer,
	clk clock.Clock, tickInterval time.Duration,
) (replication.LogReplicator, error) {
	confState := raftpb.ConfState{}
	for _, node := range initialNodes {
		confState.Nodes = append(confState.Nodes, node.ID)
	}
	return &raftLog{
		config:        *config,
		initialNodes:  initialNodes,
		storage:       openRaftStorage(db, prefix, confState),
		node:          nil,
		clk:           clk,
		tickInterval:  tickInterval,
		leaderHintSet: make(chan bool, COMMITTED_BUFFER),
		waitCommitted: make(chan replication.LogEntry, COMMITTED_BUFFER),
		stop:          make(chan struct{}),
		stopped:       make(chan struct{}),
	}, nil
}

// Start implements replication.LogReplicator
func (l *raftLog) Start(lo uint64) error {
	l.config.Storage = l.storage
	inited, err := l.storage.IsInitialized()
	if err != nil {
		return err
	}
	if inited {
		l.config.Applied = lo
		l.node = raft.RestartNode(&l.config)
	} else {
		if lo != 0 {
			panic(fmt.Errorf("storage uninitialized but state machine not fresh: lo = %d", lo))
		}
		l.node = raft.StartNode(&l.config, l.initialNodes)
	}
	go l.run()
	return nil
}

// Stop implements replication.LogReplicator
func (l *raftLog) Stop() error {
	l.stopOnce.Do(func() {
		close(l.stop)
		<-l.stopped
	})
	return nil
}

// Propose implements replication.LogReplicator
func (l *raftLog) Propose(ctx context.Context, data []byte) {
	l.node.Propose(ctx, data)
}

// Propose implements replication.LogReplicator
func (l *raftLog) ProposeConfChange(ctx context.Context, change []byte) {
	panic("raftLog.ProposeConfChange not implemented")
}

// WaitCommitted implements replication.LogReplicator
func (l *raftLog) WaitCommitted() <-chan replication.LogEntry {
	return l.waitCommitted
}

// LeaderHintSet implements replication.LogReplicator
func (l *raftLog) LeaderHintSet() <-chan bool {
	return l.leaderHintSet
}

// GetCommitted implements replication.LogReplicator
func (l *raftLog) GetCommitted(lo, hi, maxSize uint64) (ret []replication.LogEntry, err error) {
	var entries []raftpb.Entry
	entries, err = l.storage.Entries(lo, hi, maxSize)
	if err != nil {
		return
	}
	for _, entry := range entries {
		ret = append(ret, replication.LogEntry{Data: entry.Data})
	}
	return
}

// run is the CSP-style main of raftLog; all local struct fields (except
// channels) belong exclusively to run while it is running. Method invocations
// are signaled through channels.
func (l *raftLog) run() {
	defer close(l.waitCommitted)
	defer close(l.stopped)
	defer close(l.leaderHintSet)
	ticker := l.clk.Ticker(l.tickInterval)
	for {
		select {
		case <-l.stop:
			return
		case <-ticker.C:
			l.node.Tick()
		case rd := <-l.node.Ready():
			if !raft.IsEmptySnap(rd.Snapshot) {
				log.Panicf("snapshots not supported")
			}
			l.storage.save(rd.HardState, rd.Entries)
			// TODO send(rd.Messages)
			for _, entry := range rd.CommittedEntries {
				switch entry.Type {
				case raftpb.EntryConfChange:
					var cc raftpb.ConfChange
					cc.Unmarshal(entry.Data)
					l.node.ApplyConfChange(cc)
					l.waitCommitted <- replication.LogEntry{Reconfiguration: entry.Data}
				default:
					l.waitCommitted <- replication.LogEntry{Data: entry.Data}
				}
				l.node.Advance() // let Raft proceed
			}
			leaderHint := rd.SoftState.RaftState == raft.StateLeader
			if l.leaderHint != leaderHint {
				l.leaderHint = leaderHint
				l.leaderHintSet <- leaderHint
			}
		}
	}
}

// Needs to be threadsafe; right now, carries no in-memory mutable state
type raftStorage struct {
	hardStateKey   []byte
	confStateKey   []byte
	entryKeyPrefix []byte
	db             kv.DB
	initialConf    raftpb.ConfState
}

var _ raft.Storage = (*raftStorage)(nil)

func openRaftStorage(db kv.DB, prefix []byte, initialConf raftpb.ConfState) *raftStorage {
	return &raftStorage{
		hardStateKey:   append(append([]byte{}, prefix...), HARDSTATE_KEY...),
		confStateKey:   append(append([]byte{}, prefix...), CONFSTATE_KEY...),
		entryKeyPrefix: append(append([]byte{}, prefix...), ENTRY_KEY_PREFIX...),
		db:             db,
		initialConf:    initialConf,
	}
}

// Returns whether an existing state has been persisted to the storage
func (s *raftStorage) IsInitialized() (bool, error) {
	switch _, err := s.db.Get(s.hardStateKey); err {
	case s.db.ErrNotFound():
		return false, nil
	case nil:
		return true, nil
	default:
		return false, err
	}
}

// InitialState implements the raft.Storage interface
func (s *raftStorage) InitialState() (hardState raftpb.HardState, confState raftpb.ConfState, err error) {
	// Restore the confState if we can, otherwise use s.initialConf
	var confStateBytes []byte
	confStateBytes, err = s.db.Get(s.confStateKey)
	if err == s.db.ErrNotFound() {
		confState = s.initialConf
	} else if err != nil {
		return
	}
	err = confState.Unmarshal(confStateBytes)
	if err != nil {
		return
	}
	var hardStateBytes []byte
	hardStateBytes, err = s.db.Get(s.hardStateKey)
	if err == s.db.ErrNotFound() {
		err = nil
		return
	} else if err != nil {
		return
	}
	err = hardState.Unmarshal(hardStateBytes)
	if err != nil {
		return
	}
	return
}

func (s *raftStorage) getEntryKey(nr uint64) (key []byte) {
	key = make([]byte, len(s.entryKeyPrefix)+8)
	copy(key, s.entryKeyPrefix)
	binary.BigEndian.PutUint64(key[len(s.entryKeyPrefix):], nr)
	return
}

// Entries implements the raft.Storage interface
func (s *raftStorage) Entries(lo, hi, maxSize uint64) (entries []raftpb.Entry, err error) {
	it := s.db.NewIterator(&kv.Range{s.getEntryKey(lo), s.getEntryKey(hi)})
	defer it.Release()
	entries = make([]raftpb.Entry, 0)
	sizeSoFar := uint64(0)
	for ok := it.First(); ok; ok = it.Next() {
		var entry raftpb.Entry
		err = entry.Unmarshal(it.Value())
		if err != nil {
			return
		}
		sizeSoFar += uint64(entry.Size())
		// Only stop if we already have at least one entry
		if sizeSoFar > maxSize && len(entries) > 0 {
			break
		}
		entries = append(entries, entry)
		if sizeSoFar >= maxSize {
			break
		}
	}
	err = it.Error()
	return
}

// Term implements the raft.Storage interface
func (s *raftStorage) Term(i uint64) (uint64, error) {
	entries, err := s.Entries(i, i+1, math.MaxUint64)
	if err != nil {
		return 0, err
	}
	if len(entries) != 1 {
		log.Panicf("number of entries with index %d not 1: %d", i, len(entries))
	}
	return entries[0].Term, nil
}

// LastIndex implements the raft.Storage interface
func (s *raftStorage) LastIndex() (uint64, error) {
	it := s.db.NewIterator(kv.BytesPrefix(s.entryKeyPrefix))
	defer it.Release()
	if !it.Last() {
		return 0, it.Error()
	}
	indexPortion := it.Key()[len(s.entryKeyPrefix):]
	return binary.BigEndian.Uint64(indexPortion), it.Error()
}

// FirstIndex implements the raft.Storage interface
func (s *raftStorage) FirstIndex() (uint64, error) {
	// Start at index 1 to be consistent with etcd/raft's MemoryStorage
	// (not sure if this is actually necessary)
	return 1, nil
}

// Snapshot implements the raft.Storage interface
func (s *raftStorage) Snapshot() (raftpb.Snapshot, error) {
	return raftpb.Snapshot{}, nil
}

// Don't call this multiple times concurrently
func (s *raftStorage) save(state raftpb.HardState, entries []raftpb.Entry) error {
	wb := s.db.NewBatch()
	stateBytes, err := state.Marshal()
	if err != nil {
		return err
	}
	wb.Put(s.hardStateKey, stateBytes)
	if len(entries) > 0 {
		lastIndex, err := s.LastIndex()
		if err != nil {
			return err
		}
		if entries[0].Index > lastIndex+1 {
			panic(fmt.Errorf("missing log entries [last: %d, append at: %d]", lastIndex, entries[0].Index))
		}
		// clear all old entries past the new index, if any
		for ix := entries[0].Index; ix <= lastIndex; ix++ {
			wb.Delete(s.getEntryKey(ix))
		}
		// append the new entries
		for _, entry := range entries {
			entryBytes, err := entry.Marshal()
			if err != nil {
				return err
			}
			wb.Put(s.getEntryKey(entry.Index), entryBytes)
		}
	}
	err = s.db.Write(wb)
	return err
}
<commit_msg>raftlog: implement RaftServer and add static cast check<commit_after>// Copyright 2014-2015 The Dename Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
// 	http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.

package raftlog

import (
	"encoding/binary"
	"fmt"
	"log"
	"math"
	"sync"
	"time"

	"github.com/andres-erbsen/clock"
	"github.com/coreos/etcd/raft"
	"github.com/coreos/etcd/raft/raftpb"

	"github.com/yahoo/coname/server/kv"
	"github.com/yahoo/coname/server/replication"
	"github.com/yahoo/coname/server/replication/raftlog/proto"
	"golang.org/x/net/context"
)

const (
	HARDSTATE_KEY    = "HS"
	CONFSTATE_KEY    = "CS"
	ENTRY_KEY_PREFIX = "E"
	COMMITTED_BUFFER = 10 // It's fine to let commit run asynchronously ahead of apply
)

type raftLog struct {
	config       raft.Config
	initialNodes []raft.Peer
	storage      *raftStorage
	node         raft.Node

	clk          clock.Clock
	tickInterval time.Duration

	waitCommitted chan replication.LogEntry

	leaderHintSet chan bool
	leaderHint    bool

	currentPeers map[uint64]proto.RaftClient

	stopOnce sync.Once
	stop     chan struct{}
	stopped  chan struct{}
}

var _ replication.LogReplicator = (*raftLog)(nil)
var _ proto.RaftServer = (*raftLog)(nil)

func (l *raftLog) Step(ctx context.Context, msg *raftpb.Message) (*proto.Nothing, error) {
	return &proto.Nothing{}, l.node.Step(ctx, *msg)
}

// New initializes a replication.LogReplicator using an already open kv.DB.
// TODO: config.Applied and config.Storage are useless for the caller, and
// initialNodes and tickInterval are included; may want our own config struct
func New(
	db kv.DB, prefix []byte, config *raft.Config, initialNodes []raft.Peer,
	clk clock.Clock, tickInterval time.Duration,
) (replication.LogReplicator, error) {
	confState := raftpb.ConfState{}
	for _, node := range initialNodes {
		confState.Nodes = append(confState.Nodes, node.ID)
	}
	return &raftLog{
		config:        *config,
		initialNodes:  initialNodes,
		storage:       openRaftStorage(db, prefix, confState),
		node:          nil,
		clk:           clk,
		tickInterval:  tickInterval,
		leaderHintSet: make(chan bool, COMMITTED_BUFFER),
		waitCommitted: make(chan replication.LogEntry, COMMITTED_BUFFER),
		stop:          make(chan struct{}),
		stopped:       make(chan struct{}),
	}, nil
}

// Start implements replication.LogReplicator
func (l *raftLog) Start(lo uint64) error {
	l.config.Storage = l.storage
	inited, err := l.storage.IsInitialized()
	if err != nil {
		return err
	}
	if inited {
		l.config.Applied = lo
		l.node = raft.RestartNode(&l.config)
	} else {
		if lo != 0 {
			panic(fmt.Errorf("storage uninitialized but state machine not fresh: lo = %d", lo))
		}
		l.node = raft.StartNode(&l.config, l.initialNodes)
	}
	go l.run()
	return nil
}

// Stop implements replication.LogReplicator
func (l *raftLog) Stop() error {
	l.stopOnce.Do(func() {
		close(l.stop)
		<-l.stopped
	})
	return nil
}

// Propose implements replication.LogReplicator
func (l *raftLog) Propose(ctx context.Context, data []byte) {
	l.node.Propose(ctx, data)
}

// Propose implements replication.LogReplicator
func (l *raftLog) ProposeConfChange(ctx context.Context, change []byte) {
	panic("raftLog.ProposeConfChange not implemented")
}

// WaitCommitted implements replication.LogReplicator
func (l *raftLog) WaitCommitted() <-chan replication.LogEntry {
	return l.waitCommitted
}

// LeaderHintSet implements replication.LogReplicator
func (l *raftLog) LeaderHintSet() <-chan bool {
	return l.leaderHintSet
}

// GetCommitted implements replication.LogReplicator
func (l *raftLog) GetCommitted(lo, hi, maxSize uint64) (ret []replication.LogEntry, err error) {
	var entries []raftpb.Entry
	entries, err = l.storage.Entries(lo, hi, maxSize)
	if err != nil {
		return
	}
	for _, entry := range entries {
		ret = append(ret, replication.LogEntry{Data: entry.Data})
	}
	return
}

// run is the CSP-style main of raftLog; all local struct fields (except
// channels) belong exclusively to run while it is running. Method invocations
// are signaled through channels.
func (l *raftLog) run() {
	defer close(l.waitCommitted)
	defer close(l.stopped)
	defer close(l.leaderHintSet)
	ticker := l.clk.Ticker(l.tickInterval)
	for {
		select {
		case <-l.stop:
			return
		case <-ticker.C:
			l.node.Tick()
		case rd := <-l.node.Ready():
			if !raft.IsEmptySnap(rd.Snapshot) {
				log.Panicf("snapshots not supported")
			}
			l.storage.save(rd.HardState, rd.Entries)
			// TODO send(rd.Messages)
			for _, entry := range rd.CommittedEntries {
				switch entry.Type {
				case raftpb.EntryConfChange:
					var cc raftpb.ConfChange
					cc.Unmarshal(entry.Data)
					l.node.ApplyConfChange(cc)
					l.waitCommitted <- replication.LogEntry{Reconfiguration: entry.Data}
				default:
					l.waitCommitted <- replication.LogEntry{Data: entry.Data}
				}
				l.node.Advance() // let Raft proceed
			}
			leaderHint := rd.SoftState.RaftState == raft.StateLeader
			if l.leaderHint != leaderHint {
				l.leaderHint = leaderHint
				l.leaderHintSet <- leaderHint
			}
		}
	}
}

// Needs to be threadsafe; right now, carries no in-memory mutable state
type raftStorage struct {
	hardStateKey   []byte
	confStateKey   []byte
	entryKeyPrefix []byte
	db             kv.DB
	initialConf    raftpb.ConfState
}

var _ raft.Storage = (*raftStorage)(nil)

func openRaftStorage(db kv.DB, prefix []byte, initialConf raftpb.ConfState) *raftStorage {
	return &raftStorage{
		hardStateKey:   append(append([]byte{}, prefix...), HARDSTATE_KEY...),
		confStateKey:   append(append([]byte{}, prefix...), CONFSTATE_KEY...),
		entryKeyPrefix: append(append([]byte{}, prefix...), ENTRY_KEY_PREFIX...),
		db:             db,
		initialConf:    initialConf,
	}
}

// Returns whether an existing state has been persisted to the storage
func (s *raftStorage) IsInitialized() (bool, error) {
	switch _, err := s.db.Get(s.hardStateKey); err {
	case s.db.ErrNotFound():
		return false, nil
	case nil:
		return true, nil
	default:
		return false, err
	}
}

// InitialState implements the raft.Storage interface
func (s *raftStorage) InitialState() (hardState raftpb.HardState, confState raftpb.ConfState, err error) {
	// Restore the confState if we can, otherwise use s.initialConf
	var confStateBytes []byte
	confStateBytes, err = s.db.Get(s.confStateKey)
	if err == s.db.ErrNotFound() {
		confState = s.initialConf
	} else if err != nil {
		return
	}
	err = confState.Unmarshal(confStateBytes)
	if err != nil {
		return
	}
	var hardStateBytes []byte
	hardStateBytes, err = s.db.Get(s.hardStateKey)
	if err == s.db.ErrNotFound() {
		err = nil
		return
	} else if err != nil {
		return
	}
	err = hardState.Unmarshal(hardStateBytes)
	if err != nil {
		return
	}
	return
}

func (s *raftStorage) getEntryKey(nr uint64) (key []byte) {
	key = make([]byte, len(s.entryKeyPrefix)+8)
	copy(key, s.entryKeyPrefix)
	binary.BigEndian.PutUint64(key[len(s.entryKeyPrefix):], nr)
	return
}

// Entries implements the raft.Storage interface
func (s *raftStorage) Entries(lo, hi, maxSize uint64) (entries []raftpb.Entry, err error) {
	it := s.db.NewIterator(&kv.Range{s.getEntryKey(lo), s.getEntryKey(hi)})
	defer it.Release()
	entries = make([]raftpb.Entry, 0)
	sizeSoFar := uint64(0)
	for ok := it.First(); ok; ok = it.Next() {
		var entry raftpb.Entry
		err = entry.Unmarshal(it.Value())
		if err != nil {
			return
		}
		sizeSoFar += uint64(entry.Size())
		// Only stop if we already have at least one entry
		if sizeSoFar > maxSize && len(entries) > 0 {
			break
		}
		entries = append(entries, entry)
		if sizeSoFar >= maxSize {
			break
		}
	}
	err = it.Error()
	return
}

// Term implements the raft.Storage interface
func (s *raftStorage) Term(i uint64) (uint64, error) {
	entries, err := s.Entries(i, i+1, math.MaxUint64)
	if err != nil {
		return 0, err
	}
	if len(entries) != 1 {
		log.Panicf("number of entries with index %d not 1: %d", i, len(entries))
	}
	return entries[0].Term, nil
}

// LastIndex implements the raft.Storage interface
func (s *raftStorage) LastIndex() (uint64, error) {
	it := s.db.NewIterator(kv.BytesPrefix(s.entryKeyPrefix))
	defer it.Release()
	if !it.Last() {
		return 0, it.Error()
	}
	indexPortion := it.Key()[len(s.entryKeyPrefix):]
	return binary.BigEndian.Uint64(indexPortion), it.Error()
}

// FirstIndex implements the raft.Storage interface
func (s *raftStorage) FirstIndex() (uint64, error) {
	// Start at index 1 to be consistent with etcd/raft's MemoryStorage
	// (not sure if this is actually necessary)
	return 1, nil
}

// Snapshot implements the raft.Storage interface
func (s *raftStorage) Snapshot() (raftpb.Snapshot, error) {
	return raftpb.Snapshot{}, nil
}

// Don't call this multiple times concurrently
func (s *raftStorage) save(state raftpb.HardState, entries []raftpb.Entry) error {
	wb := s.db.NewBatch()
	stateBytes, err := state.Marshal()
	if err != nil {
		return err
	}
	wb.Put(s.hardStateKey, stateBytes)
	if len(entries) > 0 {
		lastIndex, err := s.LastIndex()
		if err != nil {
			return err
		}
		if entries[0].Index > lastIndex+1 {
			panic(fmt.Errorf("missing log entries [last: %d, append at: %d]", lastIndex, entries[0].Index))
		}
		// clear all old entries past the new index, if any
		for ix := entries[0].Index; ix <= lastIndex; ix++ {
			wb.Delete(s.getEntryKey(ix))
		}
		// append the new entries
		for _, entry := range entries {
			entryBytes, err := entry.Marshal()
			if err != nil {
				return err
			}
			wb.Put(s.getEntryKey(entry.Index), entryBytes)
		}
	}
	err = s.db.Write(wb)
	return err
}
<|endoftext|>