<commit_before>package builder

import (
	"github.com/appc/spec/schema"
	"github.com/blablacar/dgr/dgr"
	"github.com/blablacar/dgr/spec"
	"github.com/blablacar/dgr/utils"
	"github.com/ghodss/yaml"
	"github.com/n0rad/go-erlog/data"
	"github.com/n0rad/go-erlog/errs"
	"github.com/n0rad/go-erlog/logs"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
)

const SH_FUNCTIONS = `
execute_files() {
  fdir=$1
  [ -d "$fdir" ] || return 0

  for file in "$fdir"/*; do
    [ -e "$file" ] && {
     	[ -x "$file" ] || chmod +x "$file"
		isLevelEnabled 4 && echo -e "\e[1m\e[32mRunning script -> $file\e[0m"
     	"$file"
    }
  done
}

levelFromString() {
	case ` + "`echo ${1} | awk '{print toupper($0)}'`" + ` in
		"FATAL") echo 0; return 0 ;;
		"PANIC") echo 1; return 0 ;;
		"ERROR") echo 2; return 0 ;;
		"WARN"|"WARNING") echo 3; return 0 ;;
		"INFO") echo 4; return 0 ;;
		"DEBUG") echo 5; return 0 ;;
		"TRACE") echo 6; return 0 ;;
		*) echo 4 ;;
	esac
}

isLevelEnabled() {
	l=$(levelFromString $1)

	if [ $l -le $log_level ]; then
		return 0
	fi
	return 1
}

export log_level=$(levelFromString ${LOG_LEVEL:-INFO})
`

const BUILD_SCRIPT = `#!/dgr/bin/busybox sh
set -e
. /dgr/bin/functions.sh
isLevelEnabled "debug" && set -x

export TARGET=$( dirname $0 )
export ROOTFS=%%ROOTFS%%
export TERM=xterm

execute_files "$ROOTFS/dgr/runlevels/inherit-build-early"
execute_files "$TARGET/runlevels/build"
`

const BUILD_SCRIPT_LATE = `#!/dgr/bin/busybox sh
set -e
. /dgr/bin/functions.sh
isLevelEnabled "debug" && set -x

export TARGET=$( dirname $0 )
export ROOTFS=%%ROOTFS%%
export TERM=xterm

execute_files "$TARGET/runlevels/build-late"
execute_files "$ROOTFS/dgr/runlevels/inherit-build-late"
`

const PRESTART = `#!/dgr/bin/busybox sh
set -e
. /dgr/bin/functions.sh
isLevelEnabled "debug" && set -x

BASEDIR=${0%/*}
dgr_PATH=/dgr

execute_files ${dgr_PATH}/runlevels/prestart-early

if [ -z ${LOG_LEVEL} ]; then
	${BASEDIR}/templater -o TEMPLATER_OVERRIDE -t / /dgr
else
	${BASEDIR}/templater -o TEMPLATER_OVERRIDE -L "${LOG_LEVEL}" -t / /dgr
fi

#if [ -d ${dgr_PATH}/attributes ]; then
#	echo "$CONFD_OVERRIDE"
#    ${BASEDIR}/attributes-merger -i ${dgr_PATH}/attributes -e CONFD_OVERRIDE
#    export CONFD_DATA=$(cat attributes.json)
#fi
#${BASEDIR}/confd -onetime -config-file=${dgr_PATH}/prestart/confd.toml

execute_files ${dgr_PATH}/runlevels/prestart-late
`
const BUILD_SETUP = `#!/bin/sh
set -e
. "${TARGET}/rootfs/dgr/bin/functions.sh"
isLevelEnabled "debug" && set -x

execute_files "${BASEDIR}/runlevels/build-setup"
`

const PATH_BIN = "/bin"
const PATH_TESTS = "/tests"
const PATH_INSTALLED = "/installed"
const PATH_MANIFEST = "/manifest"
const PATH_IMAGE_ACI = "/image.aci"
const PATH_IMAGE_ACI_ZIP = "/image-zip.aci"
const PATH_ROOTFS = "/rootfs"
const PATH_TARGET = "/target"
const PATH_DGR = "/dgr"
const PATH_ACI_MANIFEST = "/aci-manifest.yml"
const PATH_RUNLEVELS = "/runlevels"
const PATH_PRESTART_EARLY = "/prestart-early"
const PATH_PRESTART_LATE = "/prestart-late"
const PATH_INHERIT_BUILD_LATE = "/inherit-build-late"
const PATH_INHERIT_BUILD_EARLY = "/inherit-build-early"
const PATH_ATTRIBUTES = "/attributes"
const PATH_FILES = "/files"
const PATH_BUILD_LATE = "/build-late"
const PATH_BUILD_SETUP = "/build-setup"
const PATH_BUILD = "/build"
const PATH_TEMPLATES = "/templates"

type Aci struct {
	fields          data.Fields
	path            string
	target          string
	rootfs          string
	podName         *spec.ACFullname
	manifest        spec.AciManifest
	args            BuildArgs
	FullyResolveDep bool
}

func NewAciWithManifest(path string, args BuildArgs, manifest spec.AciManifest) (*Aci, error) {
	if manifest.NameAndVersion == "" {
		logs.WithField("path", path).Fatal("name is mandatory in manifest")
	}

	fields := data.WithField("aci", manifest.NameAndVersion.String())
	logs.WithF(fields).WithFields(data.Fields{"args": args, "path": path, "manifest": manifest}).Debug("New aci")

	fullPath, err := filepath.Abs(path)
	if err != nil {
		return nil, errs.WithEF(err, fields, "Cannot get fullpath of project")
	}

	target := fullPath + PATH_TARGET
	if dgr.Home.Config.TargetWorkDir != "" {
		currentAbsDir, err := filepath.Abs(dgr.Home.Config.TargetWorkDir + "/" + manifest.NameAndVersion.ShortName())
		if err != nil {
			return nil, errs.WithEF(err, fields.WithField("path", path), "Invalid target path")
		}
		target = currentAbsDir
	}

	aci := &Aci{
		fields:          fields,
		args:            args,
		path:            fullPath,
		manifest:        manifest,
		target:          target,
		rootfs:          target + PATH_ROOTFS,
		FullyResolveDep: true,
	}

	aci.checkCompatibilityVersions()
	aci.checkLatestVersions()
	return aci, nil
}

func NewAci(path string, args BuildArgs) (*Aci, error) {
	manifest, err := readAciManifest(path + PATH_ACI_MANIFEST)
	if err != nil {
		manifest2, err2 := readAciManifest(path + "/cnt-manifest.yml")
		if err2 != nil {
			return nil, errs.WithEF(err, data.WithField("path", path+PATH_ACI_MANIFEST).WithField("err2", err2), "Cannot read manifest")
		}
		logs.WithField("old", "cnt-manifest.yml").WithField("new", "aci-manifest.yml").Warn("You are using the old aci configuration file")
		manifest = manifest2
	}
	return NewAciWithManifest(path, args, *manifest)
}

//////////////////////////////////////////////////////////////////

func readAciManifest(manifestPath string) (*spec.AciManifest, error) {
	manifest := spec.AciManifest{Aci: spec.AciDefinition{}}

	source, err := ioutil.ReadFile(manifestPath)
	if err != nil {
		return nil, err
	}
	err = yaml.Unmarshal([]byte(source), &manifest)
	if err != nil {
		return nil, errs.WithE(err, "Cannot unmarshall manifest")
	}

	return &manifest, nil
}

func (aci *Aci) tarAci(zip bool) {
	target := PATH_IMAGE_ACI[1:]
	if zip {
		target = PATH_IMAGE_ACI_ZIP[1:]
	}
	dir, _ := os.Getwd()
	logs.WithField("path", aci.target).Debug("chdir")
	os.Chdir(aci.target)
	utils.Tar(zip, target, PATH_MANIFEST[1:], PATH_ROOTFS[1:])
	logs.WithField("path", dir).Debug("chdir")
	os.Chdir(dir)
}

func (aci *Aci) checkCompatibilityVersions() {
	froms, err := aci.manifest.GetFroms()
	if err != nil {
		logs.WithEF(err, aci.fields).Fatal("Invalid from")
	}
	for _, from := range froms {
		if from == "" {
			continue
		}

		fromFields := aci.fields.WithField("dependency", from.String())

		err := utils.ExecCmd("rkt", "--insecure-options=image", "fetch", from.String())
		if err != nil {
			logs.WithEF(err, fromFields).Fatal("Cannot fetch from")
		}
		out, err := utils.ExecCmdGetOutput("rkt", "image", "cat-manifest", from.String())
		if err != nil {
			logs.WithEF(err, fromFields).Fatal("Cannot find dependency")
		}

		version, ok := loadManifest(out).Annotations.Get("dgr-version")
		if !ok {
			version, ok = loadManifest(out).Annotations.Get("cnt-version")
		}
		var val int
		if ok {
			val, err = strconv.Atoi(version)
			if err != nil {
				logs.WithEF(err, fromFields).WithField("version", version).Fatal("Failed to parse dgr-version from manifest")
			}
		}
		if !ok || val < 55 {
			logs.WithF(aci.fields).
				WithField("from", from).
				WithField("require", ">=55").
				Error("from aci was not build with a compatible version of dgr")
		}
	}

	for _, dep := range aci.manifest.Aci.Dependencies {
		depFields := aci.fields.WithField("dependency", dep.String())
		err := utils.ExecCmd("rkt", "--insecure-options=image", "fetch", dep.String())
		if err != nil {
			logs.WithEF(err, depFields).Fatal("Cannot fetch dependency")
		}
		out, err := utils.ExecCmdGetOutput("rkt", "image", "cat-manifest", dep.String())
		if err != nil {
			logs.WithEF(err, depFields).Fatal("Cannot find dependency")
		}

		version, ok := loadManifest(out).Annotations.Get("dgr-version")
		if !ok {
			version, ok = loadManifest(out).Annotations.Get("cnt-version")
		}
		var val int
		if ok {
			val, err = strconv.Atoi(version)
			if err != nil {
				logs.WithEF(err, depFields).WithField("version", version).Fatal("Failed to parse dgr-version from manifest")
			}
		}
		if !ok || val < 55 {
			logs.WithF(aci.fields).
				WithField("dependency", dep).
				WithField("require", ">=55").
				Error("dependency aci was not build with a compatible version of dgr")
		}
	}
}

func loadManifest(content string) schema.ImageManifest {
	im := schema.ImageManifest{}
	err := im.UnmarshalJSON([]byte(content))
	if err != nil {
		logs.WithE(err).WithField("content", content).Fatal("Failed to read manifest content")
	}
	return im
}

func (aci *Aci) checkLatestVersions() {
	froms, err := aci.manifest.GetFroms()
	if err != nil {
		logs.WithEF(err, aci.fields).Fatal("Invalid from")
	}
	for _, from := range froms {
		if from == "" {
			continue
		}

		version, _ := from.LatestVersion()
		logs.WithField("version", from.Name()+":"+version).Debug("Discovered from latest verion")
		if version != "" && utils.Version(from.Version()).LessThan(utils.Version(version)) {
			logs.WithField("newer", from.Name()+":"+version).
				WithField("current", from.String()).
				Warn("Newer 'from' version")
		}
	}
	for _, dep := range aci.manifest.Aci.Dependencies {
		if dep.Version() == "" {
			continue
		}
		version, _ := dep.LatestVersion()
		if version != "" && utils.Version(dep.Version()).LessThan(utils.Version(version)) {
			logs.WithField("newer", dep.Name()+":"+version).
				WithField("current", dep.String()).
				Warn("Newer 'dependency' version")
		}
	}
}
<commit_msg>remove useless cnt-version check<commit_after>package builder

import (
	"github.com/appc/spec/schema"
	"github.com/blablacar/dgr/dgr"
	"github.com/blablacar/dgr/spec"
	"github.com/blablacar/dgr/utils"
	"github.com/ghodss/yaml"
	"github.com/n0rad/go-erlog/data"
	"github.com/n0rad/go-erlog/errs"
	"github.com/n0rad/go-erlog/logs"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
)

const SH_FUNCTIONS = `
execute_files() {
  fdir=$1
  [ -d "$fdir" ] || return 0

  for file in "$fdir"/*; do
    [ -e "$file" ] && {
     	[ -x "$file" ] || chmod +x "$file"
		isLevelEnabled 4 && echo -e "\e[1m\e[32mRunning script -> $file\e[0m"
     	"$file"
    }
  done
}

levelFromString() {
	case ` + "`echo ${1} | awk '{print toupper($0)}'`" + ` in
		"FATAL") echo 0; return 0 ;;
		"PANIC") echo 1; return 0 ;;
		"ERROR") echo 2; return 0 ;;
		"WARN"|"WARNING") echo 3; return 0 ;;
		"INFO") echo 4; return 0 ;;
		"DEBUG") echo 5; return 0 ;;
		"TRACE") echo 6; return 0 ;;
		*) echo 4 ;;
	esac
}

isLevelEnabled() {
	l=$(levelFromString $1)

	if [ $l -le $log_level ]; then
		return 0
	fi
	return 1
}

export log_level=$(levelFromString ${LOG_LEVEL:-INFO})
`

const BUILD_SCRIPT = `#!/dgr/bin/busybox sh
set -e
. /dgr/bin/functions.sh
isLevelEnabled "debug" && set -x

export TARGET=$( dirname $0 )
export ROOTFS=%%ROOTFS%%
export TERM=xterm

execute_files "$ROOTFS/dgr/runlevels/inherit-build-early"
execute_files "$TARGET/runlevels/build"
`

const BUILD_SCRIPT_LATE = `#!/dgr/bin/busybox sh
set -e
. /dgr/bin/functions.sh
isLevelEnabled "debug" && set -x

export TARGET=$( dirname $0 )
export ROOTFS=%%ROOTFS%%
export TERM=xterm

execute_files "$TARGET/runlevels/build-late"
execute_files "$ROOTFS/dgr/runlevels/inherit-build-late"
`

const PRESTART = `#!/dgr/bin/busybox sh
set -e
. /dgr/bin/functions.sh
isLevelEnabled "debug" && set -x

BASEDIR=${0%/*}
dgr_PATH=/dgr

execute_files ${dgr_PATH}/runlevels/prestart-early

if [ -z ${LOG_LEVEL} ]; then
	${BASEDIR}/templater -o TEMPLATER_OVERRIDE -t / /dgr
else
	${BASEDIR}/templater -o TEMPLATER_OVERRIDE -L "${LOG_LEVEL}" -t / /dgr
fi

#if [ -d ${dgr_PATH}/attributes ]; then
#	echo "$CONFD_OVERRIDE"
#    ${BASEDIR}/attributes-merger -i ${dgr_PATH}/attributes -e CONFD_OVERRIDE
#    export CONFD_DATA=$(cat attributes.json)
#fi
#${BASEDIR}/confd -onetime -config-file=${dgr_PATH}/prestart/confd.toml

execute_files ${dgr_PATH}/runlevels/prestart-late
`
const BUILD_SETUP = `#!/bin/sh
set -e
. "${TARGET}/rootfs/dgr/bin/functions.sh"
isLevelEnabled "debug" && set -x

execute_files "${BASEDIR}/runlevels/build-setup"
`

const PATH_BIN = "/bin"
const PATH_TESTS = "/tests"
const PATH_INSTALLED = "/installed"
const PATH_MANIFEST = "/manifest"
const PATH_IMAGE_ACI = "/image.aci"
const PATH_IMAGE_ACI_ZIP = "/image-zip.aci"
const PATH_ROOTFS = "/rootfs"
const PATH_TARGET = "/target"
const PATH_DGR = "/dgr"
const PATH_ACI_MANIFEST = "/aci-manifest.yml"
const PATH_RUNLEVELS = "/runlevels"
const PATH_PRESTART_EARLY = "/prestart-early"
const PATH_PRESTART_LATE = "/prestart-late"
const PATH_INHERIT_BUILD_LATE = "/inherit-build-late"
const PATH_INHERIT_BUILD_EARLY = "/inherit-build-early"
const PATH_ATTRIBUTES = "/attributes"
const PATH_FILES = "/files"
const PATH_BUILD_LATE = "/build-late"
const PATH_BUILD_SETUP = "/build-setup"
const PATH_BUILD = "/build"
const PATH_TEMPLATES = "/templates"

type Aci struct {
	fields          data.Fields
	path            string
	target          string
	rootfs          string
	podName         *spec.ACFullname
	manifest        spec.AciManifest
	args            BuildArgs
	FullyResolveDep bool
}

func NewAciWithManifest(path string, args BuildArgs, manifest spec.AciManifest) (*Aci, error) {
	if manifest.NameAndVersion == "" {
		logs.WithField("path", path).Fatal("name is mandatory in manifest")
	}

	fields := data.WithField("aci", manifest.NameAndVersion.String())
	logs.WithF(fields).WithFields(data.Fields{"args": args, "path": path, "manifest": manifest}).Debug("New aci")

	fullPath, err := filepath.Abs(path)
	if err != nil {
		return nil, errs.WithEF(err, fields, "Cannot get fullpath of project")
	}

	target := fullPath + PATH_TARGET
	if dgr.Home.Config.TargetWorkDir != "" {
		currentAbsDir, err := filepath.Abs(dgr.Home.Config.TargetWorkDir + "/" + manifest.NameAndVersion.ShortName())
		if err != nil {
			return nil, errs.WithEF(err, fields.WithField("path", path), "Invalid target path")
		}
		target = currentAbsDir
	}

	aci := &Aci{
		fields:          fields,
		args:            args,
		path:            fullPath,
		manifest:        manifest,
		target:          target,
		rootfs:          target + PATH_ROOTFS,
		FullyResolveDep: true,
	}

	aci.checkCompatibilityVersions()
	aci.checkLatestVersions()
	return aci, nil
}

func NewAci(path string, args BuildArgs) (*Aci, error) {
	manifest, err := readAciManifest(path + PATH_ACI_MANIFEST)
	if err != nil {
		manifest2, err2 := readAciManifest(path + "/cnt-manifest.yml")
		if err2 != nil {
			return nil, errs.WithEF(err, data.WithField("path", path+PATH_ACI_MANIFEST).WithField("err2", err2), "Cannot read manifest")
		}
		logs.WithField("old", "cnt-manifest.yml").WithField("new", "aci-manifest.yml").Warn("You are using the old aci configuration file")
		manifest = manifest2
	}
	return NewAciWithManifest(path, args, *manifest)
}

//////////////////////////////////////////////////////////////////

func readAciManifest(manifestPath string) (*spec.AciManifest, error) {
	manifest := spec.AciManifest{Aci: spec.AciDefinition{}}

	source, err := ioutil.ReadFile(manifestPath)
	if err != nil {
		return nil, err
	}
	err = yaml.Unmarshal([]byte(source), &manifest)
	if err != nil {
		return nil, errs.WithE(err, "Cannot unmarshall manifest")
	}

	return &manifest, nil
}

func (aci *Aci) tarAci(zip bool) {
	target := PATH_IMAGE_ACI[1:]
	if zip {
		target = PATH_IMAGE_ACI_ZIP[1:]
	}
	dir, _ := os.Getwd()
	logs.WithField("path", aci.target).Debug("chdir")
	os.Chdir(aci.target)
	utils.Tar(zip, target, PATH_MANIFEST[1:], PATH_ROOTFS[1:])
	logs.WithField("path", dir).Debug("chdir")
	os.Chdir(dir)
}

func (aci *Aci) checkCompatibilityVersions() {
	froms, err := aci.manifest.GetFroms()
	if err != nil {
		logs.WithEF(err, aci.fields).Fatal("Invalid from")
	}
	for _, from := range froms {
		if from == "" {
			continue
		}

		fromFields := aci.fields.WithField("dependency", from.String())

		err := utils.ExecCmd("rkt", "--insecure-options=image", "fetch", from.String())
		if err != nil {
			logs.WithEF(err, fromFields).Fatal("Cannot fetch from")
		}
		out, err := utils.ExecCmdGetOutput("rkt", "image", "cat-manifest", from.String())
		if err != nil {
			logs.WithEF(err, fromFields).Fatal("Cannot find dependency")
		}

		version, ok := loadManifest(out).Annotations.Get("dgr-version")
		var val int
		if ok {
			val, err = strconv.Atoi(version)
			if err != nil {
				logs.WithEF(err, fromFields).WithField("version", version).Fatal("Failed to parse dgr-version from manifest")
			}
		}
		if !ok || val < 55 {
			logs.WithF(aci.fields).
				WithField("from", from).
				WithField("require", ">=55").
				Error("from aci was not build with a compatible version of dgr")
		}
	}

	for _, dep := range aci.manifest.Aci.Dependencies {
		depFields := aci.fields.WithField("dependency", dep.String())
		err := utils.ExecCmd("rkt", "--insecure-options=image", "fetch", dep.String())
		if err != nil {
			logs.WithEF(err, depFields).Fatal("Cannot fetch dependency")
		}
		out, err := utils.ExecCmdGetOutput("rkt", "image", "cat-manifest", dep.String())
		if err != nil {
			logs.WithEF(err, depFields).Fatal("Cannot find dependency")
		}

		version, ok := loadManifest(out).Annotations.Get("dgr-version")
		var val int
		if ok {
			val, err = strconv.Atoi(version)
			if err != nil {
				logs.WithEF(err, depFields).WithField("version", version).Fatal("Failed to parse dgr-version from manifest")
			}
		}
		if !ok || val < 55 {
			logs.WithF(aci.fields).
				WithField("dependency", dep).
				WithField("require", ">=55").
				Error("dependency aci was not build with a compatible version of dgr")
		}
	}
}

func loadManifest(content string) schema.ImageManifest {
	im := schema.ImageManifest{}
	err := im.UnmarshalJSON([]byte(content))
	if err != nil {
		logs.WithE(err).WithField("content", content).Fatal("Failed to read manifest content")
	}
	return im
}

func (aci *Aci) checkLatestVersions() {
	froms, err := aci.manifest.GetFroms()
	if err != nil {
		logs.WithEF(err, aci.fields).Fatal("Invalid from")
	}
	for _, from := range froms {
		if from == "" {
			continue
		}

		version, _ := from.LatestVersion()
		logs.WithField("version", from.Name()+":"+version).Debug("Discovered from latest verion")
		if version != "" && utils.Version(from.Version()).LessThan(utils.Version(version)) {
			logs.WithField("newer", from.Name()+":"+version).
				WithField("current", from.String()).
				Warn("Newer 'from' version")
		}
	}
	for _, dep := range aci.manifest.Aci.Dependencies {
		if dep.Version() == "" {
			continue
		}
		version, _ := dep.LatestVersion()
		if version != "" && utils.Version(dep.Version()).LessThan(utils.Version(version)) {
			logs.WithField("newer", dep.Name()+":"+version).
				WithField("current", dep.String()).
				Warn("Newer 'dependency' version")
		}
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package compile

import (
	"path/filepath"
	"strings"
	"unicode"
)

// capitalize changes the first letter of a string to upper case.
func capitalize(s string) string {
	if len(s) == 0 {
		return s
	}
	x := []rune(s)
	return string(unicode.ToUpper(x[0])) + string(x[1:])
	// TODO(abg): surely there has to be a better way
}

// fileBaseName returns the base name of the given file without the extension.
func fileBaseName(p string) string {
	return strings.TrimSuffix(filepath.Base(p), filepath.Ext(p))
}

// splitInclude splits the given string at the first ".".
//
// If the given string doesn't have a '.', the second returned string contains
// the entirety of it.
func splitInclude(s string) (string, string) {
	if i := strings.IndexRune(s, '.'); i > 0 {
		return s[:i], s[i+1:]
	}
	return "", s
}
<commit_msg>Simpler capitalize() implementation<commit_after>// Copyright (c) 2015 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package compile

import (
	"path/filepath"
	"strings"
	"unicode"
	"unicode/utf8"
)

// capitalize changes the first letter of a string to upper case.
func capitalize(s string) string {
	if len(s) == 0 {
		return s
	}
	x, i := utf8.DecodeRuneInString(s)
	return string(unicode.ToUpper(x)) + string(s[i:])
}

// fileBaseName returns the base name of the given file without the extension.
func fileBaseName(p string) string {
	return strings.TrimSuffix(filepath.Base(p), filepath.Ext(p))
}

// splitInclude splits the given string at the first ".".
//
// If the given string doesn't have a '.', the second returned string contains
// the entirety of it.
func splitInclude(s string) (string, string) {
	if i := strings.IndexRune(s, '.'); i > 0 {
		return s[:i], s[i+1:]
	}
	return "", s
}
<|endoftext|>


<commit_before>package db

import (
	"fmt"
	"strings"
	"time"

	"github.com/pborman/uuid"

	. "github.com/starkandwayne/shield/timestamp"
)

type AnnotatedTask struct {
	UUID        string    `json:"uuid"`
	Owner       string    `json:"owner"`
	Op          string    `json:"type"`
	JobUUID     string    `json:"job_uuid"`
	ArchiveUUID string    `json:"archive_uuid"`
	Status      string    `json:"status"`
	StartedAt   Timestamp `json:"started_at"`
	StoppedAt   Timestamp `json:"stopped_at"`
	Log         string    `json:"log"`
}

type TaskFilter struct {
	ForStatus string
}

func (f *TaskFilter) Args() []interface{} {
	var args []interface{}
	if f.ForStatus != "" {
		args = append(args, f.ForStatus)
	}
	return args
}

func (f *TaskFilter) Query() string {
	wheres := []string{"t.uuid = t.uuid"}
	if f.ForStatus != "" {
		wheres = append(wheres, "status = $1")
	}
	return `
		SELECT t.uuid, t.owner, t.op, j.uuid, a.uuid,
		       t.status, t.started_at, t.stopped_at, t.log

		FROM tasks t
			INNER JOIN jobs     j    ON j.uuid = t.job_uuid
			LEFT  JOIN archives a    ON a.uuid = t.archive_uuid

		WHERE ` + strings.Join(wheres, " AND ") + `
		ORDER BY t.started_at DESC, t.uuid ASC
	`
}

func (db *DB) GetAllAnnotatedTasks(filter *TaskFilter) ([]*AnnotatedTask, error) {
	l := []*AnnotatedTask{}
	r, err := db.Query(filter.Query(), filter.Args()...)
	if err != nil {
		return l, err
	}
	defer r.Close()

	for r.Next() {
		ann := &AnnotatedTask{}

		var archive []byte
		var started, stopped *int64
		if err = r.Scan(
			&ann.UUID, &ann.Owner, &ann.Op, &ann.JobUUID, &archive,
			&ann.Status, &started, &stopped, &ann.Log); err != nil {
			return l, err
		}

		if archive != nil {
			ann.ArchiveUUID = string(archive)
		}
		if started != nil {
			ann.StartedAt = parseEpochTime(*started)
		}
		if stopped != nil {
			ann.StoppedAt = parseEpochTime(*stopped)
		}

		l = append(l, ann)
	}

	return l, nil
}

func (db *DB) GetAnnotatedTask(id uuid.UUID) (*AnnotatedTask, error) {
	r, err := db.Query(`
		SELECT t.uuid, t.owner, t.op, j.uuid, a.uuid,
		       t.status, t.started_at, t.stopped_at, t.log

		FROM tasks t
			INNER JOIN jobs     j    ON j.uuid = t.job_uuid
			LEFT  JOIN archives a    ON a.uuid = t.archive_uuid

		WHERE t.uuid = $1`, id.String())
	if err != nil {
		return nil, err
	}
	defer r.Close()

	if !r.Next() {
		return nil, nil
	}

	ann := &AnnotatedTask{}
	var archive []byte
	var started, stopped *int64
	if err = r.Scan(
		&ann.UUID, &ann.Owner, &ann.Op, &ann.JobUUID, &archive,
		&ann.Status, &started, &stopped, &ann.Log); err != nil {
		return nil, err
	}
	if archive != nil {
		ann.ArchiveUUID = string(archive)
	}
	if started != nil {
		ann.StartedAt = parseEpochTime(*started)
	}
	if stopped != nil {
		ann.StoppedAt = parseEpochTime(*stopped)
	}

	return ann, nil
}

func (db *DB) CreateBackupTask(owner string, job uuid.UUID) (uuid.UUID, error) {
	id := uuid.NewRandom()
	return id, db.Exec(
		`INSERT INTO tasks (uuid, owner, op, job_uuid, status, log, requested_at)
			VALUES ($1, $2, $3, $4, $5, $6, $7)`,
		id.String(), owner, "backup", job.String(), "pending", "", time.Now().Unix(),
	)
}

func (db *DB) CreateRestoreTask(owner string, archive, target uuid.UUID) (uuid.UUID, error) {
	id := uuid.NewRandom()
	return id, db.Exec(
		`INSERT INTO tasks (uuid, owner, op, archive_uuid, target_uuid, status, log, requested_at)
			VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`,
		id.String(), owner, "restore", archive.String(), target.String(), "pending", "", time.Now().Unix(),
	)
}

func (db *DB) StartTask(id uuid.UUID, effective time.Time) error {
	return db.Exec(
		`UPDATE tasks SET status = $1, started_at = $2 WHERE uuid = $3`,
		"running", effective.Unix(), id.String(),
	)
}

func (db *DB) updateTaskStatus(id uuid.UUID, status string, effective int64) error {
	return db.Exec(
		`UPDATE tasks SET status = $1, stopped_at = $2 WHERE uuid = $3`,
		status, effective, id.String(),
	)
}
func (db *DB) CancelTask(id uuid.UUID, effective time.Time) error {
	return db.updateTaskStatus(id, "canceled", effective.Unix())
}

func (db *DB) FailTask(id uuid.UUID, effective time.Time) error {
	return db.updateTaskStatus(id, "failed", effective.Unix())
}

func (db *DB) CompleteTask(id uuid.UUID, effective time.Time) error {
	return db.updateTaskStatus(id, "done", effective.Unix())
}

func (db *DB) UpdateTaskLog(id uuid.UUID, more string) error {
	return db.Exec(
		`UPDATE tasks SET log = log || $1 WHERE uuid = $2`,
		more, id.String(),
	)
}

func (db *DB) CreateTaskArchive(id uuid.UUID, key string, effective time.Time) error {
	// determine how long we need to keep this specific archive for
	r, err := db.Query(
		`SELECT r.expiry
			FROM retention r
				INNER JOIN jobs  j    ON r.uuid = j.retention_uuid
				INNER JOIN tasks t    ON j.uuid = t.job_uuid
			WHERE t.uuid = $1`,
		id.String(),
	)
	if err != nil {
		return err
	}
	defer r.Close()

	if !r.Next() {
		return fmt.Errorf("failed to determine expiration for task %s", id)
	}

	var expiry int
	if err := r.Scan(&expiry); err != nil {
		return err
	}
	r.Close()

	// insert an archive with all proper references, expiration, etc.
	archive_id := uuid.NewRandom()
	err = db.Exec(
		`INSERT INTO archives
			(uuid, target_uuid, store_uuid, store_key, taken_at, expires_at, notes)
			SELECT $1, t.uuid, s.uuid, $2, $3, $4, ''
				FROM tasks
					INNER JOIN jobs    j     ON j.uuid = tasks.job_uuid
					INNER JOIN targets t     ON t.uuid = j.target_uuid
					INNER JOIN stores  s     ON s.uuid = j.store_uuid
				WHERE tasks.uuid = $5`,
		archive_id.String(), key, effective.Unix(), effective.Add(time.Duration(expiry)*time.Second).Unix(), id.String(),
	)
	if err != nil {
		return err
	}

	// and finally, associate task -> archive
	return db.Exec(
		`UPDATE tasks SET archive_uuid = $1 WHERE uuid = $2`,
		archive_id.String(), id.String(),
	)
}
<commit_msg>Retrieve restore tasks in /v1/tasks<commit_after>package db

import (
	"fmt"
	"strings"
	"time"

	"github.com/pborman/uuid"

	. "github.com/starkandwayne/shield/timestamp"
)

type AnnotatedTask struct {
	UUID        string    `json:"uuid"`
	Owner       string    `json:"owner"`
	Op          string    `json:"type"`
	JobUUID     string    `json:"job_uuid"`
	ArchiveUUID string    `json:"archive_uuid"`
	Status      string    `json:"status"`
	StartedAt   Timestamp `json:"started_at"`
	StoppedAt   Timestamp `json:"stopped_at"`
	Log         string    `json:"log"`
}

type TaskFilter struct {
	ForStatus string
}

func (f *TaskFilter) Args() []interface{} {
	var args []interface{}
	if f.ForStatus != "" {
		args = append(args, f.ForStatus)
	}
	return args
}

func (f *TaskFilter) Query() string {
	wheres := []string{"t.uuid = t.uuid"}
	if f.ForStatus != "" {
		wheres = append(wheres, "status = $1")
	}
	return `
		SELECT t.uuid, t.owner, t.op, t.job_uuid, t.archive_uuid,
		       t.status, t.started_at, t.stopped_at, t.log

		FROM tasks t

		WHERE ` + strings.Join(wheres, " AND ") + `
		ORDER BY t.started_at DESC, t.uuid ASC
	`
}

func (db *DB) GetAllAnnotatedTasks(filter *TaskFilter) ([]*AnnotatedTask, error) {
	l := []*AnnotatedTask{}
	r, err := db.Query(filter.Query(), filter.Args()...)
	if err != nil {
		return l, err
	}
	defer r.Close()

	for r.Next() {
		ann := &AnnotatedTask{}

		var archive []byte
		var started, stopped *int64
		if err = r.Scan(
			&ann.UUID, &ann.Owner, &ann.Op, &ann.JobUUID, &archive,
			&ann.Status, &started, &stopped, &ann.Log); err != nil {
			return l, err
		}

		if archive != nil {
			ann.ArchiveUUID = string(archive)
		}
		if started != nil {
			ann.StartedAt = parseEpochTime(*started)
		}
		if stopped != nil {
			ann.StoppedAt = parseEpochTime(*stopped)
		}

		l = append(l, ann)
	}

	return l, nil
}

func (db *DB) GetAnnotatedTask(id uuid.UUID) (*AnnotatedTask, error) {
	r, err := db.Query(`
		SELECT t.uuid, t.owner, t.op, j.uuid, a.uuid,
		       t.status, t.started_at, t.stopped_at, t.log

		FROM tasks t
			INNER JOIN jobs     j    ON j.uuid = t.job_uuid
			LEFT  JOIN archives a    ON a.uuid = t.archive_uuid

		WHERE t.uuid = $1`, id.String())
	if err != nil {
		return nil, err
	}
	defer r.Close()

	if !r.Next() {
		return nil, nil
	}

	ann := &AnnotatedTask{}
	var archive []byte
	var started, stopped *int64
	if err = r.Scan(
		&ann.UUID, &ann.Owner, &ann.Op, &ann.JobUUID, &archive,
		&ann.Status, &started, &stopped, &ann.Log); err != nil {
		return nil, err
	}
	if archive != nil {
		ann.ArchiveUUID = string(archive)
	}
	if started != nil {
		ann.StartedAt = parseEpochTime(*started)
	}
	if stopped != nil {
		ann.StoppedAt = parseEpochTime(*stopped)
	}

	return ann, nil
}

func (db *DB) CreateBackupTask(owner string, job uuid.UUID) (uuid.UUID, error) {
	id := uuid.NewRandom()
	return id, db.Exec(
		`INSERT INTO tasks (uuid, owner, op, job_uuid, status, log, requested_at)
			VALUES ($1, $2, $3, $4, $5, $6, $7)`,
		id.String(), owner, "backup", job.String(), "pending", "", time.Now().Unix(),
	)
}

func (db *DB) CreateRestoreTask(owner string, archive, target uuid.UUID) (uuid.UUID, error) {
	id := uuid.NewRandom()
	return id, db.Exec(
		`INSERT INTO tasks (uuid, owner, op, archive_uuid, target_uuid, status, log, requested_at)
			VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`,
		id.String(), owner, "restore", archive.String(), target.String(), "pending", "", time.Now().Unix(),
	)
}

func (db *DB) StartTask(id uuid.UUID, effective time.Time) error {
	return db.Exec(
		`UPDATE tasks SET status = $1, started_at = $2 WHERE uuid = $3`,
		"running", effective.Unix(), id.String(),
	)
}

func (db *DB) updateTaskStatus(id uuid.UUID, status string, effective int64) error {
	return db.Exec(
		`UPDATE tasks SET status = $1, stopped_at = $2 WHERE uuid = $3`,
		status, effective, id.String(),
	)
}
func (db *DB) CancelTask(id uuid.UUID, effective time.Time) error {
	return db.updateTaskStatus(id, "canceled", effective.Unix())
}

func (db *DB) FailTask(id uuid.UUID, effective time.Time) error {
	return db.updateTaskStatus(id, "failed", effective.Unix())
}

func (db *DB) CompleteTask(id uuid.UUID, effective time.Time) error {
	return db.updateTaskStatus(id, "done", effective.Unix())
}

func (db *DB) UpdateTaskLog(id uuid.UUID, more string) error {
	return db.Exec(
		`UPDATE tasks SET log = log || $1 WHERE uuid = $2`,
		more, id.String(),
	)
}

func (db *DB) CreateTaskArchive(id uuid.UUID, key string, effective time.Time) error {
	// determine how long we need to keep this specific archive for
	r, err := db.Query(
		`SELECT r.expiry
			FROM retention r
				INNER JOIN jobs  j    ON r.uuid = j.retention_uuid
				INNER JOIN tasks t    ON j.uuid = t.job_uuid
			WHERE t.uuid = $1`,
		id.String(),
	)
	if err != nil {
		return err
	}
	defer r.Close()

	if !r.Next() {
		return fmt.Errorf("failed to determine expiration for task %s", id)
	}

	var expiry int
	if err := r.Scan(&expiry); err != nil {
		return err
	}
	r.Close()

	// insert an archive with all proper references, expiration, etc.
	archive_id := uuid.NewRandom()
	err = db.Exec(
		`INSERT INTO archives
			(uuid, target_uuid, store_uuid, store_key, taken_at, expires_at, notes)
			SELECT $1, t.uuid, s.uuid, $2, $3, $4, ''
				FROM tasks
					INNER JOIN jobs    j     ON j.uuid = tasks.job_uuid
					INNER JOIN targets t     ON t.uuid = j.target_uuid
					INNER JOIN stores  s     ON s.uuid = j.store_uuid
				WHERE tasks.uuid = $5`,
		archive_id.String(), key, effective.Unix(), effective.Add(time.Duration(expiry)*time.Second).Unix(), id.String(),
	)
	if err != nil {
		return err
	}

	// and finally, associate task -> archive
	return db.Exec(
		`UPDATE tasks SET archive_uuid = $1 WHERE uuid = $2`,
		archive_id.String(), id.String(),
	)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"os"
	"path"
	"runtime"
	"strconv"
	"testing"

	"github.com/dustin/go-humanize"
	"github.com/itchio/butler/cmd/apply"
	"github.com/itchio/butler/cmd/diff"
	"github.com/itchio/butler/cmd/ditto"
	"github.com/itchio/butler/cmd/sign"
	"github.com/itchio/butler/comm"
	"github.com/itchio/savior/seeksource"
	"github.com/itchio/wharf/eos"
	"github.com/itchio/wharf/pwr"
	"github.com/stretchr/testify/assert"
)

// reverse must
func mist(t *testing.T, err error) {
	if err != nil {
		assert.NoError(t, err)
		t.FailNow()
	}
}

func octal(perm os.FileMode) string {
	return strconv.FormatInt(int64(perm), 8)
}

func permFor(t *testing.T, path string) os.FileMode {
	t.Logf("Getting perm of %s", path)
	stat, err := os.Lstat(path)
	mist(t, err)
	return stat.Mode()
}

func putfile(t *testing.T, basePath string, i int, data []byte) {
	putfileEx(t, basePath, i, data, os.FileMode(0777))
}

func putfileEx(t *testing.T, basePath string, i int, data []byte, perm os.FileMode) {
	samplePath := path.Join(basePath, fmt.Sprintf("dummy%d.dat", i))
	mist(t, ioutil.WriteFile(samplePath, data, perm))
}

func TestAllTheThings(t *testing.T) {
	perm := os.FileMode(0777)
	workingDir, err := ioutil.TempDir("", "butler-tests")
	mist(t, err)
	defer os.RemoveAll(workingDir)

	sample := path.Join(workingDir, "sample")
	mist(t, os.MkdirAll(sample, perm))
	mist(t, ioutil.WriteFile(path.Join(sample, "hello.txt"), []byte("hello!"), perm))

	sample2 := path.Join(workingDir, "sample2")
	mist(t, os.MkdirAll(sample2, perm))
	for i := 0; i < 5; i++ {
		if i == 3 {
			// e.g. .gitkeep
			putfile(t, sample2, i, []byte{})
		} else {
			putfile(t, sample2, i, bytes.Repeat([]byte{0x42, 0x69}, i*200+1))
		}
	}

	sample3 := path.Join(workingDir, "sample3")
	mist(t, os.MkdirAll(sample3, perm))
	for i := 0; i < 60; i++ {
		putfile(t, sample3, i, bytes.Repeat([]byte{0x42, 0x69}, i*300+1))
	}

	sample4 := path.Join(workingDir, "sample4")
	mist(t, os.MkdirAll(sample4, perm))
	for i := 0; i < 120; i++ {
		putfile(t, sample4, i, bytes.Repeat([]byte{0x42, 0x69}, i*150+1))
	}

	sample5 := path.Join(workingDir, "sample5")
	mist(t, os.MkdirAll(sample5, perm))
	rg := rand.New(rand.NewSource(0x239487))

	for i := 0; i < 25; i++ {
		l := 1024 * (i + 2)
		// our own little twist on fizzbuzz to look out for 1-off errors
		if i%5 == 0 {
			l = int(pwr.BlockSize)
		} else if i%3 == 0 {
			l = 0
		}

		buf := make([]byte, l)
		_, err := io.CopyN(bytes.NewBuffer(buf), rg, int64(l))
		mist(t, err)
		putfile(t, sample5, i, buf)
	}

	files := map[string]string{
		"hello":     sample,
		"80-fixed":  sample2,
		"60-fixed":  sample3,
		"120-fixed": sample4,
		"random":    sample5,
		"null":      "/dev/null",
	}

	patch := path.Join(workingDir, "patch.pwr")

	comm.Configure(true, true, false, false, false, false, false)

	for _, q := range []int{1, 9} {
		t.Logf("============ Quality %d ============", q)
		compression := pwr.CompressionSettings{
			Algorithm: pwr.CompressionAlgorithm_BROTLI,
			Quality:   int32(q),
		}

		for lhs := range files {
			for rhs := range files {
				mist(t, diff.Do(&diff.Params{
					Target:      files[lhs],
					Source:      files[rhs],
					Patch:       patch,
					Compression: compression,
				}))
				stat, err := os.Lstat(patch)
				mist(t, err)
				t.Logf("%10s -> %10s = %s", lhs, rhs, humanize.IBytes(uint64(stat.Size())))
			}
		}
	}

	compression := pwr.CompressionSettings{
		Algorithm: pwr.CompressionAlgorithm_BROTLI,
		Quality:   1,
	}

	for _, filepath := range files {
		t.Logf("Signing %s\n", filepath)

		sigPath := path.Join(workingDir, "signature.pwr.sig")
		mist(t, sign.Do(filepath, sigPath, compression, false))

		sigReader, err := eos.Open(sigPath)
		mist(t, err)

		sigSource := seeksource.FromFile(sigReader)
		_, err = sigSource.Resume(nil)
		mist(t, err)

		signature, err := pwr.ReadSignature(sigSource)
		mist(t, err)

		mist(t, sigReader.Close())

		validator := &pwr.ValidatorContext{
			FailFast: true,
		}

		mist(t, validator.Validate(filepath, signature))
	}

	// K windows you just sit this one out we'll catch you on the flip side
	if runtime.GOOS != "windows" {
		// In-place preserve permissions tests
		t.Logf("In-place patching should preserve permissions")

		eperm := os.FileMode(0750)

		samplePerm1 := path.Join(workingDir, "samplePerm1")
		mist(t, os.MkdirAll(samplePerm1, perm))
		putfileEx(t, samplePerm1, 1, bytes.Repeat([]byte{0x42, 0x69}, 8192), eperm)

		assert.Equal(t, octal(eperm), octal(permFor(t, path.Join(samplePerm1, "dummy1.dat"))))

		samplePerm2 := path.Join(workingDir, "samplePerm2")
		mist(t, os.MkdirAll(samplePerm2, perm))
		putfileEx(t, samplePerm2, 1, bytes.Repeat([]byte{0x69, 0x42}, 16384), eperm)

		assert.Equal(t, octal(eperm), octal(permFor(t, path.Join(samplePerm2, "dummy1.dat"))))

		mist(t, diff.Do(&diff.Params{
			Target:      samplePerm1,
			Source:      samplePerm2,
			Patch:       patch,
			Compression: compression,
		}))
		_, err := os.Lstat(patch)
		mist(t, err)

		cave := path.Join(workingDir, "cave")
		ditto.Do(samplePerm1, cave)

		mist(t, apply.Do(&apply.Params{
			Patch:   patch,
			Target:  cave,
			Output:  cave,
			InPlace: true,
		}))
		assert.Equal(t, octal(eperm|pwr.ModeMask), octal(permFor(t, path.Join(cave, "dummy1.dat"))))
	}
}
<commit_msg>Fix tests<commit_after>package main

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"os"
	"path"
	"runtime"
	"strconv"
	"testing"

	"github.com/dustin/go-humanize"
	"github.com/itchio/butler/cmd/apply"
	"github.com/itchio/butler/cmd/diff"
	"github.com/itchio/butler/cmd/ditto"
	"github.com/itchio/butler/cmd/sign"
	"github.com/itchio/butler/comm"
	"github.com/itchio/savior/seeksource"
	"github.com/itchio/wharf/eos"
	"github.com/itchio/wharf/pwr"
	"github.com/stretchr/testify/assert"
)

// reverse must
func mist(t *testing.T, err error) {
	if err != nil {
		assert.NoError(t, err)
		t.FailNow()
	}
}

func octal(perm os.FileMode) string {
	return strconv.FormatInt(int64(perm), 8)
}

func permFor(t *testing.T, path string) os.FileMode {
	t.Logf("Getting perm of %s", path)
	stat, err := os.Lstat(path)
	mist(t, err)
	return stat.Mode()
}

func putfile(t *testing.T, basePath string, i int, data []byte) {
	putfileEx(t, basePath, i, data, os.FileMode(0777))
}

func putfileEx(t *testing.T, basePath string, i int, data []byte, perm os.FileMode) {
	samplePath := path.Join(basePath, fmt.Sprintf("dummy%d.dat", i))
	mist(t, ioutil.WriteFile(samplePath, data, perm))
}

func TestAllTheThings(t *testing.T) {
	perm := os.FileMode(0777)
	workingDir, err := ioutil.TempDir("", "butler-tests")
	mist(t, err)
	defer os.RemoveAll(workingDir)

	sample := path.Join(workingDir, "sample")
	mist(t, os.MkdirAll(sample, perm))
	mist(t, ioutil.WriteFile(path.Join(sample, "hello.txt"), []byte("hello!"), perm))

	sample2 := path.Join(workingDir, "sample2")
	mist(t, os.MkdirAll(sample2, perm))
	for i := 0; i < 5; i++ {
		if i == 3 {
			// e.g. .gitkeep
			putfile(t, sample2, i, []byte{})
		} else {
			putfile(t, sample2, i, bytes.Repeat([]byte{0x42, 0x69}, i*200+1))
		}
	}

	sample3 := path.Join(workingDir, "sample3")
	mist(t, os.MkdirAll(sample3, perm))
	for i := 0; i < 60; i++ {
		putfile(t, sample3, i, bytes.Repeat([]byte{0x42, 0x69}, i*300+1))
	}

	sample4 := path.Join(workingDir, "sample4")
	mist(t, os.MkdirAll(sample4, perm))
	for i := 0; i < 120; i++ {
		putfile(t, sample4, i, bytes.Repeat([]byte{0x42, 0x69}, i*150+1))
	}

	sample5 := path.Join(workingDir, "sample5")
	mist(t, os.MkdirAll(sample5, perm))
	rg := rand.New(rand.NewSource(0x239487))

	for i := 0; i < 25; i++ {
		l := 1024 * (i + 2)
		// our own little twist on fizzbuzz to look out for 1-off errors
		if i%5 == 0 {
			l = int(pwr.BlockSize)
		} else if i%3 == 0 {
			l = 0
		}

		buf := make([]byte, l)
		_, err := io.CopyN(bytes.NewBuffer(buf), rg, int64(l))
		mist(t, err)
		putfile(t, sample5, i, buf)
	}

	files := map[string]string{
		"hello":     sample,
		"80-fixed":  sample2,
		"60-fixed":  sample3,
		"120-fixed": sample4,
		"random":    sample5,
		"null":      "/dev/null",
	}

	patch := path.Join(workingDir, "patch.pwr")

	comm.Configure(true, true, false, false, false, false, false)

	for _, q := range []int{1, 9} {
		t.Logf("============ Quality %d ============", q)
		compression := pwr.CompressionSettings{
			Algorithm: pwr.CompressionAlgorithm_BROTLI,
			Quality:   int32(q),
		}

		for lhs := range files {
			for rhs := range files {
				mist(t, diff.Do(&diff.Params{
					Target:      files[lhs],
					Source:      files[rhs],
					Patch:       patch,
					Compression: compression,
				}))
				stat, err := os.Lstat(patch)
				mist(t, err)
				t.Logf("%10s -> %10s = %s", lhs, rhs, humanize.IBytes(uint64(stat.Size())))
			}
		}
	}

	compression := pwr.CompressionSettings{
		Algorithm: pwr.CompressionAlgorithm_BROTLI,
		Quality:   1,
	}

	for _, filepath := range files {
		t.Logf("Signing %s\n", filepath)

		sigPath := path.Join(workingDir, "signature.pwr.sig")
		mist(t, sign.Do(filepath, sigPath, compression, false))

		sigReader, err := eos.Open(sigPath)
		mist(t, err)

		sigSource := seeksource.FromFile(sigReader)
		_, err = sigSource.Resume(nil)
		mist(t, err)

		signature, err := pwr.ReadSignature(context.Background(), sigSource)
		mist(t, err)

		mist(t, sigReader.Close())

		validator := &pwr.ValidatorContext{
			FailFast: true,
		}

		mist(t, validator.Validate(context.Background(), filepath, signature))
	}

	// K windows you just sit this one out we'll catch you on the flip side
	if runtime.GOOS != "windows" {
		// In-place preserve permissions tests
		t.Logf("In-place patching should preserve permissions")

		eperm := os.FileMode(0750)

		samplePerm1 := path.Join(workingDir, "samplePerm1")
		mist(t, os.MkdirAll(samplePerm1, perm))
		putfileEx(t, samplePerm1, 1, bytes.Repeat([]byte{0x42, 0x69}, 8192), eperm)

		assert.Equal(t, octal(eperm), octal(permFor(t, path.Join(samplePerm1, "dummy1.dat"))))

		samplePerm2 := path.Join(workingDir, "samplePerm2")
		mist(t, os.MkdirAll(samplePerm2, perm))
		putfileEx(t, samplePerm2, 1, bytes.Repeat([]byte{0x69, 0x42}, 16384), eperm)

		assert.Equal(t, octal(eperm), octal(permFor(t, path.Join(samplePerm2, "dummy1.dat"))))

		mist(t, diff.Do(&diff.Params{
			Target:      samplePerm1,
			Source:      samplePerm2,
			Patch:       patch,
			Compression: compression,
		}))
		_, err := os.Lstat(patch)
		mist(t, err)

		cave := path.Join(workingDir, "cave")
		ditto.Do(samplePerm1, cave)

		mist(t, apply.Do(&apply.Params{
			Patch:   patch,
			Target:  cave,
			Output:  cave,
			InPlace: true,
		}))
		assert.Equal(t, octal(eperm|pwr.ModeMask), octal(permFor(t, path.Join(cave, "dummy1.dat"))))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"os"
	"path"
	"strconv"
	"testing"

	"github.com/dustin/go-humanize"
	"github.com/itchio/butler/comm"
	"github.com/itchio/wharf/pwr"
	"github.com/itchio/wharf/sync"
	"github.com/itchio/wharf/tlc"
	"github.com/stretchr/testify/assert"
)

// reverse must
func mist(t *testing.T, err error) {
	if err != nil {
		panic(err)
	}
}

func octal(perm os.FileMode) string {
	return strconv.FormatInt(int64(perm), 8)
}

func permFor(t *testing.T, path string) os.FileMode {
	t.Logf("Getting perm of %s", path)
	stat, err := os.Lstat(path)
	mist(t, err)
	return stat.Mode()
}

func putfile(t *testing.T, basePath string, i int, data []byte) {
	putfileEx(t, basePath, i, data, os.FileMode(0777))
}

func putfileEx(t *testing.T, basePath string, i int, data []byte, perm os.FileMode) {
	samplePath := path.Join(basePath, fmt.Sprintf("dummy%d.dat", i))
	mist(t, ioutil.WriteFile(samplePath, data, perm))
}

func shortSizeCount(hashes []sync.BlockHash) string {
	count := 0
	for _, hash := range hashes {
		if hash.ShortSize != 0 {
			count++
		}
	}
	return fmt.Sprintf("%d/%d", count, len(hashes))
}

func TestAllTheThings(t *testing.T) {
	perm := os.FileMode(0777)
	workingDir, err := ioutil.TempDir("", "butler-tests")
	mist(t, err)
	defer os.RemoveAll(workingDir)

	sample := path.Join(workingDir, "sample")
	mist(t, os.MkdirAll(sample, perm))
	mist(t, ioutil.WriteFile(path.Join(sample, "hello.txt"), []byte("hello!"), perm))

	sample2 := path.Join(workingDir, "sample2")
	mist(t, os.MkdirAll(sample2, perm))
	for i := 0; i < 5; i++ {
		if i == 3 {
			// e.g. .gitkeep
			putfile(t, sample2, i, []byte{})
		} else {
			putfile(t, sample2, i, bytes.Repeat([]byte{0x42, 0x69}, i*200+1))
		}
	}

	sample3 := path.Join(workingDir, "sample3")
	mist(t, os.MkdirAll(sample3, perm))
	for i := 0; i < 60; i++ {
		putfile(t, sample3, i, bytes.Repeat([]byte{0x42, 0x69}, i*300+1))
	}

	sample4 := path.Join(workingDir, "sample4")
	mist(t, os.MkdirAll(sample4, perm))
	for i := 0; i < 120; i++ {
		putfile(t, sample4, i, bytes.Repeat([]byte{0x42, 0x69}, i*150+1))
	}

	sample5 := path.Join(workingDir, "sample5")
	mist(t, os.MkdirAll(sample5, perm))
	rg := rand.New(rand.NewSource(0x239487))

	for i := 0; i < 25; i++ {
		l := 1024 * (i + 2)
		// our own little twist on fizzbuzz to look out for 1-off errors
		if i%5 == 0 {
			l = pwr.BlockSize
		} else if i%3 == 0 {
			l = 0
		}

		buf := make([]byte, l)
		_, err := io.CopyN(bytes.NewBuffer(buf), rg, int64(l))
		mist(t, err)
		putfile(t, sample5, i, buf)
	}

	files := map[string]string{
		"hello":     sample,
		"80-fixed":  sample2,
		"60-fixed":  sample3,
		"120-fixed": sample4,
		"random":    sample5,
		"null":      "/dev/null",
	}

	patch := path.Join(workingDir, "patch.pwr")

	comm.Configure(true, true, false, false, false, false, false)

	for _, q := range []int{1, 9} {
		t.Logf("============ Quality %d ============", q)
		compression := pwr.CompressionSettings{
			Algorithm: pwr.CompressionAlgorithm_BROTLI,
			Quality:   int32(q),
		}

		for lhs := range files {
			for rhs := range files {
				mist(t, doDiff(files[lhs], files[rhs], patch, compression))
				stat, err := os.Lstat(patch)
				mist(t, err)
				t.Logf("%10s -> %10s = %s", lhs, rhs, humanize.Bytes(uint64(stat.Size())))
			}
		}
	}

	compression := pwr.CompressionSettings{
		Algorithm: pwr.CompressionAlgorithm_BROTLI,
		Quality:   1,
	}

	for _, filepath := range files {
		t.Logf("Signing %s\n", filepath)

		sigpath := path.Join(workingDir, "signature.pwr.sig")
		mist(t, doSign(filepath, sigpath, compression, false))

		sigr, err := os.Open(sigpath)
		mist(t, err)

		readcontainer, readsig, err := pwr.ReadSignature(sigr)
		mist(t, err)

		mist(t, sigr.Close())

		computedcontainer, err := tlc.Walk(filepath, filterPaths)
		mist(t, err)

		computedsig, err := pwr.ComputeSignature(computedcontainer, computedcontainer.NewFilePool(filepath), &pwr.StateConsumer{})
		mist(t, err)

		assert.Equal(t, len(readcontainer.Files), len(computedcontainer.Files))
		for i, rf := range readcontainer.Files {
			cf := computedcontainer.Files[i]
			assert.Equal(t, *rf, *cf)
		}

		assert.Equal(t, readsig, computedsig)
		mist(t, pwr.CompareHashes(readsig, computedsig, computedcontainer))
	}

	{
		// In-place preserve permissions tests
		t.Logf("In-place patching should preserve permissions")

		eperm := os.FileMode(0750)

		samplePerm1 := path.Join(workingDir, "samplePerm1")
		mist(t, os.MkdirAll(samplePerm1, perm))
		putfileEx(t, samplePerm1, 1, bytes.Repeat([]byte{0x42, 0x69}, 8192), eperm)

		assert.Equal(t, octal(eperm), octal(permFor(t, path.Join(samplePerm1, "dummy1.dat"))))

		samplePerm2 := path.Join(workingDir, "samplePerm2")
		mist(t, os.MkdirAll(samplePerm2, perm))
		putfileEx(t, samplePerm2, 1, bytes.Repeat([]byte{0x69, 0x42}, 16384), eperm)

		assert.Equal(t, octal(eperm), octal(permFor(t, path.Join(samplePerm2, "dummy1.dat"))))

		mist(t, doDiff(samplePerm1, samplePerm2, patch, compression))
		_, err := os.Lstat(patch)
		mist(t, err)

		cave := path.Join(workingDir, "cave")
		ditto(samplePerm1, cave)

		mist(t, doApply(patch, cave, cave, true, ""))
		assert.Equal(t, octal(eperm|pwr.ModeMask), octal(permFor(t, path.Join(cave, "dummy1.dat"))))
	}
}
<commit_msg>Windows had a rough childhood, plus they had a note from their mom<commit_after>package main

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"os"
	"path"
	"runtime"
	"strconv"
	"testing"

	"github.com/dustin/go-humanize"
	"github.com/itchio/butler/comm"
	"github.com/itchio/wharf/pwr"
	"github.com/itchio/wharf/sync"
	"github.com/itchio/wharf/tlc"
	"github.com/stretchr/testify/assert"
)

// reverse must
func mist(t *testing.T, err error) {
	if err != nil {
		panic(err)
	}
}

func octal(perm os.FileMode) string {
	return strconv.FormatInt(int64(perm), 8)
}

func permFor(t *testing.T, path string) os.FileMode {
	t.Logf("Getting perm of %s", path)
	stat, err := os.Lstat(path)
	mist(t, err)
	return stat.Mode()
}

func putfile(t *testing.T, basePath string, i int, data []byte) {
	putfileEx(t, basePath, i, data, os.FileMode(0777))
}

func putfileEx(t *testing.T, basePath string, i int, data []byte, perm os.FileMode) {
	samplePath := path.Join(basePath, fmt.Sprintf("dummy%d.dat", i))
	mist(t, ioutil.WriteFile(samplePath, data, perm))
}

func shortSizeCount(hashes []sync.BlockHash) string {
	count := 0
	for _, hash := range hashes {
		if hash.ShortSize != 0 {
			count++
		}
	}
	return fmt.Sprintf("%d/%d", count, len(hashes))
}

func TestAllTheThings(t *testing.T) {
	perm := os.FileMode(0777)
	workingDir, err := ioutil.TempDir("", "butler-tests")
	mist(t, err)
	defer os.RemoveAll(workingDir)

	sample := path.Join(workingDir, "sample")
	mist(t, os.MkdirAll(sample, perm))
	mist(t, ioutil.WriteFile(path.Join(sample, "hello.txt"), []byte("hello!"), perm))

	sample2 := path.Join(workingDir, "sample2")
	mist(t, os.MkdirAll(sample2, perm))
	for i := 0; i < 5; i++ {
		if i == 3 {
			// e.g. .gitkeep
			putfile(t, sample2, i, []byte{})
		} else {
			putfile(t, sample2, i, bytes.Repeat([]byte{0x42, 0x69}, i*200+1))
		}
	}

	sample3 := path.Join(workingDir, "sample3")
	mist(t, os.MkdirAll(sample3, perm))
	for i := 0; i < 60; i++ {
		putfile(t, sample3, i, bytes.Repeat([]byte{0x42, 0x69}, i*300+1))
	}

	sample4 := path.Join(workingDir, "sample4")
	mist(t, os.MkdirAll(sample4, perm))
	for i := 0; i < 120; i++ {
		putfile(t, sample4, i, bytes.Repeat([]byte{0x42, 0x69}, i*150+1))
	}

	sample5 := path.Join(workingDir, "sample5")
	mist(t, os.MkdirAll(sample5, perm))
	rg := rand.New(rand.NewSource(0x239487))

	for i := 0; i < 25; i++ {
		l := 1024 * (i + 2)
		// our own little twist on fizzbuzz to look out for 1-off errors
		if i%5 == 0 {
			l = pwr.BlockSize
		} else if i%3 == 0 {
			l = 0
		}

		buf := make([]byte, l)
		_, err := io.CopyN(bytes.NewBuffer(buf), rg, int64(l))
		mist(t, err)
		putfile(t, sample5, i, buf)
	}

	files := map[string]string{
		"hello":     sample,
		"80-fixed":  sample2,
		"60-fixed":  sample3,
		"120-fixed": sample4,
		"random":    sample5,
		"null":      "/dev/null",
	}

	patch := path.Join(workingDir, "patch.pwr")

	comm.Configure(true, true, false, false, false, false, false)

	for _, q := range []int{1, 9} {
		t.Logf("============ Quality %d ============", q)
		compression := pwr.CompressionSettings{
			Algorithm: pwr.CompressionAlgorithm_BROTLI,
			Quality:   int32(q),
		}

		for lhs := range files {
			for rhs := range files {
				mist(t, doDiff(files[lhs], files[rhs], patch, compression))
				stat, err := os.Lstat(patch)
				mist(t, err)
				t.Logf("%10s -> %10s = %s", lhs, rhs, humanize.Bytes(uint64(stat.Size())))
			}
		}
	}

	compression := pwr.CompressionSettings{
		Algorithm: pwr.CompressionAlgorithm_BROTLI,
		Quality:   1,
	}

	for _, filepath := range files {
		t.Logf("Signing %s\n", filepath)

		sigpath := path.Join(workingDir, "signature.pwr.sig")
		mist(t, doSign(filepath, sigpath, compression, false))

		sigr, err := os.Open(sigpath)
		mist(t, err)

		readcontainer, readsig, err := pwr.ReadSignature(sigr)
		mist(t, err)

		mist(t, sigr.Close())

		computedcontainer, err := tlc.Walk(filepath, filterPaths)
		mist(t, err)

		computedsig, err := pwr.ComputeSignature(computedcontainer, computedcontainer.NewFilePool(filepath), &pwr.StateConsumer{})
		mist(t, err)

		assert.Equal(t, len(readcontainer.Files), len(computedcontainer.Files))
		for i, rf := range readcontainer.Files {
			cf := computedcontainer.Files[i]
			assert.Equal(t, *rf, *cf)
		}

		assert.Equal(t, readsig, computedsig)
		mist(t, pwr.CompareHashes(readsig, computedsig, computedcontainer))
	}

	// K windows you just sit this one out we'll catch you on the flip side
	if runtime.GOOS != "windows" {
		// In-place preserve permissions tests
		t.Logf("In-place patching should preserve permissions")

		eperm := os.FileMode(0750)

		samplePerm1 := path.Join(workingDir, "samplePerm1")
		mist(t, os.MkdirAll(samplePerm1, perm))
		putfileEx(t, samplePerm1, 1, bytes.Repeat([]byte{0x42, 0x69}, 8192), eperm)

		assert.Equal(t, octal(eperm), octal(permFor(t, path.Join(samplePerm1, "dummy1.dat"))))

		samplePerm2 := path.Join(workingDir, "samplePerm2")
		mist(t, os.MkdirAll(samplePerm2, perm))
		putfileEx(t, samplePerm2, 1, bytes.Repeat([]byte{0x69, 0x42}, 16384), eperm)

		assert.Equal(t, octal(eperm), octal(permFor(t, path.Join(samplePerm2, "dummy1.dat"))))

		mist(t, doDiff(samplePerm1, samplePerm2, patch, compression))
		_, err := os.Lstat(patch)
		mist(t, err)

		cave := path.Join(workingDir, "cave")
		ditto(samplePerm1, cave)

		mist(t, doApply(patch, cave, cave, true, ""))
		assert.Equal(t, octal(eperm|pwr.ModeMask), octal(permFor(t, path.Join(cave, "dummy1.dat"))))
	}
}
<|endoftext|>


<commit_before>package queue

import (
	"fmt"
	"testing"
	"time"

	peer "github.com/jbenet/go-ipfs/peer"
	u "github.com/jbenet/go-ipfs/util"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
)

func newPeer(id string) *peer.Peer {
	return &peer.Peer{ID: peer.ID(id)}
}

func TestQueue(t *testing.T) {

	p1 := newPeer("11140beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a31")
	p2 := newPeer("11140beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a32")
	p3 := newPeer("11140beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33")
	p4 := newPeer("11140beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a34")
	p5 := newPeer("11140beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a31")

	// these are the peer.IDs' XORKeySpace Key values:
	// [228 47 151 130 156 102 222 232 218 31 132 94 170 208 80 253 120 103 55 35 91 237 48 157 81 245 57 247 66 150 9 40]
	// [26 249 85 75 54 49 25 30 21 86 117 62 85 145 48 175 155 194 210 216 58 14 241 143 28 209 129 144 122 28 163 6]
	// [78 135 26 216 178 181 224 181 234 117 2 248 152 115 255 103 244 34 4 152 193 88 9 225 8 127 216 158 226 8 236 246]
	// [125 135 124 6 226 160 101 94 192 57 39 12 18 79 121 140 190 154 147 55 44 83 101 151 63 255 94 179 51 203 241 51]

	pq := NewXORDistancePQ(u.Key("11140beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a31"))
	pq.Enqueue(p3)
	pq.Enqueue(p1)
	pq.Enqueue(p2)
	pq.Enqueue(p4)
	pq.Enqueue(p5)
	pq.Enqueue(p1)

	// should come out as: p1, p4, p3, p2

	if d := pq.Dequeue(); d != p1 && d != p5 {
		t.Error("ordering failed")
	}

	if d := pq.Dequeue(); d != p1 && d != p5 {
		t.Error("ordering failed")
	}

	if d := pq.Dequeue(); d != p1 && d != p5 {
		t.Error("ordering failed")
	}

	if pq.Dequeue() != p4 {
		t.Error("ordering failed")
	}

	if pq.Dequeue() != p3 {
		t.Error("ordering failed")
	}

	if pq.Dequeue() != p2 {
		t.Error("ordering failed")
	}

}

func newPeerTime(t time.Time) *peer.Peer {
	s := fmt.Sprintf("hmmm time: %v", t)
	h, _ := u.Hash([]byte(s))
	return &peer.Peer{ID: peer.ID(h)}
}

func TestSyncQueue(t *testing.T) {
	ctx, _ := context.WithTimeout(context.Background(), time.Second*2)

	pq := NewXORDistancePQ(u.Key("11140beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a31"))
	cq := NewChanQueue(ctx, pq)

	max := 100000
	consumerN := 10
	countsIn := make([]int, consumerN*2)
	countsOut := make([]int, consumerN)

	produce := func(p int) {
		tick := time.Tick(time.Millisecond)
		for i := 0; i < max; i++ {
			select {
			case tim := <-tick:
				countsIn[p]++
				cq.EnqChan <- newPeerTime(tim)
			case <-ctx.Done():
				return
			}
		}
	}

	consume := func(c int) {
		for {
			select {
			case <-cq.DeqChan:
				countsOut[c]++
			case <-ctx.Done():
				return
			}
		}
	}

	// make n * 2 producers and n consumers
	for i := 0; i < consumerN; i++ {
		go produce(i)
		go produce(consumerN + i)
		go consume(i)
	}

	select {
	case <-ctx.Done():
	}

	sum := func(ns []int) int {
		total := 0
		for _, n := range ns {
			total += n
		}
		return total
	}

	if sum(countsIn) != sum(countsOut) {
		t.Errorf("didnt get all of them out: %d/%d", countsOut, countsIn)
	}
}
<commit_msg>increment time for queue test<commit_after>package queue

import (
	"fmt"
	"testing"
	"time"

	peer "github.com/jbenet/go-ipfs/peer"
	u "github.com/jbenet/go-ipfs/util"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
)

func newPeer(id string) *peer.Peer {
	return &peer.Peer{ID: peer.ID(id)}
}

func TestQueue(t *testing.T) {

	p1 := newPeer("11140beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a31")
	p2 := newPeer("11140beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a32")
	p3 := newPeer("11140beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33")
	p4 := newPeer("11140beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a34")
	p5 := newPeer("11140beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a31")

	// these are the peer.IDs' XORKeySpace Key values:
	// [228 47 151 130 156 102 222 232 218 31 132 94 170 208 80 253 120 103 55 35 91 237 48 157 81 245 57 247 66 150 9 40]
	// [26 249 85 75 54 49 25 30 21 86 117 62 85 145 48 175 155 194 210 216 58 14 241 143 28 209 129 144 122 28 163 6]
	// [78 135 26 216 178 181 224 181 234 117 2 248 152 115 255 103 244 34 4 152 193 88 9 225 8 127 216 158 226 8 236 246]
	// [125 135 124 6 226 160 101 94 192 57 39 12 18 79 121 140 190 154 147 55 44 83 101 151 63 255 94 179 51 203 241 51]

	pq := NewXORDistancePQ(u.Key("11140beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a31"))
	pq.Enqueue(p3)
	pq.Enqueue(p1)
	pq.Enqueue(p2)
	pq.Enqueue(p4)
	pq.Enqueue(p5)
	pq.Enqueue(p1)

	// should come out as: p1, p4, p3, p2

	if d := pq.Dequeue(); d != p1 && d != p5 {
		t.Error("ordering failed")
	}

	if d := pq.Dequeue(); d != p1 && d != p5 {
		t.Error("ordering failed")
	}

	if d := pq.Dequeue(); d != p1 && d != p5 {
		t.Error("ordering failed")
	}

	if pq.Dequeue() != p4 {
		t.Error("ordering failed")
	}

	if pq.Dequeue() != p3 {
		t.Error("ordering failed")
	}

	if pq.Dequeue() != p2 {
		t.Error("ordering failed")
	}

}

func newPeerTime(t time.Time) *peer.Peer {
	s := fmt.Sprintf("hmmm time: %v", t)
	h, _ := u.Hash([]byte(s))
	return &peer.Peer{ID: peer.ID(h)}
}

func TestSyncQueue(t *testing.T) {
	ctx, _ := context.WithTimeout(context.Background(), time.Second*4)

	pq := NewXORDistancePQ(u.Key("11140beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a31"))
	cq := NewChanQueue(ctx, pq)

	max := 100000
	consumerN := 10
	countsIn := make([]int, consumerN*2)
	countsOut := make([]int, consumerN)

	produce := func(p int) {
		tick := time.Tick(time.Millisecond)
		for i := 0; i < max; i++ {
			select {
			case tim := <-tick:
				countsIn[p]++
				cq.EnqChan <- newPeerTime(tim)
			case <-ctx.Done():
				return
			}
		}
	}

	consume := func(c int) {
		for {
			select {
			case <-cq.DeqChan:
				countsOut[c]++
			case <-ctx.Done():
				return
			}
		}
	}

	// make n * 2 producers and n consumers
	for i := 0; i < consumerN; i++ {
		go produce(i)
		go produce(consumerN + i)
		go consume(i)
	}

	select {
	case <-ctx.Done():
	}

	sum := func(ns []int) int {
		total := 0
		for _, n := range ns {
			total += n
		}
		return total
	}

	if sum(countsIn) != sum(countsOut) {
		t.Errorf("didnt get all of them out: %d/%d", countsOut, countsIn)
	}
}
<|endoftext|>


<commit_before>package cache

import (
	"github.com/karlseguin/garnish/gc"
	"hash/fnv"
	"math/rand"
	"time"
)

const (
	BUCKETS     = 16
	BUCKET_MASK = BUCKETS - 1
)

type Entry struct {
	gc.CachedResponse
	Primary   string
	Secondary string
	next      *Entry
	prev      *Entry
	size      int
}

type Cache struct {
	list        *List
	maxSize     int
	size        int
	buckets     []*bucket
	persist     chan persist
	deletables  chan *Entry
	promotables chan *Entry
}

func New(maxSize int) *Cache {)
	c := &Cache{
		maxSize:     maxSize,
		list:        NewList(),
		buckets:     make([]*bucket, BUCKETS),
		persist:     make(chan persist),
		deletables:  make(chan *Entry, 1024),
		promotables: make(chan *Entry, 1024),
	}
	for i := 0; i < BUCKETS; i++ {
		c.buckets[i] = &bucket{lookup: make(map[string]map[string]*Entry)}
	}
	go c.worker()
	return c
}

func (c *Cache) Get(primary, secondary string) gc.CachedResponse {
	bucket := c.bucket(primary)
	response := bucket.get(primary, secondary)
	if response == nil {
		return nil
	}
	c.promotables <- response
	return response
}

func (c *Cache) Set(primary string, secondary string, response gc.CachedResponse) {
	entry := &Entry{
		Primary:        primary,
		Secondary:      secondary,
		CachedResponse: response,
		size:           response.Size(),
	}
	c.set(entry)
}

func (c *Cache) set(entry *Entry) {
	existing := c.bucket(entry.Primary).set(entry.Primary, entry.Secondary, entry)
	if existing != nil {
		c.deletables <- existing
	}
	c.promotables <- entry
}

func (c *Cache) Delete(primary string, secondary string) bool {
	return c.bucket(primary).delete(primary, secondary, c.deletables)
}

func (c *Cache) DeleteAll(primary string) bool {
	return c.bucket(primary).deleteAll(primary, c.deletables)
}

func (c *Cache) Save(path string, count int, cutoff time.Duration) error {
	p := persist{
		path:   path,
		count:  count,
		cutoff: cutoff,
		done:   make(chan error),
	}
	c.persist <- p
	return <-p.done
}

func (c *Cache) Load(path string) error {
	entries, err := loadFromFile(path)
	if err != nil {
		return err
	}
	expires := time.Now().Add(time.Second * 60)
	for _, entry := range entries {
		entry.Expire(expires.Add(time.Duration(rand.Intn(120)) * time.Second))
		c.set(entry)
	}
	return nil
}

func (c *Cache) bucket(key string) *bucket {
	h := fnv.New32a()
	h.Write([]byte(key))
	return c.buckets[h.Sum32()&BUCKET_MASK]
}

func (c *Cache) worker() {
	for {
		select {
		case entry := <-c.promotables:
			if entry.prev == nil { //new item
				c.size += entry.size
				if c.size > c.maxSize {
					c.gc()
				}
			}
			c.list.PushToFront(entry)
		case entry := <-c.deletables:
			if entry.prev != nil {
				c.list.Remove(entry)
				c.size -= entry.size
			}
		case p := <-c.persist:
			i := 0
			cutoff := time.Now().Add(p.cutoff)
			entries := make([]*Entry, p.count)
			entry := c.list.head.next
			for i < p.count && entry != c.list.tail {
				if entry.Expires().After(cutoff) {
					entries[i] = entry
					i++
				}
				entry = entry.next
			}
			entries = entries[:i]
			go p.persist(entries)
		}
	}
}

func (c *Cache) gc() {
	for i := 0; i < 1000; i++ {
		entry := c.list.tail.prev
		if entry == nil {
			return
		}
		primary := entry.Primary
		if len(primary) == 0 {
			return
		}
		c.bucket(primary).delete(primary, entry.Secondary, nil)
		c.list.Remove(entry)
		c.size -= entry.size
	}
}
<commit_msg>ugh x 2<commit_after>package cache

import (
	"github.com/karlseguin/garnish/gc"
	"hash/fnv"
	"math/rand"
	"time"
)

const (
	BUCKETS     = 16
	BUCKET_MASK = BUCKETS - 1
)

type Entry struct {
	gc.CachedResponse
	Primary   string
	Secondary string
	next      *Entry
	prev      *Entry
	size      int
}

type Cache struct {
	list        *List
	maxSize     int
	size        int
	buckets     []*bucket
	persist     chan persist
	deletables  chan *Entry
	promotables chan *Entry
}

func New(maxSize int) *Cache {
	c := &Cache{
		maxSize:     maxSize,
		list:        NewList(),
		buckets:     make([]*bucket, BUCKETS),
		persist:     make(chan persist),
		deletables:  make(chan *Entry, 1024),
		promotables: make(chan *Entry, 1024),
	}
	for i := 0; i < BUCKETS; i++ {
		c.buckets[i] = &bucket{lookup: make(map[string]map[string]*Entry)}
	}
	go c.worker()
	return c
}

func (c *Cache) Get(primary, secondary string) gc.CachedResponse {
	bucket := c.bucket(primary)
	response := bucket.get(primary, secondary)
	if response == nil {
		return nil
	}
	c.promotables <- response
	return response
}

func (c *Cache) Set(primary string, secondary string, response gc.CachedResponse) {
	entry := &Entry{
		Primary:        primary,
		Secondary:      secondary,
		CachedResponse: response,
		size:           response.Size(),
	}
	c.set(entry)
}

func (c *Cache) set(entry *Entry) {
	existing := c.bucket(entry.Primary).set(entry.Primary, entry.Secondary, entry)
	if existing != nil {
		c.deletables <- existing
	}
	c.promotables <- entry
}

func (c *Cache) Delete(primary string, secondary string) bool {
	return c.bucket(primary).delete(primary, secondary, c.deletables)
}

func (c *Cache) DeleteAll(primary string) bool {
	return c.bucket(primary).deleteAll(primary, c.deletables)
}

func (c *Cache) Save(path string, count int, cutoff time.Duration) error {
	p := persist{
		path:   path,
		count:  count,
		cutoff: cutoff,
		done:   make(chan error),
	}
	c.persist <- p
	return <-p.done
}

func (c *Cache) Load(path string) error {
	entries, err := loadFromFile(path)
	if err != nil {
		return err
	}
	expires := time.Now().Add(time.Second * 60)
	for _, entry := range entries {
		entry.Expire(expires.Add(time.Duration(rand.Intn(120)) * time.Second))
		c.set(entry)
	}
	return nil
}

func (c *Cache) bucket(key string) *bucket {
	h := fnv.New32a()
	h.Write([]byte(key))
	return c.buckets[h.Sum32()&BUCKET_MASK]
}

func (c *Cache) worker() {
	for {
		select {
		case entry := <-c.promotables:
			if entry.prev == nil { //new item
				c.size += entry.size
				if c.size > c.maxSize {
					c.gc()
				}
			}
			c.list.PushToFront(entry)
		case entry := <-c.deletables:
			if entry.prev != nil {
				c.list.Remove(entry)
				c.size -= entry.size
			}
		case p := <-c.persist:
			i := 0
			cutoff := time.Now().Add(p.cutoff)
			entries := make([]*Entry, p.count)
			entry := c.list.head.next
			for i < p.count && entry != c.list.tail {
				if entry.Expires().After(cutoff) {
					entries[i] = entry
					i++
				}
				entry = entry.next
			}
			entries = entries[:i]
			go p.persist(entries)
		}
	}
}

func (c *Cache) gc() {
	for i := 0; i < 1000; i++ {
		entry := c.list.tail.prev
		if entry == nil {
			return
		}
		primary := entry.Primary
		if len(primary) == 0 {
			return
		}
		c.bucket(primary).delete(primary, entry.Secondary, nil)
		c.list.Remove(entry)
		c.size -= entry.size
	}
}
<|endoftext|>


<commit_before>package metrics

import(
	"fmt"
	"sort"
	"time"
	hdr "github.com/codahale/hdrhistogram"
	histutil "github.com/skypies/util/histogram"
)

// {{{ Metric{}

type Metric struct {
	Name string
	wh *hdr.WindowedHistogram   // Holds the most recent 60 individual minutes
	prevMinute *hdr.Snapshot    // Snapshot of the most recent completed minute
	snaps []*hdr.Snapshot       // Holds the previous 24 hours
}

func (m *Metric)String() string {
	asciify := func(h *hdr.Histogram) string {
		return histutil.HDR2ASCII(h, 40, 0, 2000)
	}

	prevMinStr := "[gathering ...]"
	if m.prevMinute != nil {
		prevMinStr = asciify(hdr.Import(m.prevMinute))
	}

	str := fmt.Sprintf("%s\n   1m: %s\n  60m: %s\n", m.Name, prevMinStr, asciify(m.wh.Merge()))

	for i,snap := range m.snaps {
		str += fmt.Sprintf(" +%02dh: %s\n", i+1, asciify(hdr.Import(snap)))
	}

	return str
}

func NewMetric(name string) *Metric {
	// 60 underlying 'grams; one per minute; cover an hour.
	// Covers values 0-10M (milliseconds, so 0-10000s).
	// Only 2 sig figs - but that's enough for us.
	return &Metric{Name:name, wh:hdr.NewWindowed(60, 0,10000000, 2), snaps:[]*hdr.Snapshot{}}
}

func (m *Metric)RecordValue(v int64) {
	m.wh.Current.RecordValue(v) // ignore error
}

func (m *Metric)RotateMinute() {
	m.prevMinute = m.wh.Current.Export()
	m.wh.Rotate()
}

func (m *Metric)StoreSnapshot() {
	snap := m.wh.Merge().Export()
	m.snaps = append([]*hdr.Snapshot{snap}, m.snaps[:23]...)
//	m.snaps = append(m.snaps, snap)
//	if len(m.snaps) > 24 {
//		m.snaps = m.snaps[:24]
//	}
}

// }}}

// {{{ Metrics{}

type Metrics struct {
	m map[string]*Metric

	// State to help decide when to rotate from minute to minute, and when to take a snap
	lastRotation   time.Time
	numRotations   int
}

// }}}

// {{{ NewMetrics

func NewMetrics() Metrics {
	return Metrics{
		m: map[string]*Metric{},
		lastRotation: time.Now(),
		numRotations: 0,
	}
}

// }}}
// {{{ m.RecordValue

func (m *Metrics)RecordValue(name string, val int64) {
	// Do we need to roll things ? We rotate every minute, and take snapshots every hour.
	minsSinceLastRotation := int(time.Since(m.lastRotation).Minutes())
	for i:=0; i<minsSinceLastRotation; i++ {
		m.lastRotation = time.Now()
		m.numRotations++

		for name,_ := range m.m {
			m.m[name].RotateMinute()
		}

		if m.numRotations >= 60 {
			for name,_ := range m.m {
				m.m[name].StoreSnapshot()
			}
			m.numRotations = 0
		}
	}

	if _,exists := m.m[name]; !exists {
		m.m[name] = NewMetric(name)
	}

	m.m[name].RecordValue(val)
}

// }}}
// {{{ m.String

func (m *Metrics)String() string {
	names := []string{}
	for name,_ := range m.m { names = append(names,name) }
	sort.Strings(names)

	str := ""
	for _,name := range names {
		str += fmt.Sprintf("%s\n", m.m[name])
	}
	return str
}

// }}}

// {{{ -------------------------={ E N D }=----------------------------------

// Local variables:
// folded-file: t
// end:

// }}}
<commit_msg>Meetrics don't always have 24 slots ...<commit_after>package metrics

import(
	"fmt"
	"sort"
	"time"
	hdr "github.com/codahale/hdrhistogram"
	histutil "github.com/skypies/util/histogram"
)

// {{{ Metric{}

type Metric struct {
	Name string
	wh *hdr.WindowedHistogram   // Holds the most recent 60 individual minutes
	prevMinute *hdr.Snapshot    // Snapshot of the most recent completed minute
	snaps []*hdr.Snapshot       // Holds the previous 24 hours
}

func (m *Metric)String() string {
	asciify := func(h *hdr.Histogram) string {
		return histutil.HDR2ASCII(h, 40, 0, 2000)
	}

	prevMinStr := "[gathering ...]"
	if m.prevMinute != nil {
		prevMinStr = asciify(hdr.Import(m.prevMinute))
	}

	str := fmt.Sprintf("%s\n   1m: %s\n  60m: %s\n", m.Name, prevMinStr, asciify(m.wh.Merge()))

	for i,snap := range m.snaps {
		str += fmt.Sprintf(" +%02dh: %s\n", i+1, asciify(hdr.Import(snap)))
	}

	return str
}

func NewMetric(name string) *Metric {
	// 60 underlying 'grams; one per minute; cover an hour.
	// Covers values 0-10M (milliseconds, so 0-10000s).
	// Only 2 sig figs - but that's enough for us.
	return &Metric{Name:name, wh:hdr.NewWindowed(60, 0,10000000, 2), snaps:[]*hdr.Snapshot{}}
}

func (m *Metric)RecordValue(v int64) {
	m.wh.Current.RecordValue(v) // ignore error
}

func (m *Metric)RotateMinute() {
	m.prevMinute = m.wh.Current.Export()
	m.wh.Rotate()
}

func (m *Metric)StoreSnapshot() {
	snap := m.wh.Merge().Export()
	m.snaps = append([]*hdr.Snapshot{snap}, m.snaps...)
//	m.snaps = append(m.snaps, snap)
	if len(m.snaps) > 24 {
		m.snaps = m.snaps[:24]
	}
}

// }}}

// {{{ Metrics{}

type Metrics struct {
	m map[string]*Metric

	// State to help decide when to rotate from minute to minute, and when to take a snap
	lastRotation   time.Time
	numRotations   int
}

// }}}

// {{{ NewMetrics

func NewMetrics() Metrics {
	return Metrics{
		m: map[string]*Metric{},
		lastRotation: time.Now(),
		numRotations: 0,
	}
}

// }}}
// {{{ m.RecordValue

func (m *Metrics)RecordValue(name string, val int64) {
	// Do we need to roll things ? We rotate every minute, and take snapshots every hour.
	minsSinceLastRotation := int(time.Since(m.lastRotation).Minutes())
	for i:=0; i<minsSinceLastRotation; i++ {
		m.lastRotation = time.Now()
		m.numRotations++

		for name,_ := range m.m {
			m.m[name].RotateMinute()
		}

		if m.numRotations >= 60 {
			for name,_ := range m.m {
				m.m[name].StoreSnapshot()
			}
			m.numRotations = 0
		}
	}

	if _,exists := m.m[name]; !exists {
		m.m[name] = NewMetric(name)
	}

	m.m[name].RecordValue(val)
}

// }}}
// {{{ m.String

func (m *Metrics)String() string {
	names := []string{}
	for name,_ := range m.m { names = append(names,name) }
	sort.Strings(names)

	str := ""
	for _,name := range names {
		str += fmt.Sprintf("%s\n", m.m[name])
	}
	return str
}

// }}}

// {{{ -------------------------={ E N D }=----------------------------------

// Local variables:
// folded-file: t
// end:

// }}}
<|endoftext|>


<commit_before>package cargo

import (
	"container/list"
	"errors"
	"strings"
	"time"

	"code.google.com/p/go-uuid/uuid"

	"github.com/marcusolsson/goddd/location"
)

func NextTrackingId() TrackingId {
	return TrackingId(strings.Split(strings.ToUpper(uuid.New()), "-")[0])
}

// RoutingStatus
type RoutingStatus int

const (
	NotRouted RoutingStatus = iota
	Misrouted
	Routed
)

// TransportStatus
type TransportStatus int

const (
	NotReceived TransportStatus = iota
	InPort
	OnboardCarrier
	Claimed
	Unknown
)

func (s TransportStatus) String() string {
	switch s {
	case InPort:
		return "In port"
	default:
		return ""
	}
}

// RouteSpecification describes where a cargo origin and destination
// is, and the arrival deadline.
type RouteSpecification struct {
	Origin          location.Location
	Destination     location.Location
	ArrivalDeadline time.Time
}

func (s RouteSpecification) IsSatisfiedBy(itinerary Itinerary) bool {
	return itinerary.Legs != nil &&
		s.Origin.UNLocode == itinerary.InitialDepartureLocation().UNLocode &&
		s.Destination.UNLocode == itinerary.FinalArrivalLocation().UNLocode
}

type Leg struct {
	LoadLocation   location.Location
	UnloadLocation location.Location
}

// Itinerary
type Itinerary struct {
	Legs []Leg
}

func (i *Itinerary) InitialDepartureLocation() location.Location {
	if i.Legs == nil || len(i.Legs) == 0 {
		return location.UnknownLocation
	}
	return i.Legs[0].LoadLocation;
}

func (i *Itinerary) FinalArrivalLocation() location.Location {
	if i.Legs == nil || len(i.Legs) == 0 {
		return location.UnknownLocation
	}
	return i.Legs[len(i.Legs)-1].LoadLocation;
}

// Delivery is the actual transportation of the cargo, as opposed to
// the customer requirement (RouteSpecification) and the plan
// (Itinerary).
type Delivery struct {
	LastEvent         HandlingEvent
	LastKnownLocation location.Location
	Itinerary
	RouteSpecification
	RoutingStatus
	TransportStatus
}

// UpdateOnRouting creates a new delivery snapshot to reflect changes
// in routing, i.e.  when the route specification or the itinerary has
// changed but no additional handling of the cargo has been performed.
func (d *Delivery) UpdateOnRouting(routeSpecification RouteSpecification, itinerary Itinerary) Delivery {
	return newDelivery(d.LastEvent, itinerary, routeSpecification)
}

// DerivedFrom creates a new delivery snapshot based on the complete
// handling history of a cargo, as well as its route specification and
// itinerary.
func DeriveDeliveryFrom(routeSpecification RouteSpecification, itinerary Itinerary, history HandlingHistory) Delivery {
	lastEvent, _ := history.MostRecentlyCompletedEvent()
	return newDelivery(lastEvent, itinerary, routeSpecification)
}

func newDelivery(lastEvent HandlingEvent, itinerary Itinerary, routeSpecification RouteSpecification) Delivery {

	var (
		routingStatus     = calculateRoutingStatus(itinerary, routeSpecification)
		TransportStatus   = calculateTransportStatus(lastEvent)
		lastKnownLocation = calculateLastKnownLocation(lastEvent)
	)

	return Delivery{
		LastEvent:          lastEvent,
		Itinerary:          itinerary,
		RouteSpecification: routeSpecification,
		RoutingStatus:      routingStatus,
		TransportStatus:    TransportStatus,
		LastKnownLocation:  lastKnownLocation,
	}
}

func calculateRoutingStatus(itinerary Itinerary, routeSpecification RouteSpecification) RoutingStatus {
	if itinerary.Legs == nil {
		return NotRouted
	} else {
		if routeSpecification.IsSatisfiedBy(itinerary) {
			return Routed
		} else {
			return Misrouted
		}
	}

}

func calculateTransportStatus(event HandlingEvent) TransportStatus {
	zero := HandlingEvent{}
	if event == zero {
		return NotReceived
	}

	switch event.Type {
	case Load:
		return OnboardCarrier
	case Unload:
	case Receive:
	case Customs:
		return InPort
	case Claim:
		return Claimed
	default:
		return Unknown
	}

	return Unknown
}

func calculateLastKnownLocation(event HandlingEvent) location.Location {
	zero := HandlingEvent{}
	if event == zero {
		return event.Location
	} else {
		return location.UnknownLocation
	}
}

// TrackingId uniquely identifies a particular cargo.
type TrackingId string

type Equaler interface {
	Equal(Equaler) bool
}

// Cargo is the central class in the domain model,
type Cargo struct {
	TrackingId         TrackingId
	Origin             location.Location
	RouteSpecification RouteSpecification
	Itinerary          Itinerary
	Delivery
}

// NewCargo creates a new cargo in a consistent state.
func NewCargo(trackingId TrackingId, routeSpecification RouteSpecification) *Cargo {
	emptyItinerary := Itinerary{}

	emptyHistory := HandlingHistory{}
	emptyHistory.HandlingEvents = list.New()

	// Cargo origin never changes, even if the route specification
	// changes. However, at creation, cargo orgin can be derived
	// from the initial route specification.
	return &Cargo{
		TrackingId:         trackingId,
		Origin:             routeSpecification.Origin,
		RouteSpecification: routeSpecification,
		Delivery:           DeriveDeliveryFrom(routeSpecification, emptyItinerary, emptyHistory),
	}
}

// Assert that Cargo implements the Equaler interface.
var _ Equaler = &Cargo{}

func (c *Cargo) Equal(e Equaler) bool {
	return c.TrackingId == e.(*Cargo).TrackingId
}

// SpecifyNewRoute specifies a new route for this cargo.
func (c *Cargo) SpecifyNewRoute(routeSpecification RouteSpecification) {
	c.RouteSpecification = routeSpecification
	c.Delivery = c.Delivery.UpdateOnRouting(c.RouteSpecification, c.Itinerary)

}

// AssignToRoute attaches a new itinerary to this cargo.
func (c *Cargo) AssignToRoute(itinerary Itinerary) {
	c.Itinerary = itinerary
	c.Delivery = c.Delivery.UpdateOnRouting(c.RouteSpecification, c.Itinerary)
}

// Updates all aspects of the cargo aggregate status based on the
// current route specification, itinerary and handling of the cargo.
// When either of those three changes, i.e. when a new route is
// specified for the cargo, the cargo is assigned to a route or when
// the cargo is handled, the status must be re-calculated.
//
// RouteSpecification and Itinerary are both inside the Cargo
// aggregate, so changes to them cause the status to be updated
// synchronously, but changes to the delivery history (when a cargo is
// handled) cause the status update to happen asynchronously since
// HandlingEvent is in a different aggregate.
func (c *Cargo) DeriveDeliveryProgress(history HandlingHistory) {
	c.Delivery = DeriveDeliveryFrom(c.RouteSpecification, c.Itinerary, history)
}

// CargoRepository
type CargoRepository interface {
	Store(cargo Cargo) error
	// Finds a cargo using given id.
	Find(trackingId TrackingId) (Cargo, error)
	FindAll() []Cargo
}

var (
	ErrUnknownCargo = errors.New("Unknown cargo")
)

type cargoRepository struct {
	cargos map[TrackingId]Cargo
}

func (r *cargoRepository) Store(cargo Cargo) error {
	r.cargos[cargo.TrackingId] = cargo

	return nil
}

func (r *cargoRepository) Find(trackingId TrackingId) (Cargo, error) {

	if val, ok := r.cargos[trackingId]; ok {
		return val, nil
	}

	return Cargo{}, ErrUnknownCargo
}

func (r *cargoRepository) FindAll() []Cargo {
	c := make([]Cargo, 0, len(r.cargos))
	for _, val := range r.cargos {
		c = append(c, val)
	}
	return c
}

func NewCargoRepository() CargoRepository {
	return &cargoRepository{
		cargos: make(map[TrackingId]Cargo),
	}
}
<commit_msg>Add String() to RoutingStatus.<commit_after>package cargo

import (
	"container/list"
	"errors"
	"strings"
	"time"

	"code.google.com/p/go-uuid/uuid"

	"github.com/marcusolsson/goddd/location"
)

func NextTrackingId() TrackingId {
	return TrackingId(strings.Split(strings.ToUpper(uuid.New()), "-")[0])
}

// RoutingStatus
type RoutingStatus int

const (
	NotRouted RoutingStatus = iota
	Misrouted
	Routed
)

func (s RoutingStatus) String() string {
	switch s {
	case NotRouted:
		return "Not routed"
	case Misrouted:
		return "Misrouted"
	case Routed:
		return "Routed"
	default:
		return ""
	}
}

// TransportStatus
type TransportStatus int

const (
	NotReceived TransportStatus = iota
	InPort
	OnboardCarrier
	Claimed
	Unknown
)

func (s TransportStatus) String() string {
	switch s {
	case InPort:
		return "In port"
	default:
		return ""
	}
}

// RouteSpecification describes where a cargo origin and destination
// is, and the arrival deadline.
type RouteSpecification struct {
	Origin          location.Location
	Destination     location.Location
	ArrivalDeadline time.Time
}

func (s RouteSpecification) IsSatisfiedBy(itinerary Itinerary) bool {
	return itinerary.Legs != nil &&
		s.Origin.UNLocode == itinerary.InitialDepartureLocation().UNLocode &&
		s.Destination.UNLocode == itinerary.FinalArrivalLocation().UNLocode
}

type Leg struct {
	LoadLocation   location.Location
	UnloadLocation location.Location
}

// Itinerary
type Itinerary struct {
	Legs []Leg
}

func (i *Itinerary) InitialDepartureLocation() location.Location {
	if i.Legs == nil || len(i.Legs) == 0 {
		return location.UnknownLocation
	}
	return i.Legs[0].LoadLocation
}

func (i *Itinerary) FinalArrivalLocation() location.Location {
	if i.Legs == nil || len(i.Legs) == 0 {
		return location.UnknownLocation
	}
	return i.Legs[len(i.Legs)-1].LoadLocation
}

// Delivery is the actual transportation of the cargo, as opposed to
// the customer requirement (RouteSpecification) and the plan
// (Itinerary).
type Delivery struct {
	LastEvent         HandlingEvent
	LastKnownLocation location.Location
	Itinerary
	RouteSpecification
	RoutingStatus
	TransportStatus
}

// UpdateOnRouting creates a new delivery snapshot to reflect changes
// in routing, i.e.  when the route specification or the itinerary has
// changed but no additional handling of the cargo has been performed.
func (d *Delivery) UpdateOnRouting(routeSpecification RouteSpecification, itinerary Itinerary) Delivery {
	return newDelivery(d.LastEvent, itinerary, routeSpecification)
}

// DerivedFrom creates a new delivery snapshot based on the complete
// handling history of a cargo, as well as its route specification and
// itinerary.
func DeriveDeliveryFrom(routeSpecification RouteSpecification, itinerary Itinerary, history HandlingHistory) Delivery {
	lastEvent, _ := history.MostRecentlyCompletedEvent()
	return newDelivery(lastEvent, itinerary, routeSpecification)
}

func newDelivery(lastEvent HandlingEvent, itinerary Itinerary, routeSpecification RouteSpecification) Delivery {

	var (
		routingStatus     = calculateRoutingStatus(itinerary, routeSpecification)
		TransportStatus   = calculateTransportStatus(lastEvent)
		lastKnownLocation = calculateLastKnownLocation(lastEvent)
	)

	return Delivery{
		LastEvent:          lastEvent,
		Itinerary:          itinerary,
		RouteSpecification: routeSpecification,
		RoutingStatus:      routingStatus,
		TransportStatus:    TransportStatus,
		LastKnownLocation:  lastKnownLocation,
	}
}

func calculateRoutingStatus(itinerary Itinerary, routeSpecification RouteSpecification) RoutingStatus {
	if itinerary.Legs == nil {
		return NotRouted
	} else {
		if routeSpecification.IsSatisfiedBy(itinerary) {
			return Routed
		} else {
			return Misrouted
		}
	}

}

func calculateTransportStatus(event HandlingEvent) TransportStatus {
	zero := HandlingEvent{}
	if event == zero {
		return NotReceived
	}

	switch event.Type {
	case Load:
		return OnboardCarrier
	case Unload:
	case Receive:
	case Customs:
		return InPort
	case Claim:
		return Claimed
	default:
		return Unknown
	}

	return Unknown
}

func calculateLastKnownLocation(event HandlingEvent) location.Location {
	zero := HandlingEvent{}
	if event == zero {
		return event.Location
	} else {
		return location.UnknownLocation
	}
}

// TrackingId uniquely identifies a particular cargo.
type TrackingId string

type Equaler interface {
	Equal(Equaler) bool
}

// Cargo is the central class in the domain model,
type Cargo struct {
	TrackingId         TrackingId
	Origin             location.Location
	RouteSpecification RouteSpecification
	Itinerary          Itinerary
	Delivery
}

// NewCargo creates a new cargo in a consistent state.
func NewCargo(trackingId TrackingId, routeSpecification RouteSpecification) *Cargo {
	emptyItinerary := Itinerary{}

	emptyHistory := HandlingHistory{}
	emptyHistory.HandlingEvents = list.New()

	// Cargo origin never changes, even if the route specification
	// changes. However, at creation, cargo orgin can be derived
	// from the initial route specification.
	return &Cargo{
		TrackingId:         trackingId,
		Origin:             routeSpecification.Origin,
		RouteSpecification: routeSpecification,
		Delivery:           DeriveDeliveryFrom(routeSpecification, emptyItinerary, emptyHistory),
	}
}

// Assert that Cargo implements the Equaler interface.
var _ Equaler = &Cargo{}

func (c *Cargo) Equal(e Equaler) bool {
	return c.TrackingId == e.(*Cargo).TrackingId
}

// SpecifyNewRoute specifies a new route for this cargo.
func (c *Cargo) SpecifyNewRoute(routeSpecification RouteSpecification) {
	c.RouteSpecification = routeSpecification
	c.Delivery = c.Delivery.UpdateOnRouting(c.RouteSpecification, c.Itinerary)

}

// AssignToRoute attaches a new itinerary to this cargo.
func (c *Cargo) AssignToRoute(itinerary Itinerary) {
	c.Itinerary = itinerary
	c.Delivery = c.Delivery.UpdateOnRouting(c.RouteSpecification, c.Itinerary)
}

// Updates all aspects of the cargo aggregate status based on the
// current route specification, itinerary and handling of the cargo.
// When either of those three changes, i.e. when a new route is
// specified for the cargo, the cargo is assigned to a route or when
// the cargo is handled, the status must be re-calculated.
//
// RouteSpecification and Itinerary are both inside the Cargo
// aggregate, so changes to them cause the status to be updated
// synchronously, but changes to the delivery history (when a cargo is
// handled) cause the status update to happen asynchronously since
// HandlingEvent is in a different aggregate.
func (c *Cargo) DeriveDeliveryProgress(history HandlingHistory) {
	c.Delivery = DeriveDeliveryFrom(c.RouteSpecification, c.Itinerary, history)
}

// CargoRepository
type CargoRepository interface {
	Store(cargo Cargo) error
	// Finds a cargo using given id.
	Find(trackingId TrackingId) (Cargo, error)
	FindAll() []Cargo
}

var (
	ErrUnknownCargo = errors.New("Unknown cargo")
)

type cargoRepository struct {
	cargos map[TrackingId]Cargo
}

func (r *cargoRepository) Store(cargo Cargo) error {
	r.cargos[cargo.TrackingId] = cargo

	return nil
}

func (r *cargoRepository) Find(trackingId TrackingId) (Cargo, error) {

	if val, ok := r.cargos[trackingId]; ok {
		return val, nil
	}

	return Cargo{}, ErrUnknownCargo
}

func (r *cargoRepository) FindAll() []Cargo {
	c := make([]Cargo, 0, len(r.cargos))
	for _, val := range r.cargos {
		c = append(c, val)
	}
	return c
}

func NewCargoRepository() CargoRepository {
	return &cargoRepository{
		cargos: make(map[TrackingId]Cargo),
	}
}
<|endoftext|>