<commit_before>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package misc

import (
	"fmt"
	"strings"

	"github.com/coreos/mantle/kola/cluster"
	"github.com/coreos/mantle/kola/register"
	"github.com/coreos/mantle/platform"
)

func init() {
	register.Register(&register.Test{
		Run:         DeadLinks,
		ClusterSize: 1,
		Name:        "coreos.filesystem.deadlinks",
		UserData:    `#cloud-config`,
	})
	register.Register(&register.Test{
		Run:         SUIDFiles,
		ClusterSize: 1,
		Name:        "coreos.filesystem.suid",
		UserData:    `#cloud-config`,
	})
	register.Register(&register.Test{
		Run:         SGIDFiles,
		ClusterSize: 1,
		Name:        "coreos.filesystem.sgid",
		UserData:    `#cloud-config`,
	})
	register.Register(&register.Test{
		Run:         WritableFiles,
		ClusterSize: 1,
		Name:        "coreos.filesystem.writablefiles",
		UserData:    `#cloud-config`,
	})
	register.Register(&register.Test{
		Run:         WritableDirs,
		ClusterSize: 1,
		Name:        "coreos.filesystem.writabledirs",
		UserData:    `#cloud-config`,
	})
	register.Register(&register.Test{
		Run:         StickyDirs,
		ClusterSize: 1,
		Name:        "coreos.filesystem.stickydirs",
		UserData:    `#cloud-config`,
	})
}

func sugidFiles(m platform.Machine, validfiles []string, mode string) error {
	badfiles := make([]string, 0, 0)

	command := fmt.Sprintf("sudo find / -ignore_readdir_race -path /sys -prune -o -path /proc -prune -o -path /var/lib/rkt -prune -o -type f -perm -%v -print", mode)

	output, err := m.SSH(command)
	if err != nil {
		return fmt.Errorf("Failed to run find: output %s, status: %v", output, err)
	}

	if string(output) == "" {
		return nil
	}

	files := strings.Split(string(output), "\n")
	for _, file := range files {
		var valid bool

		for _, validfile := range validfiles {
			if file == validfile {
				valid = true
			}
		}
		if valid != true {
			badfiles = append(badfiles, file)
		}
	}

	if len(badfiles) != 0 {
		return fmt.Errorf("Unknown SUID or SGID files found: %v", badfiles)
	}

	return nil
}

func DeadLinks(c cluster.TestCluster) error {
	m := c.Machines()[0]

	ignore := []string{
		"/dev",
		"/proc",
		"/run/udev/watch",
		"/sys",
		"/var/lib/docker",
		"/var/lib/rkt",
	}

	command := fmt.Sprintf("sudo find / -ignore_readdir_race -path %s -prune -o -xtype l -print", strings.Join(ignore, " -prune -o -path "))

	output, err := m.SSH(command)
	if err != nil {
		return fmt.Errorf("Failed to run %v: output %s, status: %v", command, output, err)
	}

	if string(output) != "" {
		return fmt.Errorf("Dead symbolic links found: %v", strings.Split(string(output), "\n"))
	}

	return nil
}

func SUIDFiles(c cluster.TestCluster) error {
	m := c.Machines()[0]

	validfiles := []string{
		"/usr/bin/chage",
		"/usr/bin/chfn",
		"/usr/bin/chsh",
		"/usr/bin/expiry",
		"/usr/bin/gpasswd",
		"/usr/bin/ksu",
		"/usr/bin/man",
		"/usr/bin/mandb",
		"/usr/bin/mount",
		"/usr/bin/newgrp",
		"/usr/bin/passwd",
		"/usr/bin/pkexec",
		"/usr/bin/umount",
		"/usr/bin/su",
		"/usr/bin/sudo",
		"/usr/lib/polkit-1/polkit-agent-helper-1",
		"/usr/lib64/polkit-1/polkit-agent-helper-1",
		"/usr/libexec/dbus-daemon-launch-helper",
		"/usr/sbin/mount.nfs",
		"/usr/sbin/unix_chkpwd",
	}

	return sugidFiles(m, validfiles, "4000")
}

func SGIDFiles(c cluster.TestCluster) error {
	m := c.Machines()[0]

	validfiles := []string{}

	return sugidFiles(m, validfiles, "2000")
}

func WritableFiles(c cluster.TestCluster) error {
	m := c.Machines()[0]

	output, err := m.SSH("sudo find / -ignore_readdir_race -path /sys -prune -o -path /proc -prune -o -path /var/lib/rkt -prune -o -type f -perm -0002 -print")
	if err != nil {
		return fmt.Errorf("Failed to run find: output %s, status: %v", output, err)
	}

	if string(output) != "" {
		return fmt.Errorf("Unknown writable files found: %v", output)
	}

	return nil
}

func WritableDirs(c cluster.TestCluster) error {
	m := c.Machines()[0]

	output, err := m.SSH("sudo find / -ignore_readdir_race -path /sys -prune -o -path /proc -prune -o -path /var/lib/rkt -prune -o -type d -perm -0002 -a ! -perm -1000 -print")
	if err != nil {
		return fmt.Errorf("Failed to run find: output %s, status: %v", output, err)
	}

	if string(output) != "" {
		return fmt.Errorf("Unknown writable directories found: %v", output)
	}

	return nil
}

// The default permissions for the root of a tmpfs are 1777
// https://github.com/coreos/bugs/issues/1812
func StickyDirs(c cluster.TestCluster) error {
	m := c.Machines()[0]

	ignore := []string{
		// don't descend into these
		"/proc",
		"/sys",

		// should be sticky, and may have sticky children
		"/dev/mqueue",
		"/dev/shm",
		"/media",
		"/tmp",
		"/var/tmp",
	}

	command := fmt.Sprintf("sudo find / -ignore_readdir_race -path %s -prune -o -type d -perm /1000 -print", strings.Join(ignore, " -prune -o -path "))

	output, err := m.SSH(command)
	if err != nil {
		return fmt.Errorf("Failed to run find: output %s, status: %v", output, err)
	}

	if string(output) != "" {
		return fmt.Errorf("Unknown sticky directories found: %v", output)
	}

	return nil
}
<commit_msg>kola/tests/misc: allow container runtimes to have sticky directories<commit_after>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package misc

import (
	"fmt"
	"strings"

	"github.com/coreos/mantle/kola/cluster"
	"github.com/coreos/mantle/kola/register"
	"github.com/coreos/mantle/platform"
)

func init() {
	register.Register(&register.Test{
		Run:         DeadLinks,
		ClusterSize: 1,
		Name:        "coreos.filesystem.deadlinks",
		UserData:    `#cloud-config`,
	})
	register.Register(&register.Test{
		Run:         SUIDFiles,
		ClusterSize: 1,
		Name:        "coreos.filesystem.suid",
		UserData:    `#cloud-config`,
	})
	register.Register(&register.Test{
		Run:         SGIDFiles,
		ClusterSize: 1,
		Name:        "coreos.filesystem.sgid",
		UserData:    `#cloud-config`,
	})
	register.Register(&register.Test{
		Run:         WritableFiles,
		ClusterSize: 1,
		Name:        "coreos.filesystem.writablefiles",
		UserData:    `#cloud-config`,
	})
	register.Register(&register.Test{
		Run:         WritableDirs,
		ClusterSize: 1,
		Name:        "coreos.filesystem.writabledirs",
		UserData:    `#cloud-config`,
	})
	register.Register(&register.Test{
		Run:         StickyDirs,
		ClusterSize: 1,
		Name:        "coreos.filesystem.stickydirs",
		UserData:    `#cloud-config`,
	})
}

func sugidFiles(m platform.Machine, validfiles []string, mode string) error {
	badfiles := make([]string, 0, 0)

	command := fmt.Sprintf("sudo find / -ignore_readdir_race -path /sys -prune -o -path /proc -prune -o -path /var/lib/rkt -prune -o -type f -perm -%v -print", mode)

	output, err := m.SSH(command)
	if err != nil {
		return fmt.Errorf("Failed to run find: output %s, status: %v", output, err)
	}

	if string(output) == "" {
		return nil
	}

	files := strings.Split(string(output), "\n")
	for _, file := range files {
		var valid bool

		for _, validfile := range validfiles {
			if file == validfile {
				valid = true
			}
		}
		if valid != true {
			badfiles = append(badfiles, file)
		}
	}

	if len(badfiles) != 0 {
		return fmt.Errorf("Unknown SUID or SGID files found: %v", badfiles)
	}

	return nil
}

func DeadLinks(c cluster.TestCluster) error {
	m := c.Machines()[0]

	ignore := []string{
		"/dev",
		"/proc",
		"/run/udev/watch",
		"/sys",
		"/var/lib/docker",
		"/var/lib/rkt",
	}

	command := fmt.Sprintf("sudo find / -ignore_readdir_race -path %s -prune -o -xtype l -print", strings.Join(ignore, " -prune -o -path "))

	output, err := m.SSH(command)
	if err != nil {
		return fmt.Errorf("Failed to run %v: output %s, status: %v", command, output, err)
	}

	if string(output) != "" {
		return fmt.Errorf("Dead symbolic links found: %v", strings.Split(string(output), "\n"))
	}

	return nil
}

func SUIDFiles(c cluster.TestCluster) error {
	m := c.Machines()[0]

	validfiles := []string{
		"/usr/bin/chage",
		"/usr/bin/chfn",
		"/usr/bin/chsh",
		"/usr/bin/expiry",
		"/usr/bin/gpasswd",
		"/usr/bin/ksu",
		"/usr/bin/man",
		"/usr/bin/mandb",
		"/usr/bin/mount",
		"/usr/bin/newgrp",
		"/usr/bin/passwd",
		"/usr/bin/pkexec",
		"/usr/bin/umount",
		"/usr/bin/su",
		"/usr/bin/sudo",
		"/usr/lib/polkit-1/polkit-agent-helper-1",
		"/usr/lib64/polkit-1/polkit-agent-helper-1",
		"/usr/libexec/dbus-daemon-launch-helper",
		"/usr/sbin/mount.nfs",
		"/usr/sbin/unix_chkpwd",
	}

	return sugidFiles(m, validfiles, "4000")
}

func SGIDFiles(c cluster.TestCluster) error {
	m := c.Machines()[0]

	validfiles := []string{}

	return sugidFiles(m, validfiles, "2000")
}

func WritableFiles(c cluster.TestCluster) error {
	m := c.Machines()[0]

	output, err := m.SSH("sudo find / -ignore_readdir_race -path /sys -prune -o -path /proc -prune -o -path /var/lib/rkt -prune -o -type f -perm -0002 -print")
	if err != nil {
		return fmt.Errorf("Failed to run find: output %s, status: %v", output, err)
	}

	if string(output) != "" {
		return fmt.Errorf("Unknown writable files found: %v", output)
	}

	return nil
}

func WritableDirs(c cluster.TestCluster) error {
	m := c.Machines()[0]

	output, err := m.SSH("sudo find / -ignore_readdir_race -path /sys -prune -o -path /proc -prune -o -path /var/lib/rkt -prune -o -type d -perm -0002 -a ! -perm -1000 -print")
	if err != nil {
		return fmt.Errorf("Failed to run find: output %s, status: %v", output, err)
	}

	if string(output) != "" {
		return fmt.Errorf("Unknown writable directories found: %v", output)
	}

	return nil
}

// The default permissions for the root of a tmpfs are 1777
// https://github.com/coreos/bugs/issues/1812
func StickyDirs(c cluster.TestCluster) error {
	m := c.Machines()[0]

	ignore := []string{
		// don't descend into these
		"/proc",
		"/sys",
		"/var/lib/docker",
		"/var/lib/rkt",

		// should be sticky, and may have sticky children
		"/dev/mqueue",
		"/dev/shm",
		"/media",
		"/tmp",
		"/var/tmp",
	}

	command := fmt.Sprintf("sudo find / -ignore_readdir_race -path %s -prune -o -type d -perm /1000 -print", strings.Join(ignore, " -prune -o -path "))

	output, err := m.SSH(command)
	if err != nil {
		return fmt.Errorf("Failed to run find: output %s, status: %v", output, err)
	}

	if string(output) != "" {
		return fmt.Errorf("Unknown sticky directories found: %v", output)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"os"

	entry "github.com/stackanetes/kubernetes-entrypoint/entrypoint"

	"github.com/stackanetes/kubernetes-entrypoint/logger"
	command "github.com/stackanetes/kubernetes-entrypoint/util/command"
	"github.com/stackanetes/kubernetes-entrypoint/util/env"
	//restclient "k8s.io/kubernetes/pkg/client/restclient"
	//Register resolvers
	_ "github.com/stackanetes/kubernetes-entrypoint/dependencies/config"
	_ "github.com/stackanetes/kubernetes-entrypoint/dependencies/container"
	_ "github.com/stackanetes/kubernetes-entrypoint/dependencies/daemonset"
	_ "github.com/stackanetes/kubernetes-entrypoint/dependencies/job"
	_ "github.com/stackanetes/kubernetes-entrypoint/dependencies/service"
	_ "github.com/stackanetes/kubernetes-entrypoint/dependencies/socket"
)

func main() {
	//var client cli.ClientInterface
	var comm []string
	var entrypoint *entry.Entrypoint
	var err error
	if entrypoint, err = entry.New(nil); err != nil {
		logger.Error.Printf("Creating entrypoint failed: %v", err)
		os.Exit(1)
	}

	entrypoint.Resolve()

	if comm = env.SplitEnvToList("COMMAND", " "); len(comm) == 0 {
		// TODO(DTadrzak): we should consider other options to handle whether pod
		// is an init-container
		logger.Warning.Printf("COMMAND env is empty")
		os.Exit(0)
	}

	if err = command.Execute(comm); err != nil {
		logger.Error.Printf("Cannot execute command: %v", err)
		os.Exit(1)
	}
}
<commit_msg>Main entrypoint cleanup<commit_after>package main

import (
	"os"

	entry "github.com/stackanetes/kubernetes-entrypoint/entrypoint"

	_ "github.com/stackanetes/kubernetes-entrypoint/dependencies/config"
	_ "github.com/stackanetes/kubernetes-entrypoint/dependencies/container"
	_ "github.com/stackanetes/kubernetes-entrypoint/dependencies/daemonset"
	_ "github.com/stackanetes/kubernetes-entrypoint/dependencies/job"
	_ "github.com/stackanetes/kubernetes-entrypoint/dependencies/service"
	_ "github.com/stackanetes/kubernetes-entrypoint/dependencies/socket"
	"github.com/stackanetes/kubernetes-entrypoint/logger"
	command "github.com/stackanetes/kubernetes-entrypoint/util/command"
	"github.com/stackanetes/kubernetes-entrypoint/util/env"
)

func main() {
	var comm []string
	var entrypoint *entry.Entrypoint
	var err error
	if entrypoint, err = entry.New(nil); err != nil {
		logger.Error.Printf("Creating entrypoint failed: %v", err)
		os.Exit(1)
	}

	entrypoint.Resolve()

	if comm = env.SplitEnvToList("COMMAND", " "); len(comm) == 0 {
		// TODO(DTadrzak): we should consider other options to handle whether pod
		// is an init-container
		logger.Warning.Printf("COMMAND env is empty")
		os.Exit(0)
	}

	if err = command.Execute(comm); err != nil {
		logger.Error.Printf("Cannot execute command: %v", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package kubernetes

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/hashicorp/terraform/helper/schema"

	"k8s.io/apimachinery/pkg/api/resource"
	apiValidation "k8s.io/apimachinery/pkg/api/validation"
	utilValidation "k8s.io/apimachinery/pkg/util/validation"
)

func validateAnnotations(value interface{}, key string) (ws []string, es []error) {
	m := value.(map[string]interface{})
	for k, _ := range m {
		errors := utilValidation.IsQualifiedName(strings.ToLower(k))
		if len(errors) > 0 {
			for _, e := range errors {
				es = append(es, fmt.Errorf("%s (%q) %s", key, k, e))
			}
		}

		if isInternalKey(k) {
			es = append(es, fmt.Errorf("%s: %q is internal Kubernetes annotation", key, k))
		}
	}
	return
}

func validateName(value interface{}, key string) (ws []string, es []error) {
	v := value.(string)

	errors := apiValidation.NameIsDNSLabel(v, false)
	if len(errors) > 0 {
		for _, err := range errors {
			es = append(es, fmt.Errorf("%s %s", key, err))
		}
	}
	return
}

func validateGenerateName(value interface{}, key string) (ws []string, es []error) {
	v := value.(string)

	errors := apiValidation.NameIsDNSLabel(v, true)
	if len(errors) > 0 {
		for _, err := range errors {
			es = append(es, fmt.Errorf("%s %s", key, err))
		}
	}
	return
}

func validateLabels(value interface{}, key string) (ws []string, es []error) {
	m := value.(map[string]interface{})
	for k, v := range m {
		for _, msg := range utilValidation.IsQualifiedName(k) {
			es = append(es, fmt.Errorf("%s (%q) %s", key, k, msg))
		}
		val := v.(string)
		for _, msg := range utilValidation.IsValidLabelValue(val) {
			es = append(es, fmt.Errorf("%s (%q) %s", key, val, msg))
		}
	}
	return
}

func validatePortNum(value interface{}, key string) (ws []string, es []error) {
	errors := utilValidation.IsValidPortNum(value.(int))
	if len(errors) > 0 {
		for _, err := range errors {
			es = append(es, fmt.Errorf("%s %s", key, err))
		}
	}
	return
}

func validatePortName(value interface{}, key string) (ws []string, es []error) {
	errors := utilValidation.IsValidPortName(value.(string))
	if len(errors) > 0 {
		for _, err := range errors {
			es = append(es, fmt.Errorf("%s %s", key, err))
		}
	}
	return
}
func validatePortNumOrName(value interface{}, key string) (ws []string, es []error) {
	switch value.(type) {
	case string:
		intVal, err := strconv.Atoi(value.(string))
		if err != nil {
			return validatePortName(value, key)
		}
		return validatePortNum(intVal, key)
	case int:
		return validatePortNum(value, key)

	default:
		es = append(es, fmt.Errorf("%s must be defined of type string or int on the schema", key))
		return
	}
}

func validateResourceList(value interface{}, key string) (ws []string, es []error) {
	m := value.(map[string]interface{})
	for k, value := range m {
		if _, ok := value.(int); ok {
			continue
		}

		if v, ok := value.(string); ok {
			_, err := resource.ParseQuantity(v)
			if err != nil {
				es = append(es, fmt.Errorf("%s.%s (%q): %s", key, k, v, err))
			}
			continue
		}

		err := "Value can be either string or int"
		es = append(es, fmt.Errorf("%s.%s (%#v): %s", key, k, value, err))
	}
	return
}

func validateResourceQuantity(value interface{}, key string) (ws []string, es []error) {
	if v, ok := value.(string); ok {
		_, err := resource.ParseQuantity(v)
		if err != nil {
			es = append(es, fmt.Errorf("%s.%s : %s", key, v, err))
		}
	}
	return
}

func validatePositiveInteger(value interface{}, key string) (ws []string, es []error) {
	v := value.(int)
	if v <= 0 {
		es = append(es, fmt.Errorf("%s must be greater than 0", key))
	}
	return
}

func validateDNSPolicy(value interface{}, key string) (ws []string, es []error) {
	v := value.(string)
	if v != "ClusterFirst" && v != "Default" {
		es = append(es, fmt.Errorf("%s must be either ClusterFirst or Default", key))
	}
	return
}

func validateRestartPolicy(value interface{}, key string) (ws []string, es []error) {
	v := value.(string)
	switch v {
	case "Always", "OnFailure", "Never":
		return
	default:
		es = append(es, fmt.Errorf("%s must be one of Always, OnFailure or Never ", key))
	}
	return
}

func validateTerminationGracePeriodSeconds(value interface{}, key string) (ws []string, es []error) {
	v := value.(int)
	if v < 0 {
		es = append(es, fmt.Errorf("%s must be greater than or equal to 0", key))
	}
	return
}

func validateModeBits(value interface{}, key string) (ws []string, es []error) {
	v := value.(int)
	if v < 0 || v > 0777 {
		es = append(es, fmt.Errorf("%s (%#o) expects octal notation (a value between 0 and 0777)", key, v))
	}
	return
}

func validateAttributeValueDoesNotContain(searchString string) schema.SchemaValidateFunc {
	return func(v interface{}, k string) (ws []string, errors []error) {
		input := v.(string)
		if strings.Contains(input, searchString) {
			errors = append(errors, fmt.Errorf(
				"%q must not contain %q",
				k, searchString))
		}
		return
	}
}

func validateAttributeValueIsIn(validValues []string) schema.SchemaValidateFunc {
	return func(v interface{}, k string) (ws []string, errors []error) {
		input := v.(string)
		isValid := false
		for _, s := range validValues {
			if s == input {
				isValid = true
				break
			}
		}
		if !isValid {
			errors = append(errors, fmt.Errorf(
				"%q must contain a value from %#v, got %q",
				k, validValues, input))
		}
		return

	}
}
<commit_msg>Label validation fails for interpolated values.<commit_after>package kubernetes

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/hashicorp/terraform/helper/schema"

	"k8s.io/apimachinery/pkg/api/resource"
	apiValidation "k8s.io/apimachinery/pkg/api/validation"
	utilValidation "k8s.io/apimachinery/pkg/util/validation"
)

func validateAnnotations(value interface{}, key string) (ws []string, es []error) {
	m := value.(map[string]interface{})
	for k, _ := range m {
		errors := utilValidation.IsQualifiedName(strings.ToLower(k))
		if len(errors) > 0 {
			for _, e := range errors {
				es = append(es, fmt.Errorf("%s (%q) %s", key, k, e))
			}
		}

		if isInternalKey(k) {
			es = append(es, fmt.Errorf("%s: %q is internal Kubernetes annotation", key, k))
		}
	}
	return
}

func validateName(value interface{}, key string) (ws []string, es []error) {
	v := value.(string)

	errors := apiValidation.NameIsDNSLabel(v, false)
	if len(errors) > 0 {
		for _, err := range errors {
			es = append(es, fmt.Errorf("%s %s", key, err))
		}
	}
	return
}

func validateGenerateName(value interface{}, key string) (ws []string, es []error) {
	v := value.(string)

	errors := apiValidation.NameIsDNSLabel(v, true)
	if len(errors) > 0 {
		for _, err := range errors {
			es = append(es, fmt.Errorf("%s %s", key, err))
		}
	}
	return
}

func validateLabels(value interface{}, key string) (ws []string, es []error) {
	m := value.(map[string]interface{})
	for k, _ := range m {
		for _, msg := range utilValidation.IsQualifiedName(k) {
			es = append(es, fmt.Errorf("%s (%q) %s", key, k, msg))
		}
	}
	return
}

func validatePortNum(value interface{}, key string) (ws []string, es []error) {
	errors := utilValidation.IsValidPortNum(value.(int))
	if len(errors) > 0 {
		for _, err := range errors {
			es = append(es, fmt.Errorf("%s %s", key, err))
		}
	}
	return
}

func validatePortName(value interface{}, key string) (ws []string, es []error) {
	errors := utilValidation.IsValidPortName(value.(string))
	if len(errors) > 0 {
		for _, err := range errors {
			es = append(es, fmt.Errorf("%s %s", key, err))
		}
	}
	return
}
func validatePortNumOrName(value interface{}, key string) (ws []string, es []error) {
	switch value.(type) {
	case string:
		intVal, err := strconv.Atoi(value.(string))
		if err != nil {
			return validatePortName(value, key)
		}
		return validatePortNum(intVal, key)
	case int:
		return validatePortNum(value, key)

	default:
		es = append(es, fmt.Errorf("%s must be defined of type string or int on the schema", key))
		return
	}
}

func validateResourceList(value interface{}, key string) (ws []string, es []error) {
	m := value.(map[string]interface{})
	for k, value := range m {
		if _, ok := value.(int); ok {
			continue
		}

		if v, ok := value.(string); ok {
			_, err := resource.ParseQuantity(v)
			if err != nil {
				es = append(es, fmt.Errorf("%s.%s (%q): %s", key, k, v, err))
			}
			continue
		}

		err := "Value can be either string or int"
		es = append(es, fmt.Errorf("%s.%s (%#v): %s", key, k, value, err))
	}
	return
}

func validateResourceQuantity(value interface{}, key string) (ws []string, es []error) {
	if v, ok := value.(string); ok {
		_, err := resource.ParseQuantity(v)
		if err != nil {
			es = append(es, fmt.Errorf("%s.%s : %s", key, v, err))
		}
	}
	return
}

func validatePositiveInteger(value interface{}, key string) (ws []string, es []error) {
	v := value.(int)
	if v <= 0 {
		es = append(es, fmt.Errorf("%s must be greater than 0", key))
	}
	return
}

func validateDNSPolicy(value interface{}, key string) (ws []string, es []error) {
	v := value.(string)
	if v != "ClusterFirst" && v != "Default" {
		es = append(es, fmt.Errorf("%s must be either ClusterFirst or Default", key))
	}
	return
}

func validateRestartPolicy(value interface{}, key string) (ws []string, es []error) {
	v := value.(string)
	switch v {
	case "Always", "OnFailure", "Never":
		return
	default:
		es = append(es, fmt.Errorf("%s must be one of Always, OnFailure or Never ", key))
	}
	return
}

func validateTerminationGracePeriodSeconds(value interface{}, key string) (ws []string, es []error) {
	v := value.(int)
	if v < 0 {
		es = append(es, fmt.Errorf("%s must be greater than or equal to 0", key))
	}
	return
}

func validateModeBits(value interface{}, key string) (ws []string, es []error) {
	v := value.(int)
	if v < 0 || v > 0777 {
		es = append(es, fmt.Errorf("%s (%#o) expects octal notation (a value between 0 and 0777)", key, v))
	}
	return
}

func validateAttributeValueDoesNotContain(searchString string) schema.SchemaValidateFunc {
	return func(v interface{}, k string) (ws []string, errors []error) {
		input := v.(string)
		if strings.Contains(input, searchString) {
			errors = append(errors, fmt.Errorf(
				"%q must not contain %q",
				k, searchString))
		}
		return
	}
}

func validateAttributeValueIsIn(validValues []string) schema.SchemaValidateFunc {
	return func(v interface{}, k string) (ws []string, errors []error) {
		input := v.(string)
		isValid := false
		for _, s := range validValues {
			if s == input {
				isValid = true
				break
			}
		}
		if !isValid {
			errors = append(errors, fmt.Errorf(
				"%q must contain a value from %#v, got %q",
				k, validValues, input))
		}
		return

	}
}
<|endoftext|>


<commit_before>package common

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/signal"
	"strings"
)

var (
	GoPath = os.Getenv("GOPATH")
)

func TrapSignal(cb func()) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	signal.Notify(c, os.Kill)
	go func() {
		for sig := range c {
			fmt.Printf("captured %v, exiting...\n", sig)
			if cb != nil {
				cb()
			}
			os.Exit(1)
		}
	}()
	select {}
}

func Exit(s string) {
	fmt.Printf(s + "\n")
	os.Exit(1)
}

func EnsureDir(dir string, mode os.FileMode) error {
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		err := os.MkdirAll(dir, mode)
		if err != nil {
			return fmt.Errorf("Could not create directory %v. %v", dir, err)
		}
	}
	return nil
}

func IsDirEmpty(name string) (bool, error) {
	f, err := os.Open(name)
	if err != nil {
		if os.IsNotExist(err) {
			return true, err
		}
		// Otherwise perhaps a permission
		// error or some other error.
		return false, err
	}
	defer f.Close()

	_, err = f.Readdirnames(1) // Or f.Readdir(1)
	if err == io.EOF {
		return true, nil
	}
	return false, err // Either not empty or error, suits both cases
}

func FileExists(filePath string) bool {
	_, err := os.Stat(filePath)
	return !os.IsNotExist(err)
}

func ReadFile(filePath string) ([]byte, error) {
	return ioutil.ReadFile(filePath)
}

func MustReadFile(filePath string) []byte {
	fileBytes, err := ioutil.ReadFile(filePath)
	if err != nil {
		Exit(Fmt("MustReadFile failed: %v", err))
		return nil
	}
	return fileBytes
}

func WriteFile(filePath string, contents []byte, mode os.FileMode) error {
	err := ioutil.WriteFile(filePath, contents, mode)
	if err != nil {
		return err
	}
	// fmt.Printf("File written to %v.\n", filePath)
	return nil
}

func MustWriteFile(filePath string, contents []byte, mode os.FileMode) {
	err := WriteFile(filePath, contents, mode)
	if err != nil {
		Exit(Fmt("MustWriteFile failed: %v", err))
	}
}

// WriteFileAtomic writes newBytes to temp and atomically moves to filePath
// when everything else succeeds.
func WriteFileAtomic(filePath string, newBytes []byte, mode os.FileMode) error {
	f, err := ioutil.TempFile("", "")
	if err != nil {
		return err
	}
	_, err = f.Write(newBytes)
	if err == nil {
		err = f.Sync()
	}
	if closeErr := f.Close(); err == nil {
		err = closeErr
	}
	if permErr := os.Chmod(f.Name(), mode); err == nil {
		err = permErr
	}
	if err == nil {
		err = os.Rename(f.Name(), filePath)
	}
	// any err should result in full cleanup
	if err != nil {
		os.Remove(f.Name())
	}
	return err
}

//--------------------------------------------------------------------------------

func Tempfile(prefix string) (*os.File, string) {
	file, err := ioutil.TempFile("", prefix)
	if err != nil {
		PanicCrisis(err)
	}
	return file, file.Name()
}

func Tempdir(prefix string) (*os.File, string) {
	tempDir := os.TempDir() + "/" + prefix + RandStr(12)
	err := EnsureDir(tempDir, 0700)
	if err != nil {
		panic(Fmt("Error creating temp dir: %v", err))
	}
	dir, err := os.Open(tempDir)
	if err != nil {
		panic(Fmt("Error opening temp dir: %v", err))
	}
	return dir, tempDir
}

//--------------------------------------------------------------------------------

func Prompt(prompt string, defaultValue string) (string, error) {
	fmt.Print(prompt)
	reader := bufio.NewReader(os.Stdin)
	line, err := reader.ReadString('\n')
	if err != nil {
		return defaultValue, err
	} else {
		line = strings.TrimSpace(line)
		if line == "" {
			return defaultValue, nil
		}
		return line, nil
	}
}
<commit_msg>common: WriteFileAtomic use tempfile in current dir<commit_after>package common

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/signal"
	"path/filepath"
	"strings"
)

var (
	GoPath = os.Getenv("GOPATH")
)

func TrapSignal(cb func()) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	signal.Notify(c, os.Kill)
	go func() {
		for sig := range c {
			fmt.Printf("captured %v, exiting...\n", sig)
			if cb != nil {
				cb()
			}
			os.Exit(1)
		}
	}()
	select {}
}

func Exit(s string) {
	fmt.Printf(s + "\n")
	os.Exit(1)
}

func EnsureDir(dir string, mode os.FileMode) error {
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		err := os.MkdirAll(dir, mode)
		if err != nil {
			return fmt.Errorf("Could not create directory %v. %v", dir, err)
		}
	}
	return nil
}

func IsDirEmpty(name string) (bool, error) {
	f, err := os.Open(name)
	if err != nil {
		if os.IsNotExist(err) {
			return true, err
		}
		// Otherwise perhaps a permission
		// error or some other error.
		return false, err
	}
	defer f.Close()

	_, err = f.Readdirnames(1) // Or f.Readdir(1)
	if err == io.EOF {
		return true, nil
	}
	return false, err // Either not empty or error, suits both cases
}

func FileExists(filePath string) bool {
	_, err := os.Stat(filePath)
	return !os.IsNotExist(err)
}

func ReadFile(filePath string) ([]byte, error) {
	return ioutil.ReadFile(filePath)
}

func MustReadFile(filePath string) []byte {
	fileBytes, err := ioutil.ReadFile(filePath)
	if err != nil {
		Exit(Fmt("MustReadFile failed: %v", err))
		return nil
	}
	return fileBytes
}

func WriteFile(filePath string, contents []byte, mode os.FileMode) error {
	err := ioutil.WriteFile(filePath, contents, mode)
	if err != nil {
		return err
	}
	// fmt.Printf("File written to %v.\n", filePath)
	return nil
}

func MustWriteFile(filePath string, contents []byte, mode os.FileMode) {
	err := WriteFile(filePath, contents, mode)
	if err != nil {
		Exit(Fmt("MustWriteFile failed: %v", err))
	}
}

// WriteFileAtomic writes newBytes to temp and atomically moves to filePath
// when everything else succeeds.
func WriteFileAtomic(filePath string, newBytes []byte, mode os.FileMode) error {
	dir := filepath.Dir(filePath)
	f, err := ioutil.TempFile(dir, "")
	if err != nil {
		return err
	}
	_, err = f.Write(newBytes)
	if err == nil {
		err = f.Sync()
	}
	if closeErr := f.Close(); err == nil {
		err = closeErr
	}
	if permErr := os.Chmod(f.Name(), mode); err == nil {
		err = permErr
	}
	if err == nil {
		err = os.Rename(f.Name(), filePath)
	}
	// any err should result in full cleanup
	if err != nil {
		os.Remove(f.Name())
	}
	return err
}

//--------------------------------------------------------------------------------

func Tempfile(prefix string) (*os.File, string) {
	file, err := ioutil.TempFile("", prefix)
	if err != nil {
		PanicCrisis(err)
	}
	return file, file.Name()
}

func Tempdir(prefix string) (*os.File, string) {
	tempDir := os.TempDir() + "/" + prefix + RandStr(12)
	err := EnsureDir(tempDir, 0700)
	if err != nil {
		panic(Fmt("Error creating temp dir: %v", err))
	}
	dir, err := os.Open(tempDir)
	if err != nil {
		panic(Fmt("Error opening temp dir: %v", err))
	}
	return dir, tempDir
}

//--------------------------------------------------------------------------------

func Prompt(prompt string, defaultValue string) (string, error) {
	fmt.Print(prompt)
	reader := bufio.NewReader(os.Stdin)
	line, err := reader.ReadString('\n')
	if err != nil {
		return defaultValue, err
	} else {
		line = strings.TrimSpace(line)
		if line == "" {
			return defaultValue, nil
		}
		return line, nil
	}
}
<|endoftext|>


<commit_before>package conf

import (
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"

	flag "github.com/juju/gnuflag"
	"gopkg.in/yaml.v2"
)

var (
	// F flag prints full request
	F = flag.Bool("f", false, "Print full results, ie. no more '...'")

	// L flag limit results to a number
	L = flag.Int("l", 5, "Result list limit. Defaults to 5")

	// H flag specifies the host to connect to
	S = flag.String("s", "ok-b.org", "Server to connect to")

	H = flag.Bool("h", false, "Display help")

	Help = flag.Bool("help", false, "Display help, same as -h")

	// P flag enables private search
	P = flag.Bool("p", false, "Private search. Your search won't leave a trace. Pinky promise. Don't use this all the time if you want to see the search result relevancy improved")

	// D flag enables debug mode
	D = flag.Bool("d", false, "Debug mode")
	// DontPipe
	DontPipe = flag.Bool("dontpipe", false, "Flag for internal use - ignore this")
	// Version flag displays current version
	Version = flag.Bool("version", false, "Print version number")
	// V flag displays current version
	V = flag.Bool("v", false, "Print version number")
)
var (
	// EditFile borg edit file.
	EditFile string
	// ConfigFile borg config file.
	ConfigFile string
	// QueryFile borg query file.
	QueryFile string
)

func init() {
	borgDir := borgDir()

	EditFile = filepath.Join(borgDir, "edit")
	ConfigFile = filepath.Join(borgDir, "config.yml")
	QueryFile = filepath.Join(borgDir, "query")

	os.Mkdir(borgDir, os.ModePerm)
	os.Create(EditFile)
	if _, err := os.Stat(ConfigFile); os.IsNotExist(err) {
		os.Create(ConfigFile)
	}
	if _, err := os.Stat(QueryFile); os.IsNotExist(err) {
		os.Create(QueryFile)
	}
}

func borgDir() string {
	if xdgConfigHome := os.Getenv("XDG_CONFIG_HOME"); xdgConfigHome != "" {
		return filepath.Join(xdgConfigHome, "borg")
	}

	usr, err := user.Current()
	if err != nil {
		panic(err)
	}
	return filepath.Join(usr.HomeDir, ".borg")
}

// Config file
type Config struct {
	Token       string
	DefaultTags []string
	Editor      string
	PipeTo      string
}

// Save config
func (c Config) Save() error {
	bs, err := yaml.Marshal(c)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(QueryFile, bs, os.ModePerm)
}

// Get config
func Get() (Config, error) {
	bs, err := ioutil.ReadFile(QueryFile)
	if err != nil {
		panic(err)
	}
	c := &Config{}
	err = yaml.Unmarshal(bs, c)
	if err != nil {
		return *c, err
	}
	if len(c.Editor) == 0 {
		c.Editor = "vim"
	}
	return *c, nil
}
<commit_msg>conf: Add backward compatibility for $HOME/.borg<commit_after>package conf

import (
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"

	flag "github.com/juju/gnuflag"
	"gopkg.in/yaml.v2"
)

var (
	// F flag prints full request
	F = flag.Bool("f", false, "Print full results, ie. no more '...'")

	// L flag limit results to a number
	L = flag.Int("l", 5, "Result list limit. Defaults to 5")

	// H flag specifies the host to connect to
	S = flag.String("s", "ok-b.org", "Server to connect to")

	H = flag.Bool("h", false, "Display help")

	Help = flag.Bool("help", false, "Display help, same as -h")

	// P flag enables private search
	P = flag.Bool("p", false, "Private search. Your search won't leave a trace. Pinky promise. Don't use this all the time if you want to see the search result relevancy improved")

	// D flag enables debug mode
	D = flag.Bool("d", false, "Debug mode")
	// DontPipe
	DontPipe = flag.Bool("dontpipe", false, "Flag for internal use - ignore this")
	// Version flag displays current version
	Version = flag.Bool("version", false, "Print version number")
	// V flag displays current version
	V = flag.Bool("v", false, "Print version number")
)
var (
	// EditFile borg edit file.
	EditFile string
	// ConfigFile borg config file.
	ConfigFile string
	// QueryFile borg query file.
	QueryFile string
)

func init() {
	borgDir := borgDir()

	EditFile = filepath.Join(borgDir, "edit")
	ConfigFile = filepath.Join(borgDir, "config.yml")
	QueryFile = filepath.Join(borgDir, "query")

	os.Mkdir(borgDir, os.ModePerm)
	os.Create(EditFile)
	if _, err := os.Stat(ConfigFile); os.IsNotExist(err) {
		os.Create(ConfigFile)
	}
	if _, err := os.Stat(QueryFile); os.IsNotExist(err) {
		os.Create(QueryFile)
	}
}

func borgDir() string {
	usr, err := user.Current()
	if err != nil {
		panic(err)
	}
	dir := filepath.Join(usr.HomeDir, ".borg")

	if _, err := os.Stat(dir); os.IsNotExist(err) {
		if xdgConfigHome := os.Getenv("XDG_CONFIG_HOME"); xdgConfigHome != "" {
			dir = filepath.Join(xdgConfigHome, "borg")
		}
	}

	return dir
}

// Config file
type Config struct {
	Token       string
	DefaultTags []string
	Editor      string
	PipeTo      string
}

// Save config
func (c Config) Save() error {
	bs, err := yaml.Marshal(c)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(QueryFile, bs, os.ModePerm)
}

// Get config
func Get() (Config, error) {
	bs, err := ioutil.ReadFile(QueryFile)
	if err != nil {
		panic(err)
	}
	c := &Config{}
	err = yaml.Unmarshal(bs, c)
	if err != nil {
		return *c, err
	}
	if len(c.Editor) == 0 {
		c.Editor = "vim"
	}
	return *c, nil
}
<|endoftext|>


<commit_before>package datareader

import (
	"bytes"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"testing"
)

const (
	generateColumnize = false
)

// Not really a test function, used to generate md5 sums for the results.
func TestGenerateColumnize(t *testing.T) {

	if !generateColumnize {
		return
	}

	ms := make(map[string][16]byte)

	all_test_files := getFilenames()

	for _, f := range all_test_files {
		for _, mode := range []string{"text", "binary"} {
			m := columnizeBase(f, mode)
			k := f + "::" + mode
			ms[k] = m
		}
	}

	b, err := json.Marshal(ms)
	if err != nil {
		panic(err)
	}

	cf, err := os.Create(filepath.Join("test_files", "columnize_checksums.json"))
	if err != nil {
		panic(err)
	}

	if _, err := cf.Write(b); err != nil {
		panic(err)
	}
	cf.Close()
}

func columnizeBase(fname, mode string) [16]byte {

	outpath := filepath.Join("test_files", "tmp", "cols")
	os.RemoveAll(outpath)
	if err := os.MkdirAll(outpath, os.ModeDir); err != nil {
		panic(err)
	}

	cmdName := filepath.Join(os.Getenv("GOBIN"), "columnize")
	infile := filepath.Join("test_files", "tmp", "cols", fname)
	args := []string{fmt.Sprintf("-in=%s", infile), fmt.Sprintf("-out=%s", outpath),
		fmt.Sprintf("-mode=%s", mode)}
	_, err := exec.Command(cmdName, args...).Output()
	if err != nil {
		panic(err)
	}

	files, _ := ioutil.ReadDir(outpath)
	fileNames := make([]string, 0, 10)
	for _, v := range files {
		fileNames = append(fileNames, v.Name())
	}
	sort.Strings(fileNames)

	var buf bytes.Buffer
	for _, f := range fileNames {
		if strings.HasPrefix(f, ".") {
			continue
		}
		gname := filepath.Join("test_files", "tmp", "cols", f)
		g, err := os.Open(gname)
		if err != nil {
			panic(err)
		}
		defer g.Close()
		ba, err := ioutil.ReadAll(g)
		if err != nil {
			panic(err)
		}
		buf.Write(ba)
	}
	m := md5.Sum(buf.Bytes())

	return m
}

func TestColumnize1(t *testing.T) {

	if generateColumnize {
		return
	}

	cf, err := os.Open(filepath.Join("test_files", "columnize_checksums.json"))
	if err != nil {
		panic(err)
	}
	defer cf.Close()

	var checksum map[string][]byte
	b, err := ioutil.ReadAll(cf)
	if err != nil {
		panic(err)
	}
	if err := json.Unmarshal(b, &checksum); err != nil {
		panic(err)
	}

	allTestFiles := getFilenames()

	for _, f := range allTestFiles {
		for _, mode := range []string{"text", "binary"} {

			m := columnizeBase(f, mode)
			k := f + "::" + mode
			m1 := checksum[k]

			for j := range m {
				if m[j] != m1[j] {
					t.Fail()
				}
			}
		}

	}
}
<commit_msg>update<commit_after>package datareader

import (
	"bytes"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"testing"
)

const (
	generateColumnize = false
)

// Not really a test function, used to generate md5 sums for the results.
func TestGenerateColumnize(t *testing.T) {

	if !generateColumnize {
		return
	}

	ms := make(map[string][16]byte)

	all_test_files := getFilenames()

	for _, f := range all_test_files {
		for _, mode := range []string{"text", "binary"} {
			m := columnizeBase(f, mode)
			k := f + "::" + mode
			ms[k] = m
		}
	}

	b, err := json.Marshal(ms)
	if err != nil {
		panic(err)
	}

	cf, err := os.Create(filepath.Join("test_files", "columnize_checksums.json"))
	if err != nil {
		panic(err)
	}

	if _, err := cf.Write(b); err != nil {
		panic(err)
	}
	cf.Close()
}

func columnizeBase(fname, mode string) [16]byte {

	// Clear the workspace and set up the subdirectories.
	outpath := filepath.Join("test_files", "tmp", "cols")
	if err := os.RemoveAll(outpath); err != nil {
		panic(err)
	}
	if err := os.MkdirAll(outpath, os.ModeDir); err != nil {
		panic(err)
	}

	// Run columnize on the file
	cmdName := filepath.Join(os.Getenv("GOBIN"), "columnize")
	infile := filepath.Join("test_files", "tmp", "cols", fname)
	args := []string{
		fmt.Sprintf("-in=%s", infile),
		fmt.Sprintf("-out=%s", outpath),
		fmt.Sprintf("-mode=%s", mode),
	}
	if _, err := exec.Command(cmdName, args...).Output(); err != nil {
		panic(err)
	}

	files, err := ioutil.ReadDir(outpath)
	if err != nil {
		panic(err)
	}
	var fileNames []string
	for _, v := range files {
		fileNames = append(fileNames, v.Name())
	}
	sort.Strings(fileNames)

	var buf bytes.Buffer
	for _, f := range fileNames {
		if strings.HasPrefix(f, ".") {
			continue
		}

		gname := filepath.Join("test_files", "tmp", "cols", f)
		g, err := os.Open(gname)
		if err != nil {
			panic(err)
		}
		defer g.Close()

		ba, err := ioutil.ReadAll(g)
		if err != nil {
			panic(err)
		}

		if _, err := buf.Write(ba); err != nil {
			panic(err)
		}
	}

	return md5.Sum(buf.Bytes())
}

func TestColumnize1(t *testing.T) {

	if generateColumnize {
		return
	}

	cf, err := os.Open(filepath.Join("test_files", "columnize_checksums.json"))
	if err != nil {
		panic(err)
	}
	defer cf.Close()

	// Read the stored checksums
	var checksum map[string][]byte
	b, err := ioutil.ReadAll(cf)
	if err != nil {
		panic(err)
	}
	if err := json.Unmarshal(b, &checksum); err != nil {
		panic(err)
	}

	allTestFiles := getFilenames()

	for _, f := range allTestFiles {
		for _, mode := range []string{"text", "binary"} {

			m := columnizeBase(f, mode)
			m1 := checksum[f+"::"+mode]

			for j := range m {
				if m[j] != m1[j] {
					t.Fail()
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"log"
	"strings"

	"github.com/mitchellh/packer/builder/amazon/chroot"
	"github.com/mitchellh/packer/builder/amazon/ebs"
	"github.com/mitchellh/packer/builder/amazon/instance"
	"github.com/mitchellh/packer/builder/digitalocean"
	"github.com/mitchellh/packer/builder/docker"
	filebuilder "github.com/mitchellh/packer/builder/file"
	"github.com/mitchellh/packer/builder/googlecompute"
	"github.com/mitchellh/packer/builder/null"
	"github.com/mitchellh/packer/builder/openstack"
	parallelsiso "github.com/mitchellh/packer/builder/parallels/iso"
	parallelspvm "github.com/mitchellh/packer/builder/parallels/pvm"
	"github.com/mitchellh/packer/builder/qemu"
	virtualboxiso "github.com/mitchellh/packer/builder/virtualbox/iso"
	virtualboxovf "github.com/mitchellh/packer/builder/virtualbox/ovf"
	vmwareiso "github.com/mitchellh/packer/builder/vmware/iso"
	vmwarevmx "github.com/mitchellh/packer/builder/vmware/vmx"
	"github.com/mitchellh/packer/packer"
	"github.com/mitchellh/packer/packer/plugin"
	"github.com/mitchellh/packer/post-processor/artifice"
	"github.com/mitchellh/packer/post-processor/atlas"
	"github.com/mitchellh/packer/post-processor/compress"
	"github.com/mitchellh/packer/post-processor/docker-import"
	"github.com/mitchellh/packer/post-processor/docker-push"
	"github.com/mitchellh/packer/post-processor/docker-save"
	"github.com/mitchellh/packer/post-processor/docker-tag"
	"github.com/mitchellh/packer/post-processor/vagrant"
	"github.com/mitchellh/packer/post-processor/vagrant-cloud"
	"github.com/mitchellh/packer/post-processor/vsphere"
	"github.com/mitchellh/packer/provisioner/ansible-local"
	"github.com/mitchellh/packer/provisioner/chef-client"
	"github.com/mitchellh/packer/provisioner/chef-solo"
	fileprovisioner "github.com/mitchellh/packer/provisioner/file"
	"github.com/mitchellh/packer/provisioner/powershell"
	"github.com/mitchellh/packer/provisioner/puppet-masterless"
	"github.com/mitchellh/packer/provisioner/puppet-server"
	"github.com/mitchellh/packer/provisioner/salt-masterless"
	"github.com/mitchellh/packer/provisioner/shell"
	shelllocal "github.com/mitchellh/packer/provisioner/shell-local"
	"github.com/mitchellh/packer/provisioner/windows-restart"
	windowsshell "github.com/mitchellh/packer/provisioner/windows-shell"
)

type PluginCommand struct {
	Meta
}

var Builders = map[string]packer.Builder{
	"amazon-chroot":   new(chroot.Builder),
	"amazon-ebs":      new(ebs.Builder),
	"amazon-instance": new(instance.Builder),
	"digitalocean":    new(digitalocean.Builder),
	"docker":          new(docker.Builder),
	"file":            new(filebuilder.Builder),
	"googlecompute":   new(googlecompute.Builder),
	"null":            new(null.Builder),
	"openstack":       new(openstack.Builder),
	"parallels-iso":   new(parallelsiso.Builder),
	"parallels-pvm":   new(parallelspvm.Builder),
	"qemu":            new(qemu.Builder),
	"virtualbox-iso":  new(virtualboxiso.Builder),
	"virtualbox-ovf":  new(virtualboxovf.Builder),
	"vmware-iso":      new(vmwareiso.Builder),
	"vmware-vmx":      new(vmwarevmx.Builder),
}

var Provisioners = map[string]packer.Provisioner{
	"ansible-local":     new(ansiblelocal.Provisioner),
	"chef-client":       new(chefclient.Provisioner),
	"chef-solo":         new(chefsolo.Provisioner),
	"file":              new(fileprovisioner.Provisioner),
	"powershell":        new(powershell.Provisioner),
	"puppet-masterless": new(puppetmasterless.Provisioner),
	"puppet-server":     new(puppetserver.Provisioner),
	"salt-masterless":   new(saltmasterless.Provisioner),
	"shell":             new(shell.Provisioner),
	"shell-local":       new(shelllocal.Provisioner),
	"windows-restart":   new(restart.Provisioner),
	"windows-shell":     new(windowsshell.Provisioner),
}

var PostProcessors = map[string]packer.PostProcessor{
	"artifice":      new(artifice.PostProcessor),
	"atlas":         new(atlas.PostProcessor),
	"compress":      new(compress.PostProcessor),
	"docker-import": new(dockerimport.PostProcessor),
	"docker-push":   new(dockerpush.PostProcessor),
	"docker-save":   new(dockersave.PostProcessor),
	"docker-tag":    new(dockertag.PostProcessor),
	"vagrant":       new(vagrant.PostProcessor),
	"vagrant-cloud": new(vagrantcloud.PostProcessor),
	"vsphere":       new(vsphere.PostProcessor),
}

func (c *PluginCommand) Run(args []string) int {
	// This is an internal call so we're not going to do much error checking.
	// If there's a problem we'll usually just crash.
	log.Printf("args: %#v", args)
	if len(args) != 1 {
		c.Ui.Error("Wrong number of args")
		return 1
	}

	// Plugin should be called like "packer-builder-amazon-ebs" so we'll take it
	// apart.
	parts := strings.Split(args[0], "-")
	pluginType := parts[1]
	pluginName := ""
	// Post-processor is split so we'll so some magic here. We could use a
	// regexp but this is simpler.
	if pluginType == "post" {
		pluginType = strings.Join(parts[1:2], "-")
		pluginName = strings.Join(parts[3:], "-")
	} else {
		pluginName = strings.Join(parts[2:], "-")
	}

	server, err := plugin.Server()
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error starting plugin server: %s", err))
		return 1
	}

	if pluginType == "builder" {
		builder, found := Builders[pluginName]
		if !found {
			c.Ui.Error(fmt.Sprintf("Could not load builder: %s", pluginName))
			return 1
		}
		server.RegisterBuilder(builder)
	} else if pluginType == "provisioner" {
		provisioner, found := Provisioners[pluginName]
		if !found {
			c.Ui.Error(fmt.Sprintf("Could not load provisioner: %s", pluginName))
			return 1
		}
		server.RegisterProvisioner(provisioner)
	} else if pluginType == "post-processor" {
		postProcessor, found := PostProcessors[pluginName]
		if !found {
			c.Ui.Error(fmt.Sprintf("Could not load post-processor: %s", pluginName))
			return 1
		}
		server.RegisterPostProcessor(postProcessor)
	}

	server.Serve()

	return 0
}

func (*PluginCommand) Help() string {
	helpText := `
Usage: packer plugin PLUGIN

  Runs an internally-compiled version of a plugin from the packer binary.

  NOTE: this is an internal command and you should not call it yourself.
`

	return strings.TrimSpace(helpText)
}

func (c *PluginCommand) Synopsis() string {
	return "internal plugin command"
}
<commit_msg>Replace string splitting with a regexp; this is clearer and less code because of the post-processor case<commit_after>package command

import (
	"fmt"
	"log"
	"regexp"
	"strings"

	"github.com/mitchellh/packer/builder/amazon/chroot"
	"github.com/mitchellh/packer/builder/amazon/ebs"
	"github.com/mitchellh/packer/builder/amazon/instance"
	"github.com/mitchellh/packer/builder/digitalocean"
	"github.com/mitchellh/packer/builder/docker"
	filebuilder "github.com/mitchellh/packer/builder/file"
	"github.com/mitchellh/packer/builder/googlecompute"
	"github.com/mitchellh/packer/builder/null"
	"github.com/mitchellh/packer/builder/openstack"
	parallelsiso "github.com/mitchellh/packer/builder/parallels/iso"
	parallelspvm "github.com/mitchellh/packer/builder/parallels/pvm"
	"github.com/mitchellh/packer/builder/qemu"
	virtualboxiso "github.com/mitchellh/packer/builder/virtualbox/iso"
	virtualboxovf "github.com/mitchellh/packer/builder/virtualbox/ovf"
	vmwareiso "github.com/mitchellh/packer/builder/vmware/iso"
	vmwarevmx "github.com/mitchellh/packer/builder/vmware/vmx"
	"github.com/mitchellh/packer/packer"
	"github.com/mitchellh/packer/packer/plugin"
	"github.com/mitchellh/packer/post-processor/artifice"
	"github.com/mitchellh/packer/post-processor/atlas"
	"github.com/mitchellh/packer/post-processor/compress"
	"github.com/mitchellh/packer/post-processor/docker-import"
	"github.com/mitchellh/packer/post-processor/docker-push"
	"github.com/mitchellh/packer/post-processor/docker-save"
	"github.com/mitchellh/packer/post-processor/docker-tag"
	"github.com/mitchellh/packer/post-processor/vagrant"
	"github.com/mitchellh/packer/post-processor/vagrant-cloud"
	"github.com/mitchellh/packer/post-processor/vsphere"
	"github.com/mitchellh/packer/provisioner/ansible-local"
	"github.com/mitchellh/packer/provisioner/chef-client"
	"github.com/mitchellh/packer/provisioner/chef-solo"
	fileprovisioner "github.com/mitchellh/packer/provisioner/file"
	"github.com/mitchellh/packer/provisioner/powershell"
	"github.com/mitchellh/packer/provisioner/puppet-masterless"
	"github.com/mitchellh/packer/provisioner/puppet-server"
	"github.com/mitchellh/packer/provisioner/salt-masterless"
	"github.com/mitchellh/packer/provisioner/shell"
	shelllocal "github.com/mitchellh/packer/provisioner/shell-local"
	"github.com/mitchellh/packer/provisioner/windows-restart"
	windowsshell "github.com/mitchellh/packer/provisioner/windows-shell"
)

type PluginCommand struct {
	Meta
}

var Builders = map[string]packer.Builder{
	"amazon-chroot":   new(chroot.Builder),
	"amazon-ebs":      new(ebs.Builder),
	"amazon-instance": new(instance.Builder),
	"digitalocean":    new(digitalocean.Builder),
	"docker":          new(docker.Builder),
	"file":            new(filebuilder.Builder),
	"googlecompute":   new(googlecompute.Builder),
	"null":            new(null.Builder),
	"openstack":       new(openstack.Builder),
	"parallels-iso":   new(parallelsiso.Builder),
	"parallels-pvm":   new(parallelspvm.Builder),
	"qemu":            new(qemu.Builder),
	"virtualbox-iso":  new(virtualboxiso.Builder),
	"virtualbox-ovf":  new(virtualboxovf.Builder),
	"vmware-iso":      new(vmwareiso.Builder),
	"vmware-vmx":      new(vmwarevmx.Builder),
}

var Provisioners = map[string]packer.Provisioner{
	"ansible-local":     new(ansiblelocal.Provisioner),
	"chef-client":       new(chefclient.Provisioner),
	"chef-solo":         new(chefsolo.Provisioner),
	"file":              new(fileprovisioner.Provisioner),
	"powershell":        new(powershell.Provisioner),
	"puppet-masterless": new(puppetmasterless.Provisioner),
	"puppet-server":     new(puppetserver.Provisioner),
	"salt-masterless":   new(saltmasterless.Provisioner),
	"shell":             new(shell.Provisioner),
	"shell-local":       new(shelllocal.Provisioner),
	"windows-restart":   new(restart.Provisioner),
	"windows-shell":     new(windowsshell.Provisioner),
}

var PostProcessors = map[string]packer.PostProcessor{
	"artifice":      new(artifice.PostProcessor),
	"atlas":         new(atlas.PostProcessor),
	"compress":      new(compress.PostProcessor),
	"docker-import": new(dockerimport.PostProcessor),
	"docker-push":   new(dockerpush.PostProcessor),
	"docker-save":   new(dockersave.PostProcessor),
	"docker-tag":    new(dockertag.PostProcessor),
	"vagrant":       new(vagrant.PostProcessor),
	"vagrant-cloud": new(vagrantcloud.PostProcessor),
	"vsphere":       new(vsphere.PostProcessor),
}

var pluginRegexp = regexp.MustCompile("packer-(builder|post-processor|provisioner)-(.+)")

func (c *PluginCommand) Run(args []string) int {
	// This is an internal call (users should not call this directly) so we're
	// not going to do much input validation. If there's a problem we'll often
	// just crash. Error handling should be added to facilitate debugging.
	log.Printf("args: %#v", args)
	if len(args) != 1 {
		c.Ui.Error("Wrong number of args")
		return 1
	}

	// Plugin will match something like "packer-builder-amazon-ebs"
	parts := pluginRegexp.FindStringSubmatch(args[0])
	if len(parts) != 3 {
		c.Ui.Error(fmt.Sprintf("Error parsing plugin argument [DEBUG]: %#v", parts))
		return 1
	}
	pluginType := parts[1] // capture group 1 (builder|post-processor|provisioner)
	pluginName := parts[2] // capture group 2 (.+)

	server, err := plugin.Server()
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error starting plugin server: %s", err))
		return 1
	}

	if pluginType == "builder" {
		builder, found := Builders[pluginName]
		if !found {
			c.Ui.Error(fmt.Sprintf("Could not load builder: %s", pluginName))
			return 1
		}
		server.RegisterBuilder(builder)
	} else if pluginType == "provisioner" {
		provisioner, found := Provisioners[pluginName]
		if !found {
			c.Ui.Error(fmt.Sprintf("Could not load provisioner: %s", pluginName))
			return 1
		}
		server.RegisterProvisioner(provisioner)
	} else if pluginType == "post-processor" {
		postProcessor, found := PostProcessors[pluginName]
		if !found {
			c.Ui.Error(fmt.Sprintf("Could not load post-processor: %s", pluginName))
			return 1
		}
		server.RegisterPostProcessor(postProcessor)
	}

	server.Serve()

	return 0
}

func (*PluginCommand) Help() string {
	helpText := `
Usage: packer plugin PLUGIN

  Runs an internally-compiled version of a plugin from the packer binary.

  NOTE: this is an internal command and you should not call it yourself.
`

	return strings.TrimSpace(helpText)
}

func (c *PluginCommand) Synopsis() string {
	return "internal plugin command"
}
<|endoftext|>


<commit_before>package bind

import (
    "testing"
    "bitgirder/assert"
    "bitgirder/objpath"
    mg "mingle"
    mgRct "mingle/reactor"
    "mingle/parser"
    "time"
    "fmt"
)

func visitF1Struct( nm string, f1 int32, vc VisitContext ) error {
    out := vc.Destination
    return mgRct.VisitValue( parser.MustStruct( nm, "f1", f1 ), out )
}

type S1 struct {
    f1 int32
    f2 []int32
}

func ( s *S1 ) VisitValue( vc VisitContext ) error {
    qn := mkQn( "mingle:bind@v1/S1" )
    f := func() error {
        if err := VisitFieldValue( vc, mkId( "f1" ), s.f1 ); err != nil {
            return err
        }
        return VisitFieldFunc( vc, mkId( "f2" ), func() error {
            lt := asType( "Int32*" ).( *mg.ListTypeReference )
            elt := func( i int ) interface{} { return s.f2[ i ] }
            return VisitListValue( vc, lt, len( s.f2 ), elt )
        })
    }
    return VisitStruct( vc, qn, f )
}

type E1 string

const (
    E1V1 = E1( "v1" )
    E1V2 = E1( "v2" )
)

func ( e E1 ) VisitValue( vc VisitContext ) error {
    me := parser.MustEnum( "mingle:bind@v1/E1", string( e ) )
    ve := mgRct.NewValueEvent( me )
    return vc.Destination.ProcessEvent( ve )
}

type unregisteredType int

type failOnVisitType int

type customVisitable struct {
    f1 int32
    f2 []int32
}

func visitCustomVisitable( cv customVisitable, vc VisitContext ) error {
    qn := mkQn( "mingle:bind@v1/CustomVisitable" )
    return VisitStruct( vc, qn, func() error {
        err := VisitFieldFunc( vc, mkId( "f1" ), func() error {
            return vc.EventSender().Value( mg.Int32( int32( cv.f1 ) ) )
        })
        if err != nil { return err }
        lt := asType( "Int32*" ).( *mg.ListTypeReference )
        return VisitFieldFunc( vc, mkId( "f2" ), func() error {
            return VisitListFunc( vc, lt, len( cv.f2 ), func( i int ) error {
                return vc.EventSender().Value( mg.Int32( cv.f2[ i ] ) )
            })
        })
    })
}

func visitOkTestFunc( val interface{}, vc VisitContext ) ( error, bool ) {
    switch v := val.( type ) {
    case customVisitable: return visitCustomVisitable( v, vc ), true
    case failOnVisitType: return NewVisitError( vc.Path, "test-failure" ), true
    }
    return nil, false
}

// one-time guard for ensureTestBuilderFactories()
var didEnsureTestBuilderFactories = false

// we would otherwise do this in an init() block, except we don't want to deal
// with the possibility that this would run before the default domain itself is
// initialized (dependent packages won't have this concern)
func ensureTestBuilderFactories() {
    if didEnsureTestBuilderFactories { return }
    didEnsureTestBuilderFactories = true
    reg := NewRegistry()
    regsByDomain.Put( domainPackageBindTest, reg )
    addPrimBindings( reg )
    reg.MustAddValue(
        mkQn( "mingle:bind@v1/S1" ),
        CheckedStructFactory(
            reg,
            func() interface{} { return &S1{} },
            &CheckedFieldSetter{
                Field: mkId( "f1" ),
                Type: mg.TypeInt32,
                Assign: func( acc, val interface{} ) {
                    acc.( *S1 ).f1 = val.( int32 )
                },
            },
            &CheckedFieldSetter{
                Field: mkId( "f2" ),
                StartField: CheckedListFieldStarter(
                    func() interface{} { return make( []int32, 0, 4 ) },
                    ListElementFactoryFuncForType( mg.TypeInt32 ),
                    func( l, val interface{} ) interface{} {
                        return append( l.( []int32 ), val.( int32 ) )
                    },
                ),
                Assign: func( acc, val interface{} ) {
                    acc.( *S1 ).f2 = val.( []int32 )
                },
            },
        ),
    )
    reg.MustAddValue(
        mkQn( "mingle:bind@v1/E1" ),
        func() mgRct.BuilderFactory {
            res := mgRct.NewFunctionsBuilderFactory()
            res.ValueFunc = func(
                ve *mgRct.ValueEvent ) ( interface{}, error, bool ) {

                if e, ok := ve.Val.( *mg.Enum ); ok {
                    return E1( e.Value.ExternalForm() ), nil, true
                }
                return nil, nil, false
            }
            return res
        }(),
    )
    reg.AddVisitValueOkFunc( visitOkTestFunc )
}

var (
    tm1 = mg.MustTimestamp( "2013-10-19T02:47:00-08:00" )
)

func getDefaultValBindTestValues() *mg.IdentifierMap {
    res := mg.NewIdentifierMap()
    res.Put( mkId( "null-val" ), nil )
    res.Put( mkId( "true-val" ), true )
    res.Put( mkId( "buffer-val1" ), []byte{ 0 } )
    res.Put( mkId( "string-val1" ), "s" )
    res.Put( mkId( "int32-val1" ), int32( 1 ) )
    res.Put( mkId( "int64-val1" ), int64( 1 ) )
    res.Put( mkId( "uint32-val1" ), uint32( 1 ) )
    res.Put( mkId( "uint64-val1" ), uint64( 1 ) )
    res.Put( mkId( "float32-val1" ), float32( 1.0 ) )
    res.Put( mkId( "float64-val1" ), float64( 1.0 ) )
    res.Put( mkId( "time-val1" ), time.Time( tm1 ) )
    res.Put( mkId( "s1-val1" ), &S1{ f1: 1, f2: []int32{ 0, 1, 2 } } )
    res.Put( mkId( "e1-val1" ), E1V1 )
    res.Put(
        mkId( "custom-visitable-val1" ),
        customVisitable{ f1: 1, f2: []int32{ 0, 1, 2 } },
    )
    res.Put( mkId( "unregistered-type-val1" ), unregisteredType( 1 ) )
    res.Put( mkId( "fail-on-visit-type-val1" ), failOnVisitType( 1 ) )
    return res
}

func getDefaultValBindTests( tests []*BindTest ) []*BindTest {
    p := mg.MakeTestIdPath
    addTest := func( t *BindTest ) {
        t.Domain = domainPackageBindTest
        tests = append( tests, t )
    }
    addOk := func( in mg.Value, id string ) {
        boundId := mkId( id )
        addTest( &BindTest{ Mingle: in, BoundId: boundId } )
        addTest(
            &BindTest{
                Mingle: in,
                BoundId: boundId,
                Type: mg.TypeValue,
                StrictTypeMatching: true,
                Direction: BindTestDirectionIn,
            },
        )
    }
    addOk( mg.NullVal, "null-val" )
    addOk( mg.Boolean( true ), "true-val" )
    addOk( mg.Buffer( []byte{ 0 } ), "buffer-val1" )
    addOk( mg.String( "s" ), "string-val1" )
    addOk( mg.Int32( 1 ), "int32-val1" )
    addOk( mg.Int64( 1 ), "int64-val1" )
    addOk( mg.Uint32( 1 ), "uint32-val1" )
    addOk( mg.Uint64( 1 ), "uint64-val1" )
    addOk( mg.Float32( 1.0 ), "float32-val1" )
    addOk( mg.Float64( 1.0 ), "float64-val1" )
    addOk( tm1, "time-val1" )
    s1V1 := parser.MustStruct( "mingle:bind@v1/S1",
        "f1", int32( 1 ),
        "f2", mg.MustList( asType( "Int32*" ),
            int32( 0 ), int32( 1 ), int32( 2 ),
        ),
    )
    e1V1 := parser.MustEnum( "mingle:bind@v1/E1", "v1" )
    addOk( s1V1, "s1-val1" )
    addOk( e1V1, "e1-val1" )
    addTest(
        &BindTest{
            Mingle: parser.MustStruct( "mingle:bind@v1/CustomVisitable",
                "f1", int32( 1 ),
                "f2", mg.MustList( asType( "Int32*" ),
                    int32( 0 ), int32( 1 ), int32( 2 ),
                ),
            ),
            BoundId: mkId( "custom-visitable-val1" ),
            Direction: BindTestDirectionOut,
        },
    )
    addInErr := func( in mg.Value, path objpath.PathNode, msg string ) {
        addTest(
            &BindTest{
                Mingle: in,
                Error: NewBindError( path, msg ),
                Direction: BindTestDirectionIn,
            },
        )
    }
    addInErr(
        parser.MustStruct( "mingle:bind@v1/Bad" ),
        nil,
        "unhandled value: mingle:bind@v1/Bad",
    )
    addInErr(
        parser.MustEnum( "mingle:bind@v1/Bad", "e1" ),
        nil,
        "unhandled value: mingle:bind@v1/Bad",
    )
    addInErr(
        parser.MustStruct( "mingle:bind@v1/S1", "f1", int64( 1 ) ),
        p( 1 ),
        "unhandled value: mingle:core@v1/Int64",
    )
    addInErr(
        parser.MustStruct( "mingle:bind@v1/S1",
            "f1", parser.MustStruct( "mingle:bind@v1/S1", "f1", int32( 1 ) ),
        ),
        p( 1 ),
        "unhandled value: mingle:bind@v1/S1",
    )
    addInErr(
        parser.MustStruct( "mingle:bind@v1/S1",
            "f1", parser.MustEnum( "mingle:bind@v1/E1", "v1" ),
        ),
        p( 1 ),
        "unhandled value: mingle:bind@v1/E1",
    )
    addInErr(
        parser.MustStruct( "mingle:bind@v1/S1",
            "f1", mg.MustList( asType( "mingle:bind@v1/S1*" ),
                parser.MustStruct( "mingle:bind@v1/S1" ),
            ),
        ),
        p( 1 ),
        "unhandled value: mingle:bind@v1/S1*",
    )
    addInErr(
        parser.MustStruct( "mingle:bind@v1/S1", "f1", mg.EmptySymbolMap() ),
        p( 1 ),
        "unhandled value: mingle:core@v1/SymbolMap",
    )
    addInErr(
        mg.MustList( asType( "mingle:bind@v1/Bad*" ) ),
        nil,
        "unhandled value: mingle:bind@v1/Bad*",
    )
    addVisitErr := func( boundId string, path objpath.PathNode, msg string ) {
        addTest(
            &BindTest{
                BoundId: mkId( boundId ),
                Direction: BindTestDirectionOut,
                Error: NewVisitError( path, msg ),
            },
        )
    }
    addVisitErr(
        "unregistered-type-val1",
        nil,
        "unknown type for visit: bind.unregisteredType",
    )
    addVisitErr( "fail-on-visit-type-val1", nil, "test-failure" )
    return tests
}

func getBindTests() []*BindTest {
    return getDefaultValBindTests( []*BindTest{} )
}

type defaultBindTestCallInterface struct {
    boundVals *mg.IdentifierMap
}

func ( c defaultBindTestCallInterface ) BoundValues() *mg.IdentifierMap {
    return c.boundVals
}

func ( c defaultBindTestCallInterface ) CreateReactors(
    _ *BindTest ) []interface{} {

    return []interface{}{}
}

func TestBind( t *testing.T ) {
    ensureTestBuilderFactories()
    iface := defaultBindTestCallInterface{ getDefaultValBindTestValues() }
    AssertBindTests( getBindTests(), iface, assert.NewPathAsserter( t ) )
}

func TestRegistryAccessors( t *testing.T ) {
    a := assert.NewPathAsserter( t )
    reg := NewRegistry()
    chkGetType := func(
        key interface{}, expctOk bool, bf mgRct.BuilderFactory ) {

        ta := a.Descend( fmt.Sprint( a ) )
        var act mgRct.BuilderFactory
        var ok bool
        switch v := key.( type ) {
        case mg.TypeReference: act, ok = reg.BuilderFactoryForType( v )
        case *mg.QualifiedTypeName: act, ok = reg.BuilderFactoryForName( v )
        default: a.Fatalf( "unhandled key: %T", key )
        }
        ta.Descend( "ok" ).Equal( expctOk, ok )
        if ok { ta.Descend( "bf" ).Equal( bf, act ) }
    }
    strBld := mgRct.NewFunctionsBuilderFactory()
    reg.MustAddValue( mg.QnameString, strBld )
    chkGetType( mg.TypeString, true, strBld )
    chkGetType( mg.QnameString, true, strBld )
    chkGetType( asType( `String~"a"` ), true, strBld )
    chkGetType( mg.TypeInt32, false, nil )
    chkGetType( mg.QnameInt32, false, nil )
}
<commit_msg>fixing error in mingle/bind test<commit_after>package bind

import (
    "testing"
    "bitgirder/assert"
    "bitgirder/objpath"
    mg "mingle"
    mgRct "mingle/reactor"
    "mingle/parser"
    "time"
    "fmt"
)

func visitF1Struct( nm string, f1 int32, vc VisitContext ) error {
    out := vc.Destination
    return mgRct.VisitValue( parser.MustStruct( nm, "f1", f1 ), out )
}

type S1 struct {
    f1 int32
    f2 []int32
}

func ( s *S1 ) VisitValue( vc VisitContext ) error {
    qn := mkQn( "mingle:bind@v1/S1" )
    f := func() error {
        if err := VisitFieldValue( vc, mkId( "f1" ), s.f1 ); err != nil {
            return err
        }
        return VisitFieldFunc( vc, mkId( "f2" ), func() error {
            lt := asType( "Int32*" ).( *mg.ListTypeReference )
            elt := func( i int ) interface{} { return s.f2[ i ] }
            return VisitListValue( vc, lt, len( s.f2 ), elt )
        })
    }
    return VisitStruct( vc, qn, f )
}

type E1 string

const (
    E1V1 = E1( "v1" )
    E1V2 = E1( "v2" )
)

func ( e E1 ) VisitValue( vc VisitContext ) error {
    me := parser.MustEnum( "mingle:bind@v1/E1", string( e ) )
    ve := mgRct.NewValueEvent( me )
    return vc.Destination.ProcessEvent( ve )
}

type unregisteredType int

type failOnVisitType int

type customVisitable struct {
    f1 int32
    f2 []int32
}

func visitCustomVisitable( cv customVisitable, vc VisitContext ) error {
    qn := mkQn( "mingle:bind@v1/CustomVisitable" )
    return VisitStruct( vc, qn, func() error {
        err := VisitFieldFunc( vc, mkId( "f1" ), func() error {
            return vc.EventSender().Value( mg.Int32( int32( cv.f1 ) ) )
        })
        if err != nil { return err }
        lt := asType( "Int32*" ).( *mg.ListTypeReference )
        return VisitFieldFunc( vc, mkId( "f2" ), func() error {
            return VisitListFunc( vc, lt, len( cv.f2 ), func( i int ) error {
                return vc.EventSender().Value( mg.Int32( cv.f2[ i ] ) )
            })
        })
    })
}

func visitOkTestFunc( val interface{}, vc VisitContext ) ( error, bool ) {
    switch v := val.( type ) {
    case customVisitable: return visitCustomVisitable( v, vc ), true
    case failOnVisitType: return NewVisitError( vc.Path, "test-failure" ), true
    }
    return nil, false
}

// one-time guard for ensureTestBuilderFactories()
var didEnsureTestBuilderFactories = false

// we would otherwise do this in an init() block, except we don't want to deal
// with the possibility that this would run before the default domain itself is
// initialized (dependent packages won't have this concern)
func ensureTestBuilderFactories() {
    if didEnsureTestBuilderFactories { return }
    didEnsureTestBuilderFactories = true
    reg := NewRegistry()
    regsByDomain.Put( domainPackageBindTest, reg )
    addPrimBindings( reg )
    reg.MustAddValue(
        mkQn( "mingle:bind@v1/S1" ),
        CheckedStructFactory(
            reg,
            func() interface{} { return &S1{} },
            nil,
            &CheckedFieldSetter{
                Field: mkId( "f1" ),
                Type: mg.TypeInt32,
                Assign: func( acc, val interface{} ) {
                    acc.( *S1 ).f1 = val.( int32 )
                },
            },
            &CheckedFieldSetter{
                Field: mkId( "f2" ),
                StartField: CheckedListFieldStarter(
                    func() interface{} { return make( []int32, 0, 4 ) },
                    ListElementFactoryFuncForType( mg.TypeInt32 ),
                    func( l, val interface{} ) interface{} {
                        return append( l.( []int32 ), val.( int32 ) )
                    },
                ),
                Assign: func( acc, val interface{} ) {
                    acc.( *S1 ).f2 = val.( []int32 )
                },
            },
        ),
    )
    reg.MustAddValue(
        mkQn( "mingle:bind@v1/E1" ),
        func() mgRct.BuilderFactory {
            res := mgRct.NewFunctionsBuilderFactory()
            res.ValueFunc = func(
                ve *mgRct.ValueEvent ) ( interface{}, error, bool ) {

                if e, ok := ve.Val.( *mg.Enum ); ok {
                    return E1( e.Value.ExternalForm() ), nil, true
                }
                return nil, nil, false
            }
            return res
        }(),
    )
    reg.AddVisitValueOkFunc( visitOkTestFunc )
}

var (
    tm1 = mg.MustTimestamp( "2013-10-19T02:47:00-08:00" )
)

func getDefaultValBindTestValues() *mg.IdentifierMap {
    res := mg.NewIdentifierMap()
    res.Put( mkId( "null-val" ), nil )
    res.Put( mkId( "true-val" ), true )
    res.Put( mkId( "buffer-val1" ), []byte{ 0 } )
    res.Put( mkId( "string-val1" ), "s" )
    res.Put( mkId( "int32-val1" ), int32( 1 ) )
    res.Put( mkId( "int64-val1" ), int64( 1 ) )
    res.Put( mkId( "uint32-val1" ), uint32( 1 ) )
    res.Put( mkId( "uint64-val1" ), uint64( 1 ) )
    res.Put( mkId( "float32-val1" ), float32( 1.0 ) )
    res.Put( mkId( "float64-val1" ), float64( 1.0 ) )
    res.Put( mkId( "time-val1" ), time.Time( tm1 ) )
    res.Put( mkId( "s1-val1" ), &S1{ f1: 1, f2: []int32{ 0, 1, 2 } } )
    res.Put( mkId( "e1-val1" ), E1V1 )
    res.Put(
        mkId( "custom-visitable-val1" ),
        customVisitable{ f1: 1, f2: []int32{ 0, 1, 2 } },
    )
    res.Put( mkId( "unregistered-type-val1" ), unregisteredType( 1 ) )
    res.Put( mkId( "fail-on-visit-type-val1" ), failOnVisitType( 1 ) )
    return res
}

func getDefaultValBindTests( tests []*BindTest ) []*BindTest {
    p := mg.MakeTestIdPath
    addTest := func( t *BindTest ) {
        t.Domain = domainPackageBindTest
        tests = append( tests, t )
    }
    addOk := func( in mg.Value, id string ) {
        boundId := mkId( id )
        addTest( &BindTest{ Mingle: in, BoundId: boundId } )
        addTest(
            &BindTest{
                Mingle: in,
                BoundId: boundId,
                Type: mg.TypeValue,
                StrictTypeMatching: true,
                Direction: BindTestDirectionIn,
            },
        )
    }
    addOk( mg.NullVal, "null-val" )
    addOk( mg.Boolean( true ), "true-val" )
    addOk( mg.Buffer( []byte{ 0 } ), "buffer-val1" )
    addOk( mg.String( "s" ), "string-val1" )
    addOk( mg.Int32( 1 ), "int32-val1" )
    addOk( mg.Int64( 1 ), "int64-val1" )
    addOk( mg.Uint32( 1 ), "uint32-val1" )
    addOk( mg.Uint64( 1 ), "uint64-val1" )
    addOk( mg.Float32( 1.0 ), "float32-val1" )
    addOk( mg.Float64( 1.0 ), "float64-val1" )
    addOk( tm1, "time-val1" )
    s1V1 := parser.MustStruct( "mingle:bind@v1/S1",
        "f1", int32( 1 ),
        "f2", mg.MustList( asType( "Int32*" ),
            int32( 0 ), int32( 1 ), int32( 2 ),
        ),
    )
    e1V1 := parser.MustEnum( "mingle:bind@v1/E1", "v1" )
    addOk( s1V1, "s1-val1" )
    addOk( e1V1, "e1-val1" )
    addTest(
        &BindTest{
            Mingle: parser.MustStruct( "mingle:bind@v1/CustomVisitable",
                "f1", int32( 1 ),
                "f2", mg.MustList( asType( "Int32*" ),
                    int32( 0 ), int32( 1 ), int32( 2 ),
                ),
            ),
            BoundId: mkId( "custom-visitable-val1" ),
            Direction: BindTestDirectionOut,
        },
    )
    addInErr := func( in mg.Value, path objpath.PathNode, msg string ) {
        addTest(
            &BindTest{
                Mingle: in,
                Error: NewBindError( path, msg ),
                Direction: BindTestDirectionIn,
            },
        )
    }
    addInErr(
        parser.MustStruct( "mingle:bind@v1/Bad" ),
        nil,
        "unhandled value: mingle:bind@v1/Bad",
    )
    addInErr(
        parser.MustEnum( "mingle:bind@v1/Bad", "e1" ),
        nil,
        "unhandled value: mingle:bind@v1/Bad",
    )
    addInErr(
        parser.MustStruct( "mingle:bind@v1/S1", "f1", int64( 1 ) ),
        p( 1 ),
        "unhandled value: mingle:core@v1/Int64",
    )
    addInErr(
        parser.MustStruct( "mingle:bind@v1/S1",
            "f1", parser.MustStruct( "mingle:bind@v1/S1", "f1", int32( 1 ) ),
        ),
        p( 1 ),
        "unhandled value: mingle:bind@v1/S1",
    )
    addInErr(
        parser.MustStruct( "mingle:bind@v1/S1",
            "f1", parser.MustEnum( "mingle:bind@v1/E1", "v1" ),
        ),
        p( 1 ),
        "unhandled value: mingle:bind@v1/E1",
    )
    addInErr(
        parser.MustStruct( "mingle:bind@v1/S1",
            "f1", mg.MustList( asType( "mingle:bind@v1/S1*" ),
                parser.MustStruct( "mingle:bind@v1/S1" ),
            ),
        ),
        p( 1 ),
        "unhandled value: mingle:bind@v1/S1*",
    )
    addInErr(
        parser.MustStruct( "mingle:bind@v1/S1", "f1", mg.EmptySymbolMap() ),
        p( 1 ),
        "unhandled value: mingle:core@v1/SymbolMap",
    )
    addInErr(
        mg.MustList( asType( "mingle:bind@v1/Bad*" ) ),
        nil,
        "unhandled value: mingle:bind@v1/Bad*",
    )
    addVisitErr := func( boundId string, path objpath.PathNode, msg string ) {
        addTest(
            &BindTest{
                BoundId: mkId( boundId ),
                Direction: BindTestDirectionOut,
                Error: NewVisitError( path, msg ),
            },
        )
    }
    addVisitErr(
        "unregistered-type-val1",
        nil,
        "unknown type for visit: bind.unregisteredType",
    )
    addVisitErr( "fail-on-visit-type-val1", nil, "test-failure" )
    return tests
}

func getBindTests() []*BindTest {
    return getDefaultValBindTests( []*BindTest{} )
}

type defaultBindTestCallInterface struct {
    boundVals *mg.IdentifierMap
}

func ( c defaultBindTestCallInterface ) BoundValues() *mg.IdentifierMap {
    return c.boundVals
}

func ( c defaultBindTestCallInterface ) CreateReactors(
    _ *BindTest ) []interface{} {

    return []interface{}{}
}

func TestBind( t *testing.T ) {
    ensureTestBuilderFactories()
    iface := defaultBindTestCallInterface{ getDefaultValBindTestValues() }
    AssertBindTests( getBindTests(), iface, assert.NewPathAsserter( t ) )
}

func TestRegistryAccessors( t *testing.T ) {
    a := assert.NewPathAsserter( t )
    reg := NewRegistry()
    chkGetType := func(
        key interface{}, expctOk bool, bf mgRct.BuilderFactory ) {

        ta := a.Descend( fmt.Sprint( a ) )
        var act mgRct.BuilderFactory
        var ok bool
        switch v := key.( type ) {
        case mg.TypeReference: act, ok = reg.BuilderFactoryForType( v )
        case *mg.QualifiedTypeName: act, ok = reg.BuilderFactoryForName( v )
        default: a.Fatalf( "unhandled key: %T", key )
        }
        ta.Descend( "ok" ).Equal( expctOk, ok )
        if ok { ta.Descend( "bf" ).Equal( bf, act ) }
    }
    strBld := mgRct.NewFunctionsBuilderFactory()
    reg.MustAddValue( mg.QnameString, strBld )
    chkGetType( mg.TypeString, true, strBld )
    chkGetType( mg.QnameString, true, strBld )
    chkGetType( asType( `String~"a"` ), true, strBld )
    chkGetType( mg.TypeInt32, false, nil )
    chkGetType( mg.QnameInt32, false, nil )
}
<|endoftext|>


<commit_before>package byline

import (
	"bufio"
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"regexp"
)

var (
	// ErrOmitLine - error for Map*Err/AWKMode, for omitting current line
	ErrOmitLine = errors.New("ErrOmitLine")

	// default field separator
	defaultFS = regexp.MustCompile(`\s+`)
	// default line separator
	defaultRS byte = '\n'
	// for Grep* methods
	nullBytes = []byte{}
	// bytes.Buffer growth to this limit
	bufferSizeLimit = 512
)

// Reader - line by line Reader
type Reader struct {
	scanner     *bufio.Scanner
	buffer      bytes.Buffer
	existsData  bool
	filterFuncs []func(line []byte) ([]byte, error)
	awkVars     AWKVars
}

// AWKVars - settings for AWK mode, see man awk
type AWKVars struct {
	NR int            // number of current line (begin from 1)
	NF int            // fields count in curent line
	RS byte           // record separator, default is '\n'
	FS *regexp.Regexp // field separator, default is `\s+`
}

// NewReader - get new line by line Reader
func NewReader(reader io.Reader) *Reader {
	lr := &Reader{
		scanner:    bufio.NewScanner(reader),
		existsData: true,
		awkVars: AWKVars{
			RS: defaultRS,
			FS: defaultFS,
		},
	}

	lr.scanner.Split(lr.scanLinesBySep)
	lr.buffer.Grow(bufferSizeLimit)

	return lr
}

func (lr *Reader) scanLinesBySep(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}
	if i := bytes.IndexByte(data, lr.awkVars.RS); i >= 0 {
		// We have a full RS-terminated line.
		return i + 1, data[0 : i+1], nil
	}
	// If we're at EOF, we have a final, non-terminated line. Return it.
	if atEOF {
		return len(data), data, nil
	}

	// Request more data.
	return 0, nil, nil
}

// Read - implement io.Reader interface
func (lr *Reader) Read(p []byte) (n int, err error) {
	var (
		bufErr, filterErr error
		lineBytes         []byte
	)

	for lr.existsData && bufErr == nil && lr.buffer.Len() < bufferSizeLimit {
		if lr.existsData = lr.scanner.Scan(); !lr.existsData {
			break
		}

		lineBytes = lr.scanner.Bytes()
		lr.awkVars.NR++

		for _, filterFunc := range lr.filterFuncs {
			lineBytes, filterErr = filterFunc(lineBytes)
			if filterErr != nil {
				switch filterErr {
				case ErrOmitLine:
					lineBytes = nullBytes
				default:
					bufErr = filterErr
				}
				break
			}
		}

		_, _ = lr.buffer.Write(lineBytes) // #nosec - err always is nil
	}

	if !lr.existsData && bufErr == nil {
		bufErr = lr.scanner.Err()
	}

	n, err = lr.buffer.Read(p)
	if err != nil && bufErr == nil {
		bufErr = err
	}

	return n, bufErr
}

// Map - set filter function for process each line
func (lr *Reader) Map(filterFn func([]byte) []byte) *Reader {
	return lr.MapErr(func(line []byte) ([]byte, error) {
		return filterFn(line), nil
	})
}

// MapErr - set filter function for process each line, returns error if needed (io.EOF for example)
func (lr *Reader) MapErr(filterFn func([]byte) ([]byte, error)) *Reader {
	lr.filterFuncs = append(lr.filterFuncs, filterFn)
	return lr
}

// MapString - set filter function for process each line as string
func (lr *Reader) MapString(filterFn func(string) string) *Reader {
	return lr.MapErr(func(line []byte) ([]byte, error) {
		return []byte(filterFn(string(line))), nil
	})
}

// MapStringErr - set filter function for process each line as string, returns error if needed (io.EOF for example)
func (lr *Reader) MapStringErr(filterFn func(string) (string, error)) *Reader {
	return lr.MapErr(func(line []byte) ([]byte, error) {
		newString, err := filterFn(string(line))
		return []byte(newString), err
	})
}

// Each - processing each line.
// Do not save the value of the byte slice, since it can change in the next filter-steps.
func (lr *Reader) Each(filterFn func([]byte)) *Reader {
	return lr.MapErr(func(line []byte) ([]byte, error) {
		filterFn(line)
		return line, nil
	})
}

// EachString - processing each line as string
func (lr *Reader) EachString(filterFn func(string)) *Reader {
	return lr.MapErr(func(line []byte) ([]byte, error) {
		filterFn(string(line))
		return line, nil
	})
}

// Grep - grep lines by func
func (lr *Reader) Grep(filterFn func([]byte) bool) *Reader {
	return lr.MapErr(func(line []byte) ([]byte, error) {
		if filterFn(line) {
			return line, nil
		}

		return nullBytes, ErrOmitLine
	})
}

// GrepString - grep lines as string by func
func (lr *Reader) GrepString(filterFn func(string) bool) *Reader {
	return lr.Grep(func(line []byte) bool {
		return filterFn(string(line))
	})
}

// GrepByRegexp - grep lines by regexp
func (lr *Reader) GrepByRegexp(re *regexp.Regexp) *Reader {
	return lr.Grep(func(line []byte) bool {
		return re.Match(line)
	})
}

// SetRS - set lines (records) separator
func (lr *Reader) SetRS(rs byte) *Reader {
	lr.awkVars.RS = rs
	return lr
}

// SetFS - set field separator for AWK mode
func (lr *Reader) SetFS(fs *regexp.Regexp) *Reader {
	lr.awkVars.FS = fs
	return lr
}

// AWKMode - process lines with AWK like mode
func (lr *Reader) AWKMode(filterFn func(line string, fields []string, vars AWKVars) (string, error)) *Reader {
	return lr.MapErr(func(line []byte) ([]byte, error) {
		addRS := false
		RS := []byte{lr.awkVars.RS}
		if bytes.HasSuffix(line, RS) {
			addRS = true
			line = bytes.TrimSuffix(line, RS)
		}

		lineStr := string(line)
		fields := lr.awkVars.FS.Split(lineStr, -1)
		lr.awkVars.NF = len(fields)
		result, err := filterFn(lineStr, fields, lr.awkVars)
		if err != nil {
			return nullBytes, err
		}

		resultBytes := []byte(result)
		if !bytes.HasSuffix(resultBytes, RS) && addRS {
			resultBytes = append(resultBytes, lr.awkVars.RS)
		}
		return resultBytes, nil
	})
}

// Discard - read all content from Reader for side effect from filter functions
func (lr *Reader) Discard() error {
	_, err := io.Copy(ioutil.Discard, lr)
	return err
}

// ReadAllSlice - read all content from Reader by lines to slice of []byte
func (lr *Reader) ReadAllSlice() ([][]byte, error) {
	result := [][]byte{}
	err := lr.Map(func(line []byte) []byte {
		result = append(result, line)
		return nullBytes
	}).Discard()

	return result, err
}

// ReadAll - read all content from Reader to slice of bytes
func (lr *Reader) ReadAll() ([]byte, error) {
	return ioutil.ReadAll(lr)
}

// ReadAllSliceString - read all content from Reader to string slice by lines
func (lr *Reader) ReadAllSliceString() ([]string, error) {
	result := []string{}
	err := lr.MapString(func(line string) string {
		result = append(result, line)
		return ""
	}).Discard()

	return result, err
}

// ReadAllString - read all content from Reader to one string
func (lr *Reader) ReadAllString() (string, error) {
	result, err := ioutil.ReadAll(lr)
	return string(result), err
}
<commit_msg>Updated docs<commit_after>package byline

import (
	"bufio"
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"regexp"
)

var (
	// ErrOmitLine - error for Map*Err/AWKMode, for omitting current line
	ErrOmitLine = errors.New("ErrOmitLine")

	// default field separator
	defaultFS = regexp.MustCompile(`\s+`)
	// default line separator
	defaultRS byte = '\n'
	// for Grep* methods
	nullBytes = []byte{}
	// bytes.Buffer growth to this limit
	bufferSizeLimit = 512
)

// Reader - line by line Reader
type Reader struct {
	scanner     *bufio.Scanner
	buffer      bytes.Buffer
	existsData  bool
	filterFuncs []func(line []byte) ([]byte, error)
	awkVars     AWKVars
}

// AWKVars - settings for AWK mode, see man awk
type AWKVars struct {
	NR int            // number of the current line (begin from 1)
	NF int            // number of fields in the current line
	RS byte           // record separator, default is '\n'
	FS *regexp.Regexp // field separator, default is `\s+`
}

// NewReader - get new line by line Reader
func NewReader(reader io.Reader) *Reader {
	lr := &Reader{
		scanner:    bufio.NewScanner(reader),
		existsData: true,
		awkVars: AWKVars{
			RS: defaultRS,
			FS: defaultFS,
		},
	}

	lr.scanner.Split(lr.scanLinesBySep)
	lr.buffer.Grow(bufferSizeLimit)

	return lr
}

func (lr *Reader) scanLinesBySep(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}
	if i := bytes.IndexByte(data, lr.awkVars.RS); i >= 0 {
		// We have a full RS-terminated line.
		return i + 1, data[0 : i+1], nil
	}
	// If we're at EOF, we have a final, non-terminated line. Return it.
	if atEOF {
		return len(data), data, nil
	}

	// Request more data.
	return 0, nil, nil
}

// Read - implement io.Reader interface
func (lr *Reader) Read(p []byte) (n int, err error) {
	var (
		bufErr, filterErr error
		lineBytes         []byte
	)

	for lr.existsData && bufErr == nil && lr.buffer.Len() < bufferSizeLimit {
		if lr.existsData = lr.scanner.Scan(); !lr.existsData {
			break
		}

		lineBytes = lr.scanner.Bytes()
		lr.awkVars.NR++

		for _, filterFunc := range lr.filterFuncs {
			lineBytes, filterErr = filterFunc(lineBytes)
			if filterErr != nil {
				switch filterErr {
				case ErrOmitLine:
					lineBytes = nullBytes
				default:
					bufErr = filterErr
				}
				break
			}
		}

		_, _ = lr.buffer.Write(lineBytes) // #nosec - err always is nil
	}

	if !lr.existsData && bufErr == nil {
		bufErr = lr.scanner.Err()
	}

	n, err = lr.buffer.Read(p)
	if err != nil && bufErr == nil {
		bufErr = err
	}

	return n, bufErr
}

// Map - set filter function for process each line
func (lr *Reader) Map(filterFn func([]byte) []byte) *Reader {
	return lr.MapErr(func(line []byte) ([]byte, error) {
		return filterFn(line), nil
	})
}

// MapErr - set filter function for process each line, returns error if needed (io.EOF for example)
func (lr *Reader) MapErr(filterFn func([]byte) ([]byte, error)) *Reader {
	lr.filterFuncs = append(lr.filterFuncs, filterFn)
	return lr
}

// MapString - set filter function for process each line as string
func (lr *Reader) MapString(filterFn func(string) string) *Reader {
	return lr.MapErr(func(line []byte) ([]byte, error) {
		return []byte(filterFn(string(line))), nil
	})
}

// MapStringErr - set filter function for process each line as string, returns error if needed (io.EOF for example)
func (lr *Reader) MapStringErr(filterFn func(string) (string, error)) *Reader {
	return lr.MapErr(func(line []byte) ([]byte, error) {
		newString, err := filterFn(string(line))
		return []byte(newString), err
	})
}

// Each - processing each line.
// Do not save the value of the byte slice, since it can change in the next filter-steps.
func (lr *Reader) Each(filterFn func([]byte)) *Reader {
	return lr.MapErr(func(line []byte) ([]byte, error) {
		filterFn(line)
		return line, nil
	})
}

// EachString - processing each line as string
func (lr *Reader) EachString(filterFn func(string)) *Reader {
	return lr.MapErr(func(line []byte) ([]byte, error) {
		filterFn(string(line))
		return line, nil
	})
}

// Grep - grep lines by func
func (lr *Reader) Grep(filterFn func([]byte) bool) *Reader {
	return lr.MapErr(func(line []byte) ([]byte, error) {
		if filterFn(line) {
			return line, nil
		}

		return nullBytes, ErrOmitLine
	})
}

// GrepString - grep lines as string by func
func (lr *Reader) GrepString(filterFn func(string) bool) *Reader {
	return lr.Grep(func(line []byte) bool {
		return filterFn(string(line))
	})
}

// GrepByRegexp - grep lines by regexp
func (lr *Reader) GrepByRegexp(re *regexp.Regexp) *Reader {
	return lr.Grep(func(line []byte) bool {
		return re.Match(line)
	})
}

// SetRS - set lines (records) separator
func (lr *Reader) SetRS(rs byte) *Reader {
	lr.awkVars.RS = rs
	return lr
}

// SetFS - set field separator for AWK mode
func (lr *Reader) SetFS(fs *regexp.Regexp) *Reader {
	lr.awkVars.FS = fs
	return lr
}

// AWKMode - process lines with AWK like mode
func (lr *Reader) AWKMode(filterFn func(line string, fields []string, vars AWKVars) (string, error)) *Reader {
	return lr.MapErr(func(line []byte) ([]byte, error) {
		addRS := false
		RS := []byte{lr.awkVars.RS}
		if bytes.HasSuffix(line, RS) {
			addRS = true
			line = bytes.TrimSuffix(line, RS)
		}

		lineStr := string(line)
		fields := lr.awkVars.FS.Split(lineStr, -1)
		lr.awkVars.NF = len(fields)
		result, err := filterFn(lineStr, fields, lr.awkVars)
		if err != nil {
			return nullBytes, err
		}

		resultBytes := []byte(result)
		if !bytes.HasSuffix(resultBytes, RS) && addRS {
			resultBytes = append(resultBytes, lr.awkVars.RS)
		}
		return resultBytes, nil
	})
}

// Discard - read all content from Reader for side effect from filter functions
func (lr *Reader) Discard() error {
	_, err := io.Copy(ioutil.Discard, lr)
	return err
}

// ReadAllSlice - read all content from Reader by lines to slice of []byte
func (lr *Reader) ReadAllSlice() ([][]byte, error) {
	result := [][]byte{}
	err := lr.Map(func(line []byte) []byte {
		result = append(result, line)
		return nullBytes
	}).Discard()

	return result, err
}

// ReadAll - read all content from Reader to slice of bytes
func (lr *Reader) ReadAll() ([]byte, error) {
	return ioutil.ReadAll(lr)
}

// ReadAllSliceString - read all content from Reader to string slice by lines
func (lr *Reader) ReadAllSliceString() ([]string, error) {
	result := []string{}
	err := lr.MapString(func(line string) string {
		result = append(result, line)
		return ""
	}).Discard()

	return result, err
}

// ReadAllString - read all content from Reader to one string
func (lr *Reader) ReadAllString() (string, error) {
	result, err := ioutil.ReadAll(lr)
	return string(result), err
}
<|endoftext|>


<commit_before>package splitter

import (
	"unicode"
	"unicode/utf8"
)

// SentenceSplitter is a tiny sentence splitter for japanese texts.
type SentenceSplitter struct {
	Delim               []rune
	Follower            []rune
	SkipWhiteSpace      bool
	DoubleLineFeedSplit bool
	MaxRuneLen          int
}

var (
	// default sentence splitter
	splitter = &SentenceSplitter{
		Delim:               []rune{'。', '．'},
		Follower:            []rune{'」', '』'},
		SkipWhiteSpace:      true,
		DoubleLineFeedSplit: true,
		MaxRuneLen:          256,
	}
)

// ScanSentences is a split function for a Scanner that returns each sentece of text.
func ScanSentences(data []byte, atEOF bool) (advance int, token []byte, err error) {
	return splitter.ScanSentences(data, atEOF)
}

func (s SentenceSplitter) isDelim(r rune) bool {
	for _, d := range s.Delim {
		if r == d {
			return true
		}
	}
	return false
}

func (s SentenceSplitter) isFollower(r rune) bool {
	for _, d := range s.Follower {
		if r == d {
			return true
		}
	}
	return false
}

// ScanSentences is a split function for a Scanner that returns each sentece of text.
func (s SentenceSplitter) ScanSentences(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}
	var (
		start, end, rcount int
		head, nn           bool
	)
	head = true
	for p := 0; p < len(data); {
		r, size := utf8.DecodeRune(data[p:])
		if s.SkipWhiteSpace && unicode.IsSpace(r) {
			p += size
			if head {
				start, end = p, p
			} else if s.DoubleLineFeedSplit && r == '\n' {
				if nn {
					return p, data[start:end], nil
				}
				nn = true
			}
			continue
		}
		head, nn = false, false // clear flags
		if end != p {
			for i := 0; i < size; i++ {
				data[end+i] = data[p+i]
			}
		}
		p += size
		end += size
		rcount++
		if !s.isDelim(r) && rcount < s.MaxRuneLen {
			continue
		}
		// split
		nn = false
		for p < len(data) {
			r, size := utf8.DecodeRune(data[p:])
			if s.SkipWhiteSpace && unicode.IsSpace(r) {
				p += size
				if s.DoubleLineFeedSplit && r == '\n' {
					if nn {
						return p, data[start:end], nil
					}
					nn = true
				}
			} else if s.isDelim(r) || s.isFollower(r) {
				if end != p {
					for i := 0; i < size; i++ {
						data[end+i] = data[p+i]
					}
				}
				p += size
				end += size
			} else {
				break
			}
		}
		return p, data[start:end], nil
	}
	if !atEOF {
		// Request more data
		for i := end; i < len(data); i++ {
			data[i] = ' '
		}
		return start, nil, nil
	}
	// If we're at EOF, we have a final, non-terminated line. Return it.
	return len(data), data[start:end], nil

}
<commit_msg>Rename default sentence splitter<commit_after>package splitter

import (
	"unicode"
	"unicode/utf8"
)

// SentenceSplitter is a tiny sentence splitter for japanese texts.
type SentenceSplitter struct {
	Delim               []rune
	Follower            []rune
	SkipWhiteSpace      bool
	DoubleLineFeedSplit bool
	MaxRuneLen          int
}

var (
	// default sentence splitter
	defaultSpliter = &SentenceSplitter{
		Delim:               []rune{'。', '．'},
		Follower:            []rune{'」', '』'},
		SkipWhiteSpace:      true,
		DoubleLineFeedSplit: true,
		MaxRuneLen:          256,
	}
)

// ScanSentences is a split function for a Scanner that returns each sentece of text.
func ScanSentences(data []byte, atEOF bool) (advance int, token []byte, err error) {
	return defaultSpliter.ScanSentences(data, atEOF)
}

func (s SentenceSplitter) isDelim(r rune) bool {
	for _, d := range s.Delim {
		if r == d {
			return true
		}
	}
	return false
}

func (s SentenceSplitter) isFollower(r rune) bool {
	for _, d := range s.Follower {
		if r == d {
			return true
		}
	}
	return false
}

// ScanSentences is a split function for a Scanner that returns each sentece of text.
func (s SentenceSplitter) ScanSentences(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}
	var (
		start, end, rcount int
		head, nn           bool
	)
	head = true
	for p := 0; p < len(data); {
		r, size := utf8.DecodeRune(data[p:])
		if s.SkipWhiteSpace && unicode.IsSpace(r) {
			p += size
			if head {
				start, end = p, p
			} else if s.DoubleLineFeedSplit && r == '\n' {
				if nn {
					return p, data[start:end], nil
				}
				nn = true
			}
			continue
		}
		head, nn = false, false // clear flags
		if end != p {
			for i := 0; i < size; i++ {
				data[end+i] = data[p+i]
			}
		}
		p += size
		end += size
		rcount++
		if !s.isDelim(r) && rcount < s.MaxRuneLen {
			continue
		}
		// split
		nn = false
		for p < len(data) {
			r, size := utf8.DecodeRune(data[p:])
			if s.SkipWhiteSpace && unicode.IsSpace(r) {
				p += size
				if s.DoubleLineFeedSplit && r == '\n' {
					if nn {
						return p, data[start:end], nil
					}
					nn = true
				}
			} else if s.isDelim(r) || s.isFollower(r) {
				if end != p {
					for i := 0; i < size; i++ {
						data[end+i] = data[p+i]
					}
				}
				p += size
				end += size
			} else {
				break
			}
		}
		return p, data[start:end], nil
	}
	if !atEOF {
		// Request more data
		for i := end; i < len(data); i++ {
			data[i] = ' '
		}
		return start, nil, nil
	}
	// If we're at EOF, we have a final, non-terminated line. Return it.
	return len(data), data[start:end], nil

}
<|endoftext|>


<commit_before>package arn

import (
	"github.com/aerogo/api"
	"github.com/aerogo/nano"
	"github.com/animenotifier/jikan"
	"github.com/animenotifier/kitsu"
	"github.com/animenotifier/mal"
)

// Session ...
type Session map[string]interface{}

// Node represents the database node.
var Node = nano.New(5000)

// DB is the main database client.
var DB = Node.Namespace("arn").RegisterTypes(
	(*Analytics)(nil),
	(*Anime)(nil),
	(*AnimeCharacters)(nil),
	(*AnimeEpisodes)(nil),
	(*AnimeRelations)(nil),
	(*AnimeList)(nil),
	(*Character)(nil),
	(*Company)(nil),
	(*DraftIndex)(nil),
	(*EditLogEntry)(nil),
	(*EmailToUser)(nil),
	(*FacebookToUser)(nil),
	(*GoogleToUser)(nil),
	(*Group)(nil),
	(*GroupPost)(nil),
	(*IDList)(nil),
	(*IgnoreAnimeDifference)(nil),
	(*Inventory)(nil),
	(*NickToUser)(nil),
	(*Notification)(nil),
	(*PayPalPayment)(nil),
	(*Post)(nil),
	(*Purchase)(nil),
	(*PushSubscriptions)(nil),
	(*Quote)(nil),
	(*Session)(nil),
	(*Settings)(nil),
	(*ShopItem)(nil),
	(*SoundTrack)(nil),
	(*Thread)(nil),
	(*TwitterToUser)(nil),
	(*User)(nil),
	(*UserFollows)(nil),
	(*UserNotifications)(nil),
)

// MAL is the client for the MyAnimeList database.
var MAL = Node.Namespace("mal").RegisterTypes(
	(*mal.Anime)(nil),
)

// Kitsu is the client for the Kitsu database.
var Kitsu = Node.Namespace("kitsu").RegisterTypes(
	(*kitsu.Anime)(nil),
	(*kitsu.Mapping)(nil),
	(*kitsu.Character)(nil),
)

// API ...
var API = api.New("/api/", DB)

// init ...
func init() {
	Node.Namespace("jikan").RegisterTypes(
		(*jikan.Anime)(nil),
		(*jikan.Character)(nil),
	)
}
<commit_msg>Jikan not required anymore<commit_after>package arn

import (
	"github.com/aerogo/api"
	"github.com/aerogo/nano"
	"github.com/animenotifier/kitsu"
	"github.com/animenotifier/mal"
)

// Session ...
type Session map[string]interface{}

// Node represents the database node.
var Node = nano.New(5000)

// DB is the main database client.
var DB = Node.Namespace("arn").RegisterTypes(
	(*Analytics)(nil),
	(*Anime)(nil),
	(*AnimeCharacters)(nil),
	(*AnimeEpisodes)(nil),
	(*AnimeRelations)(nil),
	(*AnimeList)(nil),
	(*Character)(nil),
	(*Company)(nil),
	(*DraftIndex)(nil),
	(*EditLogEntry)(nil),
	(*EmailToUser)(nil),
	(*FacebookToUser)(nil),
	(*GoogleToUser)(nil),
	(*Group)(nil),
	(*GroupPost)(nil),
	(*IDList)(nil),
	(*IgnoreAnimeDifference)(nil),
	(*Inventory)(nil),
	(*NickToUser)(nil),
	(*Notification)(nil),
	(*PayPalPayment)(nil),
	(*Post)(nil),
	(*Purchase)(nil),
	(*PushSubscriptions)(nil),
	(*Quote)(nil),
	(*Session)(nil),
	(*Settings)(nil),
	(*ShopItem)(nil),
	(*SoundTrack)(nil),
	(*Thread)(nil),
	(*TwitterToUser)(nil),
	(*User)(nil),
	(*UserFollows)(nil),
	(*UserNotifications)(nil),
)

// MAL is the client for the MyAnimeList database.
var MAL = Node.Namespace("mal").RegisterTypes(
	(*mal.Anime)(nil),
)

// Kitsu is the client for the Kitsu database.
var Kitsu = Node.Namespace("kitsu").RegisterTypes(
	(*kitsu.Anime)(nil),
	(*kitsu.Mapping)(nil),
	(*kitsu.Character)(nil),
)

// API ...
var API = api.New("/api/", DB)
<|endoftext|>