<commit_before>package resources

import "github.com/jinzhu/gorm"

const (
	SpectrumScale    string = "spectrum-scale"
	SpectrumScaleNFS string = "spectrum-scale-nfs"
	SoftlayerNFS     string = "softlayer-nfs"
	SCBE             string = "scbe"
)

type UbiquityServerConfig struct {
	Port                int
	LogPath             string
	ConfigPath          string
	SpectrumScaleConfig SpectrumScaleConfig
	ScbeConfig          ScbeConfig
	BrokerConfig        BrokerConfig
	DefaultBackend      string
}

// TODO we should consider to move dedicated backend structs to the backend resource file instead of this one.
type SpectrumScaleConfig struct {
	DefaultFilesystemName string
	NfsServerAddr         string
	SshConfig             SshConfig
	RestConfig            RestConfig
	ForceDelete           bool
}

type CredentialInfo struct {
	UserName string `json:"username"`
	Password string `json:"password"`
	Group    string `json:"group"`
}

type ConnectionInfo struct {
	CredentialInfo CredentialInfo
	Port           int
	ManagementIP   string
	SkipVerifySSL  bool
}

type ScbeConfig struct {
	ConfigPath           string // TODO consider to remove later
	ConnectionInfo       ConnectionInfo
	DefaultService       string // SCBE storage service to be used by default if not mentioned by plugin
	DefaultVolumeSize    string // The default volume size in case not specified by user
	DefaultFilesystem    string // The default filesystem to create on new volumes
	UbiquityInstanceName string // Prefix for the volume name in the storage side (max length 15 char)
}

const UbiquityInstanceNameMaxSize = 15
const DefaultForScbeConfigParamDefaultVolumeSize = "1"    // if customer don't mention size, then the default is 1gb
const DefaultForScbeConfigParamDefaultFilesystem = "ext4" // if customer don't mention fstype, then the default is ext4
const PathToMountUbiquityBlockDevices = "/ubiquity/%s"    // %s is the WWN of the volume # TODO this should be moved to docker plugin side

type SshConfig struct {
	User string
	Host string
	Port string
}

type RestConfig struct {
	Endpoint string
	User     string
	Password string
	Hostname string
}

type SpectrumNfsRemoteConfig struct {
	ClientConfig string
}

type BrokerConfig struct {
	ConfigPath string
	Port       int //for CF Service broker
}

type UbiquityPluginConfig struct {
	DockerPlugin            UbiquityDockerPluginConfig
	LogPath                 string
	UbiquityServer          UbiquityServerConnectionInfo
	SpectrumNfsRemoteConfig SpectrumNfsRemoteConfig
	Backends                []string
}
type UbiquityDockerPluginConfig struct {
	//Address          string
	Port             int
	PluginsDirectory string
}

type UbiquityServerConnectionInfo struct {
	Address string
	Port    int
}

//go:generate counterfeiter -o ../fakes/fake_storage_client.go . StorageClient

type StorageClient interface {
	Activate(activateRequest ActivateRequest) error
	CreateVolume(createVolumeRequest CreateVolumeRequest) error
	RemoveVolume(removeVolumeRequest RemoveVolumeRequest) error
	ListVolumes(listVolumeRequest ListVolumesRequest) ([]Volume, error)
	GetVolume(getVolumeRequest GetVolumeRequest) (Volume, error)
	GetVolumeConfig(getVolumeConfigRequest GetVolumeConfigRequest) (map[string]interface{}, error)
	Attach(attachRequest AttachRequest) (string, error)
	Detach(detachRequest DetachRequest) error
}

//go:generate counterfeiter -o ../fakes/fake_mounter.go . Mounter

type Mounter interface {
	Mount(mountRequest MountRequest) (string, error)
	Unmount(unmountRequest UnmountRequest) error
	ActionAfterDetach(request AfterDetachRequest) error
}

type ActivateRequest struct {
	Backends []string
	Opts     map[string]string
}

type CreateVolumeRequest struct {
	Name    string
	Backend string
	Opts    map[string]interface{}
}

type RemoveVolumeRequest struct {
	Name string
}

type ListVolumesRequest struct {
	//TODO add filter
	Backends []string
}

type AttachRequest struct {
	Name string
	Host string
}

type DetachRequest struct {
	Name string
	Host string
}
type GetVolumeRequest struct {
	Name string
}
type GetVolumeConfigRequest struct {
	Name string
}
type ActivateResponse struct {
	Implements []string
	Err        string
}

type GenericResponse struct {
	Err string
}

type GenericRequest struct {
	Name string
}

type MountRequest struct {
	Mountpoint   string
	VolumeConfig map[string]interface{}
}
type UnmountRequest struct {
	VolumeConfig map[string]interface{}
}
type AfterDetachRequest struct {
	VolumeConfig map[string]interface{}
}
type AttachResponse struct {
	Mountpoint string
	Err        string
}

type MountResponse struct {
	Mountpoint string
	Err        string
}

type GetResponse struct {
	Volume Volume
	Err    string
}
type DockerGetResponse struct {
	Volume Volume
	Err    string
}

type Volume struct {
	gorm.Model
	Name       string
	Backend    string
	Mountpoint string
}

type GetConfigResponse struct {
	VolumeConfig map[string]interface{}
	Err          string
}

type ListResponse struct {
	Volumes []Volume
	Err     string
}

type FlexVolumeResponse struct {
	Status  string `json:"status"`
	Message string `json:"message"`
	Device  string `json:"device"`
}

type FlexVolumeMountRequest struct {
	MountPath   string                 `json:"mountPath"`
	MountDevice string                 `json:"name"`
	Opts        map[string]interface{} `json:"opts"`
}

type FlexVolumeUnmountRequest struct {
	MountPath string `json:"mountPath"`
}

type FlexVolumeAttachRequest struct {
	Name string            `json:"name"`
	Host string            `json:"host"`
	Opts map[string]string `json:"opts"`
}

type FlexVolumeDetachRequest struct {
	Name string `json:"name"`
	Host string `json:"host"`
}
<commit_msg> #40: Add more details to the docker volume inspect<commit_after>package resources

import "github.com/jinzhu/gorm"

const (
	SpectrumScale    string = "spectrum-scale"
	SpectrumScaleNFS string = "spectrum-scale-nfs"
	SoftlayerNFS     string = "softlayer-nfs"
	SCBE             string = "scbe"
)

type UbiquityServerConfig struct {
	Port                int
	LogPath             string
	ConfigPath          string
	SpectrumScaleConfig SpectrumScaleConfig
	ScbeConfig          ScbeConfig
	BrokerConfig        BrokerConfig
	DefaultBackend      string
}

// TODO we should consider to move dedicated backend structs to the backend resource file instead of this one.
type SpectrumScaleConfig struct {
	DefaultFilesystemName string
	NfsServerAddr         string
	SshConfig             SshConfig
	RestConfig            RestConfig
	ForceDelete           bool
}

type CredentialInfo struct {
	UserName string `json:"username"`
	Password string `json:"password"`
	Group    string `json:"group"`
}

type ConnectionInfo struct {
	CredentialInfo CredentialInfo
	Port           int
	ManagementIP   string
	SkipVerifySSL  bool
}

type ScbeConfig struct {
	ConfigPath           string // TODO consider to remove later
	ConnectionInfo       ConnectionInfo
	DefaultService       string // SCBE storage service to be used by default if not mentioned by plugin
	DefaultVolumeSize    string // The default volume size in case not specified by user
	DefaultFilesystem    string // The default filesystem to create on new volumes
	UbiquityInstanceName string // Prefix for the volume name in the storage side (max length 15 char)
}

const UbiquityInstanceNameMaxSize = 15
const DefaultForScbeConfigParamDefaultVolumeSize = "1"    // if customer don't mention size, then the default is 1gb
const DefaultForScbeConfigParamDefaultFilesystem = "ext4" // if customer don't mention fstype, then the default is ext4
const PathToMountUbiquityBlockDevices = "/ubiquity/%s"    // %s is the WWN of the volume # TODO this should be moved to docker plugin side

type SshConfig struct {
	User string
	Host string
	Port string
}

type RestConfig struct {
	Endpoint string
	User     string
	Password string
	Hostname string
}

type SpectrumNfsRemoteConfig struct {
	ClientConfig string
}

type BrokerConfig struct {
	ConfigPath string
	Port       int //for CF Service broker
}

type UbiquityPluginConfig struct {
	DockerPlugin            UbiquityDockerPluginConfig
	LogPath                 string
	UbiquityServer          UbiquityServerConnectionInfo
	SpectrumNfsRemoteConfig SpectrumNfsRemoteConfig
	Backends                []string
}
type UbiquityDockerPluginConfig struct {
	//Address          string
	Port             int
	PluginsDirectory string
}

type UbiquityServerConnectionInfo struct {
	Address string
	Port    int
}

//go:generate counterfeiter -o ../fakes/fake_storage_client.go . StorageClient

type StorageClient interface {
	Activate(activateRequest ActivateRequest) error
	CreateVolume(createVolumeRequest CreateVolumeRequest) error
	RemoveVolume(removeVolumeRequest RemoveVolumeRequest) error
	ListVolumes(listVolumeRequest ListVolumesRequest) ([]Volume, error)
	GetVolume(getVolumeRequest GetVolumeRequest) (Volume, error)
	GetVolumeConfig(getVolumeConfigRequest GetVolumeConfigRequest) (map[string]interface{}, error)
	Attach(attachRequest AttachRequest) (string, error)
	Detach(detachRequest DetachRequest) error
}

//go:generate counterfeiter -o ../fakes/fake_mounter.go . Mounter

type Mounter interface {
	Mount(mountRequest MountRequest) (string, error)
	Unmount(unmountRequest UnmountRequest) error
	ActionAfterDetach(request AfterDetachRequest) error
}

type ActivateRequest struct {
	Backends []string
	Opts     map[string]string
}

type CreateVolumeRequest struct {
	Name    string
	Backend string
	Opts    map[string]interface{}
}

type RemoveVolumeRequest struct {
	Name string
}

type ListVolumesRequest struct {
	//TODO add filter
	Backends []string
}

type AttachRequest struct {
	Name string
	Host string
}

type DetachRequest struct {
	Name string
	Host string
}
type GetVolumeRequest struct {
	Name string
}
type GetVolumeConfigRequest struct {
	Name string
}
type ActivateResponse struct {
	Implements []string
	Err        string
}

type GenericResponse struct {
	Err string
}

type GenericRequest struct {
	Name string
}

type MountRequest struct {
	Mountpoint   string
	VolumeConfig map[string]interface{}
}
type UnmountRequest struct {
	VolumeConfig map[string]interface{}
}
type AfterDetachRequest struct {
	VolumeConfig map[string]interface{}
}
type AttachResponse struct {
	Mountpoint string
	Err        string
}

type MountResponse struct {
	Mountpoint string
	Err        string
}

type GetResponse struct {
	Volume Volume
	Err    string
}

type DockerGetResponse struct {
	Volume map[string]interface{}
	Err    string
}

type Volume struct {
	gorm.Model
	Name       string
	Backend    string
	Mountpoint string
}

type GetConfigResponse struct {
	VolumeConfig map[string]interface{}
	Err          string
}

type ListResponse struct {
	Volumes []Volume
	Err     string
}

type FlexVolumeResponse struct {
	Status  string `json:"status"`
	Message string `json:"message"`
	Device  string `json:"device"`
}

type FlexVolumeMountRequest struct {
	MountPath   string                 `json:"mountPath"`
	MountDevice string                 `json:"name"`
	Opts        map[string]interface{} `json:"opts"`
}

type FlexVolumeUnmountRequest struct {
	MountPath string `json:"mountPath"`
}

type FlexVolumeAttachRequest struct {
	Name string            `json:"name"`
	Host string            `json:"host"`
	Opts map[string]string `json:"opts"`
}

type FlexVolumeDetachRequest struct {
	Name string `json:"name"`
	Host string `json:"host"`
}
<|endoftext|>


<commit_before>// Copyright 2016 tsuru-client authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package installer

import (
	"bytes"
	"io/ioutil"
	"net/http"
	"os"

	"github.com/tsuru/tsuru/cmd"
	"github.com/tsuru/tsuru/cmd/cmdtest"

	"gopkg.in/check.v1"
)

func (s *S) TestParseConfigDefaultConfig(c *check.C) {
	dmConfig, err := parseConfigFile("")
	c.Assert(err, check.IsNil)
	c.Assert(dmConfig, check.DeepEquals, defaultDockerMachineConfig)
}

func (s *S) TestParseConfigFileNotExists(c *check.C) {
	_, err := parseConfigFile("not-exist-conf.yml")
	c.Assert(err, check.NotNil)
}

func (s *S) TestParseConfigFile(c *check.C) {
	conf := `
name: tsuru-test
ca-path: /tmp/certs
driver:
    name: amazonec2
    options:
        opt1: option1-value
`
	err := ioutil.WriteFile("/tmp/config.yml", []byte(conf), 0644)
	if err != nil {
		c.Fatal("Failed to write config file for test")
	}
	defer os.Remove("/tmp/config.yml")
	expected := &DockerMachineConfig{
		DriverName: "amazonec2",
		DriverOpts: map[string]interface{}{
			"opt1": "option1-value",
		},
		CAPath: "/tmp/certs",
		Name:   "tsuru-test",
	}
	dmConfig, err := parseConfigFile("/tmp/config.yml")
	c.Assert(err, check.IsNil)
	c.Assert(dmConfig, check.DeepEquals, expected)
}

func (s *S) TestInstallInfo(c *check.C) {
	c.Assert((&Install{}).Info(), check.NotNil)
}

func (s *S) TestInstallCommandFlags(c *check.C) {
	command := Install{}
	flags := command.Flags()
	c.Assert(flags, check.NotNil)
	flags.Parse(true, []string{"-c", "my-conf.yml"})
	config := flags.Lookup("c")
	usage := "Configuration file"
	c.Check(config, check.NotNil)
	c.Check(config.Name, check.Equals, "c")
	c.Check(config.Usage, check.Equals, usage)
	c.Check(config.Value.String(), check.Equals, "my-conf.yml")
	c.Check(config.DefValue, check.Equals, "")
	config = flags.Lookup("config")
	c.Check(config, check.NotNil)
	c.Check(config.Name, check.Equals, "config")
	c.Check(config.Usage, check.Equals, usage)
	c.Check(config.Value.String(), check.Equals, "my-conf.yml")
	c.Check(config.DefValue, check.Equals, "")
}

func (s *S) TestInstallTargetAlreadyExists(c *check.C) {
	var stdout, stderr bytes.Buffer
	manager := cmd.NewManager("test", "1.0.0", "Supported-Tsuru", &stdout, &stderr, os.Stdin, nil)
	command := Install{}
	command.Flags().Parse(true, []string{"-c", "./testdata/wrong-conf.yml"})
	context := cmd.Context{
		Stdout: &stdout,
		Stderr: &stderr,
	}
	trans := cmdtest.ConditionalTransport{
		Transport: cmdtest.Transport{Message: "Ok", Status: http.StatusOK},
		CondFunc: func(r *http.Request) bool {
			return true
		},
	}
	client := cmd.NewClient(&http.Client{Transport: &trans}, nil, manager)
	expectedErr := "tsuru target \"test\" already exists"
	err := command.Run(&context, client)
	c.Assert(err, check.NotNil)
	c.Assert(expectedErr, check.Equals, err.Error())
}

func (s *S) TestUninstallInfo(c *check.C) {
	c.Assert((&Uninstall{}).Info(), check.NotNil)
}

func (s *S) TestUninstallCommandFlags(c *check.C) {
	command := Uninstall{}
	flags := command.Flags()
	c.Assert(flags, check.NotNil)
	flags.Parse(true, []string{"-c", "my-conf.yml"})
	config := flags.Lookup("c")
	usage := "Configuration file"
	c.Check(config, check.NotNil)
	c.Check(config.Name, check.Equals, "c")
	c.Check(config.Usage, check.Equals, usage)
	c.Check(config.Value.String(), check.Equals, "my-conf.yml")
	c.Check(config.DefValue, check.Equals, "")
	config = flags.Lookup("config")
	c.Check(config, check.NotNil)
	c.Check(config.Name, check.Equals, "config")
	c.Check(config.Usage, check.Equals, usage)
	c.Check(config.Value.String(), check.Equals, "my-conf.yml")
	c.Check(config.DefValue, check.Equals, "")
}
<commit_msg>tsuru/installer/install_test: add and remove tsuru target.<commit_after>// Copyright 2016 tsuru-client authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package installer

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"

	"github.com/tsuru/tsuru/cmd"

	"gopkg.in/check.v1"
)

func (s *S) TestParseConfigDefaultConfig(c *check.C) {
	dmConfig, err := parseConfigFile("")
	c.Assert(err, check.IsNil)
	c.Assert(dmConfig, check.DeepEquals, defaultDockerMachineConfig)
}

func (s *S) TestParseConfigFileNotExists(c *check.C) {
	_, err := parseConfigFile("not-exist-conf.yml")
	c.Assert(err, check.NotNil)
}

func (s *S) TestParseConfigFile(c *check.C) {
	conf := `
name: tsuru-test
ca-path: /tmp/certs
driver:
    name: amazonec2
    options:
        opt1: option1-value
`
	err := ioutil.WriteFile("/tmp/config.yml", []byte(conf), 0644)
	if err != nil {
		c.Fatal("Failed to write config file for test")
	}
	defer os.Remove("/tmp/config.yml")
	expected := &DockerMachineConfig{
		DriverName: "amazonec2",
		DriverOpts: map[string]interface{}{
			"opt1": "option1-value",
		},
		CAPath: "/tmp/certs",
		Name:   "tsuru-test",
	}
	dmConfig, err := parseConfigFile("/tmp/config.yml")
	c.Assert(err, check.IsNil)
	c.Assert(dmConfig, check.DeepEquals, expected)
}

func (s *S) TestInstallInfo(c *check.C) {
	c.Assert((&Install{}).Info(), check.NotNil)
}

func (s *S) TestInstallCommandFlags(c *check.C) {
	command := Install{}
	flags := command.Flags()
	c.Assert(flags, check.NotNil)
	flags.Parse(true, []string{"-c", "my-conf.yml"})
	config := flags.Lookup("c")
	usage := "Configuration file"
	c.Check(config, check.NotNil)
	c.Check(config.Name, check.Equals, "c")
	c.Check(config.Usage, check.Equals, usage)
	c.Check(config.Value.String(), check.Equals, "my-conf.yml")
	c.Check(config.DefValue, check.Equals, "")
	config = flags.Lookup("config")
	c.Check(config, check.NotNil)
	c.Check(config.Name, check.Equals, "config")
	c.Check(config.Usage, check.Equals, usage)
	c.Check(config.Value.String(), check.Equals, "my-conf.yml")
	c.Check(config.DefValue, check.Equals, "")
}

func (s *S) TestInstallTargetAlreadyExists(c *check.C) {
	var stdout, stderr bytes.Buffer
	manager := cmd.BuildBaseManager("uninstall-client", "0.0.0", "", nil)
	client := cmd.NewClient(&http.Client{}, nil, manager)
	context := cmd.Context{
		Args:   []string{"test", fmt.Sprintf("%s:8080", "1.2.3.4")},
		Stdout: &stdout,
		Stderr: &stderr,
	}
	targetadd := manager.Commands["target-add"]
	t, ok := targetadd.(cmd.FlaggedCommand)
	c.Assert(ok, check.Equals, true)
	err := t.Flags().Parse(true, []string{"-s"})
	c.Assert(err, check.IsNil)
	err = t.Run(&context, client)
	c.Assert(err, check.IsNil)
	defer func(manager *cmd.Manager) {
		client := cmd.NewClient(&http.Client{}, nil, manager)
		context := cmd.Context{
			Args:   []string{"test"},
			Stdout: os.Stdout,
			Stderr: os.Stderr,
		}
		targetrm := manager.Commands["target-remove"]
		targetrm.Run(&context, client)
	}(manager)
	command := Install{}
	command.Flags().Parse(true, []string{"-c", "./testdata/wrong-conf.yml"})
	context = cmd.Context{
		Stdout: &stdout,
		Stderr: &stderr,
	}
	expectedErr := "tsuru target \"test\" already exists"
	err = command.Run(&context, client)
	c.Assert(err, check.NotNil)
	c.Assert(expectedErr, check.Equals, err.Error())
}

func (s *S) TestUninstallInfo(c *check.C) {
	c.Assert((&Uninstall{}).Info(), check.NotNil)
}

func (s *S) TestUninstallCommandFlags(c *check.C) {
	command := Uninstall{}
	flags := command.Flags()
	c.Assert(flags, check.NotNil)
	flags.Parse(true, []string{"-c", "my-conf.yml"})
	config := flags.Lookup("c")
	usage := "Configuration file"
	c.Check(config, check.NotNil)
	c.Check(config.Name, check.Equals, "c")
	c.Check(config.Usage, check.Equals, usage)
	c.Check(config.Value.String(), check.Equals, "my-conf.yml")
	c.Check(config.DefValue, check.Equals, "")
	config = flags.Lookup("config")
	c.Check(config, check.NotNil)
	c.Check(config.Name, check.Equals, "config")
	c.Check(config.Usage, check.Equals, usage)
	c.Check(config.Value.String(), check.Equals, "my-conf.yml")
	c.Check(config.DefValue, check.Equals, "")
}
<|endoftext|>


<commit_before>package zfs

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"os/exec"
)

type DatasetMapping interface {
	Map(source DatasetPath) (target DatasetPath, err error)
}

func ZFSListMapping(mapping DatasetMapping) (datasets []DatasetPath, err error) {

	if mapping == nil {
		panic("mapping must not be nil")
	}

	var lines [][]string
	lines, err = ZFSList([]string{"name"}, "-r", "-t", "filesystem,volume")

	datasets = make([]DatasetPath, len(lines))

	for i, line := range lines {

		var path DatasetPath
		if path, err = NewDatasetPath(line[0]); err != nil {
			return
		}

		_, mapErr := mapping.Map(path)
		if mapErr != nil && err != NoMatchError {
			return nil, err
		}

		if mapErr == nil {
			datasets[i] = path
		}

	}

	return
}

type GlobMapping struct {
	PrefixPath DatasetPath
	TargetRoot DatasetPath
}

var NoMatchError error = errors.New("no match found in mapping")

func (m GlobMapping) Map(source DatasetPath) (target DatasetPath, err error) {

	if len(source) < len(m.PrefixPath) {
		err = NoMatchError
		return
	}

	target = make([]string, 0, len(source)+len(m.TargetRoot))
	target = append(target, m.TargetRoot...)

	for si, sc := range source {
		target = append(target, sc)
		if si < len(m.PrefixPath) {
			if sc != m.PrefixPath[si] {
				err = NoMatchError
				return
			}
			continue
		}
	}

	return
}

type ComboMapping struct {
	Mappings []DatasetMapping
}

func (m ComboMapping) Map(source DatasetPath) (target DatasetPath, err error) {
	for _, sm := range m.Mappings {
		target, err = sm.Map(source)
		if err == nil {
			return target, err
		}
	}
	return nil, NoMatchError
}

type DirectMapping struct {
	Source DatasetPath
	Target DatasetPath
}

func (m DirectMapping) Map(source DatasetPath) (target DatasetPath, err error) {

	if m.Source == nil {
		return m.Target, nil
	}

	if len(m.Source) != len(source) {
		return nil, NoMatchError
	}

	for i, c := range source {
		if c != m.Source[i] {
			return nil, NoMatchError
		}
	}

	return m.Target, nil
}

type ExecMapping struct {
	Name string
	Args []string
}

func NewExecMapping(name string, args ...string) (m *ExecMapping) {
	m = &ExecMapping{
		Name: name,
		Args: args,
	}
	return
}

func (m ExecMapping) Map(source DatasetPath) (target DatasetPath, err error) {

	var stdin io.Writer
	var stdout io.Reader

	cmd := exec.Command(m.Name, m.Args...)

	if stdin, err = cmd.StdinPipe(); err != nil {
		return
	}

	if stdout, err = cmd.StdoutPipe(); err != nil {
		return
	}

	resp := bufio.NewScanner(stdout)

	if err = cmd.Start(); err != nil {
		return
	}

	go func() {
		err := cmd.Wait()
		if err != nil {
			panic(err)
			// fmt.Printf("error: %v\n", err) // TODO
		}
	}()

	if _, err = io.WriteString(stdin, source.ToString()+"\n"); err != nil {
		return
	}

	if !resp.Scan() {
		err = errors.New(fmt.Sprintf("unexpected end of file: %v", resp.Err()))
		return
	}

	t := resp.Text()

	switch {
	case t == "NOMAP":
		return nil, NoMatchError
	}

	target = toDatasetPath(t) // TODO discover garbage?

	return
}
<commit_msg>zfs: ZFSListMapping fix broken logic<commit_after>package zfs

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"os/exec"
)

type DatasetMapping interface {
	Map(source DatasetPath) (target DatasetPath, err error)
}

func ZFSListMapping(mapping DatasetMapping) (datasets []DatasetPath, err error) {

	if mapping == nil {
		panic("mapping must not be nil")
	}

	var lines [][]string
	lines, err = ZFSList([]string{"name"}, "-r", "-t", "filesystem,volume")

	datasets = make([]DatasetPath, 0, len(lines))

	for _, line := range lines {

		var path DatasetPath
		if path, err = NewDatasetPath(line[0]); err != nil {
			return
		}

		_, mapErr := mapping.Map(path)
		if mapErr != nil && mapErr != NoMatchError {
			return nil, mapErr
		}

		if mapErr == nil {
			datasets = append(datasets, path)
		}

	}

	return
}

type GlobMapping struct {
	PrefixPath DatasetPath
	TargetRoot DatasetPath
}

var NoMatchError error = errors.New("no match found in mapping")

func (m GlobMapping) Map(source DatasetPath) (target DatasetPath, err error) {

	if len(source) < len(m.PrefixPath) {
		err = NoMatchError
		return
	}

	target = make([]string, 0, len(source)+len(m.TargetRoot))
	target = append(target, m.TargetRoot...)

	for si, sc := range source {
		target = append(target, sc)
		if si < len(m.PrefixPath) {
			if sc != m.PrefixPath[si] {
				err = NoMatchError
				return
			}
			continue
		}
	}

	return
}

type ComboMapping struct {
	Mappings []DatasetMapping
}

func (m ComboMapping) Map(source DatasetPath) (target DatasetPath, err error) {
	for _, sm := range m.Mappings {
		target, err = sm.Map(source)
		if err == nil {
			return target, err
		}
	}
	return nil, NoMatchError
}

type DirectMapping struct {
	Source DatasetPath
	Target DatasetPath
}

func (m DirectMapping) Map(source DatasetPath) (target DatasetPath, err error) {

	if m.Source == nil {
		return m.Target, nil
	}

	if len(m.Source) != len(source) {
		return nil, NoMatchError
	}

	for i, c := range source {
		if c != m.Source[i] {
			return nil, NoMatchError
		}
	}

	return m.Target, nil
}

type ExecMapping struct {
	Name string
	Args []string
}

func NewExecMapping(name string, args ...string) (m *ExecMapping) {
	m = &ExecMapping{
		Name: name,
		Args: args,
	}
	return
}

func (m ExecMapping) Map(source DatasetPath) (target DatasetPath, err error) {

	var stdin io.Writer
	var stdout io.Reader

	cmd := exec.Command(m.Name, m.Args...)

	if stdin, err = cmd.StdinPipe(); err != nil {
		return
	}

	if stdout, err = cmd.StdoutPipe(); err != nil {
		return
	}

	resp := bufio.NewScanner(stdout)

	if err = cmd.Start(); err != nil {
		return
	}

	go func() {
		err := cmd.Wait()
		if err != nil {
			panic(err)
			// fmt.Printf("error: %v\n", err) // TODO
		}
	}()

	if _, err = io.WriteString(stdin, source.ToString()+"\n"); err != nil {
		return
	}

	if !resp.Scan() {
		err = errors.New(fmt.Sprintf("unexpected end of file: %v", resp.Err()))
		return
	}

	t := resp.Text()

	switch {
	case t == "NOMAP":
		return nil, NoMatchError
	}

	target = toDatasetPath(t) // TODO discover garbage?

	return
}
<|endoftext|>


<commit_before>package listutil

import (
	"strings"

	"github.com/grokify/gotilla/type/maputil"
)

func ListStringsToLowerUniqueSorted(list []string) []string {
	myMap := map[string]bool{}
	for _, myString := range list {
		myMap[myString] = true
	}
	listOut := maputil.StringKeysToLowerSorted(myMap)
	return listOut
}

func Include(haystack []string, needle string) bool {
	for _, try := range haystack {
		if try == needle {
			return true
		}
	}
	return false
}

func IncludeCaseInsensitive(haystack []string, needle string) bool {
	needleLower := strings.ToLower(needle)
	for _, try := range haystack {
		if strings.ToLower(try) == needleLower {
			return true
		}
	}
	return false
}
<commit_msg>add listutil.SplitCount<commit_after>package listutil

import (
	"strings"

	"github.com/grokify/gotilla/type/maputil"
)

func ListStringsToLowerUniqueSorted(list []string) []string {
	myMap := map[string]bool{}
	for _, myString := range list {
		myMap[myString] = true
	}
	listOut := maputil.StringKeysToLowerSorted(myMap)
	return listOut
}

func Include(haystack []string, needle string) bool {
	for _, try := range haystack {
		if try == needle {
			return true
		}
	}
	return false
}

func IncludeCaseInsensitive(haystack []string, needle string) bool {
	needleLower := strings.ToLower(needle)
	for _, try := range haystack {
		if strings.ToLower(try) == needleLower {
			return true
		}
	}
	return false
}

func SplitCount(slice []string, size int) [][]string {
	slices := [][]string{}
	if size < 1 {
		return slices
	}
	current := []string{}
	for _, item := range slice {
		current = append(current, item)
		if len(current) == size {
			slices = append(slices, current)
			current = []string{}
		}
	}
	if len(current) > 0 {
		slices = append(slices, current)
	}
	return slices
}
<|endoftext|>


<commit_before>package revocation

import (
	"bytes"
	"crypto/rand"
	"crypto/sha256"
	"fmt"
	"math"
	"sync"

	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/fastsha256"
)

const (
	maxIndex = 1<<64 - 1
)

// chainFragment...
type chainBranch struct {
	index uint64
	hash  [32]byte
}

// HyperShaChain...
// * https://github.com/rustyrussell/ccan/blob/master/ccan/crypto/shachain/design.txt
type HyperShaChain struct {
	sync.RWMutex

	lastChainIndex uint64
	numValid       uint64

	chainBranches [64]chainBranch

	lastHash wire.ShaHash
}

// NewHyperShaChain
//  * used to track their pre-images
func NewHyperShaChain() *HyperShaChain {
	return &HyperShaChain{lastChainIndex: 0, numValid: 0}
}

// NewHyperShaChainFromSeed...
//  * used to derive your own pre-images
func NewHyperShaChainFromSeed(seed *[32]byte, deriveTo uint64) (*HyperShaChain, error) {
	var shaSeed [32]byte

	// If no seed is specified, generate a new one.
	if seed == nil {
		_, err := rand.Read(shaSeed[:])
		if err != nil {
			return nil, err
		}
	} else {
		shaSeed = *seed
	}

	// The last possible value in the chain is our starting index.
	start := uint64(maxIndex)
	stop := deriveTo

	curHash := derive(start, stop, shaSeed)

	// TODO(roasbeef): from/to or static size?
	return &HyperShaChain{lastChainIndex: deriveTo, lastHash: curHash}, nil
}

// derive...
func derive(from, to uint64, startingHash [32]byte) [32]byte {
	nextHash := startingHash

	numBranches := from ^ to
	toDerive := uint64(math.Log2(float64(numBranches))) // uh.....
	for i := toDerive - 1; i >= 0; i-- {
		if (numBranches>>i)&1 == 1 {
			// Flip the ith bit, then hash the current state to
			// advance down the tree.
			nextHash[i/8] ^= (1 << (i % 8))
			nextHash = sha256.Sum256(nextHash[:])
		}
	}

	return nextHash
}

// canDerive...
func canDerive(from, to uint64) bool {
	return ^from&to == 1
}

// getHash...
// index should be commitment #
func (h *HyperShaChain) GetHash(index uint64) (*[32]byte, error) {
	for i := uint64(0); i < h.numValid; i++ {
		/* If we can get from key to index only by resetting bits,
		 * we can derive from it => index has no bits key doesn't. */
		if !canDerive(h.chainBranches[i].index, index) {
			continue
		}

		nextHash := derive(h.chainBranches[i].index, index,
			h.chainBranches[i].hash)

		return &nextHash, nil
	}

	return nil, fmt.Errorf("unable to derive hash # %v", index)
}

// addHash
func (h *HyperShaChain) AddNextHash(hash [32]byte) error {
	// Hashes for a remote chain must be added in order.
	nextIdx := h.lastChainIndex + 1
	if nextIdx != h.lastChainIndex+1 || nextIdx == 0 && h.numValid != 0 {
		return fmt.Errorf("shachain values must be added in order, attempted"+
			"to add index %v, chain is at %v", nextIdx, h.lastChainIndex)
	}

	i := uint64(0)
	for ; i < h.numValid; i++ {
		if canDerive(nextIdx, h.chainBranches[i].index) {
			// Ensure we can actually derive this value.
			derivation := derive(nextIdx, h.chainBranches[i].index, hash)
			if !bytes.Equal(derivation[:], h.chainBranches[i].hash[:]) {
				// TODO(roasbeef): better err message
				return fmt.Errorf("chain corruption")
			}
			break
		}
	}

	h.chainBranches[i].index = nextIdx
	copy(h.chainBranches[i].hash[:], hash[:])
	copy(h.lastHash[:], hash[:])
	h.numValid = i + 1
	h.lastChainIndex = nextIdx
	return nil
}

// CurrentPreImage...
func (h *HyperShaChain) CurrentPreImage() *wire.ShaHash {
	h.RLock()
	defer h.RUnlock()
	return &h.lastHash
}

// CurrentRevocationHash...
// TODO(roasbeef): *wire.ShaHash vs [wire.HashSize]byte ?
func (h *HyperShaChain) CurrentRevocationHash() [wire.HashSize]byte {
	h.RLock()
	defer h.RUnlock()
	return fastsha256.Sum256(h.lastHash[:])
}

// LocatePreImage...
// Alice just broadcasted an old commitment tx, we need the revocation hash to
// claim the funds so we don't get cheated. However, we aren't storing all the
// pre-images in memory. So which shachain index # did she broadcast?
func (h *HyperShaChain) LocatePreImage(outputScript []byte) (uint64, *[32]byte) {
	// TODO(roasbeef): parallel goroutine divide and conquer?
	//  * need to know which side it is? also proper keys?
	//  * guess and check till script template matches the p2sh hash
	return 0, nil
}

// MarshallBinary...
func (h *HyperShaChain) Encode(b bytes.Buffer) error {
	return nil
}

// UnmarshallBinary...
func (h *HyperShaChain) Decode(b bytes.Buffer) error {
	return nil
}
<commit_msg>revocation: fix integer underflow infinite loop bug in sha chain<commit_after>package revocation

import (
	"bytes"
	"crypto/rand"
	"crypto/sha256"
	"fmt"
	"math"
	"sync"

	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/fastsha256"
)

const (
	maxIndex = 1<<64 - 1
)

// chainFragment...
type chainBranch struct {
	index uint64
	hash  [32]byte
}

// HyperShaChain...
// * https://github.com/rustyrussell/ccan/blob/master/ccan/crypto/shachain/design.txt
type HyperShaChain struct {
	sync.RWMutex

	lastChainIndex uint64
	numValid       uint64

	chainBranches [64]chainBranch

	lastHash wire.ShaHash
}

// NewHyperShaChain
//  * used to track their pre-images
func NewHyperShaChain() *HyperShaChain {
	return &HyperShaChain{lastChainIndex: 0, numValid: 0}
}

// NewHyperShaChainFromSeed...
//  * used to derive your own pre-images
func NewHyperShaChainFromSeed(seed *[32]byte, deriveTo uint64) (*HyperShaChain, error) {
	var shaSeed [32]byte

	// If no seed is specified, generate a new one.
	if seed == nil {
		_, err := rand.Read(shaSeed[:])
		if err != nil {
			return nil, err
		}
	} else {
		shaSeed = *seed
	}

	// The last possible value in the chain is our starting index.
	start := uint64(maxIndex)
	stop := deriveTo

	curHash := derive(start, stop, shaSeed)

	// TODO(roasbeef): from/to or static size?
	return &HyperShaChain{lastChainIndex: deriveTo, lastHash: curHash}, nil
}

// derive...
func derive(from, to uint64, startingHash [32]byte) [32]byte {
	nextHash := startingHash

	numBranches := from ^ to
	toDerive := uint64(math.Log2(float64(numBranches))) // uh.....
	for i := int(toDerive - 1); i >= 0; i-- {
		if (numBranches>>uint(i))&1 == 1 {
			// Flip the ith bit, then hash the current state to
			// advance down the tree.
			nextHash[i/8] ^= (1 << (uint(i) % 8))
			nextHash = sha256.Sum256(nextHash[:])
		}
	}

	return nextHash
}

// canDerive...
func canDerive(from, to uint64) bool {
	return ^from&to == 1
}

// getHash...
// index should be commitment #
func (h *HyperShaChain) GetHash(index uint64) (*[32]byte, error) {
	for i := uint64(0); i < h.numValid; i++ {
		/* If we can get from key to index only by resetting bits,
		 * we can derive from it => index has no bits key doesn't. */
		if !canDerive(h.chainBranches[i].index, index) {
			continue
		}

		nextHash := derive(h.chainBranches[i].index, index,
			h.chainBranches[i].hash)

		return &nextHash, nil
	}

	return nil, fmt.Errorf("unable to derive hash # %v", index)
}

// addHash
func (h *HyperShaChain) AddNextHash(hash [32]byte) error {
	// Hashes for a remote chain must be added in order.
	nextIdx := h.lastChainIndex + 1
	if nextIdx != h.lastChainIndex+1 || nextIdx == 0 && h.numValid != 0 {
		return fmt.Errorf("shachain values must be added in order, attempted"+
			"to add index %v, chain is at %v", nextIdx, h.lastChainIndex)
	}

	i := uint64(0)
	for ; i < h.numValid; i++ {
		if canDerive(nextIdx, h.chainBranches[i].index) {
			// Ensure we can actually derive this value.
			derivation := derive(nextIdx, h.chainBranches[i].index, hash)
			if !bytes.Equal(derivation[:], h.chainBranches[i].hash[:]) {
				// TODO(roasbeef): better err message
				return fmt.Errorf("chain corruption")
			}
			break
		}
	}

	h.chainBranches[i].index = nextIdx
	copy(h.chainBranches[i].hash[:], hash[:])
	copy(h.lastHash[:], hash[:])
	h.numValid = i + 1
	h.lastChainIndex = nextIdx
	return nil
}

// CurrentPreImage...
func (h *HyperShaChain) CurrentPreImage() *wire.ShaHash {
	h.RLock()
	defer h.RUnlock()
	return &h.lastHash
}

// CurrentRevocationHash...
// TODO(roasbeef): *wire.ShaHash vs [wire.HashSize]byte ?
func (h *HyperShaChain) CurrentRevocationHash() [wire.HashSize]byte {
	h.RLock()
	defer h.RUnlock()
	return fastsha256.Sum256(h.lastHash[:])
}

// LocatePreImage...
// Alice just broadcasted an old commitment tx, we need the revocation hash to
// claim the funds so we don't get cheated. However, we aren't storing all the
// pre-images in memory. So which shachain index # did she broadcast?
func (h *HyperShaChain) LocatePreImage(outputScript []byte) (uint64, *[32]byte) {
	// TODO(roasbeef): parallel goroutine divide and conquer?
	//  * need to know which side it is? also proper keys?
	//  * guess and check till script template matches the p2sh hash
	return 0, nil
}

// MarshallBinary...
func (h *HyperShaChain) Encode(b bytes.Buffer) error {
	return nil
}

// UnmarshallBinary...
func (h *HyperShaChain) Decode(b bytes.Buffer) error {
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Fission Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package router

import (
	"log"
	"net/http"
	"time"

	"github.com/gorilla/mux"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/rest"
	k8sCache "k8s.io/client-go/tools/cache"

	"github.com/fission/fission"
	"github.com/fission/fission/crd"
	poolmgrClient "github.com/fission/fission/poolmgr/client"
)

type HTTPTriggerSet struct {
	*functionServiceMap
	*mutableRouter
	fissionClient *crd.FissionClient
	poolmgr       *poolmgrClient.Client
	resolver      *functionReferenceResolver
	triggers      []crd.HTTPTrigger
	functions     []crd.Function
	crdClient     *rest.RESTClient
}

func makeHTTPTriggerSet(fmap *functionServiceMap, fissionClient *crd.FissionClient,
	poolmgr *poolmgrClient.Client, resolver *functionReferenceResolver, crdClient *rest.RESTClient) *HTTPTriggerSet {
	triggers := make([]crd.HTTPTrigger, 1)
	return &HTTPTriggerSet{
		functionServiceMap: fmap,
		triggers:           triggers,
		fissionClient:      fissionClient,
		poolmgr:            poolmgr,
		resolver:           resolver,
		crdClient:          crdClient,
	}
}

func (ts *HTTPTriggerSet) subscribeRouter(mr *mutableRouter) {
	ts.mutableRouter = mr
	mr.updateRouter(ts.getRouter())

	if ts.fissionClient == nil {
		// Used in tests only.
		log.Printf("Skipping continuous trigger updates")
		return
	}
	go ts.watchTriggers()
	go ts.watchFunctions()
}

func defaultHomeHandler(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusOK)
}

func (ts *HTTPTriggerSet) getRouter() *mux.Router {
	muxRouter := mux.NewRouter()

	// HTTP triggers setup by the user
	homeHandled := false
	for _, trigger := range ts.triggers {

		// resolve function reference
		rr, err := ts.resolver.resolve(trigger.Metadata.Namespace, &trigger.Spec.FunctionReference)
		if err != nil {
			// Unresolvable function reference. Report the error via
			// the trigger's status.
			go ts.updateTriggerStatusFailed(&trigger, err)

			// Ignore this route and let it 404.
			continue
		}

		if rr.resolveResultType != resolveResultSingleFunction {
			// not implemented yet
			log.Panicf("resolve result type not implemented (%v)", rr.resolveResultType)
		}

		fh := &functionHandler{
			fmap:     ts.functionServiceMap,
			function: rr.functionMetadata,
			poolmgr:  ts.poolmgr,
		}
		muxRouter.HandleFunc(trigger.Spec.RelativeURL, fh.handler).Methods(trigger.Spec.Method)
		if trigger.Spec.RelativeURL == "/" && trigger.Spec.Method == "GET" {
			homeHandled = true
		}
	}
	if !homeHandled {
		//
		// This adds a no-op handler that returns 200-OK to make sure that the
		// "GET /" request succeeds.  This route is used by GKE Ingress (and
		// perhaps other ingress implementations) as a health check, so we don't
		// want it to be a 404 even if the user doesn't have a function mapped to
		// this route.
		//
		muxRouter.HandleFunc("/", defaultHomeHandler).Methods("GET")
	}

	// Internal triggers for each function by name. Non-http
	// triggers route into these.
	for _, function := range ts.functions {
		m := function.Metadata
		fh := &functionHandler{
			fmap:     ts.functionServiceMap,
			function: &m,
			poolmgr:  ts.poolmgr,
		}
		muxRouter.HandleFunc(fission.UrlForFunction(function.Metadata.Name), fh.handler)
	}

	return muxRouter
}

func (ts *HTTPTriggerSet) updateTriggerStatusFailed(ht *crd.HTTPTrigger, err error) {
	// TODO
}

func (ts *HTTPTriggerSet) watchTriggers() {
	// sync all http triggers
	ts.syncTriggers()

	watchlist := k8sCache.NewListWatchFromClient(ts.crdClient, "httptriggers", metav1.NamespaceDefault, fields.Everything())
	listWatch := &k8sCache.ListWatch{
		ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
			return watchlist.List(options)
		},
		WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
			return watchlist.Watch(options)
		},
	}
	resyncPeriod := 30 * time.Second
	_, controller := k8sCache.NewInformer(listWatch, &crd.HTTPTrigger{}, resyncPeriod,
		k8sCache.ResourceEventHandlerFuncs{
			AddFunc: func(obj interface{}) {
				ts.syncTriggers()
			},
			DeleteFunc: func(obj interface{}) {
				ts.syncTriggers()
			},
			UpdateFunc: func(oldObj interface{}, newObj interface{}) {
				ts.syncTriggers()
			},
		})
	stop := make(chan struct{})
	defer func() {
		stop <- struct{}{}
	}()
	controller.Run(stop)
}

func (ts *HTTPTriggerSet) watchFunctions() {
	ts.syncTriggers()

	watchlist := k8sCache.NewListWatchFromClient(ts.crdClient, "functions", metav1.NamespaceDefault, fields.Everything())
	listWatch := &k8sCache.ListWatch{
		ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
			return watchlist.List(options)
		},
		WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
			return watchlist.Watch(options)
		},
	}
	resyncPeriod := 30 * time.Second
	_, controller := k8sCache.NewInformer(listWatch, &crd.Function{}, resyncPeriod,
		k8sCache.ResourceEventHandlerFuncs{
			AddFunc: func(obj interface{}) {
				ts.syncTriggers()
			},
			DeleteFunc: func(obj interface{}) {
				ts.syncTriggers()
			},
			UpdateFunc: func(oldObj interface{}, newObj interface{}) {
				fn := newObj.(*crd.Function)
				// update resolver function reference cache
				for key, rr := range ts.resolver.copy() {
					if key.functionReference.Name == fn.Metadata.Name &&
						rr.functionMetadata.ResourceVersion != fn.Metadata.ResourceVersion {
						err := ts.resolver.delete(key.namespace, &key.functionReference)
						if err != nil {
							log.Printf("Error deleting functionReferenceResolver cache: %v", err)
						}
						break
					}
				}
				ts.syncTriggers()
			},
		})
	stop := make(chan struct{})
	defer func() {
		stop <- struct{}{}
	}()
	controller.Run(stop)
}

func (ts *HTTPTriggerSet) syncTriggers() {
	log.Printf("Syncing http triggers")

	// get triggers
	triggers, err := ts.fissionClient.HTTPTriggers(metav1.NamespaceAll).List(metav1.ListOptions{})
	if err != nil {
		log.Fatalf("Failed to get http trigger list: %v", err)
	}
	ts.triggers = triggers.Items

	// get functions
	functions, err := ts.fissionClient.Functions(metav1.NamespaceAll).List(metav1.ListOptions{})
	if err != nil {
		log.Fatalf("Failed to get function list: %v", err)
	}
	ts.functions = functions.Items

	// make a new router and use it
	ts.mutableRouter.updateRouter(ts.getRouter())
}
<commit_msg>Use k8s client store to sync functions and triggers for fast synchronization (#382)<commit_after>/*
Copyright 2016 The Fission Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package router

import (
	"log"
	"net/http"
	"time"

	"github.com/gorilla/mux"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/rest"
	k8sCache "k8s.io/client-go/tools/cache"

	"github.com/fission/fission"
	"github.com/fission/fission/crd"
	poolmgrClient "github.com/fission/fission/poolmgr/client"
)

type HTTPTriggerSet struct {
	*functionServiceMap
	*mutableRouter
	fissionClient *crd.FissionClient
	poolmgr       *poolmgrClient.Client
	resolver      *functionReferenceResolver
	triggers      []crd.HTTPTrigger
	triggerStore  k8sCache.Store
	functions     []crd.Function
	funcStore     k8sCache.Store
	crdClient     *rest.RESTClient
}

func makeHTTPTriggerSet(fmap *functionServiceMap, fissionClient *crd.FissionClient,
	poolmgr *poolmgrClient.Client, resolver *functionReferenceResolver, crdClient *rest.RESTClient) *HTTPTriggerSet {
	triggers := make([]crd.HTTPTrigger, 1)
	return &HTTPTriggerSet{
		functionServiceMap: fmap,
		triggers:           triggers,
		fissionClient:      fissionClient,
		poolmgr:            poolmgr,
		resolver:           resolver,
		crdClient:          crdClient,
	}
}

func (ts *HTTPTriggerSet) subscribeRouter(mr *mutableRouter) {
	ts.mutableRouter = mr
	mr.updateRouter(ts.getRouter())

	if ts.fissionClient == nil {
		// Used in tests only.
		log.Printf("Skipping continuous trigger updates")
		return
	}
	go ts.watchTriggers()
	go ts.watchFunctions()
}

func defaultHomeHandler(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusOK)
}

func (ts *HTTPTriggerSet) getRouter() *mux.Router {
	muxRouter := mux.NewRouter()

	// HTTP triggers setup by the user
	homeHandled := false
	for _, trigger := range ts.triggers {

		// resolve function reference
		rr, err := ts.resolver.resolve(trigger.Metadata.Namespace, &trigger.Spec.FunctionReference)
		if err != nil {
			// Unresolvable function reference. Report the error via
			// the trigger's status.
			go ts.updateTriggerStatusFailed(&trigger, err)

			// Ignore this route and let it 404.
			continue
		}

		if rr.resolveResultType != resolveResultSingleFunction {
			// not implemented yet
			log.Panicf("resolve result type not implemented (%v)", rr.resolveResultType)
		}

		fh := &functionHandler{
			fmap:     ts.functionServiceMap,
			function: rr.functionMetadata,
			poolmgr:  ts.poolmgr,
		}
		muxRouter.HandleFunc(trigger.Spec.RelativeURL, fh.handler).Methods(trigger.Spec.Method)
		if trigger.Spec.RelativeURL == "/" && trigger.Spec.Method == "GET" {
			homeHandled = true
		}
	}
	if !homeHandled {
		//
		// This adds a no-op handler that returns 200-OK to make sure that the
		// "GET /" request succeeds.  This route is used by GKE Ingress (and
		// perhaps other ingress implementations) as a health check, so we don't
		// want it to be a 404 even if the user doesn't have a function mapped to
		// this route.
		//
		muxRouter.HandleFunc("/", defaultHomeHandler).Methods("GET")
	}

	// Internal triggers for each function by name. Non-http
	// triggers route into these.
	for _, function := range ts.functions {
		m := function.Metadata
		fh := &functionHandler{
			fmap:     ts.functionServiceMap,
			function: &m,
			poolmgr:  ts.poolmgr,
		}
		muxRouter.HandleFunc(fission.UrlForFunction(function.Metadata.Name), fh.handler)
	}

	return muxRouter
}

func (ts *HTTPTriggerSet) updateTriggerStatusFailed(ht *crd.HTTPTrigger, err error) {
	// TODO
}

func (ts *HTTPTriggerSet) watchTriggers() {
	watchlist := k8sCache.NewListWatchFromClient(ts.crdClient, "httptriggers", metav1.NamespaceDefault, fields.Everything())
	listWatch := &k8sCache.ListWatch{
		ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
			return watchlist.List(options)
		},
		WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
			return watchlist.Watch(options)
		},
	}
	resyncPeriod := 30 * time.Second
	store, controller := k8sCache.NewInformer(listWatch, &crd.HTTPTrigger{}, resyncPeriod,
		k8sCache.ResourceEventHandlerFuncs{
			AddFunc: func(obj interface{}) {
				ts.syncTriggers()
			},
			DeleteFunc: func(obj interface{}) {
				ts.syncTriggers()
			},
			UpdateFunc: func(oldObj interface{}, newObj interface{}) {
				ts.syncTriggers()
			},
		})
	ts.triggerStore = store
	stop := make(chan struct{})
	defer func() {
		stop <- struct{}{}
	}()
	controller.Run(stop)
}

func (ts *HTTPTriggerSet) watchFunctions() {
	watchlist := k8sCache.NewListWatchFromClient(ts.crdClient, "functions", metav1.NamespaceDefault, fields.Everything())
	listWatch := &k8sCache.ListWatch{
		ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
			return watchlist.List(options)
		},
		WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
			return watchlist.Watch(options)
		},
	}
	resyncPeriod := 30 * time.Second
	store, controller := k8sCache.NewInformer(listWatch, &crd.Function{}, resyncPeriod,
		k8sCache.ResourceEventHandlerFuncs{
			AddFunc: func(obj interface{}) {
				ts.syncTriggers()
			},
			DeleteFunc: func(obj interface{}) {
				ts.syncTriggers()
			},
			UpdateFunc: func(oldObj interface{}, newObj interface{}) {
				fn := newObj.(*crd.Function)
				// update resolver function reference cache
				for key, rr := range ts.resolver.copy() {
					if key.functionReference.Name == fn.Metadata.Name &&
						rr.functionMetadata.ResourceVersion != fn.Metadata.ResourceVersion {
						err := ts.resolver.delete(key.namespace, &key.functionReference)
						if err != nil {
							log.Printf("Error deleting functionReferenceResolver cache: %v", err)
						}
						break
					}
				}
				ts.syncTriggers()
			},
		})
	ts.funcStore = store
	stop := make(chan struct{})
	defer func() {
		stop <- struct{}{}
	}()
	controller.Run(stop)
}

func (ts *HTTPTriggerSet) syncTriggers() {
	log.Printf("Syncing http triggers")

	// get triggers
	latestTriggers := ts.triggerStore.List()
	triggers := make([]crd.HTTPTrigger, len(latestTriggers))
	for _, t := range latestTriggers {
		triggers = append(triggers, *t.(*crd.HTTPTrigger))
	}
	ts.triggers = triggers

	// get functions
	latestFunctions := ts.funcStore.List()
	functions := make([]crd.Function, len(latestFunctions))
	for _, f := range latestFunctions {
		functions = append(functions, *f.(*crd.Function))
	}
	ts.functions = functions

	// make a new router and use it
	ts.mutableRouter.updateRouter(ts.getRouter())
}
<|endoftext|>


<commit_before>/*
Copyright 2017, 2018 Ankyra

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package state

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/ankyra/escape-core"
	"github.com/ankyra/escape-core/state/validate"
)

type DeploymentState struct {
	Name        string                 `json:"name"`
	Release     string                 `json:"release,omitempty"`
	Stages      map[string]*StageState `json:"stages,omitempty"`
	Inputs      map[string]interface{} `json:"inputs,omitempty"`
	environment *EnvironmentState      `json:"-"`
	parent      *DeploymentState       `json:"-"`
	parentStage *StageState            `json:"-"`
}

func NewDeploymentState(env *EnvironmentState, name, release string) (*DeploymentState, error) {
	d := &DeploymentState{
		Name:        name,
		Release:     release,
		Stages:      map[string]*StageState{},
		Inputs:      map[string]interface{}{},
		environment: env,
	}
	return d, d.validateAndFix(name, env)
}

func (d *DeploymentState) GetName() string {
	return d.Name
}

func (d *DeploymentState) Summarize() *DeploymentState {
	result, _ := NewDeploymentState(d.environment, d.Name, d.Release)
	for name, stage := range d.Stages {
		result.Stages[name] = stage.Summarize()
	}
	return result
}

func (d *DeploymentState) GetRootDeploymentName() string {
	prev := d
	p := prev
	for p != nil {
		prev = p
		p = p.parent
	}
	return prev.Name
}

func (d *DeploymentState) GetRootDeploymentStage() string {
	stage := ""
	prev := d
	p := prev
	for prev.parentStage != nil {
		stage = prev.parentStage.Name
		prev = p
		p = p.parent
	}
	return stage
}

func (d *DeploymentState) GetDeploymentPath() string {
	result := []string{}
	p := d
	for p != nil {
		result = append(result, p.Name)
		p = p.parent
	}
	for i := len(result)/2 - 1; i >= 0; i-- {
		opp := len(result) - 1 - i
		result[i], result[opp] = result[opp], result[i]
	}
	return strings.Join(result, ":")
}

func (d *DeploymentState) GetReleaseId(stage string) string {
	return d.Release + "-v" + d.GetVersion(stage)
}

func (d *DeploymentState) GetVersion(stage string) string {
	return d.GetStageOrCreateNew(stage).Version
}

func (d *DeploymentState) GetEnvironmentState() *EnvironmentState {
	return d.environment
}

func (d *DeploymentState) GetDeployment(stage, deploymentName string) (*DeploymentState, error) {
	st := d.GetStageOrCreateNew(stage)
	depl, ok := st.Deployments[deploymentName]
	if !ok {
		return nil, DeploymentDoesNotExistError(deploymentName)
	}
	depl.parentStage = st
	st.Deployments[deploymentName] = depl
	return depl, nil
}

func (d *DeploymentState) GetDeploymentOrMakeNew(stage, deploymentName string) (*DeploymentState, error) {
	st := d.GetStageOrCreateNew(stage)
	depl, ok := st.Deployments[deploymentName]
	if !ok {
		newDepl, err := NewDeploymentState(d.environment, deploymentName, deploymentName)
		if err != nil {
			return nil, err
		}
		depl = newDepl
		depl.parent = d
	}
	depl.parentStage = st
	st.Deployments[deploymentName] = depl
	return depl, nil
}

func (d *DeploymentState) GetUserInputs(stage string) map[string]interface{} {
	return d.GetStageOrCreateNew(stage).UserInputs
}

func (d *DeploymentState) GetCalculatedInputs(stage string) map[string]interface{} {
	return d.GetStageOrCreateNew(stage).Inputs
}

func (d *DeploymentState) GetCalculatedOutputs(stage string) map[string]interface{} {
	return d.GetStageOrCreateNew(stage).Outputs
}

func (d *DeploymentState) UpdateInputs(stage string, inputs map[string]interface{}) error {
	d.GetStageOrCreateNew(stage).SetInputs(inputs)
	return d.Save()
}

func (d *DeploymentState) UpdateUserInputs(stage string, inputs map[string]interface{}) error {
	d.GetStageOrCreateNew(stage).SetUserInputs(inputs)
	return d.Save()
}

func (d *DeploymentState) UpdateOutputs(stage string, outputs map[string]interface{}) error {
	d.GetStageOrCreateNew(stage).SetOutputs(outputs)
	return d.Save()
}

func (d *DeploymentState) CommitVersion(stage string, metadata *core.ReleaseMetadata) error {
	d.GetStageOrCreateNew(stage).SetVersion(metadata.Version)
	d.GetStageOrCreateNew(stage).Provides = metadata.GetProvides()
	return nil
}

func (d *DeploymentState) UpdateStatus(stage string, status *Status) error {
	d.GetStageOrCreateNew(stage).Status = status
	return d.Save()
}
func (d *DeploymentState) GetStatus(stage string) *Status {
	return d.GetStageOrCreateNew(stage).Status
}

func (d *DeploymentState) IsDeployed(stage string, metadata *core.ReleaseMetadata) bool {
	return d.GetStageOrCreateNew(stage).Version == metadata.Version
}

func (d *DeploymentState) Save() error {
	return d.environment.Save(d)
}

func (p *DeploymentState) ToJson() string {
	str, err := json.MarshalIndent(p, "", "   ")
	if err != nil {
		panic(err)
	}
	return string(str)
}

func (d *DeploymentState) SetProvider(stage, name, deplName string) {
	d.GetStageOrCreateNew(stage).Providers[name] = deplName
}

func (d *DeploymentState) GetProviders(stage string) map[string]string {
	result := map[string]string{}
	d.walkStatesAndStages(stage, func(p *DeploymentState, st *StageState) {
		for key, val := range st.Providers {
			result[key] = val
		}
	})
	return result
}

func (d *DeploymentState) ConfigureProviders(metadata *core.ReleaseMetadata, stage string, extraProviders map[string]string) error {
	configuredProviders := d.GetProviders(stage)
	availableProviders := d.environment.GetProviders()
	for _, consumerCfg := range metadata.GetConsumerConfig(stage) {
		c := consumerCfg.Name
		variable := consumerCfg.VariableName
		provider, override := extraProviders[variable]
		if override {
			d.SetProvider(stage, variable, provider)
			continue
		}
		_, configured := configuredProviders[variable]
		if configured {
			continue
		}
		implementations := availableProviders[c]
		if len(implementations) == 1 {
			d.SetProvider(stage, variable, implementations[0])
		} else {
			if variable != c {
				return fmt.Errorf("Missing provider '%s' of type '%s'. This can be configured using the -p / --extra-provider flag.", variable, c)
			}
			return fmt.Errorf("Missing provider of type '%s'. This can be configured using the -p / --extra-provider flag.", c)
		}
	}
	return nil
}

func (d *DeploymentState) GetPreStepInputs(stage string) map[string]interface{} {
	result := map[string]interface{}{}
	for key, val := range d.environment.Inputs {
		result[key] = val
	}
	d.walkStatesAndStages(stage, func(p *DeploymentState, st *StageState) {
		if p.Inputs != nil {
			for key, val := range p.Inputs {
				result[key] = val
			}
		}
		if st.UserInputs != nil {
			for key, val := range st.UserInputs {
				result[key] = val
			}
		}
	})
	return result
}

func (d *DeploymentState) walkStatesAndStages(startStage string, cb func(*DeploymentState, *StageState)) {
	deps := d.getDependencyStates()
	stages := d.getDependencyStages(startStage)
	for i := len(deps) - 1; i >= 0; i-- {
		p := deps[i]
		stage := stages[i]
		cb(p, stage)
	}
}

func (d *DeploymentState) getDependencyStates() []*DeploymentState {
	deps := []*DeploymentState{}
	p := d
	for p != nil {
		deps = append(deps, p)
		p = p.parent
	}
	return deps
}

func (d *DeploymentState) getDependencyStages(startStage string) []*StageState {
	stages := []*StageState{d.GetStageOrCreateNew(startStage)}
	p := d
	for p != nil {
		stages = append(stages, p.parentStage)
		p = p.parent
	}
	return stages
}

func (d *DeploymentState) ValidateNames() error {
	if !validate.IsValidDeploymentName(d.Name) {
		return validate.InvalidDeploymentNameError(d.Name)
	}
	for name, st := range d.Stages {
		st.Name = name
		if err := st.ValidateNames(); err != nil {
			return err
		}
	}
	return nil
}

func (d *DeploymentState) validateAndFix(name string, env *EnvironmentState) error {
	if !validate.IsValidDeploymentName(name) {
		return validate.InvalidDeploymentNameError(name)
	}
	d.Name = name
	d.environment = env
	if d.Release == "" {
		d.Release = name
	}
	if d.Inputs == nil {
		d.Inputs = map[string]interface{}{}
	}
	if d.Stages == nil {
		d.Stages = map[string]*StageState{}
	}
	for name, st := range d.Stages {
		st.validateAndFix(name, env, d)
	}
	return nil
}

func (d *DeploymentState) validateAndFixSubDeployment(stage *StageState, env *EnvironmentState, parent *DeploymentState) error {
	d.parent = parent
	d.parentStage = stage
	return d.validateAndFix(d.Name, env)
}

func (d *DeploymentState) GetStageOrCreateNew(stage string) *StageState {
	st, ok := d.Stages[stage]
	if !ok || st == nil {
		st = newStage()
		d.Stages[stage] = st
	}
	st.validateAndFix(stage, d.environment, d)
	return st
}
<commit_msg>Add SetFailureStatus to DeploymentState<commit_after>/*
Copyright 2017, 2018 Ankyra

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package state

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/ankyra/escape-core"
	"github.com/ankyra/escape-core/state/validate"
)

type DeploymentState struct {
	Name        string                 `json:"name"`
	Release     string                 `json:"release,omitempty"`
	Stages      map[string]*StageState `json:"stages,omitempty"`
	Inputs      map[string]interface{} `json:"inputs,omitempty"`
	environment *EnvironmentState      `json:"-"`
	parent      *DeploymentState       `json:"-"`
	parentStage *StageState            `json:"-"`
}

func NewDeploymentState(env *EnvironmentState, name, release string) (*DeploymentState, error) {
	d := &DeploymentState{
		Name:        name,
		Release:     release,
		Stages:      map[string]*StageState{},
		Inputs:      map[string]interface{}{},
		environment: env,
	}
	return d, d.validateAndFix(name, env)
}

func (d *DeploymentState) GetName() string {
	return d.Name
}

func (d *DeploymentState) Summarize() *DeploymentState {
	result, _ := NewDeploymentState(d.environment, d.Name, d.Release)
	for name, stage := range d.Stages {
		result.Stages[name] = stage.Summarize()
	}
	return result
}

func (d *DeploymentState) GetRootDeploymentName() string {
	prev := d
	p := prev
	for p != nil {
		prev = p
		p = p.parent
	}
	return prev.Name
}

func (d *DeploymentState) GetRootDeploymentStage() string {
	stage := ""
	prev := d
	p := prev
	for prev.parentStage != nil {
		stage = prev.parentStage.Name
		prev = p
		p = p.parent
	}
	return stage
}

func (d *DeploymentState) GetDeploymentPath() string {
	result := []string{}
	p := d
	for p != nil {
		result = append(result, p.Name)
		p = p.parent
	}
	for i := len(result)/2 - 1; i >= 0; i-- {
		opp := len(result) - 1 - i
		result[i], result[opp] = result[opp], result[i]
	}
	return strings.Join(result, ":")
}

func (d *DeploymentState) GetReleaseId(stage string) string {
	return d.Release + "-v" + d.GetVersion(stage)
}

func (d *DeploymentState) GetVersion(stage string) string {
	return d.GetStageOrCreateNew(stage).Version
}

func (d *DeploymentState) GetEnvironmentState() *EnvironmentState {
	return d.environment
}

func (d *DeploymentState) GetDeployment(stage, deploymentName string) (*DeploymentState, error) {
	st := d.GetStageOrCreateNew(stage)
	depl, ok := st.Deployments[deploymentName]
	if !ok {
		return nil, DeploymentDoesNotExistError(deploymentName)
	}
	depl.parentStage = st
	st.Deployments[deploymentName] = depl
	return depl, nil
}

func (d *DeploymentState) GetDeploymentOrMakeNew(stage, deploymentName string) (*DeploymentState, error) {
	st := d.GetStageOrCreateNew(stage)
	depl, ok := st.Deployments[deploymentName]
	if !ok {
		newDepl, err := NewDeploymentState(d.environment, deploymentName, deploymentName)
		if err != nil {
			return nil, err
		}
		depl = newDepl
		depl.parent = d
	}
	depl.parentStage = st
	st.Deployments[deploymentName] = depl
	return depl, nil
}

func (d *DeploymentState) GetUserInputs(stage string) map[string]interface{} {
	return d.GetStageOrCreateNew(stage).UserInputs
}

func (d *DeploymentState) GetCalculatedInputs(stage string) map[string]interface{} {
	return d.GetStageOrCreateNew(stage).Inputs
}

func (d *DeploymentState) GetCalculatedOutputs(stage string) map[string]interface{} {
	return d.GetStageOrCreateNew(stage).Outputs
}

func (d *DeploymentState) UpdateInputs(stage string, inputs map[string]interface{}) error {
	d.GetStageOrCreateNew(stage).SetInputs(inputs)
	return d.Save()
}

func (d *DeploymentState) UpdateUserInputs(stage string, inputs map[string]interface{}) error {
	d.GetStageOrCreateNew(stage).SetUserInputs(inputs)
	return d.Save()
}

func (d *DeploymentState) UpdateOutputs(stage string, outputs map[string]interface{}) error {
	d.GetStageOrCreateNew(stage).SetOutputs(outputs)
	return d.Save()
}

func (d *DeploymentState) CommitVersion(stage string, metadata *core.ReleaseMetadata) error {
	d.GetStageOrCreateNew(stage).SetVersion(metadata.Version)
	d.GetStageOrCreateNew(stage).Provides = metadata.GetProvides()
	return nil
}

func (d *DeploymentState) SetFailureStatus(stage string, err error, statusCode StatusCode) error {
	status := NewStatus(statusCode)
	status.Data = err.Error()
	if err2 := d.UpdateStatus(stage, status); err2 != nil {
		return fmt.Errorf("Couldn't update status '%s'. Trying to set failure status, because: %s", err2.Error(), err.Error())
	}
	return err
}

func (d *DeploymentState) UpdateStatus(stage string, status *Status) error {
	d.GetStageOrCreateNew(stage).Status = status
	return d.Save()
}
func (d *DeploymentState) GetStatus(stage string) *Status {
	return d.GetStageOrCreateNew(stage).Status
}

func (d *DeploymentState) IsDeployed(stage string, metadata *core.ReleaseMetadata) bool {
	return d.GetStageOrCreateNew(stage).Version == metadata.Version
}

func (d *DeploymentState) Save() error {
	return d.environment.Save(d)
}

func (p *DeploymentState) ToJson() string {
	str, err := json.MarshalIndent(p, "", "   ")
	if err != nil {
		panic(err)
	}
	return string(str)
}

func (d *DeploymentState) SetProvider(stage, name, deplName string) {
	d.GetStageOrCreateNew(stage).Providers[name] = deplName
}

func (d *DeploymentState) GetProviders(stage string) map[string]string {
	result := map[string]string{}
	d.walkStatesAndStages(stage, func(p *DeploymentState, st *StageState) {
		for key, val := range st.Providers {
			result[key] = val
		}
	})
	return result
}

func (d *DeploymentState) ConfigureProviders(metadata *core.ReleaseMetadata, stage string, extraProviders map[string]string) error {
	configuredProviders := d.GetProviders(stage)
	availableProviders := d.environment.GetProviders()
	for _, consumerCfg := range metadata.GetConsumerConfig(stage) {
		c := consumerCfg.Name
		variable := consumerCfg.VariableName
		provider, override := extraProviders[variable]
		if override {
			d.SetProvider(stage, variable, provider)
			continue
		}
		_, configured := configuredProviders[variable]
		if configured {
			continue
		}
		implementations := availableProviders[c]
		if len(implementations) == 1 {
			d.SetProvider(stage, variable, implementations[0])
		} else {
			if variable != c {
				return fmt.Errorf("Missing provider '%s' of type '%s'. This can be configured using the -p / --extra-provider flag.", variable, c)
			}
			return fmt.Errorf("Missing provider of type '%s'. This can be configured using the -p / --extra-provider flag.", c)
		}
	}
	return nil
}

func (d *DeploymentState) GetPreStepInputs(stage string) map[string]interface{} {
	result := map[string]interface{}{}
	for key, val := range d.environment.Inputs {
		result[key] = val
	}
	d.walkStatesAndStages(stage, func(p *DeploymentState, st *StageState) {
		if p.Inputs != nil {
			for key, val := range p.Inputs {
				result[key] = val
			}
		}
		if st.UserInputs != nil {
			for key, val := range st.UserInputs {
				result[key] = val
			}
		}
	})
	return result
}

func (d *DeploymentState) walkStatesAndStages(startStage string, cb func(*DeploymentState, *StageState)) {
	deps := d.getDependencyStates()
	stages := d.getDependencyStages(startStage)
	for i := len(deps) - 1; i >= 0; i-- {
		p := deps[i]
		stage := stages[i]
		cb(p, stage)
	}
}

func (d *DeploymentState) getDependencyStates() []*DeploymentState {
	deps := []*DeploymentState{}
	p := d
	for p != nil {
		deps = append(deps, p)
		p = p.parent
	}
	return deps
}

func (d *DeploymentState) getDependencyStages(startStage string) []*StageState {
	stages := []*StageState{d.GetStageOrCreateNew(startStage)}
	p := d
	for p != nil {
		stages = append(stages, p.parentStage)
		p = p.parent
	}
	return stages
}

func (d *DeploymentState) ValidateNames() error {
	if !validate.IsValidDeploymentName(d.Name) {
		return validate.InvalidDeploymentNameError(d.Name)
	}
	for name, st := range d.Stages {
		st.Name = name
		if err := st.ValidateNames(); err != nil {
			return err
		}
	}
	return nil
}

func (d *DeploymentState) validateAndFix(name string, env *EnvironmentState) error {
	if !validate.IsValidDeploymentName(name) {
		return validate.InvalidDeploymentNameError(name)
	}
	d.Name = name
	d.environment = env
	if d.Release == "" {
		d.Release = name
	}
	if d.Inputs == nil {
		d.Inputs = map[string]interface{}{}
	}
	if d.Stages == nil {
		d.Stages = map[string]*StageState{}
	}
	for name, st := range d.Stages {
		st.validateAndFix(name, env, d)
	}
	return nil
}

func (d *DeploymentState) validateAndFixSubDeployment(stage *StageState, env *EnvironmentState, parent *DeploymentState) error {
	d.parent = parent
	d.parentStage = stage
	return d.validateAndFix(d.Name, env)
}

func (d *DeploymentState) GetStageOrCreateNew(stage string) *StageState {
	st, ok := d.Stages[stage]
	if !ok || st == nil {
		st = newStage()
		d.Stages[stage] = st
	}
	st.validateAndFix(stage, d.environment, d)
	return st
}
<|endoftext|>


<commit_before>package safehttp

import (
	"errors"
	"net/http"
)

// FileServer returns a handler that serves HTTP requests with the contents of
// the file system rooted at root.
func FileServer(root string) Handler {
	fileServer := http.FileServer(http.Dir(root))

	return HandlerFunc(func(rw ResponseWriter, req *IncomingRequest) Result {
		fileServerRW := &fileServerResponseWriter{flight: rw.(*flight), header: http.Header{}}
		fileServer.ServeHTTP(fileServerRW, req.req)
		return fileServerRW.result
	})
}

type fileServerResponseWriter struct {
	flight *flight
	result Result

	// We don't allow direct access to the flight's underlying http.Header. We
	// just copy over the contents on a call to WriteHeader, with the exception
	// of the Content-Type header.
	header http.Header

	// Once WriteHeader is called, any subsequent calls to it are no-ops.
	committed bool

	// If the first call to WriteHeader is not a 200 OK, we call
	// flight.WriteError with a 404 StatusCode and make further calls to Write
	// no-ops in order to not leak information about the filesystem.
	errored bool
}

func (fileServerRW *fileServerResponseWriter) Header() http.Header {
	return fileServerRW.header
}

func (fileServerRW *fileServerResponseWriter) Write(b []byte) (int, error) {
	if !fileServerRW.committed {
		fileServerRW.WriteHeader(int(StatusOK))
	}

	if fileServerRW.errored {
		// Let the framework handle the error
		return 0, errors.New("discarded")
	}
	return fileServerRW.flight.rw.Write(b)
}

func (fileServerRW *fileServerResponseWriter) WriteHeader(statusCode int) {
	if fileServerRW.committed {
		// We've already committed to a response. The headers and status code
		// were written. Ignore this call.
		return
	}
	fileServerRW.committed = true

	// Note: Add or Set might panic if a header has been already claimed. This
	// is intended behavior.
	headers := fileServerRW.flight.Header()
	for k, v := range fileServerRW.header {
		if len(v) == 0 {
			continue
		}
		if k == "Content-Type" {
			// Skip setting the Content-Type. The Dispatcher handles it.
			continue
		}
		headers.Del(k)
		for _, vv := range v {
			headers.Add(k, vv)
		}
	}

	if statusCode != int(StatusOK) {
		fileServerRW.errored = true
		// We are writing 404 for every error to avoid leaking information about
		// the filesystem.
		fileServerRW.result = fileServerRW.flight.WriteError(StatusNotFound)
		return
	}

	fileServerRW.result = fileServerRW.flight.Write(FileServerResponse{
		Path:        fileServerRW.flight.req.URL.Path(),
		contentType: contentType(fileServerRW.header),
	})
}

// FileServerResponse represents a FileServer response.
type FileServerResponse struct {
	// The URL path.
	Path string

	// private, to not allow modifications
	contentType string
}

// ContentType is the Content-Type of the response.
func (resp FileServerResponse) ContentType() string {
	return resp.contentType
}

func contentType(h http.Header) string {
	if len(h["Content-Type"]) > 0 {
		return h["Content-Type"][0]
	}
	// Content-Type should have been set by the http.FileServer.
	return "application/octet-stream; charset=utf-8"
}
<commit_msg>Missing copyright notices.<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 	https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package safehttp

import (
	"errors"
	"net/http"
)

// FileServer returns a handler that serves HTTP requests with the contents of
// the file system rooted at root.
func FileServer(root string) Handler {
	fileServer := http.FileServer(http.Dir(root))

	return HandlerFunc(func(rw ResponseWriter, req *IncomingRequest) Result {
		fileServerRW := &fileServerResponseWriter{flight: rw.(*flight), header: http.Header{}}
		fileServer.ServeHTTP(fileServerRW, req.req)
		return fileServerRW.result
	})
}

type fileServerResponseWriter struct {
	flight *flight
	result Result

	// We don't allow direct access to the flight's underlying http.Header. We
	// just copy over the contents on a call to WriteHeader, with the exception
	// of the Content-Type header.
	header http.Header

	// Once WriteHeader is called, any subsequent calls to it are no-ops.
	committed bool

	// If the first call to WriteHeader is not a 200 OK, we call
	// flight.WriteError with a 404 StatusCode and make further calls to Write
	// no-ops in order to not leak information about the filesystem.
	errored bool
}

func (fileServerRW *fileServerResponseWriter) Header() http.Header {
	return fileServerRW.header
}

func (fileServerRW *fileServerResponseWriter) Write(b []byte) (int, error) {
	if !fileServerRW.committed {
		fileServerRW.WriteHeader(int(StatusOK))
	}

	if fileServerRW.errored {
		// Let the framework handle the error
		return 0, errors.New("discarded")
	}
	return fileServerRW.flight.rw.Write(b)
}

func (fileServerRW *fileServerResponseWriter) WriteHeader(statusCode int) {
	if fileServerRW.committed {
		// We've already committed to a response. The headers and status code
		// were written. Ignore this call.
		return
	}
	fileServerRW.committed = true

	// Note: Add or Set might panic if a header has been already claimed. This
	// is intended behavior.
	headers := fileServerRW.flight.Header()
	for k, v := range fileServerRW.header {
		if len(v) == 0 {
			continue
		}
		if k == "Content-Type" {
			// Skip setting the Content-Type. The Dispatcher handles it.
			continue
		}
		headers.Del(k)
		for _, vv := range v {
			headers.Add(k, vv)
		}
	}

	if statusCode != int(StatusOK) {
		fileServerRW.errored = true
		// We are writing 404 for every error to avoid leaking information about
		// the filesystem.
		fileServerRW.result = fileServerRW.flight.WriteError(StatusNotFound)
		return
	}

	fileServerRW.result = fileServerRW.flight.Write(FileServerResponse{
		Path:        fileServerRW.flight.req.URL.Path(),
		contentType: contentType(fileServerRW.header),
	})
}

// FileServerResponse represents a FileServer response.
type FileServerResponse struct {
	// The URL path.
	Path string

	// private, to not allow modifications
	contentType string
}

// ContentType is the Content-Type of the response.
func (resp FileServerResponse) ContentType() string {
	return resp.contentType
}

func contentType(h http.Header) string {
	if len(h["Content-Type"]) > 0 {
		return h["Content-Type"][0]
	}
	// Content-Type should have been set by the http.FileServer.
	return "application/octet-stream; charset=utf-8"
}
<|endoftext|>


<commit_before>package queuesender

import (
	"bufio"
	"bytes"
	"crypto/tls"
	"encoding/json"
	"errors"
	"io"
	"net"
	"net/http"
	"net/url"
	"strings"
	"sync"
)

type connManagerType struct {
	refresh       func() (net.Conn, error)
	lock          sync.Mutex
	active        *connType
	connRefreshes uint64
}

// newConnManager returns a connection manager that creates connections to
// urlStr.
func newConnManager(urlStr string) (*connManagerType, error) {
	scheme, endpoint, err := extractEndpoint(urlStr)
	if err != nil {
		return nil, err
	}
	var refresh func() (net.Conn, error)
	if scheme == "https" {
		refresh = func() (net.Conn, error) {
			return tls.Dial("tcp", endpoint, nil)
		}
	}
	if refresh == nil {
		return nil, errors.New("Unsupported scheme")
	}
	return &connManagerType{refresh: refresh}, nil
}

// Get returns the current connection. If the current connection is bad,
// Get replaces it with a new connection.
func (c *connManagerType) Get() (*connType, error) {
	c.lock.Lock()
	defer c.lock.Unlock()
	if c.active != nil && !c.active.isBad() {
		return c.active.get(), nil
	}
	conn, err := c.refresh()
	if err != nil {
		return nil, err
	}
	c.active = newConn(conn)
	c.connRefreshes++
	return c.active.get(), nil
}

// Refreshes returns the number of connection refreshes. That is, the
// number of times Get() created a new connection to replace a bad connection.
func (c *connManagerType) Refreshes() uint64 {
	c.lock.Lock()
	defer c.lock.Unlock()
	return c.connRefreshes
}

// connType represents a single connection.
type connType struct {
	conn      net.Conn
	br        *bufio.Reader
	lock      sync.Mutex
	userCount int
	bad       bool
}

func newConn(conn net.Conn) *connType {
	return &connType{conn: conn, br: bufio.NewReader(conn)}
}

// Put marks this connection as unused by the caller. Each time the caller
// calls Get() on the connection manager to get a connection, it must call
// Put() on the returned connection when done with it. The caller must not
// use a connection after calling Put on it.
func (c *connType) Put() {
	c.lock.Lock()
	defer c.lock.Unlock()
	c.userCount--
	if c.userCount < 1 && c.bad {
		c.conn.Close()
	}
}

// MarkBad marks this connection as bad. Caller must call MarkBad before
// calling Put.
func (c *connType) MarkBad() {
	c.lock.Lock()
	defer c.lock.Unlock()
	c.bad = true
}

func (c *connType) isBad() bool {
	c.lock.Lock()
	defer c.lock.Unlock()
	return c.bad
}

func (c *connType) get() *connType {
	c.lock.Lock()
	defer c.lock.Unlock()
	c.userCount++
	return c
}

// Send sends given request on this connection.
func (c *connType) Send(req requestType) error {
	buffer, err := encodeJSON(req.Json)
	if err != nil {
		return err
	}
	httpreq, err := http.NewRequest("POST", req.Endpoint, buffer)
	if err != nil {
		return err
	}
	httpreq.Header.Set("Content-Type", "application/json")
	return httpreq.Write(c.conn)
}

// Read reads response from the connection returning true if the connection
// is good or false if the connection is bad. If the connection is good,
// err will be non-nil if response read is something other than a 2XX
// response.
func (c *connType) Read() (bool, error) {
	resp, err := http.ReadResponse(c.br, nil)
	if err != nil {
		return false, err
	}
	defer resp.Body.Close()
	if resp.StatusCode/100 != 2 {
		var buffer bytes.Buffer
		io.Copy(&buffer, resp.Body)
		// 408 is a special error meaning that the connection timed out
		// because we didn't send any request. 408 errors don't go with
		// any particular request. When we get one of these, assume the
		// connection is bad.
		return resp.StatusCode != 408, errors.New(resp.Status + ": " + buffer.String())
	}
	return true, nil
}

func encodeJSON(payload interface{}) (*bytes.Buffer, error) {
	result := &bytes.Buffer{}
	encoder := json.NewEncoder(result)
	if err := encoder.Encode(payload); err != nil {
		return nil, err
	}
	return result, nil
}

func extractEndpoint(urlStr string) (scheme, endpoint string, err error) {
	u, err := url.Parse(urlStr)
	if err != nil {
		return "", "", err
	}
	host := u.Host
	if u.Scheme == "https" && !strings.Contains(u.Host, ":") {
		host = host + ":443"
	}
	if u.Scheme == "http" && !strings.Contains(u.Host, ":") {
		host = host + ":80"
	}
	return u.Scheme, host, nil
}
<commit_msg>Richard's comments.<commit_after>package queuesender

import (
	"bufio"
	"bytes"
	"crypto/tls"
	"encoding/json"
	"errors"
	"io"
	"net"
	"net/http"
	"net/url"
	"strings"
	"sync"
)

type connManagerType struct {
	refresh       func() (net.Conn, error)
	lock          sync.Mutex
	active        *connType
	connRefreshes uint64
}

// newConnManager returns a connection manager that creates connections to
// urlStr.
func newConnManager(urlStr string) (*connManagerType, error) {
	scheme, endpoint, err := extractEndpoint(urlStr)
	if err != nil {
		return nil, err
	}
	var refresh func() (net.Conn, error)
	if scheme == "https" {
		refresh = func() (net.Conn, error) {
			return tls.Dial("tcp", endpoint, nil)
		}
	}
	if refresh == nil {
		return nil, errors.New("Unsupported scheme")
	}
	return &connManagerType{refresh: refresh}, nil
}

// Get returns the current connection. If the current connection is bad,
// Get replaces it with a new connection.
func (c *connManagerType) Get() (*connType, error) {
	c.lock.Lock()
	defer c.lock.Unlock()
	if c.active != nil && !c.active.isBad() {
		return c.active.get(), nil
	}
	conn, err := c.refresh()
	if err != nil {
		return nil, err
	}
	c.active = newConn(conn)
	c.connRefreshes++
	return c.active.get(), nil
}

// Refreshes returns the number of connection refreshes. That is, the
// number of times Get() created a new connection to replace a bad connection.
func (c *connManagerType) Refreshes() uint64 {
	c.lock.Lock()
	defer c.lock.Unlock()
	return c.connRefreshes
}

// connType represents a single connection.
type connType struct {
	conn      net.Conn
	br        *bufio.Reader
	lock      sync.Mutex
	userCount int
	bad       bool
}

func newConn(conn net.Conn) *connType {
	return &connType{conn: conn, br: bufio.NewReader(conn)}
}

// Put marks this connection as unused by the caller. Each time the caller
// calls Get() on the connection manager to get a connection, it must call
// Put() on the returned connection when done with it. The caller must not
// use a connection after calling Put on it.
func (c *connType) Put() {
	c.lock.Lock()
	defer c.lock.Unlock()
	c.userCount--
	if c.userCount < 0 {
		panic("userCount cannot be negative")
	}
	if c.userCount < 1 && c.bad {
		c.conn.Close()
	}
}

// MarkBad marks this connection as bad. Caller must call MarkBad before
// calling Put.
func (c *connType) MarkBad() {
	c.lock.Lock()
	defer c.lock.Unlock()
	c.bad = true
}

func (c *connType) isBad() bool {
	c.lock.Lock()
	defer c.lock.Unlock()
	return c.bad
}

func (c *connType) get() *connType {
	c.lock.Lock()
	defer c.lock.Unlock()
	c.userCount++
	return c
}

// Send sends given request on this connection.
func (c *connType) Send(req requestType) error {
	buffer, err := encodeJSON(req.Json)
	if err != nil {
		return err
	}
	httpreq, err := http.NewRequest("POST", req.Endpoint, buffer)
	if err != nil {
		return err
	}
	httpreq.Header.Set("Content-Type", "application/json")
	return httpreq.Write(c.conn)
}

// Read reads response from the connection returning true if the connection
// is good or false if the connection is bad. If the connection is good,
// err will be non-nil if response read is something other than a 2XX
// response.
func (c *connType) Read() (bool, error) {
	resp, err := http.ReadResponse(c.br, nil)
	if err != nil {
		return false, err
	}
	defer resp.Body.Close()
	if resp.StatusCode/100 != 2 {
		var buffer bytes.Buffer
		io.Copy(&buffer, resp.Body)
		// 408 is a special error meaning that the connection timed out
		// because we didn't send any request. 408 errors don't go with
		// any particular request. When we get one of these, assume the
		// connection is bad.
		return resp.StatusCode != 408, errors.New(resp.Status + ": " + buffer.String())
	}
	return true, nil
}

func encodeJSON(payload interface{}) (*bytes.Buffer, error) {
	result := &bytes.Buffer{}
	encoder := json.NewEncoder(result)
	if err := encoder.Encode(payload); err != nil {
		return nil, err
	}
	return result, nil
}

func extractEndpoint(urlStr string) (scheme, endpoint string, err error) {
	u, err := url.Parse(urlStr)
	if err != nil {
		return "", "", err
	}
	host := u.Host
	if u.Scheme == "https" && !strings.Contains(u.Host, ":") {
		host = host + ":443"
	}
	if u.Scheme == "http" && !strings.Contains(u.Host, ":") {
		host = host + ":80"
	}
	return u.Scheme, host, nil
}
<|endoftext|>