<commit_before>package carto

import (
	"encoding/json"
	"github.com/ctessum/geom"
	"github.com/ctessum/geom/encoding/geojson"
	"io"
)

type GeoJSONfeature struct {
	Type       string             `json:"type"`
	Geometry   *geojson.Geometry  `json:"geometry"`
	Properties map[string]float64 `json:"properties"`
}
type GeoJSON struct {
	Type     string `json:"type"`
	CRS      Crs    `json:"crs"`
	Features []*GeoJSONfeature
}

// Coordinate reference system. Used for GeoJSON
type Crs struct {
	Type       string   `json:"type"`
	Properties CrsProps `json:"properties"`
}

// Coordinate reference system properties.
type CrsProps struct {
	Name string `json:"name"`
}

func LoadGeoJSON(r io.Reader) (*GeoJSON, error) {
	out := new(GeoJSON)
	d := json.NewDecoder(r)
	err := d.Decode(&out)
	return out, err
}

func (g *GeoJSON) Sum(propertyName string) float64 {
	sum := 0.
	for _, f := range g.Features {
		sum += f.Properties[propertyName]
	}
	return sum
}

func (g *GeoJSON) GetProperty(propertyName string) []float64 {
	out := make([]float64, len(g.Features))
	for i, f := range g.Features {
		out[i] = f.Properties[propertyName]
	}
	return out
}

func (g *GeoJSON) GetGeometry() ([]geom.Geom, error) {
	var err error
	out := make([]geom.Geom, len(g.Features))
	for i, f := range g.Features {
		out[i], err = geojson.FromGeoJSON(f.Geometry)
		if err != nil {
			return nil, err
		}
	}
	return out, nil
}

// Convert map data to GeoJSON, where value name is a
// name for the data values being output.
func (m *MapData) ToGeoJSON(valueName string) (*GeoJSON, error) {
	var err error
	g := new(GeoJSON)
	g.Type = "FeatureCollection"
	g.CRS = Crs{"name", CrsProps{"EPSG:3857"}}
	g.Features = make([]*GeoJSONfeature, len(m.Shapes))
	for i, shape := range m.Shapes {
		f := new(GeoJSONfeature)
		f.Type = "Feature"
		f.Geometry, err = geojson.ToGeoJSON(shape)
		if err != nil {
			return nil, err
		}
		f.Properties = map[string]float64{valueName: m.Data[i]}
		g.Features[i] = f
	}
	return g, nil
}
<commit_msg>Changed geojson to match spec<commit_after>package carto

import (
	"encoding/json"
	"io"

	"github.com/ctessum/geom"
	"github.com/ctessum/geom/encoding/geojson"
)

type GeoJSONfeature struct {
	Type       string             `json:"type"`
	Geometry   *geojson.Geometry  `json:"geometry"`
	Properties map[string]float64 `json:"properties"`
}
type GeoJSON struct {
	Type     string            `json:"type"`
	CRS      Crs               `json:"crs"`
	Features []*GeoJSONfeature `json:"features"`
}

// Coordinate reference system. Used for GeoJSON
type Crs struct {
	Type       string   `json:"type"`
	Properties CrsProps `json:"properties"`
}

// Coordinate reference system properties.
type CrsProps struct {
	Name string `json:"name"`
}

func LoadGeoJSON(r io.Reader) (*GeoJSON, error) {
	out := new(GeoJSON)
	d := json.NewDecoder(r)
	err := d.Decode(&out)
	return out, err
}

func (g *GeoJSON) Sum(propertyName string) float64 {
	sum := 0.
	for _, f := range g.Features {
		sum += f.Properties[propertyName]
	}
	return sum
}

func (g *GeoJSON) GetProperty(propertyName string) []float64 {
	out := make([]float64, len(g.Features))
	for i, f := range g.Features {
		out[i] = f.Properties[propertyName]
	}
	return out
}

func (g *GeoJSON) GetGeometry() ([]geom.Geom, error) {
	var err error
	out := make([]geom.Geom, len(g.Features))
	for i, f := range g.Features {
		out[i], err = geojson.FromGeoJSON(f.Geometry)
		if err != nil {
			return nil, err
		}
	}
	return out, nil
}

// Convert map data to GeoJSON, where value name is a
// name for the data values being output.
func (m *MapData) ToGeoJSON(valueName string) (*GeoJSON, error) {
	var err error
	g := new(GeoJSON)
	g.Type = "FeatureCollection"
	g.CRS = Crs{"name", CrsProps{"EPSG:3857"}}
	g.Features = make([]*GeoJSONfeature, len(m.Shapes))
	for i, shape := range m.Shapes {
		f := new(GeoJSONfeature)
		f.Type = "Feature"
		f.Geometry, err = geojson.ToGeoJSON(shape)
		if err != nil {
			return nil, err
		}
		f.Properties = map[string]float64{valueName: m.Data[i]}
		g.Features[i] = f
	}
	return g, nil
}
<|endoftext|>


<commit_before>package main

import "time"

const (
	Timeout       = 10 * time.Minute
	CheckInterval = 10 * time.Second

	TimeoutPod = 5 * time.Minute

	ClusterName               = "e2e"
	ClusterSmallNodePoolSize  = 2
	ClusterMediumNodePoolSize = 1

	NginxName  = "e2e-nginx"
	NginxImage = "circa10a/nginx-wget" //ngnix with wget
	NginxPort  = 80
	Namespace  = "default"

	WGETRetries = 12
	WGETTimeout = 10

	PVCSize      = "1Gi"
	PVCName      = "e2e-nginx-pvc"
	PVCMountPath = "/mymount"

	PathBin                 = "/usr/bin"
	KubernikusctlBinaryName = "kubernikusctl"
)
<commit_msg>waits 10min for dvs<commit_after>package main

import "time"

const (
	Timeout       = 10 * time.Minute
	CheckInterval = 10 * time.Second

	TimeoutPod = 5 * time.Minute

	ClusterName               = "e2e"
	ClusterSmallNodePoolSize  = 2
	ClusterMediumNodePoolSize = 1

	NginxName  = "e2e-nginx"
	NginxImage = "circa10a/nginx-wget" //ngnix with wget
	NginxPort  = 80
	Namespace  = "default"

	WGETRetries = 10
	WGETTimeout = 60

	PVCSize      = "1Gi"
	PVCName      = "e2e-nginx-pvc"
	PVCMountPath = "/mymount"

	PathBin                 = "/usr/bin"
	KubernikusctlBinaryName = "kubernikusctl"
)
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"os"
	"path"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

//TODO : Consolodate this code with the code for emptyDir.
//This will require some smart.
var _ = Describe("hostDir", func() {
	var (
		c         *client.Client
		namespace *api.Namespace
	)

	BeforeEach(func() {
		var err error
		c, err = loadClient()
		Expect(err).NotTo(HaveOccurred())

		By("Building a namespace api object")
		namespace, err = createTestingNS("hostdir", c)
		Expect(err).NotTo(HaveOccurred())

		//cleanup before running the test.
		_ = os.Remove("/tmp/test-file")
	})

	AfterEach(func() {
		By(fmt.Sprintf("Destroying namespace for this suite %v", namespace.Name))
		if err := c.Namespaces().Delete(namespace.Name); err != nil {
			Failf("Couldn't delete ns %s", err)
		}
	})

	It("should give a volume the correct mode", func() {
		volumePath := "/test-volume"
		source := &api.HostPathVolumeSource{
			Path: "/tmp",
		}
		pod := testPodWithHostVol(volumePath, source)

		pod.Spec.Containers[0].Args = []string{
			fmt.Sprintf("--fs_type=%v", volumePath),
			fmt.Sprintf("--file_mode=%v", volumePath),
		}
		testContainerOutputInNamespace("emptydir r/w on tmpfs", c, pod, 0, []string{
			"mode of file \"/test-volume\": dtrwxrwxrwx", // we expect the sticky bit (mode flag t) to be set for the dir
		},
			namespace.Name)
	})

	It("should support r/w on tmpfs", func() {
		volumePath := "/test-volume"
		filePath := path.Join(volumePath, "test-file")
		source := &api.HostPathVolumeSource{
			Path: "/tmp",
		}
		pod := testPodWithHostVol(volumePath, source)

		pod.Spec.Containers[0].Args = []string{
			fmt.Sprintf("--fs_type=%v", volumePath),
			fmt.Sprintf("--rw_new_file=%v", filePath),
			fmt.Sprintf("--file_mode=%v", filePath),
		}
		testContainerOutputInNamespace("emptydir r/w on tmpfs", c, pod, 0, []string{
			"mode of file \"/test-volume/test-file\": -rw-r--r--",
			"content of file \"/test-volume/test-file\": mount-tester new file",
		}, namespace.Name,
		)
	})
})

//These constants are borrowed from the other test.
//const containerName = "test-container"
//const volumeName = "test-volume"

func mount(source *api.HostPathVolumeSource) []api.Volume {
	return []api.Volume{
		{
			Name: volumeName,
			VolumeSource: api.VolumeSource{
				HostPath: source,
			},
		},
	}
}

//TODO: To merge this with the emptyDir tests, we can make source a lambda.
func testPodWithHostVol(path string, source *api.HostPathVolumeSource) *api.Pod {
	podName := "pod-" + string(util.NewUUID())

	return &api.Pod{
		TypeMeta: api.TypeMeta{
			Kind:       "Pod",
			APIVersion: latest.Version,
		},
		ObjectMeta: api.ObjectMeta{
			Name: podName,
		},
		Spec: api.PodSpec{
			Containers: []api.Container{
				{
					Name:  containerName,
					Image: "gcr.io/google_containers/mounttest:0.2",
					VolumeMounts: []api.VolumeMount{
						{
							Name:      volumeName,
							MountPath: path,
						},
					},
				},
			},
			RestartPolicy: api.RestartPolicyNever,
			Volumes:       mount(source),
		},
	}
}
<commit_msg>Cleanup hostPath e2e tests<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"os"
	"path"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

//TODO : Consolodate this code with the code for emptyDir.
//This will require some smart.
var _ = Describe("hostPath", func() {
	var (
		c         *client.Client
		namespace *api.Namespace
	)

	BeforeEach(func() {
		var err error
		c, err = loadClient()
		Expect(err).NotTo(HaveOccurred())

		By("Building a namespace api object")
		namespace, err = createTestingNS("hostpath", c)
		Expect(err).NotTo(HaveOccurred())

		//cleanup before running the test.
		_ = os.Remove("/tmp/test-file")
	})

	AfterEach(func() {
		By(fmt.Sprintf("Destroying namespace for this suite %v", namespace.Name))
		if err := c.Namespaces().Delete(namespace.Name); err != nil {
			Failf("Couldn't delete ns %s", err)
		}
	})

	It("should give a volume the correct mode", func() {
		volumePath := "/test-volume"
		source := &api.HostPathVolumeSource{
			Path: "/tmp",
		}
		pod := testPodWithHostVol(volumePath, source)

		pod.Spec.Containers[0].Args = []string{
			fmt.Sprintf("--fs_type=%v", volumePath),
			fmt.Sprintf("--file_mode=%v", volumePath),
		}
		testContainerOutputInNamespace("hostPath mode", c, pod, 0, []string{
			"mode of file \"/test-volume\": dtrwxrwxrwx", // we expect the sticky bit (mode flag t) to be set for the dir
		},
			namespace.Name)
	})

	It("should support r/w", func() {
		volumePath := "/test-volume"
		filePath := path.Join(volumePath, "test-file")
		source := &api.HostPathVolumeSource{
			Path: "/tmp",
		}
		pod := testPodWithHostVol(volumePath, source)

		pod.Spec.Containers[0].Args = []string{
			fmt.Sprintf("--fs_type=%v", volumePath),
			fmt.Sprintf("--rw_new_file=%v", filePath),
			fmt.Sprintf("--file_mode=%v", filePath),
		}
		testContainerOutputInNamespace("hostPath r/w", c, pod, 0, []string{
			"mode of file \"/test-volume/test-file\": -rw-r--r--",
			"content of file \"/test-volume/test-file\": mount-tester new file",
		}, namespace.Name,
		)
	})
})

//These constants are borrowed from the other test.
//const containerName = "test-container"
//const volumeName = "test-volume"

func mount(source *api.HostPathVolumeSource) []api.Volume {
	return []api.Volume{
		{
			Name: volumeName,
			VolumeSource: api.VolumeSource{
				HostPath: source,
			},
		},
	}
}

//TODO: To merge this with the emptyDir tests, we can make source a lambda.
func testPodWithHostVol(path string, source *api.HostPathVolumeSource) *api.Pod {
	podName := "pod-" + string(util.NewUUID())

	return &api.Pod{
		TypeMeta: api.TypeMeta{
			Kind:       "Pod",
			APIVersion: latest.Version,
		},
		ObjectMeta: api.ObjectMeta{
			Name: podName,
		},
		Spec: api.PodSpec{
			Containers: []api.Container{
				{
					Name:  containerName,
					Image: "gcr.io/google_containers/mounttest:0.2",
					VolumeMounts: []api.VolumeMount{
						{
							Name:      volumeName,
							MountPath: path,
						},
					},
				},
			},
			RestartPolicy: api.RestartPolicyNever,
			Volumes:       mount(source),
		},
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mockiam

import (
	"fmt"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/request"
	"github.com/aws/aws-sdk-go/service/iam"
	"github.com/golang/glog"
)

func (m *MockIAM) GetInstanceProfile(request *iam.GetInstanceProfileInput) (*iam.GetInstanceProfileOutput, error) {
	m.mutex.Lock()
	defer m.mutex.Unlock()

	ip := m.InstanceProfiles[aws.StringValue(request.InstanceProfileName)]
	if ip == nil {
		return nil, awserr.New("NoSuchEntity", "No such entity", nil)
	}
	response := &iam.GetInstanceProfileOutput{
		InstanceProfile: ip,
	}
	return response, nil
}
func (m *MockIAM) GetInstanceProfileWithContext(aws.Context, *iam.GetInstanceProfileInput, ...request.Option) (*iam.GetInstanceProfileOutput, error) {
	panic("Not implemented")
	return nil, nil
}
func (m *MockIAM) GetInstanceProfileRequest(*iam.GetInstanceProfileInput) (*request.Request, *iam.GetInstanceProfileOutput) {
	panic("Not implemented")
	return nil, nil
}

func (m *MockIAM) CreateInstanceProfile(request *iam.CreateInstanceProfileInput) (*iam.CreateInstanceProfileOutput, error) {
	m.mutex.Lock()
	defer m.mutex.Unlock()

	glog.Infof("CreateInstanceProfile: %v", request)

	p := &iam.InstanceProfile{
		InstanceProfileName: request.InstanceProfileName,
		// Arn:                 request.Arn,
		// InstanceProfileId:   request.InstanceProfileId,
		Path: request.Path,
		// Roles:               request.Roles,
	}

	// TODO: Some fields
	// // The date when the instance profile was created.
	// //
	// // CreateDate is a required field
	// CreateDate *time.Time `type:"timestamp" timestampFormat:"iso8601" required:"true"`
	// // The stable and unique string identifying the instance profile. For more information
	// 	// about IDs, see IAM Identifiers (http://docs.aws.amazon.com/IAM/latest/UserGuide/Using_Identifiers.html)
	// 	// in the Using IAM guide.
	// 	//
	// 	// InstanceProfileId is a required field
	// 	InstanceProfileId *string `min:"16" type:"string" required:"true"`

	if m.InstanceProfiles == nil {
		m.InstanceProfiles = make(map[string]*iam.InstanceProfile)
	}
	m.InstanceProfiles[*p.InstanceProfileName] = p

	copy := *p
	return &iam.CreateInstanceProfileOutput{InstanceProfile: &copy}, nil
}
func (m *MockIAM) CreateInstanceProfileWithContext(aws.Context, *iam.CreateInstanceProfileInput, ...request.Option) (*iam.CreateInstanceProfileOutput, error) {
	panic("Not implemented")
	return nil, nil
}
func (m *MockIAM) CreateInstanceProfileRequest(*iam.CreateInstanceProfileInput) (*request.Request, *iam.CreateInstanceProfileOutput) {
	panic("Not implemented")
	return nil, nil
}
func (m *MockIAM) AddRoleToInstanceProfile(request *iam.AddRoleToInstanceProfileInput) (*iam.AddRoleToInstanceProfileOutput, error) {
	m.mutex.Lock()
	defer m.mutex.Unlock()

	glog.Infof("AddRoleToInstanceProfile: %v", request)

	ip := m.InstanceProfiles[aws.StringValue(request.InstanceProfileName)]
	if ip == nil {
		return nil, fmt.Errorf("InstanceProfile not found")
	}
	r := m.Roles[aws.StringValue(request.RoleName)]
	if r == nil {
		return nil, fmt.Errorf("Role not found")
	}

	ip.Roles = append(ip.Roles, r)

	return &iam.AddRoleToInstanceProfileOutput{}, nil
}

func (m *MockIAM) AddRoleToInstanceProfileWithContext(aws.Context, *iam.AddRoleToInstanceProfileInput, ...request.Option) (*iam.AddRoleToInstanceProfileOutput, error) {
	panic("Not implemented")
	return nil, nil
}

func (m *MockIAM) AddRoleToInstanceProfileRequest(*iam.AddRoleToInstanceProfileInput) (*request.Request, *iam.AddRoleToInstanceProfileOutput) {
	panic("Not implemented")
	return nil, nil
}

func (m *MockIAM) RemoveRoleFromInstanceProfile(request *iam.RemoveRoleFromInstanceProfileInput) (*iam.RemoveRoleFromInstanceProfileOutput, error) {
	m.mutex.Lock()
	defer m.mutex.Unlock()

	glog.Infof("RemoveRoleFromInstanceProfile: %v", request)

	ip := m.InstanceProfiles[aws.StringValue(request.InstanceProfileName)]
	if ip == nil {
		return nil, fmt.Errorf("InstanceProfile not found")
	}

	found := false
	var newRoles []*iam.Role
	for _, role := range ip.Roles {
		if aws.StringValue(role.RoleName) == aws.StringValue(request.RoleName) {
			found = true
			continue
		}
		newRoles = append(newRoles, role)
	}

	if !found {
		return nil, fmt.Errorf("Role not found")
	}
	ip.Roles = newRoles

	return &iam.RemoveRoleFromInstanceProfileOutput{}, nil
}

func (m *MockIAM) RemoveRoleFromInstanceProfileWithContext(aws.Context, *iam.RemoveRoleFromInstanceProfileInput, ...request.Option) (*iam.RemoveRoleFromInstanceProfileOutput, error) {
	panic("Not implemented")
	return nil, nil
}
func (m *MockIAM) RemoveRoleFromInstanceProfileRequest(*iam.RemoveRoleFromInstanceProfileInput) (*request.Request, *iam.RemoveRoleFromInstanceProfileOutput) {
	panic("Not implemented")
	return nil, nil
}

func (m *MockIAM) ListInstanceProfiles(request *iam.ListInstanceProfilesInput) (*iam.ListInstanceProfilesOutput, error) {
	m.mutex.Lock()
	defer m.mutex.Unlock()

	glog.Infof("ListInstanceProfiles: %v", request)

	if request.PathPrefix != nil {
		glog.Fatalf("MockIAM ListInstanceProfiles PathPrefix not implemented")
	}

	var instanceProfiles []*iam.InstanceProfile

	for _, ip := range m.InstanceProfiles {
		copy := *ip
		instanceProfiles = append(instanceProfiles, &copy)
	}

	response := &iam.ListInstanceProfilesOutput{
		InstanceProfiles: instanceProfiles,
	}

	return response, nil
}

func (m *MockIAM) ListInstanceProfilesWithContext(aws.Context, *iam.ListInstanceProfilesInput, ...request.Option) (*iam.ListInstanceProfilesOutput, error) {
	panic("Not implemented")
	return nil, nil
}

func (m *MockIAM) ListInstanceProfilesRequest(*iam.ListInstanceProfilesInput) (*request.Request, *iam.ListInstanceProfilesOutput) {
	panic("Not implemented")
	return nil, nil
}

func (m *MockIAM) ListInstanceProfilesPages(request *iam.ListInstanceProfilesInput, callback func(*iam.ListInstanceProfilesOutput, bool) bool) error {
	// For the mock, we just send everything in one page
	page, err := m.ListInstanceProfiles(request)
	if err != nil {
		return err
	}

	callback(page, false)

	return nil
}

func (m *MockIAM) ListInstanceProfilesPagesWithContext(aws.Context, *iam.ListInstanceProfilesInput, func(*iam.ListInstanceProfilesOutput, bool) bool, ...request.Option) error {
	panic("Not implemented")
	return nil
}

func (m *MockIAM) DeleteInstanceProfile(request *iam.DeleteInstanceProfileInput) (*iam.DeleteInstanceProfileOutput, error) {
	m.mutex.Lock()
	defer m.mutex.Unlock()

	glog.Infof("DeleteInstanceProfile: %v", request)

	id := aws.StringValue(request.InstanceProfileName)
	o := m.InstanceProfiles[id]
	if o == nil {
		return nil, fmt.Errorf("InstanceProfile %q not found", id)
	}
	delete(m.InstanceProfiles, id)

	return &iam.DeleteInstanceProfileOutput{}, nil
}

func (m *MockIAM) DeleteInstanceProfileWithContext(aws.Context, *iam.DeleteInstanceProfileInput, ...request.Option) (*iam.DeleteInstanceProfileOutput, error) {
	panic("Not implemented")
	return nil, nil
}
func (m *MockIAM) DeleteInstanceProfileRequest(*iam.DeleteInstanceProfileInput) (*request.Request, *iam.DeleteInstanceProfileOutput) {
	panic("Not implemented")
	return nil, nil
}
<commit_msg>Update iaminstanceprofile.go<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mockiam

import (
	"fmt"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/request"
	"github.com/aws/aws-sdk-go/service/iam"
	"github.com/golang/glog"
)

func (m *MockIAM) GetInstanceProfile(request *iam.GetInstanceProfileInput) (*iam.GetInstanceProfileOutput, error) {
	m.mutex.Lock()
	defer m.mutex.Unlock()

	ip := m.InstanceProfiles[aws.StringValue(request.InstanceProfileName)]
	if ip == nil {
		return nil, awserr.New("NoSuchEntity", "No such entity", nil)
	}
	response := &iam.GetInstanceProfileOutput{
		InstanceProfile: ip,
	}
	return response, nil
}
func (m *MockIAM) GetInstanceProfileWithContext(aws.Context, *iam.GetInstanceProfileInput, ...request.Option) (*iam.GetInstanceProfileOutput, error) {
	panic("Not implemented")
	return nil, nil
}
func (m *MockIAM) GetInstanceProfileRequest(*iam.GetInstanceProfileInput) (*request.Request, *iam.GetInstanceProfileOutput) {
	panic("Not implemented")
	return nil, nil
}

func (m *MockIAM) CreateInstanceProfile(request *iam.CreateInstanceProfileInput) (*iam.CreateInstanceProfileOutput, error) {
	m.mutex.Lock()
	defer m.mutex.Unlock()

	glog.Infof("CreateInstanceProfile: %v", request)

	p := &iam.InstanceProfile{
		InstanceProfileName: request.InstanceProfileName,
		// Arn:                 request.Arn,
		// InstanceProfileId:   request.InstanceProfileId,
		Path: request.Path,
		// Roles:               request.Roles,
	}

	// TODO: Some fields
	// // The date when the instance profile was created.
	// //
	// // CreateDate is a required field
	// CreateDate *time.Time `type:"timestamp" timestampFormat:"iso8601" required:"true"`
	// // The stable and unique string identifying the instance profile. For more information
	// 	// about IDs, see IAM Identifiers (https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html)
	// 	// in the Using IAM guide.
	// 	//
	// 	// InstanceProfileId is a required field
	// 	InstanceProfileId *string `min:"16" type:"string" required:"true"`

	if m.InstanceProfiles == nil {
		m.InstanceProfiles = make(map[string]*iam.InstanceProfile)
	}
	m.InstanceProfiles[*p.InstanceProfileName] = p

	copy := *p
	return &iam.CreateInstanceProfileOutput{InstanceProfile: &copy}, nil
}
func (m *MockIAM) CreateInstanceProfileWithContext(aws.Context, *iam.CreateInstanceProfileInput, ...request.Option) (*iam.CreateInstanceProfileOutput, error) {
	panic("Not implemented")
	return nil, nil
}
func (m *MockIAM) CreateInstanceProfileRequest(*iam.CreateInstanceProfileInput) (*request.Request, *iam.CreateInstanceProfileOutput) {
	panic("Not implemented")
	return nil, nil
}
func (m *MockIAM) AddRoleToInstanceProfile(request *iam.AddRoleToInstanceProfileInput) (*iam.AddRoleToInstanceProfileOutput, error) {
	m.mutex.Lock()
	defer m.mutex.Unlock()

	glog.Infof("AddRoleToInstanceProfile: %v", request)

	ip := m.InstanceProfiles[aws.StringValue(request.InstanceProfileName)]
	if ip == nil {
		return nil, fmt.Errorf("InstanceProfile not found")
	}
	r := m.Roles[aws.StringValue(request.RoleName)]
	if r == nil {
		return nil, fmt.Errorf("Role not found")
	}

	ip.Roles = append(ip.Roles, r)

	return &iam.AddRoleToInstanceProfileOutput{}, nil
}

func (m *MockIAM) AddRoleToInstanceProfileWithContext(aws.Context, *iam.AddRoleToInstanceProfileInput, ...request.Option) (*iam.AddRoleToInstanceProfileOutput, error) {
	panic("Not implemented")
	return nil, nil
}

func (m *MockIAM) AddRoleToInstanceProfileRequest(*iam.AddRoleToInstanceProfileInput) (*request.Request, *iam.AddRoleToInstanceProfileOutput) {
	panic("Not implemented")
	return nil, nil
}

func (m *MockIAM) RemoveRoleFromInstanceProfile(request *iam.RemoveRoleFromInstanceProfileInput) (*iam.RemoveRoleFromInstanceProfileOutput, error) {
	m.mutex.Lock()
	defer m.mutex.Unlock()

	glog.Infof("RemoveRoleFromInstanceProfile: %v", request)

	ip := m.InstanceProfiles[aws.StringValue(request.InstanceProfileName)]
	if ip == nil {
		return nil, fmt.Errorf("InstanceProfile not found")
	}

	found := false
	var newRoles []*iam.Role
	for _, role := range ip.Roles {
		if aws.StringValue(role.RoleName) == aws.StringValue(request.RoleName) {
			found = true
			continue
		}
		newRoles = append(newRoles, role)
	}

	if !found {
		return nil, fmt.Errorf("Role not found")
	}
	ip.Roles = newRoles

	return &iam.RemoveRoleFromInstanceProfileOutput{}, nil
}

func (m *MockIAM) RemoveRoleFromInstanceProfileWithContext(aws.Context, *iam.RemoveRoleFromInstanceProfileInput, ...request.Option) (*iam.RemoveRoleFromInstanceProfileOutput, error) {
	panic("Not implemented")
	return nil, nil
}
func (m *MockIAM) RemoveRoleFromInstanceProfileRequest(*iam.RemoveRoleFromInstanceProfileInput) (*request.Request, *iam.RemoveRoleFromInstanceProfileOutput) {
	panic("Not implemented")
	return nil, nil
}

func (m *MockIAM) ListInstanceProfiles(request *iam.ListInstanceProfilesInput) (*iam.ListInstanceProfilesOutput, error) {
	m.mutex.Lock()
	defer m.mutex.Unlock()

	glog.Infof("ListInstanceProfiles: %v", request)

	if request.PathPrefix != nil {
		glog.Fatalf("MockIAM ListInstanceProfiles PathPrefix not implemented")
	}

	var instanceProfiles []*iam.InstanceProfile

	for _, ip := range m.InstanceProfiles {
		copy := *ip
		instanceProfiles = append(instanceProfiles, &copy)
	}

	response := &iam.ListInstanceProfilesOutput{
		InstanceProfiles: instanceProfiles,
	}

	return response, nil
}

func (m *MockIAM) ListInstanceProfilesWithContext(aws.Context, *iam.ListInstanceProfilesInput, ...request.Option) (*iam.ListInstanceProfilesOutput, error) {
	panic("Not implemented")
	return nil, nil
}

func (m *MockIAM) ListInstanceProfilesRequest(*iam.ListInstanceProfilesInput) (*request.Request, *iam.ListInstanceProfilesOutput) {
	panic("Not implemented")
	return nil, nil
}

func (m *MockIAM) ListInstanceProfilesPages(request *iam.ListInstanceProfilesInput, callback func(*iam.ListInstanceProfilesOutput, bool) bool) error {
	// For the mock, we just send everything in one page
	page, err := m.ListInstanceProfiles(request)
	if err != nil {
		return err
	}

	callback(page, false)

	return nil
}

func (m *MockIAM) ListInstanceProfilesPagesWithContext(aws.Context, *iam.ListInstanceProfilesInput, func(*iam.ListInstanceProfilesOutput, bool) bool, ...request.Option) error {
	panic("Not implemented")
	return nil
}

func (m *MockIAM) DeleteInstanceProfile(request *iam.DeleteInstanceProfileInput) (*iam.DeleteInstanceProfileOutput, error) {
	m.mutex.Lock()
	defer m.mutex.Unlock()

	glog.Infof("DeleteInstanceProfile: %v", request)

	id := aws.StringValue(request.InstanceProfileName)
	o := m.InstanceProfiles[id]
	if o == nil {
		return nil, fmt.Errorf("InstanceProfile %q not found", id)
	}
	delete(m.InstanceProfiles, id)

	return &iam.DeleteInstanceProfileOutput{}, nil
}

func (m *MockIAM) DeleteInstanceProfileWithContext(aws.Context, *iam.DeleteInstanceProfileInput, ...request.Option) (*iam.DeleteInstanceProfileOutput, error) {
	panic("Not implemented")
	return nil, nil
}
func (m *MockIAM) DeleteInstanceProfileRequest(*iam.DeleteInstanceProfileInput) (*request.Request, *iam.DeleteInstanceProfileOutput) {
	panic("Not implemented")
	return nil, nil
}
<|endoftext|>


<commit_before>package twelve

import (
	"fmt"
	"strings"
)

func Verse(day int) string {
	return fmt.Sprintf("On the %v day of Christmas my true love gave to me: %s.", ordinal(day), clause(day))
}

func ordinal(day int) string {
	dayToOrdinal := map[int]string{
		1:  "first",
		2:  "second",
		3:  "third",
		4:  "fourth",
		5:  "fifth",
		6:  "sixth",
		7:  "seventh",
		8:  "eigth",
		9:  "ninth",
		10: "tenth",
		11: "eleventh",
		12: "twelfth",
	}
	return dayToOrdinal[day]
}

func clause(day int) (result string) {
	clauses := []string{}
	for i := day; i >= 1; i-- {
		if day > 1 && i == 1 {
			clauses = append(clauses, fmt.Sprintf("and %s", gift(i)))
		} else {
			clauses = append(clauses, gift(i))
		}
	}
	result = strings.Join(clauses, ", ")
	return result
}

func gift(day int) string {
	dayToGift := map[int]string{
		1: "a Partridge in a Pear Tree",
		2: "two Turtle Doves",
	}
	return dayToGift[day]
}

func Song() string {
	panic("Please implement the Song function")
}
<commit_msg>Pass verse<commit_after>package twelve

import (
	"fmt"
	"strings"
)

func Verse(day int) string {
	return fmt.Sprintf("On the %v day of Christmas my true love gave to me: %s.", ordinal(day), clause(day))
}

func ordinal(day int) string {
	dayToOrdinal := map[int]string{
		1:  "first",
		2:  "second",
		3:  "third",
		4:  "fourth",
		5:  "fifth",
		6:  "sixth",
		7:  "seventh",
		8:  "eighth",
		9:  "ninth",
		10: "tenth",
		11: "eleventh",
		12: "twelfth",
	}
	return dayToOrdinal[day]
}

func clause(day int) (result string) {
	clauses := []string{}
	for i := day; i >= 1; i-- {
		if day > 1 && i == 1 {
			clauses = append(clauses, fmt.Sprintf("and %s", gift(i)))
		} else {
			clauses = append(clauses, gift(i))
		}
	}
	result = strings.Join(clauses, ", ")
	return result
}

func gift(day int) string {
	dayToGift := map[int]string{
		12: "twelve Drummers Drumming",
		11: "eleven Pipers Piping",
		10: "ten Lords-a-Leaping",
		9:  "nine Ladies Dancing",
		8:  "eight Maids-a-Milking",
		7:  "seven Swans-a-Swimming",
		6:  "six Geese-a-Laying",
		5:  "five Gold Rings",
		4:  "four Calling Birds",
		3:  "three French Hens",
		2:  "two Turtle Doves",
		1:  "a Partridge in a Pear Tree",
	}
	return dayToGift[day]
}

func Song() string {
	panic("Please implement the Song function")
}
<|endoftext|>


<commit_before>package kubeutil

import (
	"fmt"
	"os"
	"strings"

	edlresource "github.com/PaddlePaddle/cloud/go/edl/resource"
	"k8s.io/api/core/v1"
	"k8s.io/api/extensions/v1beta1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
)

func buildConfig(kubeconfig string) (*rest.Config, error) {
	return clientcmd.BuildConfigFromFlags("", kubeconfig)
}

// CreateClient creates ClientSet and rest.RESTClient used by client.
func CreateClient(kubeconfig string) (*rest.RESTClient, *kubernetes.Clientset, error) {
	config, err := buildConfig(kubeconfig)
	if err != nil {
		return nil, nil, fmt.Errorf("init from config '%s' error: %v", kubeconfig, err)
	}

	clientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		return nil, nil, fmt.Errorf("create clientset from config '%s' error: %v", kubeconfig, err)
	}

	edlresource.RegisterTrainingJob(config)

	client, err := rest.RESTClientFor(config)
	if err != nil {
		return nil, nil, fmt.Errorf("create rest client from config '%s' error: %v", kubeconfig, err)
	}

	return client, clientset, nil
}

// FindNamespace finds whether a namespace exists.
func FindNamespace(clientset *kubernetes.Clientset, namespace string) error {
	n := v1.Namespace{}
	n.SetName(namespace)

	if _, err := clientset.CoreV1().Namespaces().Get(namespace, metav1.GetOptions{}); err != nil {
		return fmt.Errorf("get namespace '%s' error:%v", namespace, err)
	}

	return nil
}

// EnsureTPR ensure a TPR should exists and create it if not.
func EnsureTPR(clientset *kubernetes.Clientset, resourceName, apiversion string) error {
	tpr, err := clientset.ExtensionsV1beta1().ThirdPartyResources().Get(resourceName, metav1.GetOptions{})
	if err != nil {
		if errors.IsNotFound(err) {
			tpr := &v1beta1.ThirdPartyResource{
				TypeMeta: metav1.TypeMeta{
					Kind:       "ThirdPartyResource",
					APIVersion: "extensions/v1beta1",
				},

				ObjectMeta: metav1.ObjectMeta{
					Name: resourceName,
				},

				Versions: []v1beta1.APIVersion{
					{Name: apiversion},
				},
				Description: "PaddlePaddle TrainingJob operator",
			}

			_, err := clientset.ExtensionsV1beta1().ThirdPartyResources().Create(tpr)
			if err != nil {
				return err
			}
		} else {
			return err
		}
	} else {
		fmt.Printf("SKIPPING: already exists %#v\n", tpr)
	}

	return nil
}

// CreateTrainingJob try to create a training-job under namespace.
func CreateTrainingJob(restClient *rest.RESTClient, namespace string, job *edlresource.TrainingJob) error {
	var result edlresource.TrainingJob
	err := restClient.Post().
		Resource("trainingjobs").
		Namespace(namespace).
		Body(job).
		Do().Into(&result)

	if err != nil {
		fmt.Fprintf(os.Stderr, "can't create TPR extenion TrainningJob: %s\n", job)
		return err
	}

	return nil
}

// NameEscape replace characters not supported by Kubernetes.
func NameEscape(name string) string {
	name = strings.Replace(name, "@", "-", -1)
	name = strings.Replace(name, ".", "-", -1)
	name = strings.Replace(name, "_", "-", -1)
	return name
}
<commit_msg>fix build<commit_after>package kubeutil

import (
	"fmt"
	"os"
	"strings"

	edlresource "github.com/PaddlePaddle/cloud/go/edl/resource"
	"k8s.io/api/core/v1"
	"k8s.io/api/extensions/v1beta1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
)

func buildConfig(kubeconfig string) (*rest.Config, error) {
	return clientcmd.BuildConfigFromFlags("", kubeconfig)
}

// CreateClient creates ClientSet and rest.RESTClient used by client.
func CreateClient(kubeconfig string) (*rest.RESTClient, *kubernetes.Clientset, error) {
	config, err := buildConfig(kubeconfig)
	if err != nil {
		return nil, nil, fmt.Errorf("init from config '%s' error: %v", kubeconfig, err)
	}

	clientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		return nil, nil, fmt.Errorf("create clientset from config '%s' error: %v", kubeconfig, err)
	}

	edlresource.RegisterResource(config, &edlresource.TrainingJob{}, &edlresource.TrainingJobList{})

	client, err := rest.RESTClientFor(config)
	if err != nil {
		return nil, nil, fmt.Errorf("create rest client from config '%s' error: %v", kubeconfig, err)
	}

	return client, clientset, nil
}

// FindNamespace finds whether a namespace exists.
func FindNamespace(clientset *kubernetes.Clientset, namespace string) error {
	n := v1.Namespace{}
	n.SetName(namespace)

	if _, err := clientset.CoreV1().Namespaces().Get(namespace, metav1.GetOptions{}); err != nil {
		return fmt.Errorf("get namespace '%s' error:%v", namespace, err)
	}

	return nil
}

// EnsureTPR ensure a TPR should exists and create it if not.
func EnsureTPR(clientset *kubernetes.Clientset, resourceName, apiversion string) error {
	tpr, err := clientset.ExtensionsV1beta1().ThirdPartyResources().Get(resourceName, metav1.GetOptions{})
	if err != nil {
		if errors.IsNotFound(err) {
			tpr := &v1beta1.ThirdPartyResource{
				TypeMeta: metav1.TypeMeta{
					Kind:       "ThirdPartyResource",
					APIVersion: "extensions/v1beta1",
				},

				ObjectMeta: metav1.ObjectMeta{
					Name: resourceName,
				},

				Versions: []v1beta1.APIVersion{
					{Name: apiversion},
				},
				Description: "PaddlePaddle TrainingJob operator",
			}

			_, err := clientset.ExtensionsV1beta1().ThirdPartyResources().Create(tpr)
			if err != nil {
				return err
			}
		} else {
			return err
		}
	} else {
		fmt.Printf("SKIPPING: already exists %#v\n", tpr)
	}

	return nil
}

// CreateTrainingJob try to create a training-job under namespace.
func CreateTrainingJob(restClient *rest.RESTClient, namespace string, job *edlresource.TrainingJob) error {
	var result edlresource.TrainingJob
	err := restClient.Post().
		Resource("trainingjobs").
		Namespace(namespace).
		Body(job).
		Do().Into(&result)

	if err != nil {
		fmt.Fprintf(os.Stderr, "can't create TPR extenion TrainningJob: %s\n", job)
		return err
	}

	return nil
}

// NameEscape replace characters not supported by Kubernetes.
func NameEscape(name string) string {
	name = strings.Replace(name, "@", "-", -1)
	name = strings.Replace(name, ".", "-", -1)
	name = strings.Replace(name, "_", "-", -1)
	return name
}
<|endoftext|>


<commit_before>package buildcfg

import (
	"io/ioutil"
	"os"
	"testing"
)

func TestConfigRead(t *testing.T) {
	s := `docker_image: test
script:
- boom
- foo`

	err := ioutil.WriteFile("./tmp.cfg", []byte(s), 0644)
	if err != nil {
		panic(err)
	}

	defer os.Remove("./tmp.cfg")

	cfg := Read("./tmp.cfg")

	if cfg.DockerImage != "test" {
		t.Error("Expected docker-image to be test")
	}
	if len(cfg.Script) != 2 {
		t.Error("Expected exactly 2 script lines")
	}
	if cfg.Script[0] != "boom" || cfg.Script[1] != "foo" {
		t.Error("expected boom foo")
	}
}

func TestDefault(t *testing.T) {
	cfg := readCfg("/non/existing/file")

	if cfg.DockerImage != "debian" {
		t.Error("Expected docker-image to be debian")
	}
	if len(cfg.Script) != 0 {
		t.Error("Expected no script lines")
	}
}
<commit_msg>Fix failing config test<commit_after>package buildcfg

import (
	"io/ioutil"
	"os"
	"testing"
)

func TestConfigRead(t *testing.T) {
	s := `docker_image: test
script:
- boom
- foo`

	err := ioutil.WriteFile(".ci.yml", []byte(s), 0644)
	if err != nil {
		panic(err)
	}

	defer os.Remove(".ci.yml")

	cfg := Read(".", "boembats")

	if cfg.DockerImage != "test" {
		t.Error("Expected docker-image to be test")
	}
	if len(cfg.Script) != 2 {
		t.Error("Expected exactly 2 script lines")
	}
	if cfg.Script[0] != "boom" || cfg.Script[1] != "foo" {
		t.Error("expected boom foo")
	}
}

func TestDefault(t *testing.T) {
	cfg := Read("/non/existing/file", "bots")

	if cfg.DockerImage != "debian" {
		t.Error("Expected docker-image to be debian")
	}
	if len(cfg.Script) != 0 {
		t.Error("Expected no script lines")
	}
}
<|endoftext|>


<commit_before>// +build linux darwin freebsd

package mediasort

import (
	"errors"
	"io"
	"os"
	"strings"
)

func move(src, dst string) (err error) {
	err = os.Rename(src, dst)
	// cross device move
	if err != nil && strings.Contains(err.Error(), "cross-device") {
		if err = copy(src, dst); err != nil {
			return err
		}
		err = os.Remove(src)
	}
	return
}

func copy(src, dst string) (err error) {
	srcFile, err := os.Open(src)
	defer srcFile.Close()
	if err != nil {
		return
	}
	dstFile, err := os.Open(dst)
	defer dstFile.Close()
	if err != nil {
		return
	}
	_, err = io.Copy(dstFile, srcFile)
	return
}

func link(src, dst string, linkType linkType) (err error) {
	switch linkType {
	case hardLink:
		err = os.Link(src, dst)
	case symLink:
		err = os.Link(src, dst)
	default:
		err = errors.New("wrong link type, please open an issue")
	}
	return
}
<commit_msg>fix typo in link type<commit_after>// +build linux darwin freebsd

package mediasort

import (
	"errors"
	"io"
	"os"
	"strings"
)

func move(src, dst string) (err error) {
	err = os.Rename(src, dst)
	// cross device move
	if err != nil && strings.Contains(err.Error(), "cross-device") {
		if err = copy(src, dst); err != nil {
			return err
		}
		err = os.Remove(src)
	}
	return
}

func copy(src, dst string) (err error) {
	srcFile, err := os.Open(src)
	defer srcFile.Close()
	if err != nil {
		return
	}
	dstFile, err := os.Open(dst)
	defer dstFile.Close()
	if err != nil {
		return
	}
	_, err = io.Copy(dstFile, srcFile)
	return
}

func link(src, dst string, linkType linkType) (err error) {
	switch linkType {
	case hardLink:
		err = os.Link(src, dst)
	case symLink:
		err = os.Symlink(src, dst)
	default:
		err = errors.New("wrong link type, please open an issue")
	}
	return
}
<|endoftext|>


<commit_before>package digitalocean

import (
	"fmt"
	"io/ioutil"
	"time"

	"code.google.com/p/goauth2/oauth"
	"github.com/codegangsta/cli"
	"github.com/digitalocean/godo"
	"github.com/docker/machine/drivers"
	"github.com/docker/machine/log"
	"github.com/docker/machine/ssh"
	"github.com/docker/machine/state"
)

type Driver struct {
	*drivers.BaseDriver
	AccessToken       string
	DropletID         int
	DropletName       string
	Image             string
	Region            string
	SSHKeyID          int
	Size              string
	IPv6              bool
	Backups           bool
	PrivateNetworking bool
}

func init() {
	drivers.Register("digitalocean", &drivers.RegisteredDriver{
		New:            NewDriver,
		GetCreateFlags: GetCreateFlags,
	})
}

// GetCreateFlags registers the flags this driver adds to
// "docker hosts create"
func GetCreateFlags() []cli.Flag {
	return []cli.Flag{
		cli.StringFlag{
			EnvVar: "DIGITALOCEAN_ACCESS_TOKEN",
			Name:   "digitalocean-access-token",
			Usage:  "Digital Ocean access token",
		},
		cli.StringFlag{
			EnvVar: "DIGITALOCEAN_IMAGE",
			Name:   "digitalocean-image",
			Usage:  "Digital Ocean Image",
			Value:  "ubuntu-14-04-x64",
		},
		cli.StringFlag{
			EnvVar: "DIGITALOCEAN_REGION",
			Name:   "digitalocean-region",
			Usage:  "Digital Ocean region",
			Value:  "nyc3",
		},
		cli.StringFlag{
			EnvVar: "DIGITALOCEAN_SIZE",
			Name:   "digitalocean-size",
			Usage:  "Digital Ocean size",
			Value:  "512mb",
		},
		cli.BoolFlag{
			EnvVar: "DIGITALOCEAN_IPV6",
			Name:   "digitalocean-ipv6",
			Usage:  "enable ipv6 for droplet",
		},
		cli.BoolFlag{
			EnvVar: "DIGITALOCEAN_PRIVATE_NETWORKING",
			Name:   "digitalocean-private-networking",
			Usage:  "enable private networking for droplet",
		},
		cli.BoolFlag{
			EnvVar: "DIGITALOCEAN_BACKUPS",
			Name:   "digitalocean-backups",
			Usage:  "enable backups for droplet",
		},
	}
}

func NewDriver(machineName string, storePath string, caCert string, privateKey string) (drivers.Driver, error) {
	inner := drivers.NewBaseDriver(machineName, storePath, caCert, privateKey)
	return &Driver{BaseDriver: inner}, nil
}

func (d *Driver) GetSSHHostname() (string, error) {
	return d.GetIP()
}

func (d *Driver) DriverName() string {
	return "digitalocean"
}

func (d *Driver) SetConfigFromFlags(flags drivers.DriverOptions) error {
	d.AccessToken = flags.String("digitalocean-access-token")
	d.Image = flags.String("digitalocean-image")
	d.Region = flags.String("digitalocean-region")
	d.Size = flags.String("digitalocean-size")
	d.IPv6 = flags.Bool("digitalocean-ipv6")
	d.PrivateNetworking = flags.Bool("digitalocean-private-networking")
	d.Backups = flags.Bool("digitalocean-backups")
	d.SwarmMaster = flags.Bool("swarm-master")
	d.SwarmHost = flags.String("swarm-host")
	d.SwarmDiscovery = flags.String("swarm-discovery")
	d.SSHUser = "root"
	d.SSHPort = 22

	if d.AccessToken == "" {
		return fmt.Errorf("digitalocean driver requires the --digitalocean-access-token option")
	}

	return nil
}

func (d *Driver) PreCreateCheck() error {
	client := d.getClient()
	regions, _, err := client.Regions.List(nil)
	if err != nil {
		return err
	}
	for _, region := range regions {
		if region.Slug == d.Region {
			return nil
		}
	}

	return fmt.Errorf("digitalocean requires a valid region")
}

func (d *Driver) Create() error {
	log.Infof("Creating SSH key...")

	key, err := d.createSSHKey()
	if err != nil {
		return err
	}

	d.SSHKeyID = key.ID

	log.Infof("Creating Digital Ocean droplet...")

	client := d.getClient()

	createRequest := &godo.DropletCreateRequest{
		Image:             d.Image,
		Name:              d.MachineName,
		Region:            d.Region,
		Size:              d.Size,
		IPv6:              d.IPv6,
		PrivateNetworking: d.PrivateNetworking,
		Backups:           d.Backups,
		SSHKeys:           []interface{}{d.SSHKeyID},
	}

	newDroplet, _, err := client.Droplets.Create(createRequest)
	if err != nil {
		return err
	}

	d.DropletID = newDroplet.Droplet.ID

	for {
		newDroplet, _, err = client.Droplets.Get(d.DropletID)
		if err != nil {
			return err
		}
		for _, network := range newDroplet.Droplet.Networks.V4 {
			if network.Type == "public" {
				d.IPAddress = network.IPAddress
			}
		}

		if d.IPAddress != "" {
			break
		}

		time.Sleep(1 * time.Second)
	}

	log.Debugf("Created droplet ID %d, IP address %s",
		newDroplet.Droplet.ID,
		d.IPAddress)

	return nil
}

func (d *Driver) createSSHKey() (*godo.Key, error) {
	if err := ssh.GenerateSSHKey(d.GetSSHKeyPath()); err != nil {
		return nil, err
	}

	publicKey, err := ioutil.ReadFile(d.publicSSHKeyPath())
	if err != nil {
		return nil, err
	}

	createRequest := &godo.KeyCreateRequest{
		Name:      d.MachineName,
		PublicKey: string(publicKey),
	}

	key, _, err := d.getClient().Keys.Create(createRequest)
	if err != nil {
		return key, err
	}

	return key, nil
}

func (d *Driver) GetURL() (string, error) {
	ip, err := d.GetIP()
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("tcp://%s:2376", ip), nil
}

func (d *Driver) GetIP() (string, error) {
	if d.IPAddress == "" {
		return "", fmt.Errorf("IP address is not set")
	}
	return d.IPAddress, nil
}

func (d *Driver) GetState() (state.State, error) {
	droplet, _, err := d.getClient().Droplets.Get(d.DropletID)
	if err != nil {
		return state.Error, err
	}
	switch droplet.Droplet.Status {
	case "new":
		return state.Starting, nil
	case "active":
		return state.Running, nil
	case "off":
		return state.Stopped, nil
	}
	return state.None, nil
}

func (d *Driver) Start() error {
	_, _, err := d.getClient().DropletActions.PowerOn(d.DropletID)
	return err
}

func (d *Driver) Stop() error {
	_, _, err := d.getClient().DropletActions.Shutdown(d.DropletID)
	return err
}

func (d *Driver) Remove() error {
	client := d.getClient()
	if resp, err := client.Keys.DeleteByID(d.SSHKeyID); err != nil {
		if resp.StatusCode == 404 {
			log.Infof("Digital Ocean SSH key doesn't exist, assuming it is already deleted")
		} else {
			return err
		}
	}
	if resp, err := client.Droplets.Delete(d.DropletID); err != nil {
		if resp.StatusCode == 404 {
			log.Infof("Digital Ocean droplet doesn't exist, assuming it is already deleted")
		} else {
			return err
		}
	}
	return nil
}

func (d *Driver) Restart() error {
	_, _, err := d.getClient().DropletActions.Reboot(d.DropletID)
	return err
}

func (d *Driver) Kill() error {
	_, _, err := d.getClient().DropletActions.PowerOff(d.DropletID)
	return err
}

func (d *Driver) getClient() *godo.Client {
	t := &oauth.Transport{
		Token: &oauth.Token{AccessToken: d.AccessToken},
	}

	return godo.NewClient(t.Client())
}

func (d *Driver) publicSSHKeyPath() string {
	return d.GetSSHKeyPath() + ".pub"
}
<commit_msg>digitalocean: enable specifying ssh user<commit_after>package digitalocean

import (
	"fmt"
	"io/ioutil"
	"time"

	"code.google.com/p/goauth2/oauth"
	"github.com/codegangsta/cli"
	"github.com/digitalocean/godo"
	"github.com/docker/machine/drivers"
	"github.com/docker/machine/log"
	"github.com/docker/machine/ssh"
	"github.com/docker/machine/state"
)

type Driver struct {
	*drivers.BaseDriver
	AccessToken       string
	DropletID         int
	DropletName       string
	Image             string
	Region            string
	SSHKeyID          int
	Size              string
	IPv6              bool
	Backups           bool
	PrivateNetworking bool
}

func init() {
	drivers.Register("digitalocean", &drivers.RegisteredDriver{
		New:            NewDriver,
		GetCreateFlags: GetCreateFlags,
	})
}

// GetCreateFlags registers the flags this driver adds to
// "docker hosts create"
func GetCreateFlags() []cli.Flag {
	return []cli.Flag{
		cli.StringFlag{
			EnvVar: "DIGITALOCEAN_ACCESS_TOKEN",
			Name:   "digitalocean-access-token",
			Usage:  "Digital Ocean access token",
		},
		cli.StringFlag{
			EnvVar: "DIGITALOCEAN_SSH_USER",
			Name:   "digitalocean-ssh-user",
			Usage:  "Digital Ocean SSH username",
			Value:  "root",
		},
		cli.StringFlag{
			EnvVar: "DIGITALOCEAN_IMAGE",
			Name:   "digitalocean-image",
			Usage:  "Digital Ocean Image",
			Value:  "ubuntu-14-04-x64",
		},
		cli.StringFlag{
			EnvVar: "DIGITALOCEAN_REGION",
			Name:   "digitalocean-region",
			Usage:  "Digital Ocean region",
			Value:  "nyc3",
		},
		cli.StringFlag{
			EnvVar: "DIGITALOCEAN_SIZE",
			Name:   "digitalocean-size",
			Usage:  "Digital Ocean size",
			Value:  "512mb",
		},
		cli.BoolFlag{
			EnvVar: "DIGITALOCEAN_IPV6",
			Name:   "digitalocean-ipv6",
			Usage:  "enable ipv6 for droplet",
		},
		cli.BoolFlag{
			EnvVar: "DIGITALOCEAN_PRIVATE_NETWORKING",
			Name:   "digitalocean-private-networking",
			Usage:  "enable private networking for droplet",
		},
		cli.BoolFlag{
			EnvVar: "DIGITALOCEAN_BACKUPS",
			Name:   "digitalocean-backups",
			Usage:  "enable backups for droplet",
		},
	}
}

func NewDriver(machineName string, storePath string, caCert string, privateKey string) (drivers.Driver, error) {
	inner := drivers.NewBaseDriver(machineName, storePath, caCert, privateKey)
	return &Driver{BaseDriver: inner}, nil
}

func (d *Driver) GetSSHHostname() (string, error) {
	return d.GetIP()
}

func (d *Driver) DriverName() string {
	return "digitalocean"
}

func (d *Driver) SetConfigFromFlags(flags drivers.DriverOptions) error {
	d.AccessToken = flags.String("digitalocean-access-token")
	d.Image = flags.String("digitalocean-image")
	d.Region = flags.String("digitalocean-region")
	d.Size = flags.String("digitalocean-size")
	d.IPv6 = flags.Bool("digitalocean-ipv6")
	d.PrivateNetworking = flags.Bool("digitalocean-private-networking")
	d.Backups = flags.Bool("digitalocean-backups")
	d.SwarmMaster = flags.Bool("swarm-master")
	d.SwarmHost = flags.String("swarm-host")
	d.SwarmDiscovery = flags.String("swarm-discovery")
	d.SSHUser = flags.String("digitalocean-ssh-user")
	d.SSHPort = 22

	if d.AccessToken == "" {
		return fmt.Errorf("digitalocean driver requires the --digitalocean-access-token option")
	}

	return nil
}

func (d *Driver) PreCreateCheck() error {
	client := d.getClient()
	regions, _, err := client.Regions.List(nil)
	if err != nil {
		return err
	}
	for _, region := range regions {
		if region.Slug == d.Region {
			return nil
		}
	}

	return fmt.Errorf("digitalocean requires a valid region")
}

func (d *Driver) Create() error {
	log.Infof("Creating SSH key...")

	key, err := d.createSSHKey()
	if err != nil {
		return err
	}

	d.SSHKeyID = key.ID

	log.Infof("Creating Digital Ocean droplet...")

	client := d.getClient()

	createRequest := &godo.DropletCreateRequest{
		Image:             d.Image,
		Name:              d.MachineName,
		Region:            d.Region,
		Size:              d.Size,
		IPv6:              d.IPv6,
		PrivateNetworking: d.PrivateNetworking,
		Backups:           d.Backups,
		SSHKeys:           []interface{}{d.SSHKeyID},
	}

	newDroplet, _, err := client.Droplets.Create(createRequest)
	if err != nil {
		return err
	}

	d.DropletID = newDroplet.Droplet.ID

	for {
		newDroplet, _, err = client.Droplets.Get(d.DropletID)
		if err != nil {
			return err
		}
		for _, network := range newDroplet.Droplet.Networks.V4 {
			if network.Type == "public" {
				d.IPAddress = network.IPAddress
			}
		}

		if d.IPAddress != "" {
			break
		}

		time.Sleep(1 * time.Second)
	}

	log.Debugf("Created droplet ID %d, IP address %s",
		newDroplet.Droplet.ID,
		d.IPAddress)

	return nil
}

func (d *Driver) createSSHKey() (*godo.Key, error) {
	if err := ssh.GenerateSSHKey(d.GetSSHKeyPath()); err != nil {
		return nil, err
	}

	publicKey, err := ioutil.ReadFile(d.publicSSHKeyPath())
	if err != nil {
		return nil, err
	}

	createRequest := &godo.KeyCreateRequest{
		Name:      d.MachineName,
		PublicKey: string(publicKey),
	}

	key, _, err := d.getClient().Keys.Create(createRequest)
	if err != nil {
		return key, err
	}

	return key, nil
}

func (d *Driver) GetURL() (string, error) {
	ip, err := d.GetIP()
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("tcp://%s:2376", ip), nil
}

func (d *Driver) GetIP() (string, error) {
	if d.IPAddress == "" {
		return "", fmt.Errorf("IP address is not set")
	}
	return d.IPAddress, nil
}

func (d *Driver) GetState() (state.State, error) {
	droplet, _, err := d.getClient().Droplets.Get(d.DropletID)
	if err != nil {
		return state.Error, err
	}
	switch droplet.Droplet.Status {
	case "new":
		return state.Starting, nil
	case "active":
		return state.Running, nil
	case "off":
		return state.Stopped, nil
	}
	return state.None, nil
}

func (d *Driver) Start() error {
	_, _, err := d.getClient().DropletActions.PowerOn(d.DropletID)
	return err
}

func (d *Driver) Stop() error {
	_, _, err := d.getClient().DropletActions.Shutdown(d.DropletID)
	return err
}

func (d *Driver) Remove() error {
	client := d.getClient()
	if resp, err := client.Keys.DeleteByID(d.SSHKeyID); err != nil {
		if resp.StatusCode == 404 {
			log.Infof("Digital Ocean SSH key doesn't exist, assuming it is already deleted")
		} else {
			return err
		}
	}
	if resp, err := client.Droplets.Delete(d.DropletID); err != nil {
		if resp.StatusCode == 404 {
			log.Infof("Digital Ocean droplet doesn't exist, assuming it is already deleted")
		} else {
			return err
		}
	}
	return nil
}

func (d *Driver) Restart() error {
	_, _, err := d.getClient().DropletActions.Reboot(d.DropletID)
	return err
}

func (d *Driver) Kill() error {
	_, _, err := d.getClient().DropletActions.PowerOff(d.DropletID)
	return err
}

func (d *Driver) getClient() *godo.Client {
	t := &oauth.Transport{
		Token: &oauth.Token{AccessToken: d.AccessToken},
	}

	return godo.NewClient(t.Client())
}

func (d *Driver) publicSSHKeyPath() string {
	return d.GetSSHKeyPath() + ".pub"
}
<|endoftext|>


<commit_before>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package containerd

import (
	"os"
	"path/filepath"
)

const (
	defaultAddress = `\\.\pipe\containerd-containerd-test`
	testImage      = "docker.io/microsoft/nanoserver@sha256:8f78a4a7da4464973a5cd239732626141aec97e69ba3e4023357628630bc1ee2"
)

var (
	defaultRoot  = filepath.Join(os.Getenv("programfiles"), "containerd", "root-test")
	defaultState = filepath.Join(os.Getenv("programfiles"), "containerd", "state-test")
)
<commit_msg>Convert Windows CI to use Microsoft MCR image urls<commit_after>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package containerd

import (
	"os"
	"path/filepath"
)

const (
	defaultAddress = `\\.\pipe\containerd-containerd-test`
	testImage      = "mcr.microsoft.com/windows/nanoserver:sac2016"
)

var (
	defaultRoot  = filepath.Join(os.Getenv("programfiles"), "containerd", "root-test")
	defaultState = filepath.Join(os.Getenv("programfiles"), "containerd", "state-test")
)
<|endoftext|>


<commit_before>package main

import (
	_ "expvar"
	"fmt"
	"io/ioutil"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"path/filepath"
	"syscall"
	"time"

	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
	gocontext "golang.org/x/net/context"
	"google.golang.org/grpc"

	"github.com/Sirupsen/logrus"
	"github.com/containerd/containerd"
	contentapi "github.com/containerd/containerd/api/services/content"
	api "github.com/containerd/containerd/api/services/execution"
	rootfsapi "github.com/containerd/containerd/api/services/rootfs"
	"github.com/containerd/containerd/content"
	"github.com/containerd/containerd/log"
	"github.com/containerd/containerd/plugin"
	"github.com/containerd/containerd/reaper"
	"github.com/containerd/containerd/snapshot"
	"github.com/containerd/containerd/sys"
	metrics "github.com/docker/go-metrics"
	"github.com/pkg/errors"
	"github.com/urfave/cli"
)

const usage = `
                    __        _                     __
  _________  ____  / /_____ _(_)___  ___  _________/ /
 / ___/ __ \/ __ \/ __/ __ ` + "`" + `/ / __ \/ _ \/ ___/ __  /
/ /__/ /_/ / / / / /_/ /_/ / / / / /  __/ /  / /_/ /
\___/\____/_/ /_/\__/\__,_/_/_/ /_/\___/_/   \__,_/

high performance container runtime
`

var (
	conf   = defaultConfig()
	global = log.WithModule(gocontext.Background(), "containerd")
)

func init() {
	cli.VersionPrinter = func(c *cli.Context) {
		fmt.Println(c.App.Name, containerd.Package, c.App.Version)
	}
}

func main() {
	app := cli.NewApp()
	app.Name = "containerd"
	app.Version = containerd.Version
	app.Usage = usage
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "config,c",
			Usage: "path to the configuration file (Use 'default' to output the default toml)",
			Value: "/etc/containerd/config.toml",
		},
		cli.StringFlag{
			Name:  "log-level,l",
			Usage: "set the logging level [debug, info, warn, error, fatal, panic]",
		},
		cli.StringFlag{
			Name:  "state",
			Usage: "containerd state directory",
		},
		cli.StringFlag{
			Name:  "socket,s",
			Usage: "socket path for containerd's GRPC server",
		},
		cli.StringFlag{
			Name:  "root",
			Usage: "containerd root directory",
		},
	}
	app.Before = before
	app.Action = func(context *cli.Context) error {
		start := time.Now()
		// start the signal handler as soon as we can to make sure that
		// we don't miss any signals during boot
		signals := make(chan os.Signal, 2048)
		signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT, syscall.SIGUSR1, syscall.SIGCHLD)
		if conf.Subreaper {
			log.G(global).Info("setting subreaper...")
			if err := sys.SetSubreaper(1); err != nil {
				return err
			}
		}
		log.G(global).Info("starting containerd boot...")

		// load all plugins into containerd
		if err := plugin.Load(filepath.Join(conf.Root, "plugins")); err != nil {
			return err
		}
		// start debug and metrics APIs
		if err := serveDebugAPI(); err != nil {
			return err
		}
		monitor, err := loadMonitor()
		if err != nil {
			return err
		}
		runtimes, err := loadRuntimes(monitor)
		if err != nil {
			return err
		}
		store, err := resolveContentStore()
		if err != nil {
			return err
		}
		snapshotter, err := loadSnapshotter(store)
		if err != nil {
			return err
		}
		services, err := loadServices(runtimes, store, snapshotter)
		if err != nil {
			return err
		}
		// start the GRPC api with the execution service registered
		server := newGRPCServer()
		for _, service := range services {
			if err := service.Register(server); err != nil {
				return err
			}
		}
		log.G(global).Info("starting GRPC API server...")
		if err := serveGRPC(server); err != nil {
			return err
		}
		// start the prometheus metrics API for containerd
		serveMetricsAPI()

		log.G(global).Infof("containerd successfully booted in %fs", time.Since(start).Seconds())
		return handleSignals(signals, server)
	}

	if err := app.Run(os.Args); err != nil {
		fmt.Fprintf(os.Stderr, "containerd: %s\n", err)
		os.Exit(1)
	}
}

func before(context *cli.Context) error {
	if context.GlobalString("config") == "default" {
		fh, err := ioutil.TempFile("", "containerd-config.toml.")
		if err != nil {
			return err
		}
		if _, err := conf.WriteTo(fh); err != nil {
			fh.Close()
			return err
		}
		fh.Close()
		log.G(global).Infof("containerd default config written to %q", fh.Name())
		os.Exit(0)
	}
	if err := loadConfig(context.GlobalString("config")); err != nil &&
		!os.IsNotExist(err) {
		return err
	}
	// the order for config vs flag values is that flags will always override
	// the config values if they are set
	if err := setLevel(context); err != nil {
		return err
	}
	for _, v := range []struct {
		name string
		d    *string
	}{
		{
			name: "root",
			d:    &conf.Root,
		},
		{
			name: "state",
			d:    &conf.State,
		},
		{
			name: "socket",
			d:    &conf.GRPC.Socket,
		},
	} {
		if s := context.GlobalString(v.name); s != "" {
			*v.d = s
		}
	}
	return nil
}

func setLevel(context *cli.Context) error {
	l := context.GlobalString("log-level")
	if l == "" {
		l = conf.Debug.Level
	}
	if l != "" {
		lvl, err := logrus.ParseLevel(l)
		if err != nil {
			return err
		}
		logrus.SetLevel(lvl)
	}
	return nil
}

func serveMetricsAPI() {
	if conf.Metrics.Address != "" {
		log.G(global).WithField("metrics", conf.Metrics.Address).Info("starting metrics API...")
		h := newMetricsHandler()
		go func() {
			if err := http.ListenAndServe(conf.Metrics.Address, h); err != nil {
				log.G(global).WithError(err).Fatal("serve metrics API")
			}
		}()
	}
}

func newMetricsHandler() http.Handler {
	m := http.NewServeMux()
	m.Handle("/metrics", metrics.Handler())
	return m
}

func serveDebugAPI() error {
	path := conf.Debug.Socket
	if path == "" {
		return errors.New("debug socket path cannot be empty")
	}
	l, err := sys.CreateUnixSocket(path)
	if err != nil {
		return err
	}
	log.G(global).WithField("debug", path).Info("starting debug API...")
	go func() {
		defer l.Close()
		// pprof and expvars are imported and automatically register their endpoints
		// under /debug
		if err := http.Serve(l, nil); err != nil {
			log.G(global).WithError(err).Fatal("serve debug API")
		}
	}()
	return nil
}

func resolveContentStore() (*content.Store, error) {
	cp := filepath.Join(conf.Root, "content")
	return content.NewStore(cp)
}

func loadRuntimes(monitor plugin.ContainerMonitor) (map[string]containerd.Runtime, error) {
	o := make(map[string]containerd.Runtime)
	for name, rr := range plugin.Registrations() {
		if rr.Type != plugin.RuntimePlugin {
			continue
		}
		log.G(global).Infof("loading runtime plugin %q...", name)
		ic := &plugin.InitContext{
			Root:    conf.Root,
			State:   conf.State,
			Context: log.WithModule(global, fmt.Sprintf("runtime-%s", name)),
			Monitor: monitor,
		}
		if rr.Config != nil {
			if err := conf.decodePlugin(name, rr.Config); err != nil {
				return nil, err
			}
			ic.Config = rr.Config
		}
		vr, err := rr.Init(ic)
		if err != nil {
			return nil, err
		}
		o[name] = vr.(containerd.Runtime)
	}
	return o, nil
}

func loadMonitor() (plugin.ContainerMonitor, error) {
	var monitors []plugin.ContainerMonitor
	for name, m := range plugin.Registrations() {
		if m.Type != plugin.ContainerMonitorPlugin {
			continue
		}
		log.G(global).Infof("loading monitor plugin %q...", name)
		ic := &plugin.InitContext{
			Root:    conf.Root,
			State:   conf.State,
			Context: log.WithModule(global, fmt.Sprintf("monitor-%s", name)),
		}
		mm, err := m.Init(ic)
		if err != nil {
			return nil, err
		}
		monitors = append(monitors, mm.(plugin.ContainerMonitor))
	}
	if len(monitors) == 0 {
		return plugin.NewNoopMonitor(), nil
	}
	return plugin.NewMultiContainerMonitor(monitors...), nil
}

func loadSnapshotter(store *content.Store) (snapshot.Snapshotter, error) {
	for name, sr := range plugin.Registrations() {
		if sr.Type != plugin.SnapshotPlugin {
			continue
		}
		moduleName := fmt.Sprintf("snapshot-%s", conf.Snapshotter)
		if name != moduleName {
			continue
		}

		log.G(global).Infof("loading snapshot plugin %q...", name)
		ic := &plugin.InitContext{
			Root:    conf.Root,
			State:   conf.State,
			Store:   store,
			Context: log.WithModule(global, moduleName),
		}
		if sr.Config != nil {
			if err := conf.decodePlugin(name, sr.Config); err != nil {
				return nil, err
			}
			ic.Config = sr.Config
		}
		sn, err := sr.Init(ic)
		if err != nil {
			return nil, err
		}

		return sn.(snapshot.Snapshotter), nil
	}
	return nil, fmt.Errorf("snapshotter not loaded: %v", conf.Snapshotter)
}

func newGRPCServer() *grpc.Server {
	s := grpc.NewServer(
		grpc.UnaryInterceptor(interceptor),
		grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor),
	)
	return s
}

func loadServices(runtimes map[string]containerd.Runtime, store *content.Store, sn snapshot.Snapshotter) ([]plugin.Service, error) {
	var o []plugin.Service
	for name, sr := range plugin.Registrations() {
		if sr.Type != plugin.GRPCPlugin {
			continue
		}
		log.G(global).Infof("loading grpc service plugin %q...", name)
		ic := &plugin.InitContext{
			Root:        conf.Root,
			State:       conf.State,
			Context:     log.WithModule(global, fmt.Sprintf("service-%s", name)),
			Runtimes:    runtimes,
			Store:       store,
			Snapshotter: sn,
		}
		if sr.Config != nil {
			if err := conf.decodePlugin(name, sr.Config); err != nil {
				return nil, err
			}
			ic.Config = sr.Config
		}
		vs, err := sr.Init(ic)
		if err != nil {
			return nil, err
		}
		o = append(o, vs.(plugin.Service))
	}
	return o, nil
}

func serveGRPC(server *grpc.Server) error {
	path := conf.GRPC.Socket
	if path == "" {
		return errors.New("--socket path cannot be empty")
	}
	l, err := sys.CreateUnixSocket(path)
	if err != nil {
		return err
	}
	if err := os.Chown(path, conf.GRPC.Uid, conf.GRPC.Gid); err != nil {
		return err
	}
	go func() {
		defer l.Close()
		if err := server.Serve(l); err != nil {
			log.G(global).WithError(err).Fatal("serve GRPC")
		}
	}()
	return nil
}

func interceptor(ctx gocontext.Context,
	req interface{},
	info *grpc.UnaryServerInfo,
	handler grpc.UnaryHandler,
) (interface{}, error) {
	ctx = log.WithModule(ctx, "containerd")
	switch info.Server.(type) {
	case api.ContainerServiceServer:
		ctx = log.WithModule(ctx, "execution")
	case contentapi.ContentServer:
		ctx = log.WithModule(ctx, "content")
	case rootfsapi.RootFSServer:
		ctx = log.WithModule(ctx, "rootfs")
	default:
		fmt.Printf("unknown GRPC server type: %#v\n", info.Server)
	}
	return grpc_prometheus.UnaryServerInterceptor(ctx, req, info, handler)
}

func handleSignals(signals chan os.Signal, server *grpc.Server) error {
	for s := range signals {
		log.G(global).WithField("signal", s).Debug("received signal")
		switch s {
		case syscall.SIGCHLD:
			if err := reaper.Reap(); err != nil {
				log.G(global).WithError(err).Error("reap containerd processes")
			}
		default:
			server.Stop()
			return nil
		}
	}
	return nil
}
<commit_msg>containerd: missing config creates a default<commit_after>package main

import (
	_ "expvar"
	"fmt"
	"io/ioutil"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"path/filepath"
	"syscall"
	"time"

	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
	gocontext "golang.org/x/net/context"
	"google.golang.org/grpc"

	"github.com/Sirupsen/logrus"
	"github.com/containerd/containerd"
	contentapi "github.com/containerd/containerd/api/services/content"
	api "github.com/containerd/containerd/api/services/execution"
	rootfsapi "github.com/containerd/containerd/api/services/rootfs"
	"github.com/containerd/containerd/content"
	"github.com/containerd/containerd/log"
	"github.com/containerd/containerd/plugin"
	"github.com/containerd/containerd/reaper"
	"github.com/containerd/containerd/snapshot"
	"github.com/containerd/containerd/sys"
	metrics "github.com/docker/go-metrics"
	"github.com/pkg/errors"
	"github.com/urfave/cli"
)

const usage = `
                    __        _                     __
  _________  ____  / /_____ _(_)___  ___  _________/ /
 / ___/ __ \/ __ \/ __/ __ ` + "`" + `/ / __ \/ _ \/ ___/ __  /
/ /__/ /_/ / / / / /_/ /_/ / / / / /  __/ /  / /_/ /
\___/\____/_/ /_/\__/\__,_/_/_/ /_/\___/_/   \__,_/

high performance container runtime
`

var (
	conf   = defaultConfig()
	global = log.WithModule(gocontext.Background(), "containerd")
)

func init() {
	cli.VersionPrinter = func(c *cli.Context) {
		fmt.Println(c.App.Name, containerd.Package, c.App.Version)
	}
}

func main() {
	app := cli.NewApp()
	app.Name = "containerd"
	app.Version = containerd.Version
	app.Usage = usage
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "config,c",
			Usage: "path to the configuration file (Use 'default' to output the default toml)",
			Value: "/etc/containerd/config.toml",
		},
		cli.StringFlag{
			Name:  "log-level,l",
			Usage: "set the logging level [debug, info, warn, error, fatal, panic]",
		},
		cli.StringFlag{
			Name:  "state",
			Usage: "containerd state directory",
		},
		cli.StringFlag{
			Name:  "socket,s",
			Usage: "socket path for containerd's GRPC server",
		},
		cli.StringFlag{
			Name:  "root",
			Usage: "containerd root directory",
		},
	}
	app.Before = before
	app.Action = func(context *cli.Context) error {
		start := time.Now()
		// start the signal handler as soon as we can to make sure that
		// we don't miss any signals during boot
		signals := make(chan os.Signal, 2048)
		signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT, syscall.SIGUSR1, syscall.SIGCHLD)
		if conf.Subreaper {
			log.G(global).Info("setting subreaper...")
			if err := sys.SetSubreaper(1); err != nil {
				return err
			}
		}
		log.G(global).Info("starting containerd boot...")

		// load all plugins into containerd
		if err := plugin.Load(filepath.Join(conf.Root, "plugins")); err != nil {
			return err
		}
		// start debug and metrics APIs
		if err := serveDebugAPI(); err != nil {
			return err
		}
		monitor, err := loadMonitor()
		if err != nil {
			return err
		}
		runtimes, err := loadRuntimes(monitor)
		if err != nil {
			return err
		}
		store, err := resolveContentStore()
		if err != nil {
			return err
		}
		snapshotter, err := loadSnapshotter(store)
		if err != nil {
			return err
		}
		services, err := loadServices(runtimes, store, snapshotter)
		if err != nil {
			return err
		}
		// start the GRPC api with the execution service registered
		server := newGRPCServer()
		for _, service := range services {
			if err := service.Register(server); err != nil {
				return err
			}
		}
		log.G(global).Info("starting GRPC API server...")
		if err := serveGRPC(server); err != nil {
			return err
		}
		// start the prometheus metrics API for containerd
		serveMetricsAPI()

		log.G(global).Infof("containerd successfully booted in %fs", time.Since(start).Seconds())
		return handleSignals(signals, server)
	}

	if err := app.Run(os.Args); err != nil {
		fmt.Fprintf(os.Stderr, "containerd: %s\n", err)
		os.Exit(1)
	}
}

func before(context *cli.Context) error {
	if context.GlobalString("config") == "default" {
		fh, err := ioutil.TempFile("", "containerd-config.toml.")
		if err != nil {
			return err
		}
		if _, err := conf.WriteTo(fh); err != nil {
			fh.Close()
			return err
		}
		fh.Close()
		log.G(global).Infof("containerd default config written to %q", fh.Name())
		os.Exit(0)
	}
	err := loadConfig(context.GlobalString("config"))
	if err != nil && !os.IsNotExist(err) {
		return err
	} else if err != nil && os.IsNotExist(err) {
		log.G(global).Infof("config %q does not exist. Creating it.", context.GlobalString("config"))
		fh, err := os.Create(context.GlobalString("config"))
		if err != nil {
			return err
		}
		if _, err := conf.WriteTo(fh); err != nil {
			fh.Close()
			return err
		}
		fh.Close()
	}

	// the order for config vs flag values is that flags will always override
	// the config values if they are set
	if err := setLevel(context); err != nil {
		return err
	}
	for _, v := range []struct {
		name string
		d    *string
	}{
		{
			name: "root",
			d:    &conf.Root,
		},
		{
			name: "state",
			d:    &conf.State,
		},
		{
			name: "socket",
			d:    &conf.GRPC.Socket,
		},
	} {
		if s := context.GlobalString(v.name); s != "" {
			*v.d = s
		}
	}
	return nil
}

func setLevel(context *cli.Context) error {
	l := context.GlobalString("log-level")
	if l == "" {
		l = conf.Debug.Level
	}
	if l != "" {
		lvl, err := logrus.ParseLevel(l)
		if err != nil {
			return err
		}
		logrus.SetLevel(lvl)
	}
	return nil
}

func serveMetricsAPI() {
	if conf.Metrics.Address != "" {
		log.G(global).WithField("metrics", conf.Metrics.Address).Info("starting metrics API...")
		h := newMetricsHandler()
		go func() {
			if err := http.ListenAndServe(conf.Metrics.Address, h); err != nil {
				log.G(global).WithError(err).Fatal("serve metrics API")
			}
		}()
	}
}

func newMetricsHandler() http.Handler {
	m := http.NewServeMux()
	m.Handle("/metrics", metrics.Handler())
	return m
}

func serveDebugAPI() error {
	path := conf.Debug.Socket
	if path == "" {
		return errors.New("debug socket path cannot be empty")
	}
	l, err := sys.CreateUnixSocket(path)
	if err != nil {
		return err
	}
	log.G(global).WithField("debug", path).Info("starting debug API...")
	go func() {
		defer l.Close()
		// pprof and expvars are imported and automatically register their endpoints
		// under /debug
		if err := http.Serve(l, nil); err != nil {
			log.G(global).WithError(err).Fatal("serve debug API")
		}
	}()
	return nil
}

func resolveContentStore() (*content.Store, error) {
	cp := filepath.Join(conf.Root, "content")
	return content.NewStore(cp)
}

func loadRuntimes(monitor plugin.ContainerMonitor) (map[string]containerd.Runtime, error) {
	o := make(map[string]containerd.Runtime)
	for name, rr := range plugin.Registrations() {
		if rr.Type != plugin.RuntimePlugin {
			continue
		}
		log.G(global).Infof("loading runtime plugin %q...", name)
		ic := &plugin.InitContext{
			Root:    conf.Root,
			State:   conf.State,
			Context: log.WithModule(global, fmt.Sprintf("runtime-%s", name)),
			Monitor: monitor,
		}
		if rr.Config != nil {
			if err := conf.decodePlugin(name, rr.Config); err != nil {
				return nil, err
			}
			ic.Config = rr.Config
		}
		vr, err := rr.Init(ic)
		if err != nil {
			return nil, err
		}
		o[name] = vr.(containerd.Runtime)
	}
	return o, nil
}

func loadMonitor() (plugin.ContainerMonitor, error) {
	var monitors []plugin.ContainerMonitor
	for name, m := range plugin.Registrations() {
		if m.Type != plugin.ContainerMonitorPlugin {
			continue
		}
		log.G(global).Infof("loading monitor plugin %q...", name)
		ic := &plugin.InitContext{
			Root:    conf.Root,
			State:   conf.State,
			Context: log.WithModule(global, fmt.Sprintf("monitor-%s", name)),
		}
		mm, err := m.Init(ic)
		if err != nil {
			return nil, err
		}
		monitors = append(monitors, mm.(plugin.ContainerMonitor))
	}
	if len(monitors) == 0 {
		return plugin.NewNoopMonitor(), nil
	}
	return plugin.NewMultiContainerMonitor(monitors...), nil
}

func loadSnapshotter(store *content.Store) (snapshot.Snapshotter, error) {
	for name, sr := range plugin.Registrations() {
		if sr.Type != plugin.SnapshotPlugin {
			continue
		}
		moduleName := fmt.Sprintf("snapshot-%s", conf.Snapshotter)
		if name != moduleName {
			continue
		}

		log.G(global).Infof("loading snapshot plugin %q...", name)
		ic := &plugin.InitContext{
			Root:    conf.Root,
			State:   conf.State,
			Store:   store,
			Context: log.WithModule(global, moduleName),
		}
		if sr.Config != nil {
			if err := conf.decodePlugin(name, sr.Config); err != nil {
				return nil, err
			}
			ic.Config = sr.Config
		}
		sn, err := sr.Init(ic)
		if err != nil {
			return nil, err
		}

		return sn.(snapshot.Snapshotter), nil
	}
	return nil, fmt.Errorf("snapshotter not loaded: %v", conf.Snapshotter)
}

func newGRPCServer() *grpc.Server {
	s := grpc.NewServer(
		grpc.UnaryInterceptor(interceptor),
		grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor),
	)
	return s
}

func loadServices(runtimes map[string]containerd.Runtime, store *content.Store, sn snapshot.Snapshotter) ([]plugin.Service, error) {
	var o []plugin.Service
	for name, sr := range plugin.Registrations() {
		if sr.Type != plugin.GRPCPlugin {
			continue
		}
		log.G(global).Infof("loading grpc service plugin %q...", name)
		ic := &plugin.InitContext{
			Root:        conf.Root,
			State:       conf.State,
			Context:     log.WithModule(global, fmt.Sprintf("service-%s", name)),
			Runtimes:    runtimes,
			Store:       store,
			Snapshotter: sn,
		}
		if sr.Config != nil {
			if err := conf.decodePlugin(name, sr.Config); err != nil {
				return nil, err
			}
			ic.Config = sr.Config
		}
		vs, err := sr.Init(ic)
		if err != nil {
			return nil, err
		}
		o = append(o, vs.(plugin.Service))
	}
	return o, nil
}

func serveGRPC(server *grpc.Server) error {
	path := conf.GRPC.Socket
	if path == "" {
		return errors.New("--socket path cannot be empty")
	}
	l, err := sys.CreateUnixSocket(path)
	if err != nil {
		return err
	}
	if err := os.Chown(path, conf.GRPC.Uid, conf.GRPC.Gid); err != nil {
		return err
	}
	go func() {
		defer l.Close()
		if err := server.Serve(l); err != nil {
			log.G(global).WithError(err).Fatal("serve GRPC")
		}
	}()
	return nil
}

func interceptor(ctx gocontext.Context,
	req interface{},
	info *grpc.UnaryServerInfo,
	handler grpc.UnaryHandler,
) (interface{}, error) {
	ctx = log.WithModule(ctx, "containerd")
	switch info.Server.(type) {
	case api.ContainerServiceServer:
		ctx = log.WithModule(ctx, "execution")
	case contentapi.ContentServer:
		ctx = log.WithModule(ctx, "content")
	case rootfsapi.RootFSServer:
		ctx = log.WithModule(ctx, "rootfs")
	default:
		fmt.Printf("unknown GRPC server type: %#v\n", info.Server)
	}
	return grpc_prometheus.UnaryServerInterceptor(ctx, req, info, handler)
}

func handleSignals(signals chan os.Signal, server *grpc.Server) error {
	for s := range signals {
		log.G(global).WithField("signal", s).Debug("received signal")
		switch s {
		case syscall.SIGCHLD:
			if err := reaper.Reap(); err != nil {
				log.G(global).WithError(err).Error("reap containerd processes")
			}
		default:
			server.Stop()
			return nil
		}
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Package runfile implements sensorbee runfile command. This command is provided
separately from run command to reduce the footprint of runfile command itself.
For example, if sensorbee doesn't need to provide sensorbee run command and only
offers sensorbee runfile, the footprint of sensorbee command might be reduced a
lot due to low functionality of this command.
*/
package runfile

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/Sirupsen/logrus"
	"gopkg.in/sensorbee/sensorbee.v0/bql"
	"gopkg.in/sensorbee/sensorbee.v0/bql/parser"
	"gopkg.in/sensorbee/sensorbee.v0/bql/udf"
	"gopkg.in/sensorbee/sensorbee.v0/core"
	"gopkg.in/sensorbee/sensorbee.v0/data"
	"gopkg.in/sensorbee/sensorbee.v0/server/config"
	"gopkg.in/sensorbee/sensorbee.v0/server/udsstorage"
	"gopkg.in/urfave/cli.v1"
	"gopkg.in/yaml.v2"
)

// SetUp sets up a command for running single BQL file.
func SetUp() cli.Command {
	cmd := cli.Command{
		Name:        "runfile",
		Usage:       "run a BQL file",
		Description: "runfile command runs a BQL file",
		Action:      Run,
	}

	cmd.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "config, c",
			Value:  "",
			Usage:  "file path of a config file in YAML format (only logging and storage sections are used)",
			EnvVar: "SENSORBEE_CONFIG",
		},
		cli.StringFlag{
			Name:  "save-uds, s",
			Value: "",
			Usage: "save UDSs after all tuples are processed",
		},
		cli.StringFlag{
			Name:  "topology, t",
			Value: "",
			Usage: "name of the topology",
		},
	}
	return cmd
}

// Run runs "runfile" command.
func Run(c *cli.Context) error {
	// TODO: Merge this implementation with cmd/run
	if len(c.Args()) != 1 {
		cli.ShowSubcommandHelp(c)
		os.Exit(1)
	}

	emptyError := fmt.Errorf("") // to provide exit code but not error message for cli
	err := func() (retErr error) {
		conf, err := func() (*config.Config, error) {
			if c.IsSet("config") {
				p := c.String("config")
				in, err := ioutil.ReadFile(p)
				if err != nil {
					return nil, fmt.Errorf("Cannot read the config file %v: %v", p, err)
				}

				var yml map[string]interface{}
				if err := yaml.Unmarshal(in, &yml); err != nil {
					return nil, fmt.Errorf("Cannot parse the config file %v: %v", p, err)
				}
				m, err := data.NewMap(yml)
				if err != nil {
					return nil, fmt.Errorf("The config file %v has invalid values: %v", p, err)
				}
				return config.New(m)

			} else {
				// Currently there's no required parameters. However, when a required
				// parameter is added to Config, remove this else block and add a
				// default value like "/etc/sensorbee/config.yaml" to the config option.
				return config.New(data.Map{})
			}
		}()
		if err != nil {
			return err
		}

		w, err := conf.Logging.CreateWriter()
		if err != nil {
			return err
		}
		logLevel, err := logrus.ParseLevel(conf.Logging.MinLogLevel)
		if err != nil {
			return err
		}
		logger := logrus.New()
		logger.Out = w
		logger.Level = logLevel

		udsStorage, err := setUpUDSStorage(&conf.Storage.UDS)
		if err != nil {
			logger.WithField("err", err).Error("Cannot set up a storage for UDSs")
			return emptyError
		}

		logger.Info("Setting up a topology")
		bqlFile := c.Args()[0]
		topologyName := filepath.Base(bqlFile)
		topologyName = topologyName[:len(topologyName)-len(filepath.Ext(topologyName))]
		if n := c.String("topology"); n != "" {
			topologyName = n
		}

		tb, err := setUpTopology(topologyName, logger, conf, udsStorage)
		if err != nil {
			logger.WithField("err", err).Error("Cannot set up the topology")
			return emptyError
		}

		if err := setUpBQLStmt(tb, bqlFile); err != nil {
			logger.WithFields(logrus.Fields{
				"err":      err,
				"bql_file": bqlFile,
			}).Error("Cannot set up BQL statement")
			return emptyError
		}
		if c.IsSet("save-uds") {
			if err := checkStates(tb, c.String("save-uds")); err != nil {
				logger.WithField("err", err).Error("Cannot set up 'save-uds' option")
				return emptyError
			}
		}

		defer func() {
			logger.Info("Waiting for all nodes to finish processing tuples")
			if err := tb.Topology().Stop(); err != nil {
				logger.WithField("err", err).Error("Cannot stop the topology")
				retErr = emptyError
				return
			}
			logger.Info("Topology stopped")

			if c.IsSet("save-uds") {
				saveUDSList := c.String("save-uds")
				if err := saveStates(tb, saveUDSList); err != nil {
					logger.WithFields(logrus.Fields{
						"err":      err,
						"topology": tb.Topology().Name(),
					}).Error("Cannot save UDSs")
					retErr = emptyError
				}
			}

			registry := tb.Topology().Context().SharedStates
			if states, err := registry.List(); err != nil {
				logger.WithField("err", err).Error("Cannot list shared states")
				retErr = emptyError
				return
			} else if 0 < len(states) {
				logger.Info("Terminating states")
				for name := range states {
					if _, err := registry.Remove(name); err != nil {
						logger.WithFields(logrus.Fields{
							"err":   err,
							"state": name,
						}).Error("Cannot terminate state")
						retErr = emptyError
						// Continue to terminate the next state.
					}
				}
			}
		}()

		logger.WithField("config", conf.ToMap()).Info("Starting the topology")
		for name, s := range tb.Topology().Sources() {
			if err := s.Resume(); err != nil {
				logger.WithFields(logrus.Fields{
					"err":    err,
					"source": name,
				}).Error("Cannot resume the source")
				return emptyError
			}
		}

		for _, s := range tb.Topology().Sources() {
			// TODO: error check if necessary
			s.State().Wait(core.TSStopped)
		}
		logger.Info("All sources has been stopped.")
		return nil
	}()
	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}
	return nil
}

func setUpUDSStorage(conf *config.UDSStorage) (udf.UDSStorage, error) {
	// TODO: merge this implementation with server/context.go
	// Parameters are already validated in conf
	switch conf.Type {
	case "in_memory":
		return udf.NewInMemoryUDSStorage(), nil
	case "fs":
		dir, _ := data.AsString(conf.Params["dir"])
		var tempDir string
		if v, ok := conf.Params["temp_dir"]; ok {
			tempDir, _ = data.AsString(v)
		}
		return udsstorage.NewFS(dir, tempDir)
	default:
		return nil, fmt.Errorf("unsupported uds storage type: %v", conf.Type)
	}
}

func setUpTopology(name string, logger *logrus.Logger, conf *config.Config, us udf.UDSStorage) (
	*bql.TopologyBuilder, error) {
	cc := &core.ContextConfig{
		Logger: logger,
	}
	cc.Flags.DroppedTupleLog.Set(conf.Logging.LogDroppedTuples)
	cc.Flags.DestinationlessTupleLog.Set(conf.Logging.LogDestinationlessTuples)
	cc.Flags.DroppedTupleSummarization.Set(conf.Logging.SummarizeDroppedTuples)

	tp, err := core.NewDefaultTopology(core.NewContext(cc), name)
	if err != nil {
		return nil, err
	}
	tb, err := bql.NewTopologyBuilder(tp)
	if err != nil {
		return nil, fmt.Errorf("cannot create a new topology builder: %v", err)
	}
	tb.UDSStorage = us

	return tb, nil
}

func setUpBQLStmt(tb *bql.TopologyBuilder, bqlFile string) error {
	queries, err := func() (string, error) {
		f, err := os.Open(bqlFile)
		if err != nil {
			return "", err
		}
		defer f.Close()
		b, err := ioutil.ReadAll(f)
		if err != nil {
			return "", err
		}
		return string(b), nil
	}()
	if err != nil {
		return err
	}

	bp := parser.New()
	// TODO: provide better parse error reporting using ParseStmt instead of ParseStmts
	stmts, err := bp.ParseStmts(string(queries))
	if err != nil {
		return err
	}

	for _, stmt := range stmts {
		// TODO: if stmt is CREATE SOURCE, create it with PAUSED
		if n, err := tb.AddStmt(stmt); err != nil {
			tb.Topology().Context().ErrLog(err).WithField("stmt", stmt).Error(
				"Cannot add a statement to the topology")
			return err // FIXME: logger output "err" two twice
		} else if n != nil && n.Type() == core.NTSource {
			sn, _ := n.(core.SourceNode)
			if _, ok := sn.Source().(core.RewindableSource); ok {
				return fmt.Errorf(`rewindable source "%v" isn't supported`, n.Name())
			}
		}
	}
	return nil
}

func checkStates(tb *bql.TopologyBuilder, saveUDSList string) error {
	if saveUDSList == "" {
		return nil // save all UDS and pass to check
	}
	states, err := tb.Topology().Context().SharedStates.List()
	if err != nil {
		return err
	}
	notFoundFlag := false
	for _, name := range strings.Split(saveUDSList, ",") {
		if _, ok := states[name]; !ok {
			err := fmt.Errorf("the UDS is not found")
			tb.Topology().Context().ErrLog(err).WithField("uds", name).Error(
				"Invalid UDS name")
			notFoundFlag = true
		}
	}
	if notFoundFlag {
		return fmt.Errorf("the target UDS list includes invalid name")
	}
	return nil
}

func saveStates(tb *bql.TopologyBuilder, saveUDSList string) error {
	states, err := tb.Topology().Context().SharedStates.List()
	if err != nil {
		return err
	}
	// if save UDS list is empty then all UDS will be saved.
	saveUDSs := []string{}
	if saveUDSList == "" {
		for k := range states {
			saveUDSs = append(saveUDSs, k)
		}
	} else {
		saveUDSs = strings.Split(saveUDSList, ",")
		for i, n := range saveUDSs {
			saveUDSs[i] = strings.TrimSpace(n)
		}
	}

	saveErrorFlag := false
	for _, name := range saveUDSs {
		state, ok := states[name]
		if !ok {
			err := fmt.Errorf("the UDS is not found")
			tb.Topology().Context().ErrLog(err).WithField("uds", name).Error(
				"Cannot save the UDS")
			continue
		}
		target, ok := state.(core.SavableSharedState)
		if !ok {
			tb.Topology().Context().Log().WithField("uds", name).Info(
				"The UDS doesn't support Save")
			continue
		}
		// TODO get tag
		w, err := tb.UDSStorage.Save(tb.Topology().Name(), name, "default")
		if err != nil {
			tb.Topology().Context().ErrLog(err).WithField("uds", name).Error(
				"Cannot save the UDS")
			saveErrorFlag = true
			continue
		}
		err = func() error {
			defer func() {
				if err := w.Commit(); err != nil {
					tb.Topology().Context().ErrLog(err).WithField("uds", name).Error(
						"Cannot save the UDS")
					saveErrorFlag = true
				}
			}()
			// TODO get parameters
			return target.Save(tb.Topology().Context(), w, data.Map{})
		}()
		if err != nil {
			tb.Topology().Context().ErrLog(err).WithField("uds", name).Error(
				"Cannot save the UDS")
			saveErrorFlag = true
		}
	}
	if saveErrorFlag {
		return fmt.Errorf("fail to save or commit some UDSs")
	}
	return nil
}
<commit_msg>change method name to check error and improve error messages<commit_after>/*
Package runfile implements sensorbee runfile command. This command is provided
separately from run command to reduce the footprint of runfile command itself.
For example, if sensorbee doesn't need to provide sensorbee run command and only
offers sensorbee runfile, the footprint of sensorbee command might be reduced a
lot due to low functionality of this command.
*/
package runfile

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/Sirupsen/logrus"
	"gopkg.in/sensorbee/sensorbee.v0/bql"
	"gopkg.in/sensorbee/sensorbee.v0/bql/parser"
	"gopkg.in/sensorbee/sensorbee.v0/bql/udf"
	"gopkg.in/sensorbee/sensorbee.v0/core"
	"gopkg.in/sensorbee/sensorbee.v0/data"
	"gopkg.in/sensorbee/sensorbee.v0/server/config"
	"gopkg.in/sensorbee/sensorbee.v0/server/udsstorage"
	"gopkg.in/urfave/cli.v1"
	"gopkg.in/yaml.v2"
)

// SetUp sets up a command for running single BQL file.
func SetUp() cli.Command {
	cmd := cli.Command{
		Name:        "runfile",
		Usage:       "run a BQL file",
		Description: "runfile command runs a BQL file",
		Action:      Run,
	}

	cmd.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "config, c",
			Value:  "",
			Usage:  "file path of a config file in YAML format (only logging and storage sections are used)",
			EnvVar: "SENSORBEE_CONFIG",
		},
		cli.StringFlag{
			Name:  "save-uds, s",
			Value: "",
			Usage: "save UDSs after all tuples are processed",
		},
		cli.StringFlag{
			Name:  "topology, t",
			Value: "",
			Usage: "name of the topology",
		},
	}
	return cmd
}

// Run runs "runfile" command.
func Run(c *cli.Context) error {
	// TODO: Merge this implementation with cmd/run
	if len(c.Args()) != 1 {
		cli.ShowSubcommandHelp(c)
		os.Exit(1)
	}

	emptyError := fmt.Errorf("") // to provide exit code but not error message for cli
	err := func() (retErr error) {
		conf, err := func() (*config.Config, error) {
			if c.IsSet("config") {
				p := c.String("config")
				in, err := ioutil.ReadFile(p)
				if err != nil {
					return nil, fmt.Errorf("Cannot read the config file %v: %v", p, err)
				}

				var yml map[string]interface{}
				if err := yaml.Unmarshal(in, &yml); err != nil {
					return nil, fmt.Errorf("Cannot parse the config file %v: %v", p, err)
				}
				m, err := data.NewMap(yml)
				if err != nil {
					return nil, fmt.Errorf("The config file %v has invalid values: %v", p, err)
				}
				return config.New(m)

			} else {
				// Currently there's no required parameters. However, when a required
				// parameter is added to Config, remove this else block and add a
				// default value like "/etc/sensorbee/config.yaml" to the config option.
				return config.New(data.Map{})
			}
		}()
		if err != nil {
			return err
		}

		w, err := conf.Logging.CreateWriter()
		if err != nil {
			return err
		}
		logLevel, err := logrus.ParseLevel(conf.Logging.MinLogLevel)
		if err != nil {
			return err
		}
		logger := logrus.New()
		logger.Out = w
		logger.Level = logLevel

		udsStorage, err := setUpUDSStorage(&conf.Storage.UDS)
		if err != nil {
			logger.WithField("err", err).Error("Cannot set up a storage for UDSs")
			return emptyError
		}

		logger.Info("Setting up a topology")
		bqlFile := c.Args()[0]
		topologyName := filepath.Base(bqlFile)
		topologyName = topologyName[:len(topologyName)-len(filepath.Ext(topologyName))]
		if n := c.String("topology"); n != "" {
			topologyName = n
		}

		tb, err := setUpTopology(topologyName, logger, conf, udsStorage)
		if err != nil {
			logger.WithField("err", err).Error("Cannot set up the topology")
			return emptyError
		}

		if err := setUpBQLStmt(tb, bqlFile); err != nil {
			logger.WithFields(logrus.Fields{
				"err":      err,
				"bql_file": bqlFile,
			}).Error("Cannot set up BQL statement")
			return emptyError
		}
		if c.IsSet("save-uds") {
			if err := hasStates(tb, c.String("save-uds")); err != nil {
				logger.WithField("err", err).Error("Cannot set up 'save-uds' option")
				return emptyError
			}
		}

		defer func() {
			logger.Info("Waiting for all nodes to finish processing tuples")
			if err := tb.Topology().Stop(); err != nil {
				logger.WithField("err", err).Error("Cannot stop the topology")
				retErr = emptyError
				return
			}
			logger.Info("Topology stopped")

			if c.IsSet("save-uds") {
				saveUDSList := c.String("save-uds")
				if err := saveStates(tb, saveUDSList); err != nil {
					logger.WithFields(logrus.Fields{
						"err":      err,
						"topology": tb.Topology().Name(),
					}).Error("Cannot save UDSs")
					retErr = emptyError
				}
			}

			registry := tb.Topology().Context().SharedStates
			if states, err := registry.List(); err != nil {
				logger.WithField("err", err).Error("Cannot list shared states")
				retErr = emptyError
				return
			} else if 0 < len(states) {
				logger.Info("Terminating states")
				for name := range states {
					if _, err := registry.Remove(name); err != nil {
						logger.WithFields(logrus.Fields{
							"err":   err,
							"state": name,
						}).Error("Cannot terminate state")
						retErr = emptyError
						// Continue to terminate the next state.
					}
				}
			}
		}()

		logger.WithField("config", conf.ToMap()).Info("Starting the topology")
		for name, s := range tb.Topology().Sources() {
			if err := s.Resume(); err != nil {
				logger.WithFields(logrus.Fields{
					"err":    err,
					"source": name,
				}).Error("Cannot resume the source")
				return emptyError
			}
		}

		for _, s := range tb.Topology().Sources() {
			// TODO: error check if necessary
			s.State().Wait(core.TSStopped)
		}
		logger.Info("All sources has been stopped.")
		return nil
	}()
	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}
	return nil
}

func setUpUDSStorage(conf *config.UDSStorage) (udf.UDSStorage, error) {
	// TODO: merge this implementation with server/context.go
	// Parameters are already validated in conf
	switch conf.Type {
	case "in_memory":
		return udf.NewInMemoryUDSStorage(), nil
	case "fs":
		dir, _ := data.AsString(conf.Params["dir"])
		var tempDir string
		if v, ok := conf.Params["temp_dir"]; ok {
			tempDir, _ = data.AsString(v)
		}
		return udsstorage.NewFS(dir, tempDir)
	default:
		return nil, fmt.Errorf("unsupported uds storage type: %v", conf.Type)
	}
}

func setUpTopology(name string, logger *logrus.Logger, conf *config.Config, us udf.UDSStorage) (
	*bql.TopologyBuilder, error) {
	cc := &core.ContextConfig{
		Logger: logger,
	}
	cc.Flags.DroppedTupleLog.Set(conf.Logging.LogDroppedTuples)
	cc.Flags.DestinationlessTupleLog.Set(conf.Logging.LogDestinationlessTuples)
	cc.Flags.DroppedTupleSummarization.Set(conf.Logging.SummarizeDroppedTuples)

	tp, err := core.NewDefaultTopology(core.NewContext(cc), name)
	if err != nil {
		return nil, err
	}
	tb, err := bql.NewTopologyBuilder(tp)
	if err != nil {
		return nil, fmt.Errorf("cannot create a new topology builder: %v", err)
	}
	tb.UDSStorage = us

	return tb, nil
}

func setUpBQLStmt(tb *bql.TopologyBuilder, bqlFile string) error {
	queries, err := func() (string, error) {
		f, err := os.Open(bqlFile)
		if err != nil {
			return "", err
		}
		defer f.Close()
		b, err := ioutil.ReadAll(f)
		if err != nil {
			return "", err
		}
		return string(b), nil
	}()
	if err != nil {
		return err
	}

	bp := parser.New()
	// TODO: provide better parse error reporting using ParseStmt instead of ParseStmts
	stmts, err := bp.ParseStmts(string(queries))
	if err != nil {
		return err
	}

	for _, stmt := range stmts {
		// TODO: if stmt is CREATE SOURCE, create it with PAUSED
		if n, err := tb.AddStmt(stmt); err != nil {
			tb.Topology().Context().ErrLog(err).WithField("stmt", stmt).Error(
				"Cannot add a statement to the topology")
			return err // FIXME: logger output "err" two twice
		} else if n != nil && n.Type() == core.NTSource {
			sn, _ := n.(core.SourceNode)
			if _, ok := sn.Source().(core.RewindableSource); ok {
				return fmt.Errorf(`rewindable source "%v" isn't supported`, n.Name())
			}
		}
	}
	return nil
}

func hasStates(tb *bql.TopologyBuilder, saveUDSList string) error {
	if saveUDSList == "" {
		return nil
	}
	states, err := tb.Topology().Context().SharedStates.List()
	if err != nil {
		return err
	}
	missing := []string{}
	for _, name := range strings.Split(saveUDSList, ",") {
		if _, ok := states[name]; !ok {
			missing = append(missing, name)
		}
	}
	if len(missing) > 0 {
		return fmt.Errorf("the topology doesn't have UDSs: %v",
			strings.Join(missing, ","))
	}
	return nil
}

func saveStates(tb *bql.TopologyBuilder, saveUDSList string) error {
	states, err := tb.Topology().Context().SharedStates.List()
	if err != nil {
		return err
	}
	// if save UDS list is empty then all UDS will be saved.
	saveUDSs := []string{}
	if saveUDSList == "" {
		for k := range states {
			saveUDSs = append(saveUDSs, k)
		}
	} else {
		saveUDSs = strings.Split(saveUDSList, ",")
		for i, n := range saveUDSs {
			saveUDSs[i] = strings.TrimSpace(n)
		}
	}

	saveErrorFlag := false
	for _, name := range saveUDSs {
		state, ok := states[name]
		if !ok {
			err := fmt.Errorf("the UDS is not found")
			tb.Topology().Context().ErrLog(err).WithField("uds", name).Error(
				"Cannot save the UDS")
			continue
		}
		target, ok := state.(core.SavableSharedState)
		if !ok {
			tb.Topology().Context().Log().WithField("uds", name).Info(
				"The UDS doesn't support Save")
			continue
		}
		// TODO get tag
		w, err := tb.UDSStorage.Save(tb.Topology().Name(), name, "default")
		if err != nil {
			tb.Topology().Context().ErrLog(err).WithField("uds", name).Error(
				"Cannot save the UDS")
			saveErrorFlag = true
			continue
		}
		err = func() error {
			defer func() {
				if err := w.Commit(); err != nil {
					tb.Topology().Context().ErrLog(err).WithField("uds", name).Error(
						"Cannot save the UDS")
					saveErrorFlag = true
				}
			}()
			// TODO get parameters
			return target.Save(tb.Topology().Context(), w, data.Map{})
		}()
		if err != nil {
			tb.Topology().Context().ErrLog(err).WithField("uds", name).Error(
				"Cannot save the UDS")
			saveErrorFlag = true
		}
	}
	if saveErrorFlag {
		return fmt.Errorf("fail to save or commit some UDSs")
	}
	return nil
}
<|endoftext|>