<commit_before>package hclutils

import (
	"testing"

	"github.com/hashicorp/hcl"
	"github.com/hashicorp/hcl/hcl/ast"
	hcl2 "github.com/hashicorp/hcl2/hcl"
	"github.com/hashicorp/hcl2/hcldec"
	"github.com/hashicorp/nomad/helper"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/kr/pretty"
	"github.com/mitchellh/mapstructure"
	"github.com/stretchr/testify/require"
	"github.com/ugorji/go/codec"
	"github.com/zclconf/go-cty/cty"
	"github.com/zclconf/go-cty/cty/gocty"
)

var (
	dockerSpec hcldec.Spec = hcldec.ObjectSpec(map[string]hcldec.Spec{
		"image": &hcldec.AttrSpec{
			Name:     "image",
			Type:     cty.String,
			Required: true,
		},
		"args": &hcldec.AttrSpec{
			Name: "args",
			Type: cty.List(cty.String),
		},
		"pids_limit": &hcldec.AttrSpec{
			Name: "pids_limit",
			Type: cty.Number,
		},
		"port_map": &hcldec.BlockAttrsSpec{
			TypeName:    "port_map",
			ElementType: cty.String,
		},

		"devices": &hcldec.BlockListSpec{
			TypeName: "devices",
			Nested: hcldec.ObjectSpec(map[string]hcldec.Spec{
				"host_path": &hcldec.AttrSpec{
					Name: "host_path",
					Type: cty.String,
				},
				"container_path": &hcldec.AttrSpec{
					Name: "container_path",
					Type: cty.String,
				},
				"cgroup_permissions": &hcldec.DefaultSpec{
					Primary: &hcldec.AttrSpec{
						Name: "cgroup_permissions",
						Type: cty.String,
					},
					Default: &hcldec.LiteralSpec{
						Value: cty.StringVal(""),
					},
				},
			}),
		},
	},
	)
)

type dockerConfig struct {
	Image     string            `cty:"image"`
	Args      []string          `cty:"args"`
	PidsLimit *int64            `cty:"pids_limit"`
	PortMap   map[string]string `cty:"port_map"`
	Devices   []DockerDevice    `cty:"devices"`
}

type DockerDevice struct {
	HostPath          string `cty:"host_path"`
	ContainerPath     string `cty:"container_path"`
	CgroupPermissions string `cty:"cgroup_permissions"`
}

func hclConfigToInterface(t *testing.T, config string) interface{} {
	t.Helper()

	// Parse as we do in the jobspec parser
	root, err := hcl.Parse(config)
	if err != nil {
		t.Fatalf("failed to hcl parse the config: %v", err)
	}

	// Top-level item should be a list
	list, ok := root.Node.(*ast.ObjectList)
	if !ok {
		t.Fatalf("root should be an object")
	}

	var m map[string]interface{}
	if err := hcl.DecodeObject(&m, list.Items[0]); err != nil {
		t.Fatalf("failed to decode object: %v", err)
	}

	var m2 map[string]interface{}
	if err := mapstructure.WeakDecode(m, &m2); err != nil {
		t.Fatalf("failed to weak decode object: %v", err)
	}

	return m2["config"]
}

func jsonConfigToInterface(t *testing.T, config string) interface{} {
	t.Helper()

	// Decode from json
	dec := codec.NewDecoderBytes([]byte(config), structs.JsonHandle)

	var m map[string]interface{}
	err := dec.Decode(&m)
	if err != nil {
		t.Fatalf("failed to decode: %v", err)
	}

	return m["Config"]
}

func TestParseHclInterface_Hcl(t *testing.T) {
	defaultCtx := &hcl2.EvalContext{
		Functions: GetStdlibFuncs(),
	}
	variableCtx := &hcl2.EvalContext{
		Functions: GetStdlibFuncs(),
		Variables: map[string]cty.Value{
			"NOMAD_ALLOC_INDEX": cty.NumberIntVal(2),
			"NOMAD_META_hello":  cty.StringVal("world"),
		},
	}

	// XXX Useful for determining what cty thinks the type is
	//implied, err := gocty.ImpliedType(&dockerConfig{})
	//if err != nil {
	//t.Fatalf("implied type failed: %v", err)
	//}

	//t.Logf("Implied type: %v", implied.GoString())

	cases := []struct {
		name         string
		config       interface{}
		spec         hcldec.Spec
		ctx          *hcl2.EvalContext
		expected     interface{}
		expectedType interface{}
	}{
		{
			name: "single string attr",
			config: hclConfigToInterface(t, `
			config {
				image = "redis:3.2"
			}`),
			spec: dockerSpec,
			ctx:  defaultCtx,
			expected: &dockerConfig{
				Image:   "redis:3.2",
				Devices: []DockerDevice{},
			},
			expectedType: &dockerConfig{},
		},
		{
			name: "single string attr json",
			config: jsonConfigToInterface(t, `
						{
							"Config": {
								"image": "redis:3.2"
			                }
						}`),
			spec: dockerSpec,
			ctx:  defaultCtx,
			expected: &dockerConfig{
				Image:   "redis:3.2",
				Devices: []DockerDevice{},
			},
			expectedType: &dockerConfig{},
		},
		{
			name: "number attr",
			config: hclConfigToInterface(t, `
						config {
							image = "redis:3.2"
							pids_limit  = 2
						}`),
			spec: dockerSpec,
			ctx:  defaultCtx,
			expected: &dockerConfig{
				Image:     "redis:3.2",
				PidsLimit: helper.Int64ToPtr(2),
				Devices:   []DockerDevice{},
			},
			expectedType: &dockerConfig{},
		},
		{
			name: "number attr json",
			config: jsonConfigToInterface(t, `
						{
							"Config": {
								"image": "redis:3.2",
								"pids_limit": "2"
			                }
						}`),
			spec: dockerSpec,
			ctx:  defaultCtx,
			expected: &dockerConfig{
				Image:     "redis:3.2",
				PidsLimit: helper.Int64ToPtr(2),
				Devices:   []DockerDevice{},
			},
			expectedType: &dockerConfig{},
		},
		{
			name: "number attr interpolated",
			config: hclConfigToInterface(t, `
						config {
							image = "redis:3.2"
							pids_limit  = "${2 + 2}"
						}`),
			spec: dockerSpec,
			ctx:  defaultCtx,
			expected: &dockerConfig{
				Image:     "redis:3.2",
				PidsLimit: helper.Int64ToPtr(4),
				Devices:   []DockerDevice{},
			},
			expectedType: &dockerConfig{},
		},
		{
			name: "number attr interploated json",
			config: jsonConfigToInterface(t, `
						{
							"Config": {
								"image": "redis:3.2",
								"pids_limit": "${2 + 2}"
			                }
						}`),
			spec: dockerSpec,
			ctx:  defaultCtx,
			expected: &dockerConfig{
				Image:     "redis:3.2",
				PidsLimit: helper.Int64ToPtr(4),
				Devices:   []DockerDevice{},
			},
			expectedType: &dockerConfig{},
		},
		{
			name: "multi attr",
			config: hclConfigToInterface(t, `
						config {
							image = "redis:3.2"
							args = ["foo", "bar"]
						}`),
			spec: dockerSpec,
			ctx:  defaultCtx,
			expected: &dockerConfig{
				Image:   "redis:3.2",
				Args:    []string{"foo", "bar"},
				Devices: []DockerDevice{},
			},
			expectedType: &dockerConfig{},
		},
		{
			name: "multi attr json",
			config: jsonConfigToInterface(t, `
						{
							"Config": {
								"image": "redis:3.2",
								"args": ["foo", "bar"]
			                }
						}`),
			spec: dockerSpec,
			ctx:  defaultCtx,
			expected: &dockerConfig{
				Image:   "redis:3.2",
				Args:    []string{"foo", "bar"},
				Devices: []DockerDevice{},
			},
			expectedType: &dockerConfig{},
		},
		{
			name: "multi attr variables",
			config: hclConfigToInterface(t, `
						config {
							image = "redis:3.2"
							args = ["${NOMAD_META_hello}", "${NOMAD_ALLOC_INDEX}"]
							pids_limit = "${NOMAD_ALLOC_INDEX + 2}"
						}`),
			spec: dockerSpec,
			ctx:  variableCtx,
			expected: &dockerConfig{
				Image:     "redis:3.2",
				Args:      []string{"world", "2"},
				PidsLimit: helper.Int64ToPtr(4),
				Devices:   []DockerDevice{},
			},
			expectedType: &dockerConfig{},
		},
		{
			name: "multi attr variables json",
			config: jsonConfigToInterface(t, `
						{
							"Config": {
								"image": "redis:3.2",
								"args": ["foo", "bar"]
			                }
						}`),
			spec: dockerSpec,
			ctx:  defaultCtx,
			expected: &dockerConfig{
				Image:   "redis:3.2",
				Args:    []string{"foo", "bar"},
				Devices: []DockerDevice{},
			},
			expectedType: &dockerConfig{},
		},
		{
			name: "port_map",
			config: hclConfigToInterface(t, `
			config {
				image = "redis:3.2"
				port_map {
					foo = "db"
					bar = "db2"
				}
			}`),
			spec: dockerSpec,
			ctx:  defaultCtx,
			expected: &dockerConfig{
				Image: "redis:3.2",
				PortMap: map[string]string{
					"foo": "db",
					"bar": "db2",
				},
				Devices: []DockerDevice{},
			},
			expectedType: &dockerConfig{},
		},
		{
			name: "port_map json",
			config: jsonConfigToInterface(t, `
							{
								"Config": {
									"image": "redis:3.2",
									"port_map": [{
										"foo": "db",
										"bar": "db2"
									}]
				                }
							}`),
			spec: dockerSpec,
			ctx:  defaultCtx,
			expected: &dockerConfig{
				Image: "redis:3.2",
				PortMap: map[string]string{
					"foo": "db",
					"bar": "db2",
				},
				Devices: []DockerDevice{},
			},
			expectedType: &dockerConfig{},
		},
		{
			name: "devices",
			config: hclConfigToInterface(t, `
						config {
							image = "redis:3.2"
							devices = [
								{
									host_path = "/dev/sda1"
									container_path = "/dev/xvdc"
									cgroup_permissions = "r"
								},
								{
									host_path = "/dev/sda2"
									container_path = "/dev/xvdd"
								}
							]
						}`),
			spec: dockerSpec,
			ctx:  defaultCtx,
			expected: &dockerConfig{
				Image: "redis:3.2",
				Devices: []DockerDevice{
					{
						HostPath:          "/dev/sda1",
						ContainerPath:     "/dev/xvdc",
						CgroupPermissions: "r",
					},
					{
						HostPath:      "/dev/sda2",
						ContainerPath: "/dev/xvdd",
					},
				},
			},
			expectedType: &dockerConfig{},
		},
		{
			name: "devices json",
			config: jsonConfigToInterface(t, `
							{
								"Config": {
									"image": "redis:3.2",
									"devices": [
										{
											"host_path": "/dev/sda1",
											"container_path": "/dev/xvdc",
											"cgroup_permissions": "r"
										},
										{
											"host_path": "/dev/sda2",
											"container_path": "/dev/xvdd"
										}
									]
				                }
							}`),
			spec: dockerSpec,
			ctx:  defaultCtx,
			expected: &dockerConfig{
				Image: "redis:3.2",
				Devices: []DockerDevice{
					{
						HostPath:          "/dev/sda1",
						ContainerPath:     "/dev/xvdc",
						CgroupPermissions: "r",
					},
					{
						HostPath:      "/dev/sda2",
						ContainerPath: "/dev/xvdd",
					},
				},
			},
			expectedType: &dockerConfig{},
		},
	}

	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			t.Logf("Val: % #v", pretty.Formatter(c.config))
			// Parse the interface
			ctyValue, diag := ParseHclInterface(c.config, c.spec, c.ctx)
			if diag.HasErrors() {
				for _, err := range diag.Errs() {
					t.Error(err)
				}
				t.FailNow()
			}

			// Convert cty-value to go structs
			require.NoError(t, gocty.FromCtyValue(ctyValue, c.expectedType))

			require.EqualValues(t, c.expected, c.expectedType)

		})
	}
}
<commit_msg>plugins: update hclutils test<commit_after>package hclutils_test

import (
	"testing"

	"github.com/hashicorp/hcl"
	"github.com/hashicorp/hcl/hcl/ast"
	"github.com/hashicorp/hcl2/hcldec"
	"github.com/hashicorp/nomad/drivers/docker"
	"github.com/hashicorp/nomad/helper/pluginutils/hclspecutils"
	"github.com/hashicorp/nomad/helper/pluginutils/hclutils"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/plugins/drivers"
	"github.com/kr/pretty"
	"github.com/mitchellh/mapstructure"
	"github.com/stretchr/testify/require"
	"github.com/ugorji/go/codec"
	"github.com/zclconf/go-cty/cty"
)

func hclConfigToInterface(t *testing.T, config string) interface{} {
	t.Helper()

	// Parse as we do in the jobspec parser
	root, err := hcl.Parse(config)
	if err != nil {
		t.Fatalf("failed to hcl parse the config: %v", err)
	}

	// Top-level item should be a list
	list, ok := root.Node.(*ast.ObjectList)
	if !ok {
		t.Fatalf("root should be an object")
	}

	var m map[string]interface{}
	if err := hcl.DecodeObject(&m, list.Items[0]); err != nil {
		t.Fatalf("failed to decode object: %v", err)
	}

	var m2 map[string]interface{}
	if err := mapstructure.WeakDecode(m, &m2); err != nil {
		t.Fatalf("failed to weak decode object: %v", err)
	}

	return m2["config"]
}

func jsonConfigToInterface(t *testing.T, config string) interface{} {
	t.Helper()

	// Decode from json
	dec := codec.NewDecoderBytes([]byte(config), structs.JsonHandle)

	var m map[string]interface{}
	err := dec.Decode(&m)
	if err != nil {
		t.Fatalf("failed to decode: %v", err)
	}

	return m["Config"]
}

func TestParseHclInterface_Hcl(t *testing.T) {
	dockerDriver := new(docker.Driver)
	dockerSpec, err := dockerDriver.TaskConfigSchema()
	require.NoError(t, err)
	dockerDecSpec, diags := hclspecutils.Convert(dockerSpec)
	require.False(t, diags.HasErrors())

	vars := map[string]cty.Value{
		"NOMAD_ALLOC_INDEX": cty.NumberIntVal(2),
		"NOMAD_META_hello":  cty.StringVal("world"),
	}

	cases := []struct {
		name         string
		config       interface{}
		spec         hcldec.Spec
		vars         map[string]cty.Value
		expected     interface{}
		expectedType interface{}
	}{
		{
			name: "single string attr",
			config: hclConfigToInterface(t, `
			config {
				image = "redis:3.2"
			}`),
			spec: dockerDecSpec,
			expected: &docker.TaskConfig{
				Image:   "redis:3.2",
				Devices: []docker.DockerDevice{},
				Mounts:  []docker.DockerMount{},
			},
			expectedType: &docker.TaskConfig{},
		},
		{
			name: "single string attr json",
			config: jsonConfigToInterface(t, `
						{
							"Config": {
								"image": "redis:3.2"
			                }
						}`),
			spec: dockerDecSpec,
			expected: &docker.TaskConfig{
				Image:   "redis:3.2",
				Devices: []docker.DockerDevice{},
				Mounts:  []docker.DockerMount{},
			},
			expectedType: &docker.TaskConfig{},
		},
		{
			name: "number attr",
			config: hclConfigToInterface(t, `
						config {
							image = "redis:3.2"
							pids_limit  = 2
						}`),
			spec: dockerDecSpec,
			expected: &docker.TaskConfig{
				Image:     "redis:3.2",
				PidsLimit: 2,
				Devices:   []docker.DockerDevice{},
				Mounts:    []docker.DockerMount{},
			},
			expectedType: &docker.TaskConfig{},
		},
		{
			name: "number attr json",
			config: jsonConfigToInterface(t, `
						{
							"Config": {
								"image": "redis:3.2",
								"pids_limit": "2"
			                }
						}`),
			spec: dockerDecSpec,
			expected: &docker.TaskConfig{
				Image:     "redis:3.2",
				PidsLimit: 2,
				Devices:   []docker.DockerDevice{},
				Mounts:    []docker.DockerMount{},
			},
			expectedType: &docker.TaskConfig{},
		},
		{
			name: "number attr interpolated",
			config: hclConfigToInterface(t, `
						config {
							image = "redis:3.2"
							pids_limit  = "${2 + 2}"
						}`),
			spec: dockerDecSpec,
			expected: &docker.TaskConfig{
				Image:     "redis:3.2",
				PidsLimit: 4,
				Devices:   []docker.DockerDevice{},
				Mounts:    []docker.DockerMount{},
			},
			expectedType: &docker.TaskConfig{},
		},
		{
			name: "number attr interploated json",
			config: jsonConfigToInterface(t, `
						{
							"Config": {
								"image": "redis:3.2",
								"pids_limit": "${2 + 2}"
			                }
						}`),
			spec: dockerDecSpec,
			expected: &docker.TaskConfig{
				Image:     "redis:3.2",
				PidsLimit: 4,
				Devices:   []docker.DockerDevice{},
				Mounts:    []docker.DockerMount{},
			},
			expectedType: &docker.TaskConfig{},
		},
		{
			name: "multi attr",
			config: hclConfigToInterface(t, `
						config {
							image = "redis:3.2"
							args = ["foo", "bar"]
						}`),
			spec: dockerDecSpec,
			expected: &docker.TaskConfig{
				Image:   "redis:3.2",
				Args:    []string{"foo", "bar"},
				Devices: []docker.DockerDevice{},
				Mounts:  []docker.DockerMount{},
			},
			expectedType: &docker.TaskConfig{},
		},
		{
			name: "multi attr json",
			config: jsonConfigToInterface(t, `
						{
							"Config": {
								"image": "redis:3.2",
								"args": ["foo", "bar"]
			                }
						}`),
			spec: dockerDecSpec,
			expected: &docker.TaskConfig{
				Image:   "redis:3.2",
				Args:    []string{"foo", "bar"},
				Devices: []docker.DockerDevice{},
				Mounts:  []docker.DockerMount{},
			},
			expectedType: &docker.TaskConfig{},
		},
		{
			name: "multi attr variables",
			config: hclConfigToInterface(t, `
						config {
							image = "redis:3.2"
							args = ["${NOMAD_META_hello}", "${NOMAD_ALLOC_INDEX}"]
							pids_limit = "${NOMAD_ALLOC_INDEX + 2}"
						}`),
			spec: dockerDecSpec,
			vars: vars,
			expected: &docker.TaskConfig{
				Image:     "redis:3.2",
				Args:      []string{"world", "2"},
				PidsLimit: 4,
				Devices:   []docker.DockerDevice{},
				Mounts:    []docker.DockerMount{},
			},
			expectedType: &docker.TaskConfig{},
		},
		{
			name: "multi attr variables json",
			config: jsonConfigToInterface(t, `
						{
							"Config": {
								"image": "redis:3.2",
								"args": ["foo", "bar"]
			                }
						}`),
			spec: dockerDecSpec,
			expected: &docker.TaskConfig{
				Image:   "redis:3.2",
				Args:    []string{"foo", "bar"},
				Devices: []docker.DockerDevice{},
				Mounts:  []docker.DockerMount{},
			},
			expectedType: &docker.TaskConfig{},
		},
		{
			name: "port_map",
			config: hclConfigToInterface(t, `
			config {
				image = "redis:3.2"
				port_map {
					foo = 1234
					bar = 5678
				}
			}`),
			spec: dockerDecSpec,
			expected: &docker.TaskConfig{
				Image: "redis:3.2",
				PortMap: map[string]int{
					"foo": 1234,
					"bar": 5678,
				},
				Devices: []docker.DockerDevice{},
				Mounts:  []docker.DockerMount{},
			},
			expectedType: &docker.TaskConfig{},
		},
		{
			name: "port_map json",
			config: jsonConfigToInterface(t, `
							{
								"Config": {
									"image": "redis:3.2",
									"port_map": [{
										"foo": 1234,
										"bar": 5678
									}]
				                }
							}`),
			spec: dockerDecSpec,
			expected: &docker.TaskConfig{
				Image: "redis:3.2",
				PortMap: map[string]int{
					"foo": 1234,
					"bar": 5678,
				},
				Devices: []docker.DockerDevice{},
				Mounts:  []docker.DockerMount{},
			},
			expectedType: &docker.TaskConfig{},
		},
		{
			name: "devices",
			config: hclConfigToInterface(t, `
						config {
							image = "redis:3.2"
							devices = [
								{
									host_path = "/dev/sda1"
									container_path = "/dev/xvdc"
									cgroup_permissions = "r"
								},
								{
									host_path = "/dev/sda2"
									container_path = "/dev/xvdd"
								}
							]
						}`),
			spec: dockerDecSpec,
			expected: &docker.TaskConfig{
				Image: "redis:3.2",
				Devices: []docker.DockerDevice{
					{
						HostPath:          "/dev/sda1",
						ContainerPath:     "/dev/xvdc",
						CgroupPermissions: "r",
					},
					{
						HostPath:      "/dev/sda2",
						ContainerPath: "/dev/xvdd",
					},
				},
				Mounts: []docker.DockerMount{},
			},
			expectedType: &docker.TaskConfig{},
		},
		{
			name: "docker_logging",
			config: hclConfigToInterface(t, `
				config {
					image = "redis:3.2"
					network_mode = "host"
					dns_servers = ["169.254.1.1"]
					logging {
					    type = "syslog"
					    config {
						tag  = "driver-test"
					    }
					}
				}`),
			spec: dockerDecSpec,
			expected: &docker.TaskConfig{
				Image:       "redis:3.2",
				NetworkMode: "host",
				DNSServers:  []string{"169.254.1.1"},
				Logging: docker.DockerLogging{
					Type: "syslog",
					Config: map[string]string{
						"tag": "driver-test",
					},
				},
			},
			expectedType: &docker.TaskConfig{},
		},
		{
			name: "docker_json",
			config: jsonConfigToInterface(t, `
					{
						"Config": {
							"image": "redis:3.2",
							"devices": [
								{
									"host_path": "/dev/sda1",
									"container_path": "/dev/xvdc",
									"cgroup_permissions": "r"
								},
								{
									"host_path": "/dev/sda2",
									"container_path": "/dev/xvdd"
								}
							]
				}
					}`),
			spec: dockerDecSpec,
			expected: &docker.TaskConfig{
				Image: "redis:3.2",
				Devices: []docker.DockerDevice{
					{
						HostPath:          "/dev/sda1",
						ContainerPath:     "/dev/xvdc",
						CgroupPermissions: "r",
					},
					{
						HostPath:      "/dev/sda2",
						ContainerPath: "/dev/xvdd",
					},
				},
				Mounts: []docker.DockerMount{},
			},
			expectedType: &docker.TaskConfig{},
		},
	}

	for _, c := range cases {
		c := c
		t.Run(c.name, func(t *testing.T) {
			t.Logf("Val: % #v", pretty.Formatter(c.config))
			// Parse the interface
			ctyValue, diag := hclutils.ParseHclInterface(c.config, c.spec, c.vars)
			if diag.HasErrors() {
				for _, err := range diag.Errs() {
					t.Error(err)
				}
				t.FailNow()
			}

			// Test encoding
			taskConfig := &drivers.TaskConfig{}
			require.NoError(t, taskConfig.EncodeDriverConfig(ctyValue))

			// Test decoding
			require.NoError(t, taskConfig.DecodeDriverConfig(c.expectedType))

			require.EqualValues(t, c.expected, c.expectedType)

		})
	}
}
<|endoftext|>


<commit_before>package clusterconfigs

import (
	"errors"
	"fmt"
	"strconv"
	"strings"
	"github.com/radanalyticsio/oshinko-rest/models"
	kclient "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/api"
)

var defaultConfig models.NewClusterConfig = models.NewClusterConfig{
								MasterCount: 1,
	                                                        WorkerCount: 1,
								Name: "default",
								SparkMasterConfig: "",
								SparkWorkerConfig: ""}

const Defaultname = "default"
const failOnMissing = true
const allowMissing = false
const DefaultConfigPath = "/etc/oshinko-cluster-configs/"

const MasterCountMustBeOne = "Cluster configuration must have a masterCount of 1"
const WorkerCountMustBeAtLeastOne = "Cluster configuration may not have a workerCount less than 1"
const NamedConfigDoesNotExist = "Named config '%s' does not exist"
const ErrorWhileProcessing = "Error while processing %s: %s"

// This function is meant to support testability
func GetDefaultConfig() models.NewClusterConfig {
	return defaultConfig
}

func assignConfig(res *models.NewClusterConfig, src models.NewClusterConfig) {
	if src.MasterCount != 0 {
		res.MasterCount = src.MasterCount
	}
	if src.WorkerCount != 0 {
		res.WorkerCount = src.WorkerCount
	}

	if src.SparkMasterConfig != "" {
		res.SparkMasterConfig = src.SparkMasterConfig
	}
	if src.SparkWorkerConfig != "" {
		res.SparkWorkerConfig = src.SparkWorkerConfig
	}
}

func checkConfiguration(config models.NewClusterConfig) error {
	var err error
	if config.MasterCount != 1 {
		err = errors.New(MasterCountMustBeOne)
	} else if config.WorkerCount < 1 {
		err = errors.New(WorkerCountMustBeAtLeastOne)
	}
	return err
}

func getInt64(value, configmapname string) (int64, error) {
	i, err := strconv.Atoi(strings.Trim(value, "\n"))
	if err != nil {
		err = errors.New(fmt.Sprintf(ErrorWhileProcessing, configmapname, errors.New("expected integer")))
	}
	return int64(i), err
}

func process(config *models.NewClusterConfig, name, value, configmapname string) error {

	var err error

	// At present we only have a single level of configs, but if/when we have
	// nested configs then we would descend through the levels beginning here with
	// the first element in the name
	switch name {
	case "mastercount":
		config.MasterCount, err = getInt64(value, configmapname + ".mastercount")
	case "workercount":
		config.WorkerCount, err = getInt64(value, configmapname + ".workercount")
	case "sparkmasterconfig":
                config.SparkMasterConfig = strings.Trim(value, "\n")
	case "sparkworkerconfig":
                config.SparkWorkerConfig = strings.Trim(value, "\n")
	}
	return err
}

func checkForConfigMap(name string, failOnMissing bool, cm kclient.ConfigMapsInterface) (*api.ConfigMap, error) {
	cmap, err := cm.Get(name)
	if (cmap == nil || len(cmap.Data) == 0) && failOnMissing == false {
		return cmap, nil
	}
	return cmap, err
}

func readConfig(name string, res *models.NewClusterConfig, failOnMissing bool, cm kclient.ConfigMapsInterface) (err error) {
        cmap, err := checkForConfigMap(name, failOnMissing, cm)
	if err == nil && cmap != nil {
                for n, v := range (cmap.Data) {
			err = process(res, n, v, name)
			if err != nil {
				break
			}
		}
	}
	return err
}

func loadConfig(name string, cm kclient.ConfigMapsInterface) (res models.NewClusterConfig, err error) {
	// If the default config has been modified use those mods.
	res = defaultConfig
	err = readConfig(Defaultname, &res, allowMissing, cm)
	if err == nil && name != "" && name != Defaultname {
		err = readConfig(name, &res, failOnMissing, cm)
	}
	return res, err
}

func GetClusterConfig(config *models.NewClusterConfig, cm kclient.ConfigMapsInterface) (res models.NewClusterConfig, err error) {
        var name string = ""
	if config != nil {
	   name = config.Name
	}
	res, err = loadConfig(name, cm)
	if err == nil && config != nil {
		assignConfig(&res, *config)
	}

	// Check that the final configuration is valid
	if err == nil {
		err = checkConfiguration(res)
	}
	return res, err
}
<commit_msg>Get rid of configpath constant<commit_after>package clusterconfigs

import (
	"errors"
	"fmt"
	"strconv"
	"strings"
	"github.com/radanalyticsio/oshinko-rest/models"
	kclient "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/api"
)

var defaultConfig models.NewClusterConfig = models.NewClusterConfig{
								MasterCount: 1,
	                                                        WorkerCount: 1,
								Name: "default",
								SparkMasterConfig: "",
								SparkWorkerConfig: ""}

const Defaultname = "default"
const failOnMissing = true
const allowMissing = false

const MasterCountMustBeOne = "Cluster configuration must have a masterCount of 1"
const WorkerCountMustBeAtLeastOne = "Cluster configuration may not have a workerCount less than 1"
const NamedConfigDoesNotExist = "Named config '%s' does not exist"
const ErrorWhileProcessing = "Error while processing %s: %s"

// This function is meant to support testability
func GetDefaultConfig() models.NewClusterConfig {
	return defaultConfig
}

func assignConfig(res *models.NewClusterConfig, src models.NewClusterConfig) {
	if src.MasterCount != 0 {
		res.MasterCount = src.MasterCount
	}
	if src.WorkerCount != 0 {
		res.WorkerCount = src.WorkerCount
	}

	if src.SparkMasterConfig != "" {
		res.SparkMasterConfig = src.SparkMasterConfig
	}
	if src.SparkWorkerConfig != "" {
		res.SparkWorkerConfig = src.SparkWorkerConfig
	}
}

func checkConfiguration(config models.NewClusterConfig) error {
	var err error
	if config.MasterCount != 1 {
		err = errors.New(MasterCountMustBeOne)
	} else if config.WorkerCount < 1 {
		err = errors.New(WorkerCountMustBeAtLeastOne)
	}
	return err
}

func getInt64(value, configmapname string) (int64, error) {
	i, err := strconv.Atoi(strings.Trim(value, "\n"))
	if err != nil {
		err = errors.New(fmt.Sprintf(ErrorWhileProcessing, configmapname, errors.New("expected integer")))
	}
	return int64(i), err
}

func process(config *models.NewClusterConfig, name, value, configmapname string) error {

	var err error

	// At present we only have a single level of configs, but if/when we have
	// nested configs then we would descend through the levels beginning here with
	// the first element in the name
	switch name {
	case "mastercount":
		config.MasterCount, err = getInt64(value, configmapname + ".mastercount")
	case "workercount":
		config.WorkerCount, err = getInt64(value, configmapname + ".workercount")
	case "sparkmasterconfig":
                config.SparkMasterConfig = strings.Trim(value, "\n")
	case "sparkworkerconfig":
                config.SparkWorkerConfig = strings.Trim(value, "\n")
	}
	return err
}

func checkForConfigMap(name string, failOnMissing bool, cm kclient.ConfigMapsInterface) (*api.ConfigMap, error) {
	cmap, err := cm.Get(name)
	if (cmap == nil || len(cmap.Data) == 0) && failOnMissing == false {
		return cmap, nil
	}
	return cmap, err
}

func readConfig(name string, res *models.NewClusterConfig, failOnMissing bool, cm kclient.ConfigMapsInterface) (err error) {
        cmap, err := checkForConfigMap(name, failOnMissing, cm)
	if err == nil && cmap != nil {
                for n, v := range (cmap.Data) {
			err = process(res, n, v, name)
			if err != nil {
				break
			}
		}
	}
	return err
}

func loadConfig(name string, cm kclient.ConfigMapsInterface) (res models.NewClusterConfig, err error) {
	// If the default config has been modified use those mods.
	res = defaultConfig
	err = readConfig(Defaultname, &res, allowMissing, cm)
	if err == nil && name != "" && name != Defaultname {
		err = readConfig(name, &res, failOnMissing, cm)
	}
	return res, err
}

func GetClusterConfig(config *models.NewClusterConfig, cm kclient.ConfigMapsInterface) (res models.NewClusterConfig, err error) {
        var name string = ""
	if config != nil {
	   name = config.Name
	}
	res, err = loadConfig(name, cm)
	if err == nil && config != nil {
		assignConfig(&res, *config)
	}

	// Check that the final configuration is valid
	if err == nil {
		err = checkConfiguration(res)
	}
	return res, err
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package juju

import (
	"github.com/flaviamissi/go-elb/aws"
	"github.com/flaviamissi/go-elb/elb"
	"github.com/globocom/config"
	"github.com/globocom/tsuru/db"
	"github.com/globocom/tsuru/log"
	"github.com/globocom/tsuru/provision"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

// loadBalancer represents an ELB instance.
type loadBalancer struct {
	Name    string
	DNSName string
}

// ELBManager manages load balancers within Amazon Elastic Load Balancer.
//
// If juju:use-elb is true on tsuru.conf, this manager will be used for
// managing load balancers on tsuru.
//
// It uses db package and adds a new collection to tsuru's DB. The name of the
// collection is also defined in the configuration file (juju:elb-collection).
type ELBManager struct {
	e *elb.ELB
}

func (m *ELBManager) collection() *mgo.Collection {
	name, err := config.GetString("juju:elb-collection")
	if err != nil {
		log.Fatal("juju:elb-collection is undefined on config file.")
	}
	conn, err := db.Conn()
	if err != nil {
		log.Printf("[juju] Failed to connect to the database: %s", err)
		return nil
	}
	return conn.Collection(name)
}

func (m *ELBManager) elb() *elb.ELB {
	if m.e == nil {
		access, err := config.GetString("aws:access-key-id")
		if err != nil {
			log.Fatal(err)
		}
		secret, err := config.GetString("aws:secret-access-key")
		if err != nil {
			log.Fatal(err)
		}
		endpoint, err := config.GetString("juju:elb-endpoint")
		if err != nil {
			log.Fatal(err)
		}
		auth := aws.Auth{AccessKey: access, SecretKey: secret}
		region := aws.Region{ELBEndpoint: endpoint}
		m.e = elb.New(auth, region)
	}
	return m.e
}

func (m *ELBManager) vpc() bool {
	vpc, _ := config.GetBool("juju:elb-use-vpc")
	return vpc
}

func (m *ELBManager) Create(app provision.Named) error {
	options := elb.CreateLoadBalancer{
		Name: app.GetName(),
		Listeners: []elb.Listener{
			{
				InstancePort:     80,
				InstanceProtocol: "HTTP",
				LoadBalancerPort: 80,
				Protocol:         "HTTP",
			},
		},
	}
	var err error
	if m.vpc() {
		options.Subnets, err = config.GetList("juju:elb-vpc-subnets")
		if err != nil {
			log.Fatal(err)
		}
		options.SecurityGroups, err = config.GetList("juju:elb-vpc-secgroups")
		if err != nil {
			log.Fatal(err)
		}
		options.Scheme = "internal"
	} else {
		options.AvailZones, err = config.GetList("juju:elb-avail-zones")
		if err != nil {
			log.Fatal(err)
		}
	}
	resp, err := m.elb().CreateLoadBalancer(&options)
	if err != nil {
		return err
	}
	lb := loadBalancer{Name: app.GetName(), DNSName: resp.DNSName}
	return m.collection().Insert(lb)
}

func (m *ELBManager) Destroy(app provision.Named) error {
	_, err := m.elb().DeleteLoadBalancer(app.GetName())
	if err != nil {
		return err
	}
	return m.collection().Remove(bson.M{"name": app.GetName()})
}

func (m *ELBManager) Register(app provision.Named, units ...provision.Unit) error {
	ids := make([]string, len(units))
	for i, u := range units {
		ids[i] = u.InstanceId
	}
	_, err := m.elb().RegisterInstancesWithLoadBalancer(ids, app.GetName())
	return err
}

func (m *ELBManager) Deregister(app provision.Named, units ...provision.Unit) error {
	ids := make([]string, len(units))
	for i, u := range units {
		ids[i] = u.InstanceId
	}
	_, err := m.elb().DeregisterInstancesFromLoadBalancer(ids, app.GetName())
	return err
}

func (m *ELBManager) Addr(app provision.Named) (string, error) {
	var lb loadBalancer
	err := m.collection().Find(bson.M{"name": app.GetName()}).One(&lb)
	return lb.DNSName, err
}
<commit_msg>provision/juju: added docs for ELBManager methods<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package juju

import (
	"github.com/flaviamissi/go-elb/aws"
	"github.com/flaviamissi/go-elb/elb"
	"github.com/globocom/config"
	"github.com/globocom/tsuru/db"
	"github.com/globocom/tsuru/log"
	"github.com/globocom/tsuru/provision"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

// loadBalancer represents an ELB instance.
type loadBalancer struct {
	Name    string
	DNSName string
}

// ELBManager manages load balancers within Amazon Elastic Load Balancer.
//
// If juju:use-elb is true on tsuru.conf, this manager will be used for
// managing load balancers on tsuru.
//
// It uses db package and adds a new collection to tsuru's DB. The name of the
// collection is also defined in the configuration file (juju:elb-collection).
type ELBManager struct {
	e *elb.ELB
}

func (m *ELBManager) collection() *mgo.Collection {
	name, err := config.GetString("juju:elb-collection")
	if err != nil {
		log.Fatal("juju:elb-collection is undefined on config file.")
	}
	conn, err := db.Conn()
	if err != nil {
		log.Printf("[juju] Failed to connect to the database: %s", err)
		return nil
	}
	return conn.Collection(name)
}

func (m *ELBManager) elb() *elb.ELB {
	if m.e == nil {
		access, err := config.GetString("aws:access-key-id")
		if err != nil {
			log.Fatal(err)
		}
		secret, err := config.GetString("aws:secret-access-key")
		if err != nil {
			log.Fatal(err)
		}
		endpoint, err := config.GetString("juju:elb-endpoint")
		if err != nil {
			log.Fatal(err)
		}
		auth := aws.Auth{AccessKey: access, SecretKey: secret}
		region := aws.Region{ELBEndpoint: endpoint}
		m.e = elb.New(auth, region)
	}
	return m.e
}

func (m *ELBManager) vpc() bool {
	vpc, _ := config.GetBool("juju:elb-use-vpc")
	return vpc
}

// Create creates a new Elastic Load Balancing instance for the given app. The
// name of the instance will be the same as the name of the app.
func (m *ELBManager) Create(app provision.Named) error {
	options := elb.CreateLoadBalancer{
		Name: app.GetName(),
		Listeners: []elb.Listener{
			{
				InstancePort:     80,
				InstanceProtocol: "HTTP",
				LoadBalancerPort: 80,
				Protocol:         "HTTP",
			},
		},
	}
	var err error
	if m.vpc() {
		options.Subnets, err = config.GetList("juju:elb-vpc-subnets")
		if err != nil {
			log.Fatal(err)
		}
		options.SecurityGroups, err = config.GetList("juju:elb-vpc-secgroups")
		if err != nil {
			log.Fatal(err)
		}
		options.Scheme = "internal"
	} else {
		options.AvailZones, err = config.GetList("juju:elb-avail-zones")
		if err != nil {
			log.Fatal(err)
		}
	}
	resp, err := m.elb().CreateLoadBalancer(&options)
	if err != nil {
		return err
	}
	lb := loadBalancer{Name: app.GetName(), DNSName: resp.DNSName}
	return m.collection().Insert(lb)
}

// Destroy destroys an Elastic Load Balancing instance from AWS. It matches the
// name of the given app.
func (m *ELBManager) Destroy(app provision.Named) error {
	_, err := m.elb().DeleteLoadBalancer(app.GetName())
	if err != nil {
		return err
	}
	return m.collection().Remove(bson.M{"name": app.GetName()})
}

// Register adds new EC2 instances (represented as units) to a load balancer.
func (m *ELBManager) Register(app provision.Named, units ...provision.Unit) error {
	ids := make([]string, len(units))
	for i, u := range units {
		ids[i] = u.InstanceId
	}
	_, err := m.elb().RegisterInstancesWithLoadBalancer(ids, app.GetName())
	return err
}

// Deregister removes EC2 instances (represented as units) from a load
// balancer.
func (m *ELBManager) Deregister(app provision.Named, units ...provision.Unit) error {
	ids := make([]string, len(units))
	for i, u := range units {
		ids[i] = u.InstanceId
	}
	_, err := m.elb().DeregisterInstancesFromLoadBalancer(ids, app.GetName())
	return err
}

// Addr returns the dns-name of a load balancer, which is also the DNS name of
// the app.
func (m *ELBManager) Addr(app provision.Named) (string, error) {
	var lb loadBalancer
	err := m.collection().Find(bson.M{"name": app.GetName()}).One(&lb)
	return lb.DNSName, err
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package github

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"time"
)

type Logger interface {
	Printf(s string, v ...interface{})
}

type Client struct {
	// If Logger is non-nil, log all method calls with it.
	Logger Logger

	client *http.Client
	token  string
	base   string
	dry    bool
	fake   bool
}

const (
	githubBase = "https://api.github.com"
	maxRetries = 8
	retryDelay = 2 * time.Second
)

// NewClient creates a new fully operational GitHub client.
func NewClient(token string) *Client {
	return &Client{
		client: &http.Client{},
		token:  token,
		base:   githubBase,
		dry:    false,
	}
}

// NewDryRunClient creates a new client that will not perform mutating actions
// such as setting statuses or commenting, but it will still query GitHub and
// use up API tokens.
func NewDryRunClient(token string) *Client {
	return &Client{
		client: &http.Client{},
		token:  token,
		base:   githubBase,
		dry:    true,
	}
}

// NewFakeClient creates a new client that will not perform any actions at all.
func NewFakeClient() *Client {
	return &Client{
		fake: true,
		dry:  true,
	}
}

func (c *Client) log(methodName string, args ...interface{}) {
	if c.Logger == nil {
		return
	}
	var as []string
	for _, arg := range args {
		as = append(as, fmt.Sprintf("%v", arg))
	}
	c.Logger.Printf("%s(%s)", methodName, strings.Join(as, ", "))
}

var timeSleep = time.Sleep

// Retry on transport failures. Retries on 500s and retries after sleep on
// ratelimit exceeded.
func (c *Client) request(method, path string, body interface{}) (*http.Response, error) {
	var resp *http.Response
	var err error
	backoff := retryDelay
	for retries := 0; retries < maxRetries; retries++ {
		resp, err = c.doRequest(method, path, body)
		if err == nil {
			// If we are out of API tokens, sleep first. The X-RateLimit-Reset
			// header tells us the time at which we can request again.
			if resp.StatusCode == 403 && resp.Header.Get("X-RateLimit-Remaining") == "0" {
				resp.Body.Close()
				var t int
				if t, err = strconv.Atoi(resp.Header.Get("X-RateLimit-Reset")); err == nil {
					timeSleep(time.Unix(int64(t), 0).Sub(time.Now()) + time.Second)
				}
			} else if resp.StatusCode < 500 {
				break
			} else {
				resp.Body.Close()
				timeSleep(backoff)
				backoff *= 2
			}
		} else {
			timeSleep(backoff)
			backoff *= 2
		}
	}
	return resp, err
}

func (c *Client) doRequest(method, path string, body interface{}) (*http.Response, error) {
	var buf io.Reader
	if body != nil {
		b, err := json.Marshal(body)
		if err != nil {
			return nil, err
		}
		buf = bytes.NewBuffer(b)
	}
	req, err := http.NewRequest(method, path, buf)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Authorization", "Token "+c.token)
	req.Header.Add("Accept", "application/vnd.github.v3+json")
	// Disable keep-alive so that we don't get flakes when GitHub closes the
	// connection prematurely.
	// https://go-review.googlesource.com/#/c/3210/ fixed it for GET, but not
	// for POST.
	req.Close = true
	return c.client.Do(req)
}

// IsMember returns whether or not the user is a member of the org.
func (c *Client) IsMember(org, user string) (bool, error) {
	c.log("IsMember", org, user)
	if c.fake {
		return true, nil
	}
	resp, err := c.request(http.MethodGet, fmt.Sprintf("%s/orgs/%s/members/%s", c.base, org, user), nil)
	if err != nil {
		return false, err
	}
	defer resp.Body.Close()
	if resp.StatusCode == 204 {
		return true, nil
	} else if resp.StatusCode == 404 {
		return false, nil
	} else if resp.StatusCode == 302 {
		return false, fmt.Errorf("requester is not %s org member", org)
	}
	return false, fmt.Errorf("unexpected status: %s", resp.Status)
}

// CreateComment creates a comment on the issue.
func (c *Client) CreateComment(org, repo string, number int, comment string) error {
	c.log("CreateComment", org, repo, number, comment)
	if c.dry {
		return nil
	}

	ic := IssueComment{
		Body: comment,
	}
	resp, err := c.request(http.MethodPost, fmt.Sprintf("%s/repos/%s/%s/issues/%d/comments", c.base, org, repo, number), ic)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 201 {
		return fmt.Errorf("response not 201: %s", resp.Status)
	}
	return nil
}

// DeleteComment deletes the comment.
func (c *Client) DeleteComment(org, repo string, ID int) error {
	c.log("DeleteComment", org, repo, ID)
	if c.dry {
		return nil
	}

	resp, err := c.request(http.MethodDelete, fmt.Sprintf("%s/repos/%s/%s/issues/comments/%d", c.base, org, repo, ID), nil)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 204 {
		return fmt.Errorf("response not 204: %s", resp.Status)
	}
	return nil
}

// ListIssueComments returns all comments on an issue. This may use more than
// one API token.
func (c *Client) ListIssueComments(org, repo string, number int) ([]IssueComment, error) {
	c.log("ListIssueComments", org, repo, number)
	if c.fake {
		return nil, nil
	}
	nextURL := fmt.Sprintf("%s/repos/%s/%s/issues/%d/comments?per_page=100", c.base, org, repo, number)
	var comments []IssueComment
	for nextURL != "" {
		resp, err := c.request(http.MethodGet, nextURL, nil)
		if err != nil {
			return nil, err
		}
		defer resp.Body.Close()
		if resp.StatusCode < 200 || resp.StatusCode > 299 {
			return nil, fmt.Errorf("return code not 2XX: %s", resp.Status)
		}

		b, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, err
		}

		var ics []IssueComment
		if err := json.Unmarshal(b, &ics); err != nil {
			return nil, err
		}
		comments = append(comments, ics...)
		nextURL = parseLinks(resp.Header.Get("Link"))["next"]
	}
	return comments, nil
}

// GetPullRequest gets a pull request.
func (c *Client) GetPullRequest(org, repo string, number int) (*PullRequest, error) {
	c.log("GetPullRequest", org, repo, number)
	if c.fake {
		return &PullRequest{}, nil
	}
	resp, err := c.request(http.MethodGet, fmt.Sprintf("%s/repos/%s/%s/pulls/%d", c.base, org, repo, number), nil)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("response not 200: %s", resp.Status)
	}
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	var pr PullRequest
	if err := json.Unmarshal(b, &pr); err != nil {
		return nil, err
	}
	return &pr, nil
}

// CreateStatus creates or updates the status of a commit.
func (c *Client) CreateStatus(org, repo, ref string, s Status) error {
	c.log("CreateStatus", org, repo, ref, s)
	if c.dry {
		return nil
	}

	resp, err := c.request(http.MethodPost, fmt.Sprintf("%s/repos/%s/%s/statuses/%s", c.base, org, repo, ref), s)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 201 {
		return fmt.Errorf("response not 201: %s", resp.Status)
	}
	return nil
}

func (c *Client) AddLabel(org, repo string, number int, label string) error {
	c.log("AddLabel", org, repo, number, label)
	if c.dry {
		return nil
	}
	resp, err := c.request(http.MethodPost, fmt.Sprintf("%s/repos/%s/%s/issues/%d/labels", c.base, org, repo, number), []string{label})
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return fmt.Errorf("response not 200: %s", resp.Status)
	}
	return nil
}

func (c *Client) RemoveLabel(org, repo string, number int, label string) error {
	c.log("RemoveLabel", org, repo, number, label)
	if c.dry {
		return nil
	}
	resp, err := c.request(http.MethodDelete, fmt.Sprintf("%s/repos/%s/%s/issues/%d/labels/%s", c.base, org, repo, number, label), nil)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	// GitHub sometimes returns 200 for this call, which is a bug on their end.
	if resp.StatusCode != 204 && resp.StatusCode != 200 {
		return fmt.Errorf("response not 204: %s", resp.Status)
	}
	return nil
}

func (c *Client) CloseIssue(org, repo string, number int) error {
	c.log("CloseIssue", org, repo, number)
	if c.dry {
		return nil
	}
	resp, err := c.request(http.MethodPatch, fmt.Sprintf("%s/repos/%s/%s/issues/%d", c.base, org, repo, number), map[string]string{"state": "closed"})
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return fmt.Errorf("response not 200: %s", resp.Status)
	}
	return nil
}

// GetRef returns the SHA of the given ref, such as "heads/master".
func (c *Client) GetRef(org, repo, ref string) (string, error) {
	c.log("GetRef", org, repo, ref)
	if c.fake {
		return "", nil
	}
	resp, err := c.request(http.MethodGet, fmt.Sprintf("%s/repos/%s/%s/git/refs/%s", c.base, org, repo, ref), nil)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return "", fmt.Errorf("response not 200: %s", resp.Status)
	}
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	var res struct {
		Object map[string]string `json:"object"`
	}
	if err := json.Unmarshal(b, &res); err != nil {
		return "", err
	}
	return res.Object["sha"], nil
}

// FindIssues uses the github search API to find issues which match a particular query.
// TODO(foxish): we should accept map[string][]string and use net/url properly.
func (c *Client) FindIssues(query string) ([]Issue, error) {
	c.log("FindIssues", query)
	if c.fake {
		return nil, nil
	}
	resp, err := c.request(http.MethodGet, fmt.Sprintf("%s/search/issues?q=%s", c.base, query), nil)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("response not 200: %s", resp.Status)
	}

	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	var issSearchResult IssuesSearchResult
	if err := json.Unmarshal(b, &issSearchResult); err != nil {
		return nil, err
	}
	return issSearchResult.Issues, nil
}
<commit_msg>retryDelay -> initialDelay<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package github

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"time"
)

type Logger interface {
	Printf(s string, v ...interface{})
}

type Client struct {
	// If Logger is non-nil, log all method calls with it.
	Logger Logger

	client *http.Client
	token  string
	base   string
	dry    bool
	fake   bool
}

const (
	githubBase   = "https://api.github.com"
	maxRetries   = 8
	initialDelay = 2 * time.Second
)

// NewClient creates a new fully operational GitHub client.
func NewClient(token string) *Client {
	return &Client{
		client: &http.Client{},
		token:  token,
		base:   githubBase,
		dry:    false,
	}
}

// NewDryRunClient creates a new client that will not perform mutating actions
// such as setting statuses or commenting, but it will still query GitHub and
// use up API tokens.
func NewDryRunClient(token string) *Client {
	return &Client{
		client: &http.Client{},
		token:  token,
		base:   githubBase,
		dry:    true,
	}
}

// NewFakeClient creates a new client that will not perform any actions at all.
func NewFakeClient() *Client {
	return &Client{
		fake: true,
		dry:  true,
	}
}

func (c *Client) log(methodName string, args ...interface{}) {
	if c.Logger == nil {
		return
	}
	var as []string
	for _, arg := range args {
		as = append(as, fmt.Sprintf("%v", arg))
	}
	c.Logger.Printf("%s(%s)", methodName, strings.Join(as, ", "))
}

var timeSleep = time.Sleep

// Retry on transport failures. Retries on 500s and retries after sleep on
// ratelimit exceeded.
func (c *Client) request(method, path string, body interface{}) (*http.Response, error) {
	var resp *http.Response
	var err error
	backoff := initialDelay
	for retries := 0; retries < maxRetries; retries++ {
		resp, err = c.doRequest(method, path, body)
		if err == nil {
			// If we are out of API tokens, sleep first. The X-RateLimit-Reset
			// header tells us the time at which we can request again.
			if resp.StatusCode == 403 && resp.Header.Get("X-RateLimit-Remaining") == "0" {
				resp.Body.Close()
				var t int
				if t, err = strconv.Atoi(resp.Header.Get("X-RateLimit-Reset")); err == nil {
					timeSleep(time.Unix(int64(t), 0).Sub(time.Now()) + time.Second)
				}
			} else if resp.StatusCode < 500 {
				break
			} else {
				resp.Body.Close()
				timeSleep(backoff)
				backoff *= 2
			}
		} else {
			timeSleep(backoff)
			backoff *= 2
		}
	}
	return resp, err
}

func (c *Client) doRequest(method, path string, body interface{}) (*http.Response, error) {
	var buf io.Reader
	if body != nil {
		b, err := json.Marshal(body)
		if err != nil {
			return nil, err
		}
		buf = bytes.NewBuffer(b)
	}
	req, err := http.NewRequest(method, path, buf)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Authorization", "Token "+c.token)
	req.Header.Add("Accept", "application/vnd.github.v3+json")
	// Disable keep-alive so that we don't get flakes when GitHub closes the
	// connection prematurely.
	// https://go-review.googlesource.com/#/c/3210/ fixed it for GET, but not
	// for POST.
	req.Close = true
	return c.client.Do(req)
}

// IsMember returns whether or not the user is a member of the org.
func (c *Client) IsMember(org, user string) (bool, error) {
	c.log("IsMember", org, user)
	if c.fake {
		return true, nil
	}
	resp, err := c.request(http.MethodGet, fmt.Sprintf("%s/orgs/%s/members/%s", c.base, org, user), nil)
	if err != nil {
		return false, err
	}
	defer resp.Body.Close()
	if resp.StatusCode == 204 {
		return true, nil
	} else if resp.StatusCode == 404 {
		return false, nil
	} else if resp.StatusCode == 302 {
		return false, fmt.Errorf("requester is not %s org member", org)
	}
	return false, fmt.Errorf("unexpected status: %s", resp.Status)
}

// CreateComment creates a comment on the issue.
func (c *Client) CreateComment(org, repo string, number int, comment string) error {
	c.log("CreateComment", org, repo, number, comment)
	if c.dry {
		return nil
	}

	ic := IssueComment{
		Body: comment,
	}
	resp, err := c.request(http.MethodPost, fmt.Sprintf("%s/repos/%s/%s/issues/%d/comments", c.base, org, repo, number), ic)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 201 {
		return fmt.Errorf("response not 201: %s", resp.Status)
	}
	return nil
}

// DeleteComment deletes the comment.
func (c *Client) DeleteComment(org, repo string, ID int) error {
	c.log("DeleteComment", org, repo, ID)
	if c.dry {
		return nil
	}

	resp, err := c.request(http.MethodDelete, fmt.Sprintf("%s/repos/%s/%s/issues/comments/%d", c.base, org, repo, ID), nil)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 204 {
		return fmt.Errorf("response not 204: %s", resp.Status)
	}
	return nil
}

// ListIssueComments returns all comments on an issue. This may use more than
// one API token.
func (c *Client) ListIssueComments(org, repo string, number int) ([]IssueComment, error) {
	c.log("ListIssueComments", org, repo, number)
	if c.fake {
		return nil, nil
	}
	nextURL := fmt.Sprintf("%s/repos/%s/%s/issues/%d/comments?per_page=100", c.base, org, repo, number)
	var comments []IssueComment
	for nextURL != "" {
		resp, err := c.request(http.MethodGet, nextURL, nil)
		if err != nil {
			return nil, err
		}
		defer resp.Body.Close()
		if resp.StatusCode < 200 || resp.StatusCode > 299 {
			return nil, fmt.Errorf("return code not 2XX: %s", resp.Status)
		}

		b, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, err
		}

		var ics []IssueComment
		if err := json.Unmarshal(b, &ics); err != nil {
			return nil, err
		}
		comments = append(comments, ics...)
		nextURL = parseLinks(resp.Header.Get("Link"))["next"]
	}
	return comments, nil
}

// GetPullRequest gets a pull request.
func (c *Client) GetPullRequest(org, repo string, number int) (*PullRequest, error) {
	c.log("GetPullRequest", org, repo, number)
	if c.fake {
		return &PullRequest{}, nil
	}
	resp, err := c.request(http.MethodGet, fmt.Sprintf("%s/repos/%s/%s/pulls/%d", c.base, org, repo, number), nil)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("response not 200: %s", resp.Status)
	}
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	var pr PullRequest
	if err := json.Unmarshal(b, &pr); err != nil {
		return nil, err
	}
	return &pr, nil
}

// CreateStatus creates or updates the status of a commit.
func (c *Client) CreateStatus(org, repo, ref string, s Status) error {
	c.log("CreateStatus", org, repo, ref, s)
	if c.dry {
		return nil
	}

	resp, err := c.request(http.MethodPost, fmt.Sprintf("%s/repos/%s/%s/statuses/%s", c.base, org, repo, ref), s)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 201 {
		return fmt.Errorf("response not 201: %s", resp.Status)
	}
	return nil
}

func (c *Client) AddLabel(org, repo string, number int, label string) error {
	c.log("AddLabel", org, repo, number, label)
	if c.dry {
		return nil
	}
	resp, err := c.request(http.MethodPost, fmt.Sprintf("%s/repos/%s/%s/issues/%d/labels", c.base, org, repo, number), []string{label})
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return fmt.Errorf("response not 200: %s", resp.Status)
	}
	return nil
}

func (c *Client) RemoveLabel(org, repo string, number int, label string) error {
	c.log("RemoveLabel", org, repo, number, label)
	if c.dry {
		return nil
	}
	resp, err := c.request(http.MethodDelete, fmt.Sprintf("%s/repos/%s/%s/issues/%d/labels/%s", c.base, org, repo, number, label), nil)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	// GitHub sometimes returns 200 for this call, which is a bug on their end.
	if resp.StatusCode != 204 && resp.StatusCode != 200 {
		return fmt.Errorf("response not 204: %s", resp.Status)
	}
	return nil
}

func (c *Client) CloseIssue(org, repo string, number int) error {
	c.log("CloseIssue", org, repo, number)
	if c.dry {
		return nil
	}
	resp, err := c.request(http.MethodPatch, fmt.Sprintf("%s/repos/%s/%s/issues/%d", c.base, org, repo, number), map[string]string{"state": "closed"})
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return fmt.Errorf("response not 200: %s", resp.Status)
	}
	return nil
}

// GetRef returns the SHA of the given ref, such as "heads/master".
func (c *Client) GetRef(org, repo, ref string) (string, error) {
	c.log("GetRef", org, repo, ref)
	if c.fake {
		return "", nil
	}
	resp, err := c.request(http.MethodGet, fmt.Sprintf("%s/repos/%s/%s/git/refs/%s", c.base, org, repo, ref), nil)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return "", fmt.Errorf("response not 200: %s", resp.Status)
	}
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	var res struct {
		Object map[string]string `json:"object"`
	}
	if err := json.Unmarshal(b, &res); err != nil {
		return "", err
	}
	return res.Object["sha"], nil
}

// FindIssues uses the github search API to find issues which match a particular query.
// TODO(foxish): we should accept map[string][]string and use net/url properly.
func (c *Client) FindIssues(query string) ([]Issue, error) {
	c.log("FindIssues", query)
	if c.fake {
		return nil, nil
	}
	resp, err := c.request(http.MethodGet, fmt.Sprintf("%s/search/issues?q=%s", c.base, query), nil)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("response not 200: %s", resp.Status)
	}

	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	var issSearchResult IssuesSearchResult
	if err := json.Unmarshal(b, &issSearchResult); err != nil {
		return nil, err
	}
	return issSearchResult.Issues, nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris

package os

import (
	"io"
	"runtime"
	"syscall"
)

const (
	blockSize = 4096
)

func (f *File) readdir(n int) (fi []FileInfo, err error) {
	dirname := f.name
	if dirname == "" {
		dirname = "."
	}
	names, err := f.Readdirnames(n)
	fi = make([]FileInfo, 0, len(names))
	for _, filename := range names {
		fip, lerr := lstat(dirname + "/" + filename)
		if IsNotExist(lerr) {
			// File disappeared between readdir + stat.
			// Just treat it as if it didn't exist.
			continue
		}
		if lerr != nil {
			return fi, lerr
		}
		fi = append(fi, fip)
	}
	if len(fi) == 0 && err == nil && n > 0 {
		// Per File.Readdir, the slice must be non-empty or err
		// must be non-nil if n > 0.
		err = io.EOF
	}
	return fi, err
}

func (f *File) readdirnames(n int) (names []string, err error) {
	// If this file has no dirinfo, create one.
	if f.dirinfo == nil {
		f.dirinfo = new(dirInfo)
		// The buffer must be at least a block long.
		f.dirinfo.buf = make([]byte, blockSize)
	}
	d := f.dirinfo

	size := n
	if size <= 0 {
		size = 100
		n = -1
	}

	names = make([]string, 0, size) // Empty with room to grow.
	for n != 0 {
		// Refill the buffer if necessary
		if d.bufp >= d.nbuf {
			d.bufp = 0
			var errno error
			d.nbuf, errno = f.pfd.ReadDirent(d.buf)
			runtime.KeepAlive(f)
			if errno != nil {
				return names, wrapSyscallError("readdirent", errno)
			}
			if d.nbuf <= 0 {
				break // EOF
			}
		}

		// Drain the buffer
		var nb, nc int
		nb, nc, names = syscall.ParseDirent(d.buf[d.bufp:d.nbuf], n, names)
		d.bufp += nb
		n -= nc
	}
	if n >= 0 && len(names) == 0 {
		return names, io.EOF
	}
	return names, nil
}
<commit_msg>os: increase directory reading block size on Unix systems<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris

package os

import (
	"io"
	"runtime"
	"syscall"
)

const (
	// More than 5760 to work around https://golang.org/issue/24015.
	blockSize = 8192
)

func (f *File) readdir(n int) (fi []FileInfo, err error) {
	dirname := f.name
	if dirname == "" {
		dirname = "."
	}
	names, err := f.Readdirnames(n)
	fi = make([]FileInfo, 0, len(names))
	for _, filename := range names {
		fip, lerr := lstat(dirname + "/" + filename)
		if IsNotExist(lerr) {
			// File disappeared between readdir + stat.
			// Just treat it as if it didn't exist.
			continue
		}
		if lerr != nil {
			return fi, lerr
		}
		fi = append(fi, fip)
	}
	if len(fi) == 0 && err == nil && n > 0 {
		// Per File.Readdir, the slice must be non-empty or err
		// must be non-nil if n > 0.
		err = io.EOF
	}
	return fi, err
}

func (f *File) readdirnames(n int) (names []string, err error) {
	// If this file has no dirinfo, create one.
	if f.dirinfo == nil {
		f.dirinfo = new(dirInfo)
		// The buffer must be at least a block long.
		f.dirinfo.buf = make([]byte, blockSize)
	}
	d := f.dirinfo

	size := n
	if size <= 0 {
		size = 100
		n = -1
	}

	names = make([]string, 0, size) // Empty with room to grow.
	for n != 0 {
		// Refill the buffer if necessary
		if d.bufp >= d.nbuf {
			d.bufp = 0
			var errno error
			d.nbuf, errno = f.pfd.ReadDirent(d.buf)
			runtime.KeepAlive(f)
			if errno != nil {
				return names, wrapSyscallError("readdirent", errno)
			}
			if d.nbuf <= 0 {
				break // EOF
			}
		}

		// Drain the buffer
		var nb, nc int
		nb, nc, names = syscall.ParseDirent(d.buf[d.bufp:d.nbuf], n, names)
		d.bufp += nb
		n -= nc
	}
	if n >= 0 && len(names) == 0 {
		return names, io.EOF
	}
	return names, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/mg/i"
	"os"
	"strings"
)

type dirwalk struct {
	cur   string
	queue []string
	err   error
}

func Dirwalk(filename string) i.Forward {
	// remove trailing /
	strings.TrimSuffix(filename, "/")

	// construct and initialize
	var dw dirwalk
	dw.queue = []string{filename}
	dw.Next()
	return &dw
}

func (dw *dirwalk) Value() interface{} {
	return dw.cur
}

func (dw *dirwalk) Error() error {
	return dw.err
}

func (dw *dirwalk) AtEnd() bool {
	return len(dw.queue) == 0
}

func (dw *dirwalk) Next() error {
	// pop head from queue
	dw.cur, dw.queue = dw.queue[0], append(dw.queue[1:])

	// open file
	var file *os.File
	if file, dw.err = os.Open(dw.cur); dw.err != nil {
		return dw.err
	}
	defer file.Close()

	// stat file
	var stat os.FileInfo
	if stat, dw.err = file.Stat(); dw.err != nil {
		return dw.err
	}

	if stat.IsDir() {
		// read files in directory
		var files []string
		if files, dw.err = file.Readdirnames(0); dw.err != nil {
			return dw.err
		}

		// add files in directory to queue
		for _, subfile := range files {
			dw.queue = append(dw.queue, dw.cur+string(os.PathSeparator)+subfile)
		}
	}
	return dw.err
}

func hasInName(val string) i.FilterFunc {
	val = strings.ToUpper(val)
	return func(itr i.Iterator) bool {
		filename, _ := itr.Value().(string)
		return strings.Contains(strings.ToUpper(filename), val)
	}
}

func not(f i.FilterFunc) i.FilterFunc {
	return func(itr i.Iterator) bool {
		return !f(itr)
	}
}

func main() {
	if len(os.Args) != 2 {
		fmt.Printf("Usage %s NAME\n", os.Args[0])
		os.Exit(0)
	}
	i.Each(
		i.Filter(not(hasInName("example")), Dirwalk(os.Args[1])),
		func(itr i.Iterator) bool {
			fmt.Println(itr.Value())
			return true
		})
}
<commit_msg>interesting: add SetError()<commit_after>package main

import (
	"fmt"
	"github.com/mg/i"
	"os"
	"strings"
)

type dirwalk struct {
	cur   string
	queue []string
	err   error
}

func Dirwalk(filename string) i.Forward {
	// remove trailing /
	strings.TrimSuffix(filename, "/")

	// construct and initialize
	var dw dirwalk
	dw.queue = []string{filename}
	dw.Next()
	return &dw
}

func (dw *dirwalk) Value() interface{} {
	return dw.cur
}

func (dw *dirwalk) Error() error {
	return dw.err
}

func (dw *dirwalk) SetError(err error) {
	dw.err = err
}

func (dw *dirwalk) AtEnd() bool {
	return len(dw.queue) == 0
}

func (dw *dirwalk) Next() error {
	// pop head from queue
	dw.cur, dw.queue = dw.queue[0], append(dw.queue[1:])

	// open file
	var file *os.File
	if file, dw.err = os.Open(dw.cur); dw.err != nil {
		return dw.err
	}
	defer file.Close()

	// stat file
	var stat os.FileInfo
	if stat, dw.err = file.Stat(); dw.err != nil {
		return dw.err
	}

	if stat.IsDir() {
		// read files in directory
		var files []string
		if files, dw.err = file.Readdirnames(0); dw.err != nil {
			return dw.err
		}

		// add files in directory to queue
		for _, subfile := range files {
			dw.queue = append(dw.queue, dw.cur+string(os.PathSeparator)+subfile)
		}
	}
	return dw.err
}

func hasInName(val string) i.FilterFunc {
	val = strings.ToUpper(val)
	return func(itr i.Iterator) bool {
		filename, _ := itr.Value().(string)
		return strings.Contains(strings.ToUpper(filename), val)
	}
}

func not(f i.FilterFunc) i.FilterFunc {
	return func(itr i.Iterator) bool {
		return !f(itr)
	}
}

func main() {
	if len(os.Args) != 2 {
		fmt.Printf("Usage %s NAME\n", os.Args[0])
		os.Exit(0)
	}
	i.Each(
		i.Filter(not(hasInName("example")), Dirwalk(os.Args[1])),
		func(itr i.Iterator) bool {
			fmt.Println(itr.Value())
			return true
		})
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"github.com/globocom/gandalf/db"
	"github.com/globocom/gandalf/repository"
	"github.com/globocom/gandalf/user"
	"io"
	"labix.org/v2/mgo/bson"
	"log/syslog"
	"os"
	"os/exec"
	"regexp"
	"strings"
)

var log *syslog.Writer

func hasWritePermission(u *user.User, r *repository.Repository) (allowed bool) {
	for _, userName := range r.Users {
		if u.Name == userName {
			return true
		}
	}
	return false
}

func hasReadPermission(u *user.User, r *repository.Repository) (allowed bool) {
	if r.IsPublic {
		return true
	}
	for _, userName := range r.Users {
		if u.Name == userName {
			return true
		}
	}
	return false
}

// Returns the command being executed by ssh.
// When a user runs `$ git push` from his/her machine, the server
// receives a ssh command, identified by this user (by the ssh key).
// The command and it's parameters are available through the SSH_ORIGINAL_COMMAND
// environment variable. In the git push example, it would have the following value:
// SSH_ORIGINAL_COMMAND=git-receive-pack 'foo.git'
// This function is responsible for retrieving the `git-receive-pack` part of SSH_ORIGINAL_COMMAND
func action() string {
	return strings.Split(os.Getenv("SSH_ORIGINAL_COMMAND"), " ")[0]
}

// Get the repository name requested in SSH_ORIGINAL_COMMAND and retrieves
// the related document on the database and returns it.
// this function does two distinct things (maybe it should'n), it
// parses the SSH_ORIGINAL_COMMAND and returns a "validation" error if it doesn't
// matches the expected format and gets the repository from the database based on the info
// obtained by the SSH_ORIGINAL_COMMAND parse.
func requestedRepository() (repository.Repository, error) {
	r, err := regexp.Compile(`[\w-]+ '([\w-]+)\.git'`)
	if err != nil {
		panic(err)
	}
	m := r.FindStringSubmatch(os.Getenv("SSH_ORIGINAL_COMMAND"))
	if len(m) < 2 {
		return repository.Repository{}, errors.New("Cannot deduce repository name from command. You are probably trying to do something you shouldn't")
	}
	repoName := m[1]
	var repo repository.Repository
	if err = db.Session.Repository().Find(bson.M{"_id": repoName}).One(&repo); err != nil {
		return repository.Repository{}, errors.New("Repository not found")
	}
	return repo, nil
}

// Checks whether a command is a valid git command
// The following format is allowed:
//  git-([\w-]+) '([\w-]+)\.git'
func validateCmd() error {
	r, err := regexp.Compile(`git-([\w-]+) '([\w-]+)\.git'`)
	if err != nil {
		panic(err)
	}
	if m := r.FindStringSubmatch(os.Getenv("SSH_ORIGINAL_COMMAND")); len(m) < 3 {
		return errors.New("You've tried to execute some weird command, I'm deliberately denying you to do that, get over it.")
	}
	return nil
}

// Executes the SSH_ORIGINAL_COMMAND based on the condition
// defined by the `f` parameter.
// Also receives a custom error message to print to the end user and a
// stdout object, where the SSH_ORIGINAL_COMMAND output is going to be written
func executeAction(f func(*user.User, *repository.Repository) bool, errMsg string, stdout io.Writer) {
	var u user.User
	if err := db.Session.User().Find(bson.M{"_id": os.Args[1]}).One(&u); err != nil {
		log.Err("Error obtaining user. Gandalf database is probably in an inconsistent state.")
		return
	}
	repo, err := requestedRepository()
	if err != nil {
		log.Err(err.Error())
		return
	}
	if f(&u, &repo) {
		sshOrigCmd := os.Getenv("SSH_ORIGINAL_COMMAND")
		log.Info("Executing " + sshOrigCmd)
		cmdStr := strings.Split(sshOrigCmd, " ")
		cmd := exec.Command(cmdStr[0], cmdStr[1:]...)
		cmd.Stdout = stdout
		stderr := bytes.Buffer{}
		cmd.Stderr = stderr
		err = cmd.Run()
		if err != nil {
			log.Err("Got error while executing original command: " + err.Error())
			log.Err(stderr)
		}
		return
	}
	log.Err("Permission denied.")
	log.Err(errMsg)
}

func main() {
	var err error
	log, err = syslog.New(syslog.LOG_INFO, "gandalf-listener")
	if err != nil {
		panic(err.Error())
	}
	err = validateCmd()
	if err != nil {
		log.Err(err.Error())
		return
	}
	a := action()
	if a == "git-receive-pack" {
		executeAction(hasWritePermission, "You don't have access to write in this repository.", os.Stdout)
		return
	}
	if a == "git-upload-pack" {
		executeAction(hasReadPermission, "You don't have access to read this repository.", os.Stdout)
		return
	}
}
<commit_msg>bin: using pointer instead of struct<commit_after>package main

import (
	"bytes"
	"errors"
	"github.com/globocom/gandalf/db"
	"github.com/globocom/gandalf/repository"
	"github.com/globocom/gandalf/user"
	"io"
	"labix.org/v2/mgo/bson"
	"log/syslog"
	"os"
	"os/exec"
	"regexp"
	"strings"
)

var log *syslog.Writer

func hasWritePermission(u *user.User, r *repository.Repository) (allowed bool) {
	for _, userName := range r.Users {
		if u.Name == userName {
			return true
		}
	}
	return false
}

func hasReadPermission(u *user.User, r *repository.Repository) (allowed bool) {
	if r.IsPublic {
		return true
	}
	for _, userName := range r.Users {
		if u.Name == userName {
			return true
		}
	}
	return false
}

// Returns the command being executed by ssh.
// When a user runs `$ git push` from his/her machine, the server
// receives a ssh command, identified by this user (by the ssh key).
// The command and it's parameters are available through the SSH_ORIGINAL_COMMAND
// environment variable. In the git push example, it would have the following value:
// SSH_ORIGINAL_COMMAND=git-receive-pack 'foo.git'
// This function is responsible for retrieving the `git-receive-pack` part of SSH_ORIGINAL_COMMAND
func action() string {
	return strings.Split(os.Getenv("SSH_ORIGINAL_COMMAND"), " ")[0]
}

// Get the repository name requested in SSH_ORIGINAL_COMMAND and retrieves
// the related document on the database and returns it.
// this function does two distinct things (maybe it should'n), it
// parses the SSH_ORIGINAL_COMMAND and returns a "validation" error if it doesn't
// matches the expected format and gets the repository from the database based on the info
// obtained by the SSH_ORIGINAL_COMMAND parse.
func requestedRepository() (repository.Repository, error) {
	r, err := regexp.Compile(`[\w-]+ '([\w-]+)\.git'`)
	if err != nil {
		panic(err)
	}
	m := r.FindStringSubmatch(os.Getenv("SSH_ORIGINAL_COMMAND"))
	if len(m) < 2 {
		return repository.Repository{}, errors.New("Cannot deduce repository name from command. You are probably trying to do something you shouldn't")
	}
	repoName := m[1]
	var repo repository.Repository
	if err = db.Session.Repository().Find(bson.M{"_id": repoName}).One(&repo); err != nil {
		return repository.Repository{}, errors.New("Repository not found")
	}
	return repo, nil
}

// Checks whether a command is a valid git command
// The following format is allowed:
//  git-([\w-]+) '([\w-]+)\.git'
func validateCmd() error {
	r, err := regexp.Compile(`git-([\w-]+) '([\w-]+)\.git'`)
	if err != nil {
		panic(err)
	}
	if m := r.FindStringSubmatch(os.Getenv("SSH_ORIGINAL_COMMAND")); len(m) < 3 {
		return errors.New("You've tried to execute some weird command, I'm deliberately denying you to do that, get over it.")
	}
	return nil
}

// Executes the SSH_ORIGINAL_COMMAND based on the condition
// defined by the `f` parameter.
// Also receives a custom error message to print to the end user and a
// stdout object, where the SSH_ORIGINAL_COMMAND output is going to be written
func executeAction(f func(*user.User, *repository.Repository) bool, errMsg string, stdout io.Writer) {
	var u user.User
	if err := db.Session.User().Find(bson.M{"_id": os.Args[1]}).One(&u); err != nil {
		log.Err("Error obtaining user. Gandalf database is probably in an inconsistent state.")
		return
	}
	repo, err := requestedRepository()
	if err != nil {
		log.Err(err.Error())
		return
	}
	if f(&u, &repo) {
		sshOrigCmd := os.Getenv("SSH_ORIGINAL_COMMAND")
		log.Info("Executing " + sshOrigCmd)
		cmdStr := strings.Split(sshOrigCmd, " ")
		cmd := exec.Command(cmdStr[0], cmdStr[1:]...)
		cmd.Stdout = stdout
		stderr := &bytes.Buffer{}
		cmd.Stderr = stderr
		err = cmd.Run()
		if err != nil {
			log.Err("Got error while executing original command: " + err.Error())
			log.Err(stderr.String())
		}
		return
	}
	log.Err("Permission denied.")
	log.Err(errMsg)
}

func main() {
	var err error
	log, err = syslog.New(syslog.LOG_INFO, "gandalf-listener")
	if err != nil {
		panic(err.Error())
	}
	err = validateCmd()
	if err != nil {
		log.Err(err.Error())
		return
	}
	a := action()
	if a == "git-receive-pack" {
		executeAction(hasWritePermission, "You don't have access to write in this repository.", os.Stdout)
		return
	}
	if a == "git-upload-pack" {
		executeAction(hasReadPermission, "You don't have access to read this repository.", os.Stdout)
		return
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package RuntimeTest

import (
	"context"
	"fmt"
	"sync"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"

	. "github.com/onsi/gomega"
	"github.com/sirupsen/logrus"
)

var _ = Describe("RuntimeValidatedKafka", func() {

	var once sync.Once
	var logger *logrus.Entry
	var vm *helpers.SSHMeta

	var allowedTopic string = "allowedTopic"
	var disallowTopic string = "disallowTopic"
	var produceCmd string = fmt.Sprintf(
		"echo \"Message 0\" | docker exec -i client /opt/kafka/bin/kafka-console-producer.sh "+
			"--broker-list kafka:9092 --topic %s", allowedTopic)
	var MaxMessages int = 6
	var client string = "client"

	initialize := func() {
		logger = log.WithFields(logrus.Fields{"testName": "RuntimeKafka"})
		logger.Info("Starting")
		vm = helpers.CreateNewRuntimeHelper(helpers.Runtime, logger)
	}

	containers := func(mode string) {

		images := map[string]string{
			"zook":   "digitalwonderland/zookeeper",
			"client": "cilium/kafkaclient2",
		}

		switch mode {
		case "create":
			for k, v := range images {
				vm.ContainerCreate(k, v, helpers.CiliumDockerNetwork, fmt.Sprintf("-l id.%s", k))
			}
			zook, err := vm.ContainerInspectNet("zook")
			Expect(err).Should(BeNil())

			vm.ContainerCreate("kafka", "wurstmeister/kafka", helpers.CiliumDockerNetwork, fmt.Sprintf(
				"-l id.kafka -e KAFKA_ZOOKEEPER_CONNECT=%s:2181 -e KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS=20000 -e KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS=20000", zook["IPv4"]))

		case "delete":
			for k := range images {
				vm.ContainerRm(k)
			}
			vm.ContainerRm("kafka")
		}
	}

	createTopic := func(name string) {
		logger.Infof("Creating new kafka topic %s", name)
		res := vm.ContainerExec(client, fmt.Sprintf(
			"/opt/kafka/bin/kafka-topics.sh --create --zookeeper zook:2181 "+
				"--replication-factor 1 --partitions 1 --topic %s", name))
		res.ExpectSuccess()
	}
	consumerCmd := func(topic string, maxMsg int) string {
		return fmt.Sprintf("/opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server "+
			"kafka:9092 --topic %s --max-messages %d --timeout-ms 300000 --from-beginning",
			topic, maxMsg)
	}

	consumer := func(topic string, maxMsg int) *helpers.CmdRes {
		return vm.ContainerExec(client, consumerCmd(topic, maxMsg))
	}

	producer := func(topic string, message string) {
		cmd := fmt.Sprintf(
			"echo %s | docker exec -i %s /opt/kafka/bin/kafka-console-producer.sh "+
				"--broker-list kafka:9092 --topic %s",
			message, client, topic)
		vm.Exec(cmd)
	}

	// WaitKafkaBroker waits for the broker to be ready, by executing
	// a produce request on existing topics and waiting for a response from broker
	waitForKafkaBroker := func(pod string, cmd string) error {
		body := func() bool {
			res := vm.ContainerExec(pod, cmd)
			return res.WasSuccessful()
		}
		err := helpers.WithTimeout(body, "Kafka Broker not ready", &helpers.TimeoutConfig{Timeout: 150})
		return err
	}

	BeforeEach(func() {
		once.Do(initialize)
		containers("create")
		epsReady := vm.WaitEndpointsReady()
		Expect(epsReady).Should(BeTrue())
	})

	AfterEach(func() {
		vm.PolicyDelAll()
		containers("delete")
	})

	JustAfterEach(func() {
		vm.ValidateNoErrorsOnLogs(CurrentGinkgoTestDescription().Duration)
	})

	AfterFailed(func() {
		vm.ReportFailed()
	})

	It("Kafka Policy Ingress", func() {
		_, err := vm.PolicyImportAndWait(vm.GetFullPath("Policies-kafka.json"), 300)
		Expect(err).Should(BeNil())

		endPoints, err := vm.PolicyEndpointsSummary()
		Expect(err).Should(BeNil(), "Cannot get endpoint list")
		Expect(endPoints[helpers.Enabled]).To(Equal(1),
			"Check number of endpoints with policy enforcement enabled")
		Expect(endPoints[helpers.Disabled]).To(Equal(2),
			"Check number of endpoints with policy enforcement disabled")
		By("Creating kafka topics")
		createTopic(allowedTopic)
		createTopic(disallowTopic)

		By("Listing created Kafka topics")
		res := vm.ContainerExec(client,
			"/opt/kafka/bin/kafka-topics.sh --list --zookeeper zook:2181")
		res.ExpectSuccess("Cannot get kafka topics")

		// Waiting for kafka broker to be up.
		err = waitForKafkaBroker(client, produceCmd)
		Expect(err).To(BeNil(), "Kafka broker failed to come up")

		By("Allowed topic")

		By("Sending produce request on kafka topic `allowedTopic`")
		for i := 1; i <= MaxMessages-1; i++ {
			producer(allowedTopic, fmt.Sprintf("Message %d", i))
		}

		By("Sending consume request on kafka topic `allowedTopic`")
		res = consumer(allowedTopic, MaxMessages)
		res.ExpectSuccess("Failed to consume messages from kafka topic `allowedTopic`")

		Expect(res.Output().String()).Should(ContainSubstring(
			"Processed a total of %d messages", MaxMessages))
		By("Disable topic")
		res = consumer(disallowTopic, MaxMessages)
		res.ExpectFail("Kafka consumer can access to disallowTopic")
	})

	It("Kafka Policy Role Ingress", func() {
		_, err := vm.PolicyImportAndWait(vm.GetFullPath("Policies-kafka-Role.json"), 300)
		Expect(err).Should(BeNil(), "Expected nil got %s while importing policy Policies-kafka-Role.json", err)

		endPoints, err := vm.PolicyEndpointsSummary()
		Expect(err).Should(BeNil(), "Expect nil. Failed to apply policy on all endpoints with error :%s", err)
		Expect(endPoints[helpers.Enabled]).To(Equal(1), "Expected 1 endpoint to be policy enabled. Policy enforcement failed")
		Expect(endPoints[helpers.Disabled]).To(Equal(2), "Expected 2 endpoint to be policy disabled. Policy enforcement failed")

		By("Creating kafka topics")
		createTopic(allowedTopic)
		createTopic(disallowTopic)

		By("Listing created Kafka topics")
		res := vm.ContainerExec(client,
			"/opt/kafka/bin/kafka-topics.sh --list --zookeeper zook:2181")
		res.ExpectSuccess("Cannot get kafka topics")

		// Waiting for kafka broker to be up.
		err = waitForKafkaBroker(client, produceCmd)
		Expect(err).To(BeNil(), "Kafka broker failed to come up")

		By("By sending produce/consume request on topic `allowedTopic`")

		By("Sending produce request on kafka topic `allowedTopic`")
		for i := 1; i <= MaxMessages-1; i++ {
			producer(allowedTopic, fmt.Sprintf("Message %d", i))
		}

		By("Sending consume request on kafka topic `allowedTopic`")
		res = consumer(allowedTopic, MaxMessages)
		res.ExpectSuccess("Failed to consume messages from kafka topic `allowedTopic`")

		Expect(res.Output().String()).Should(ContainSubstring(
			"Processed a total of %d messages", MaxMessages))

		By("Disable topic")
		// Consumer timeout didn't work correctly, so make sure that AUTH is present in the reply
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()
		res = vm.ExecContext(ctx, fmt.Sprintf(
			"docker exec -i %s %s", client, consumerCmd(disallowTopic, MaxMessages)))
		err = res.WaitUntilMatch("{disallowTopic=TOPIC_AUTHORIZATION_FAILED}")
		Expect(err).To(BeNil(), "Traffic in disallowTopic is allowed")
	})
})
<commit_msg>test: Disable unstable Kafka runtime test<commit_after>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package RuntimeTest

import (
	"context"
	"fmt"
	"sync"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"

	. "github.com/onsi/gomega"
	"github.com/sirupsen/logrus"
)

var _ = Describe("RuntimeValidatedKafka", func() {

	var once sync.Once
	var logger *logrus.Entry
	var vm *helpers.SSHMeta

	var allowedTopic string = "allowedTopic"
	var disallowTopic string = "disallowTopic"
	var produceCmd string = fmt.Sprintf(
		"echo \"Message 0\" | docker exec -i client /opt/kafka/bin/kafka-console-producer.sh "+
			"--broker-list kafka:9092 --topic %s", allowedTopic)
	var MaxMessages int = 6
	var client string = "client"

	initialize := func() {
		logger = log.WithFields(logrus.Fields{"testName": "RuntimeKafka"})
		logger.Info("Starting")
		vm = helpers.CreateNewRuntimeHelper(helpers.Runtime, logger)
	}

	containers := func(mode string) {

		images := map[string]string{
			"zook":   "digitalwonderland/zookeeper",
			"client": "cilium/kafkaclient2",
		}

		switch mode {
		case "create":
			for k, v := range images {
				vm.ContainerCreate(k, v, helpers.CiliumDockerNetwork, fmt.Sprintf("-l id.%s", k))
			}
			zook, err := vm.ContainerInspectNet("zook")
			Expect(err).Should(BeNil())

			vm.ContainerCreate("kafka", "wurstmeister/kafka", helpers.CiliumDockerNetwork, fmt.Sprintf(
				"-l id.kafka -e KAFKA_ZOOKEEPER_CONNECT=%s:2181 -e KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS=20000 -e KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS=20000", zook["IPv4"]))

		case "delete":
			for k := range images {
				vm.ContainerRm(k)
			}
			vm.ContainerRm("kafka")
		}
	}

	createTopic := func(name string) {
		logger.Infof("Creating new kafka topic %s", name)
		res := vm.ContainerExec(client, fmt.Sprintf(
			"/opt/kafka/bin/kafka-topics.sh --create --zookeeper zook:2181 "+
				"--replication-factor 1 --partitions 1 --topic %s", name))
		res.ExpectSuccess()
	}
	consumerCmd := func(topic string, maxMsg int) string {
		return fmt.Sprintf("/opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server "+
			"kafka:9092 --topic %s --max-messages %d --timeout-ms 300000 --from-beginning",
			topic, maxMsg)
	}

	consumer := func(topic string, maxMsg int) *helpers.CmdRes {
		return vm.ContainerExec(client, consumerCmd(topic, maxMsg))
	}

	producer := func(topic string, message string) {
		cmd := fmt.Sprintf(
			"echo %s | docker exec -i %s /opt/kafka/bin/kafka-console-producer.sh "+
				"--broker-list kafka:9092 --topic %s",
			message, client, topic)
		vm.Exec(cmd)
	}

	// WaitKafkaBroker waits for the broker to be ready, by executing
	// a produce request on existing topics and waiting for a response from broker
	waitForKafkaBroker := func(pod string, cmd string) error {
		body := func() bool {
			res := vm.ContainerExec(pod, cmd)
			return res.WasSuccessful()
		}
		err := helpers.WithTimeout(body, "Kafka Broker not ready", &helpers.TimeoutConfig{Timeout: 150})
		return err
	}

	BeforeEach(func() {
		once.Do(initialize)
		containers("create")
		epsReady := vm.WaitEndpointsReady()
		Expect(epsReady).Should(BeTrue())
	})

	AfterEach(func() {
		vm.PolicyDelAll()
		containers("delete")
	})

	JustAfterEach(func() {
		vm.ValidateNoErrorsOnLogs(CurrentGinkgoTestDescription().Duration)
	})

	AfterFailed(func() {
		vm.ReportFailed()
	})

	It("Kafka Policy Ingress", func() {
		// GH-3440 Re-enable when stable
		return

		_, err := vm.PolicyImportAndWait(vm.GetFullPath("Policies-kafka.json"), 300)
		Expect(err).Should(BeNil())

		endPoints, err := vm.PolicyEndpointsSummary()
		Expect(err).Should(BeNil(), "Cannot get endpoint list")
		Expect(endPoints[helpers.Enabled]).To(Equal(1),
			"Check number of endpoints with policy enforcement enabled")
		Expect(endPoints[helpers.Disabled]).To(Equal(2),
			"Check number of endpoints with policy enforcement disabled")
		By("Creating kafka topics")
		createTopic(allowedTopic)
		createTopic(disallowTopic)

		By("Listing created Kafka topics")
		res := vm.ContainerExec(client,
			"/opt/kafka/bin/kafka-topics.sh --list --zookeeper zook:2181")
		res.ExpectSuccess("Cannot get kafka topics")

		// Waiting for kafka broker to be up.
		err = waitForKafkaBroker(client, produceCmd)
		Expect(err).To(BeNil(), "Kafka broker failed to come up")

		By("Allowed topic")

		By("Sending produce request on kafka topic `allowedTopic`")
		for i := 1; i <= MaxMessages-1; i++ {
			producer(allowedTopic, fmt.Sprintf("Message %d", i))
		}

		By("Sending consume request on kafka topic `allowedTopic`")
		res = consumer(allowedTopic, MaxMessages)
		res.ExpectSuccess("Failed to consume messages from kafka topic `allowedTopic`")

		Expect(res.Output().String()).Should(ContainSubstring(
			"Processed a total of %d messages", MaxMessages))
		By("Disable topic")
		res = consumer(disallowTopic, MaxMessages)
		res.ExpectFail("Kafka consumer can access to disallowTopic")
	})

	It("Kafka Policy Role Ingress", func() {
		// GH-3440 Re-enable when stable
		return

		_, err := vm.PolicyImportAndWait(vm.GetFullPath("Policies-kafka-Role.json"), 300)
		Expect(err).Should(BeNil(), "Expected nil got %s while importing policy Policies-kafka-Role.json", err)

		endPoints, err := vm.PolicyEndpointsSummary()
		Expect(err).Should(BeNil(), "Expect nil. Failed to apply policy on all endpoints with error :%s", err)
		Expect(endPoints[helpers.Enabled]).To(Equal(1), "Expected 1 endpoint to be policy enabled. Policy enforcement failed")
		Expect(endPoints[helpers.Disabled]).To(Equal(2), "Expected 2 endpoint to be policy disabled. Policy enforcement failed")

		By("Creating kafka topics")
		createTopic(allowedTopic)
		createTopic(disallowTopic)

		By("Listing created Kafka topics")
		res := vm.ContainerExec(client,
			"/opt/kafka/bin/kafka-topics.sh --list --zookeeper zook:2181")
		res.ExpectSuccess("Cannot get kafka topics")

		// Waiting for kafka broker to be up.
		err = waitForKafkaBroker(client, produceCmd)
		Expect(err).To(BeNil(), "Kafka broker failed to come up")

		By("By sending produce/consume request on topic `allowedTopic`")

		By("Sending produce request on kafka topic `allowedTopic`")
		for i := 1; i <= MaxMessages-1; i++ {
			producer(allowedTopic, fmt.Sprintf("Message %d", i))
		}

		By("Sending consume request on kafka topic `allowedTopic`")
		res = consumer(allowedTopic, MaxMessages)
		res.ExpectSuccess("Failed to consume messages from kafka topic `allowedTopic`")

		Expect(res.Output().String()).Should(ContainSubstring(
			"Processed a total of %d messages", MaxMessages))

		By("Disable topic")
		// Consumer timeout didn't work correctly, so make sure that AUTH is present in the reply
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()
		res = vm.ExecContext(ctx, fmt.Sprintf(
			"docker exec -i %s %s", client, consumerCmd(disallowTopic, MaxMessages)))
		err = res.WaitUntilMatch("{disallowTopic=TOPIC_AUTHORIZATION_FAILED}")
		Expect(err).To(BeNil(), "Traffic in disallowTopic is allowed")
	})
})
<|endoftext|>


<commit_before>// The unexport command unexports exported identifiers which are not imported
// by any other Go code.
package main

import (
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"log"
	"os"
	"strings"

	"golang.org/x/tools/go/loader"
	"golang.org/x/tools/go/types"
	"golang.org/x/tools/refactor/importgraph"
)

// Usage is a replacement usage function for the flags package.
func Usage() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "\tunexport [flags] -identifier T [packages]\n")
	fmt.Fprintf(os.Stderr, "Flags:\n")
	flag.PrintDefaults()
}

func main() {
	var (
		flagIdentifier = flag.String("identifier", "", "comma-separated list of identifiers names; if empty all identifiers are unexported")
	)

	log.SetPrefix("unexport: ")
	flag.Usage = Usage
	flag.Parse()

	identifiers := strings.Split(*flagIdentifier, ",")
	fmt.Printf("identifiers = %+v\n", identifiers)

	args := flag.Args()

	if err := runMain(args[0]); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	// pkg, err := build.Import(args[0], "", build.ImportComment)
	// if err != nil {
	// 	log.Fatalf("%s", err)
	// }
	// parsePackage(pkg)

}

// runMain runs the actual command. It's an helper function so we can easily
// calls defers or return errors.
func runMain(path string) error {
	ctxt := &build.Default
	prog, err := loadProgram(ctxt, map[string]bool{path: true})
	if err != nil {
		return err
	}

	var info *loader.PackageInfo
	for name, p := range prog.Imported {
		if name == path {
			info = p
			break
		}
	}

	if info == nil {
		return fmt.Errorf("import path %s couldn't be find", path)
	}

	_, rev, errors := importgraph.Build(ctxt)
	if len(errors) > 0 {
		// With a large GOPATH tree, errors are inevitable.
		// Report them but proceed.
		fmt.Fprintf(os.Stderr, "While scanning Go workspace:\n")
		for path, err := range errors {
			fmt.Fprintf(os.Stderr, "Package %q: %s.\n", path, err)
		}
	}

	// Enumerate the set of potentially affected packages.
	affectedPackages := make(map[string]bool)
	for _, obj := range exportedObjects(info) {
		// External test packages are never imported,
		// so they will never appear in the graph.
		for path := range rev.Search(obj.Pkg().Path()) {
			affectedPackages[path] = true
		}
	}

	for pkg := range affectedPackages {
		fmt.Println("\t", pkg)
	}

	return nil
}

// exportedObjects returns objects which are exported only
func exportedObjects(info *loader.PackageInfo) []types.Object {
	var objects []types.Object
	for _, obj := range info.Defs {
		if obj == nil {
			continue
		}

		if obj.Exported() {
			objects = append(objects, obj)
		}
	}

	return objects
}

func loadProgram(ctxt *build.Context, pkgs map[string]bool) (*loader.Program, error) {
	conf := loader.Config{
		Build:       ctxt,
		ParserMode:  parser.ParseComments,
		AllowErrors: false,
	}

	for pkg := range pkgs {
		conf.ImportWithTests(pkg)
	}
	return conf.Load()
}

func parsePackage(pkg *build.Package) {
	fs := token.NewFileSet()

	include := func(info os.FileInfo) bool {
		for _, name := range pkg.GoFiles {
			if name == info.Name() {
				return true
			}
		}
		for _, name := range pkg.CgoFiles {
			if name == info.Name() {
				return true
			}
		}
		return false
	}

	pkgs, err := parser.ParseDir(fs, pkg.Dir, include, parser.ParseComments)
	if err != nil {
		log.Fatal(err)
	}

	// Make sure they are all in one package.
	if len(pkgs) != 1 {
		log.Fatalf("multiple packages in directory %s", pkg.Dir)
	}

	astPkg := pkgs[pkg.Name]

	for _, f := range astPkg.Files {
		ast.FileExports(f)
		for _, decl := range f.Decls {
			switch d := decl.(type) {
			case *ast.GenDecl:
				switch d.Tok {
				case token.IMPORT:
				case token.CONST:
					for _, spec := range d.Specs {
						if v, ok := spec.(*ast.ValueSpec); ok {
							fmt.Println("Const:", v.Names)
						}
					}
				case token.VAR:
					for _, spec := range d.Specs {
						if v, ok := spec.(*ast.ValueSpec); ok {
							fmt.Println("Var:", v.Names)
						}
					}
				case token.TYPE:
					for _, spec := range d.Specs {
						if s, ok := spec.(*ast.TypeSpec); ok {
							fmt.Println("Type:", s.Name.Name)
							switch t := s.Type.(type) {
							case *ast.StructType:
								for _, l := range t.Fields.List {
									fmt.Printf("\tField: %+v\n", l.Names)
								}
							}
						}
					}
				}
			case *ast.FuncDecl:
				// methods might bound to unexported types, show only if those
				// types are exported too
				if d.Recv != nil {
					for _, l := range d.Recv.List {
						for _, n := range l.Names {
							if ast.IsExported(n.Name) {
								fmt.Printf("Func: %s\n", d.Name.Name)
							}
						}
					}
				} else {
					fmt.Printf("Func: %s\n", d.Name.Name)
				}
			}
		}
	}
}
<commit_msg>unexport: one step closer to final result ;)<commit_after>// The unexport command unexports exported identifiers which are not imported
// by any other Go code.
package main

import (
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"log"
	"os"
	"strings"

	"golang.org/x/tools/go/loader"
	"golang.org/x/tools/go/types"
	"golang.org/x/tools/refactor/importgraph"
)

// Usage is a replacement usage function for the flags package.
func Usage() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "\tunexport [flags] -identifier T [packages]\n")
	fmt.Fprintf(os.Stderr, "Flags:\n")
	flag.PrintDefaults()
}

func main() {
	var (
		flagIdentifier = flag.String("identifier", "", "comma-separated list of identifiers names; if empty all identifiers are unexported")
	)

	log.SetPrefix("unexport: ")
	flag.Usage = Usage
	flag.Parse()

	identifiers := strings.Split(*flagIdentifier, ",")
	fmt.Printf("identifiers = %+v\n", identifiers)

	args := flag.Args()

	if err := runMain(args[0]); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

// runMain runs the actual command. It's an helper function so we can easily
// calls defers or return errors.
func runMain(path string) error {
	ctxt := &build.Default
	prog, err := loadProgram(ctxt, map[string]bool{path: true})
	if err != nil {
		return err
	}

	_, rev, errors := importgraph.Build(ctxt)
	if len(errors) > 0 {
		// With a large GOPATH tree, errors are inevitable.
		// Report them but proceed.
		fmt.Fprintf(os.Stderr, "While scanning Go workspace:\n")
		for path, err := range errors {
			fmt.Fprintf(os.Stderr, "Package %q: %s.\n", path, err)
		}
	}

	// Enumerate the set of potentially affected packages.
	possiblePackages := make(map[string]bool)
	for _, obj := range findExportedObjects(prog, path) {
		// External test packages are never imported,
		// so they will never appear in the graph.
		for path := range rev.Search(obj.Pkg().Path()) {
			possiblePackages[path] = true
		}
	}

	fmt.Println("Possible affected packages:")
	for pkg := range possiblePackages {
		fmt.Println("\t", pkg)
	}

	// reload the program with all possible packages to fetch the packageinfo's
	globalProg, err := loadProgram(ctxt, possiblePackages)
	if err != nil {
		return err
	}

	objsToUpdate := make(map[types.Object]bool, 0)
	objects := findExportedObjects(globalProg, path)

	fmt.Println("Exported identififers are:")
	for _, obj := range objects {
		fmt.Println("\t", obj)
	}

	for _, info := range globalProg.Imported {
		safeObjects := filterObjects(info, objects)
		for _, obj := range safeObjects {
			objsToUpdate[obj] = true
		}
	}

	fmt.Println("Safe to unexport identifiers are:")
	for obj := range objsToUpdate {
		fmt.Println("\t", obj)
	}

	var nidents int
	var filesToUpdate = make(map[*token.File]bool)
	for _, info := range globalProg.Imported {
		for id, obj := range info.Defs {
			if objsToUpdate[obj] {
				nidents++
				id.Name = strings.ToLower(obj.Name())
				filesToUpdate[globalProg.Fset.File(id.Pos())] = true
			}
		}
		for id, obj := range info.Uses {
			if objsToUpdate[obj] {
				nidents++
				id.Name = strings.ToLower(obj.Name())
				filesToUpdate[globalProg.Fset.File(id.Pos())] = true
			}
		}
	}

	return nil
}

// filterObjects filters the given objects and returns objects which are not in use by the given info package
func filterObjects(info *loader.PackageInfo, exported map[*ast.Ident]types.Object) map[*ast.Ident]types.Object {
	filtered := make(map[*ast.Ident]types.Object, 0)
	for id, ex := range exported {
		if !hasUse(info, ex) {
			filtered[id] = ex
		}
	}

	return filtered
}

// hasUse returns true if the given obj is part of the use in info
func hasUse(info *loader.PackageInfo, obj types.Object) bool {
	for _, o := range info.Uses {
		if o == obj {
			return true
		}
	}
	return false
}

// exportedObjects returns objects which are exported only
func exportedObjects(info *loader.PackageInfo) map[*ast.Ident]types.Object {
	objects := make(map[*ast.Ident]types.Object, 0)
	for id, obj := range info.Defs {
		if obj == nil {
			continue
		}

		if obj.Exported() {
			objects[id] = obj
		}
	}

	return objects
}

func findExportedObjects(prog *loader.Program, path string) map[*ast.Ident]types.Object {
	var pkgObj *types.Package
	for pkg := range prog.AllPackages {
		if pkg.Path() == path {
			pkgObj = pkg
			break
		}
	}

	info := prog.AllPackages[pkgObj]
	return exportedObjects(info)
}

func loadProgram(ctxt *build.Context, pkgs map[string]bool) (*loader.Program, error) {
	conf := loader.Config{
		Build:       ctxt,
		ParserMode:  parser.ParseComments,
		AllowErrors: false,
	}

	for pkg := range pkgs {
		conf.ImportWithTests(pkg)
	}
	return conf.Load()
}
<|endoftext|>