<commit_before>package mux

import (
	"net/http"

	muxpath "github.com/gogolfing/mux/path"
)

type Route struct {
	path         string
	children     []*Route
	routeHandler *routeHandler
}

func newRoute(path string, children ...*Route) *Route {
	return &Route{
		path,
		children,
		nil,
	}
}

func (route *Route) Delete(handler http.Handler) *Route {
	return route.Handle(handler)
}

func (route *Route) DeleteFunc(handlerFunc http.HandlerFunc) *Route {
	return route.HandleFunc(handlerFunc)
}

func (route *Route) Get(handler http.Handler) *Route {
	return route.Handle(handler)
}

func (route *Route) GetFunc(handlerFunc http.HandlerFunc) *Route {
	return route.HandleFunc(handlerFunc)
}

func (route *Route) Post(handler http.Handler) *Route {
	return route.Handle(handler)
}

func (route *Route) PostFunc(handlerFunc http.HandlerFunc) *Route {
	return route.HandleFunc(handlerFunc)
}

func (route *Route) Put(handler http.Handler) *Route {
	return route.Handle(handler)
}

func (route *Route) PutFunc(handlerFunc http.HandlerFunc) *Route {
	return route.HandleFunc(handlerFunc)
}

func (route *Route) SubRoute(path string) *Route {
	return route.insertSubRoute(path)
}

func (route *Route) HandleFunc(handlerFunc http.HandlerFunc, methods ...string) *Route {
	return route.Handle(http.HandlerFunc(handlerFunc), methods...)
}

func (route *Route) Handle(handler http.Handler, methods ...string) *Route {
	if route.routeHandler == nil {
		route.routeHandler = &routeHandler{}
	}
	route.routeHandler.handle(handler, methods...)
	return route
}

func (route *Route) getHandler(r *http.Request) (http.Handler, error) {
	if route.routeHandler == nil {
		return nil, ErrNotFound
	}
	return route.routeHandler.getHandler(r)
}

func (route *Route) insertSubRoute(path string) *Route {
	parent, found, remainingPath := route.findSubRoute(path)
	if len(remainingPath) == 0 {
		return found
	}
	if found == nil {
		return parent.insertChildPath(remainingPath)
	}
	found, remainingPath = parent.splitChild(remainingPath)
	return found.insertChildPath(remainingPath)
}

func (route *Route) insertChildPath(path string) *Route {
	if len(path) == 0 {
		return route
	}
	index, _ := route.indexOfCommonPrefixChild(path)
	return route.insertChildAtIndex(newRoute(path), ^index)
}

func (route *Route) splitChild(path string) (*Route, string) {
	oldChild, index, prefix := route.findChildWithCommonPrefix(path)
	newChild := newRoute(prefix, oldChild)
	route.children[index] = newChild
	oldChild.path = oldChild.path[len(prefix):]
	return newChild, path[len(prefix):]
}

func (route *Route) findSubRoute(path string) (*Route, *Route, string) {
	parent := route
	child, _, prefix := parent.findChildWithCommonPrefix(path)
	for child != nil && len(path) > 0 && len(prefix) == len(child.path) {
		path = path[len(prefix):]
		if len(path) == 0 {
			break
		}
		parent = child
		child, _, prefix = parent.findChildWithCommonPrefix(path)
	}
	return parent, child, path
}

func (route *Route) findChildWithCommonPrefix(path string) (*Route, int, string) {
	index, prefix := route.indexOfCommonPrefixChild(path)
	if index >= 0 {
		return route.children[index], index, prefix
	}
	return nil, index, prefix
}

func (route *Route) indexOfCommonPrefixChild(path string) (int, string) {
	low, high := 0, len(route.children)
	for low < high {
		mid := (low + high) >> 1
		comparison, prefix := muxpath.CompareIgnorePrefix(path, route.children[mid].path)
		if len(prefix) > 0 {
			return mid, prefix
		} else if comparison == 0 {
			return mid, path
		} else if comparison < 0 {
			high = mid
		} else { //comparison must be > 0.
			low = mid + 1
		}
	}
	return ^high, ""
}

func (route *Route) insertChildAtIndex(child *Route, index int) *Route {
	if index < 0 || index > len(route.children) {
		return nil
	}
	if route.children == nil {
		route.children = []*Route{child}
		return child
	}
	before := route.children[:index]
	after := route.children[index:]
	route.children = make([]*Route, 0, len(before)+1+len(after))
	route.children = append(route.children, before...)
	route.children = append(route.children, child)
	route.children = append(route.children, after...)
	return child
}

func (route *Route) Methods() []string {
	return route.routeHandler.methods()
}
<commit_msg>Adds methods to mux handle calls.<commit_after>package mux

import (
	"net/http"

	muxpath "github.com/gogolfing/mux/path"
)

type Route struct {
	path         string
	children     []*Route
	routeHandler *routeHandler
}

func newRoute(path string, children ...*Route) *Route {
	return &Route{
		path,
		children,
		nil,
	}
}

func (route *Route) Delete(handler http.Handler) *Route {
	return route.Handle(handler, "DELETE")
}

func (route *Route) DeleteFunc(handlerFunc http.HandlerFunc) *Route {
	return route.HandleFunc(handlerFunc, "DELETE")
}

func (route *Route) Get(handler http.Handler) *Route {
	return route.Handle(handler, "GET")
}

func (route *Route) GetFunc(handlerFunc http.HandlerFunc) *Route {
	return route.HandleFunc(handlerFunc, "GET")
}

func (route *Route) Post(handler http.Handler) *Route {
	return route.Handle(handler, "POST")
}

func (route *Route) PostFunc(handlerFunc http.HandlerFunc) *Route {
	return route.HandleFunc(handlerFunc, "POST")
}

func (route *Route) Put(handler http.Handler) *Route {
	return route.Handle(handler, "PUT")
}

func (route *Route) PutFunc(handlerFunc http.HandlerFunc) *Route {
	return route.HandleFunc(handlerFunc, "PUT")
}

func (route *Route) SubRoute(path string) *Route {
	return route.insertSubRoute(path)
}

func (route *Route) HandleFunc(handlerFunc http.HandlerFunc, methods ...string) *Route {
	return route.Handle(http.HandlerFunc(handlerFunc), methods...)
}

func (route *Route) Handle(handler http.Handler, methods ...string) *Route {
	if route.routeHandler == nil {
		route.routeHandler = &routeHandler{}
	}
	route.routeHandler.handle(handler, methods...)
	return route
}

func (route *Route) getHandler(r *http.Request) (http.Handler, error) {
	if route.routeHandler == nil {
		return nil, ErrNotFound
	}
	return route.routeHandler.getHandler(r)
}

func (route *Route) insertSubRoute(path string) *Route {
	parent, found, remainingPath := route.findSubRoute(path)
	if len(remainingPath) == 0 {
		return found
	}
	if found == nil {
		return parent.insertChildPath(remainingPath)
	}
	found, remainingPath = parent.splitChild(remainingPath)
	return found.insertChildPath(remainingPath)
}

func (route *Route) insertChildPath(path string) *Route {
	if len(path) == 0 {
		return route
	}
	index, _ := route.indexOfCommonPrefixChild(path)
	return route.insertChildAtIndex(newRoute(path), ^index)
}

func (route *Route) splitChild(path string) (*Route, string) {
	oldChild, index, prefix := route.findChildWithCommonPrefix(path)
	newChild := newRoute(prefix, oldChild)
	route.children[index] = newChild
	oldChild.path = oldChild.path[len(prefix):]
	return newChild, path[len(prefix):]
}

func (route *Route) findSubRoute(path string) (*Route, *Route, string) {
	parent := route
	child, _, prefix := parent.findChildWithCommonPrefix(path)
	for child != nil && len(path) > 0 && len(prefix) == len(child.path) {
		path = path[len(prefix):]
		if len(path) == 0 {
			break
		}
		parent = child
		child, _, prefix = parent.findChildWithCommonPrefix(path)
	}
	return parent, child, path
}

func (route *Route) findChildWithCommonPrefix(path string) (*Route, int, string) {
	index, prefix := route.indexOfCommonPrefixChild(path)
	if index >= 0 {
		return route.children[index], index, prefix
	}
	return nil, index, prefix
}

func (route *Route) indexOfCommonPrefixChild(path string) (int, string) {
	low, high := 0, len(route.children)
	for low < high {
		mid := (low + high) >> 1
		comparison, prefix := muxpath.CompareIgnorePrefix(path, route.children[mid].path)
		if len(prefix) > 0 {
			return mid, prefix
		} else if comparison == 0 {
			return mid, path
		} else if comparison < 0 {
			high = mid
		} else { //comparison must be > 0.
			low = mid + 1
		}
	}
	return ^high, ""
}

func (route *Route) insertChildAtIndex(child *Route, index int) *Route {
	if index < 0 || index > len(route.children) {
		return nil
	}
	if route.children == nil {
		route.children = []*Route{child}
		return child
	}
	before := route.children[:index]
	after := route.children[index:]
	route.children = make([]*Route, 0, len(before)+1+len(after))
	route.children = append(route.children, before...)
	route.children = append(route.children, child)
	route.children = append(route.children, after...)
	return child
}

func (route *Route) Methods() []string {
	return route.routeHandler.methods()
}
<|endoftext|>


<commit_before>package livequery

import (
	"encoding/json"

	"golang.org/x/net/websocket"
)

var server *liveQueryServer

// Run ...
func Run(args map[string]string) {
	server = &liveQueryServer{}
	server.initServer(args)
	server.run()
}

type liveQueryServer struct {
	pattern    string
	addr       string
	subscriber subscriber
}

// initServer 初始化 liveQuery 服务
func (l *liveQueryServer) initServer(args map[string]string) {
	l.pattern = args["pattern"]
	l.addr = args["addr"]

	l.subscriber = createSubscriber("", "")
	l.subscriber.subscribe("afterSave")
	l.subscriber.subscribe("afterDelete")
	var h HandlerType
	h = func(args ...string) {
		channel := args[0]
		messageStr := args[1]
		var message M
		err := json.Unmarshal([]byte(messageStr), &message)
		if err != nil {
			return
		}
		l.inflateParseObject(message)
		if channel == "afterSave" {
			l.onAfterSave(message)
		} else if channel == "afterDelete" {
			l.onAfterDelete(message)
		} else {

		}
	}
	l.subscriber.on("message", h)
}

// run 启动 WebSocket 服务
func (l *liveQueryServer) run() {
	runWebSocketServer(l.pattern, l.addr, l)
}

// onConnect 当有客户端连接成功时调用
func (l *liveQueryServer) onConnect(ws *websocket.Conn) {

}

// onMessage 当接收到客户端发来的消息时调用
func (l *liveQueryServer) onMessage(ws *websocket.Conn, msg interface{}) {

}

// onDisconnect 当客户端断开时调用
func (l *liveQueryServer) onDisconnect(ws *websocket.Conn) {

}

// inflateParseObject 展开对象
func (l *liveQueryServer) inflateParseObject(message M) {

}

// onAfterDelete 有对象删除时调用
func (l *liveQueryServer) onAfterDelete(message M) {

}

// onAfterSave 有对象保存时调用
func (l *liveQueryServer) onAfterSave(message M) {

}
<commit_msg>添加 onMessage<commit_after>package livequery

import (
	"encoding/json"

	"golang.org/x/net/websocket"
)

var server *liveQueryServer

// Run ...
func Run(args map[string]string) {
	server = &liveQueryServer{}
	server.initServer(args)
	server.run()
}

type liveQueryServer struct {
	pattern    string
	addr       string
	subscriber subscriber
}

// initServer 初始化 liveQuery 服务
func (l *liveQueryServer) initServer(args map[string]string) {
	l.pattern = args["pattern"]
	l.addr = args["addr"]

	l.subscriber = createSubscriber("", "")
	l.subscriber.subscribe("afterSave")
	l.subscriber.subscribe("afterDelete")
	var h HandlerType
	h = func(args ...string) {
		channel := args[0]
		messageStr := args[1]
		var message M
		err := json.Unmarshal([]byte(messageStr), &message)
		if err != nil {
			return
		}
		l.inflateParseObject(message)
		if channel == "afterSave" {
			l.onAfterSave(message)
		} else if channel == "afterDelete" {
			l.onAfterDelete(message)
		} else {

		}
	}
	l.subscriber.on("message", h)
}

// run 启动 WebSocket 服务
func (l *liveQueryServer) run() {
	runWebSocketServer(l.pattern, l.addr, l)
}

// onConnect 当有客户端连接成功时调用
func (l *liveQueryServer) onConnect(ws *websocket.Conn) {

}

// onMessage 当接收到客户端发来的消息时调用
func (l *liveQueryServer) onMessage(ws *websocket.Conn, msg interface{}) {
	var request M
	if message, ok := msg.(string); ok {
		err := json.Unmarshal([]byte(message), &request)
		if err != nil {
			return
		}
	}

	op := request["op"].(string)
	if op == "" {
		return
	}

	switch op {
	case "connect":
		l.handleConnect(ws, request)
	case "subscribe":
		l.handleSubscribe(ws, request)
	case "unsubscribe":
		l.handleUnsubscribe(ws, request)
	default:

	}
}

// onDisconnect 当客户端断开时调用
func (l *liveQueryServer) onDisconnect(ws *websocket.Conn) {

}

// inflateParseObject 展开对象
func (l *liveQueryServer) inflateParseObject(message M) {

}

// onAfterDelete 有对象删除时调用
func (l *liveQueryServer) onAfterDelete(message M) {

}

// onAfterSave 有对象保存时调用
func (l *liveQueryServer) onAfterSave(message M) {

}

// handleConnect 处理客户端 Connect 操作
func (l *liveQueryServer) handleConnect(ws *websocket.Conn, request M) {

}

// handleSubscribe 处理客户端 Subscribe 操作
func (l *liveQueryServer) handleSubscribe(ws *websocket.Conn, request M) {

}

// handleUnsubscribe 处理客户端 Unsubscribe 操作
func (l *liveQueryServer) handleUnsubscribe(ws *websocket.Conn, request M) {

}
<|endoftext|>


<commit_before>package test

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"testing"
	"text/template"
	"time"

	"github.com/GoogleCloudPlatform/terraform-validator/converters/google"
)

const (
	samplePolicyPath       = "../testdata/sample_policies"
	defaultAncestry        = "organization/12345/folder/67890"
	defaultOrganization    = "12345"
	defaultFolder          = "67890"
	defaultProject         = "foobar"
	defaultProviderVersion = "4.4.0"
)

var (
	data      *testData
	tfvBinary string
	tmpDir    = os.TempDir()
)

// testData represents the full dataset that is used for templating terraform
// configs. It contains Google API resources that are expected to be returned
// after converting the terraform plan.
type testData struct {
	// is not nil - Terraform 12 version used
	TFVersion string
	// provider "google"
	Provider map[string]string
	Project  map[string]string
	Time     map[string]string
	OrgID    string
	FolderID string
	Ancestry string
}

// init initializes the variables used for testing. As tests rely on
// environment variables, the parsing of those are only done once.
func init() {
	// don't raise errors in glog

	cwd, err := os.Getwd()
	if err != nil {
		log.Fatalf("cannot get current directory: %v", err)
	}
	tfvBinary = filepath.Join(cwd, "..", "bin", "terraform-validator")
	project, ok := os.LookupEnv("TEST_PROJECT")
	if !ok {
		log.Printf("Missing required env var TEST_PROJECT. Default (%s) will be used.", defaultProject)
		project = defaultProject
	}
	org, ok := os.LookupEnv("TEST_ORG_ID")
	if !ok {
		log.Printf("Missing required env var TEST_ORG_ID. Default (%s) will be used.", defaultOrganization)
		org = defaultOrganization
	}
	folder, ok := os.LookupEnv("TEST_FOLDER_ID")
	if !ok {
		log.Printf("Missing required env var TEST_FOLDER_ID. Default (%s) will be used.", defaultFolder)
		folder = defaultFolder
	}
	credentials, ok := os.LookupEnv("TEST_CREDENTIALS")
	if ok {
		// Make credentials path relative to repo root rather than
		// test/ dir if it is a relative path.
		if !filepath.IsAbs(credentials) {
			credentials = filepath.Join(cwd, "..", credentials)
		}
	} else {
		log.Printf("missing env var TEST_CREDENTIALS, will try to use Application Default Credentials")
	}
	ancestry, ok := os.LookupEnv("TEST_ANCESTRY")
	if !ok {
		log.Printf("Missing required env var TEST_ANCESTRY. Default (%s) will be used.", defaultAncestry)
		ancestry = defaultAncestry
	}
	providerVersion := defaultProviderVersion
	//As time is not information in terraform resource data, time is fixed for testing purposes
	fixedTime := time.Date(2021, time.April, 14, 15, 16, 17, 0, time.UTC)
	data = &testData{
		TFVersion: "0.12",
		Provider: map[string]string{
			"version":     providerVersion,
			"project":     project,
			"credentials": credentials,
		},
		Time: map[string]string{
			"RFC3339Nano": fixedTime.Format(time.RFC3339Nano),
		},
		Project: map[string]string{
			"Name":               "My Project Name",
			"ProjectId":          "my-project-id",
			"BillingAccountName": "012345-567890-ABCDEF",
			"Number":             "1234567890",
		},
		OrgID:    org,
		FolderID: folder,
		Ancestry: ancestry,
	}
}

func generateTestFiles(t *testing.T, sourceDir string, targetDir string, selector string) {
	funcMap := template.FuncMap{
		"pastLastSlash": func(s string) string {
			split := strings.Split(s, "/")
			return split[len(split)-1]
		},
	}
	tmpls, err := template.New("").Funcs(funcMap).
		ParseGlob(filepath.Join(sourceDir, selector))
	if err != nil {
		t.Fatalf("generateTestFiles: %v", err)
	}
	for _, tmpl := range tmpls.Templates() {
		if tmpl.Name() == "" {
			continue // Skip base template.
		}
		path := filepath.Join(targetDir, tmpl.Name())
		f, err := os.Create(path)
		if err != nil {
			t.Fatalf("creating terraform file %v: %v", path, err)
		}
		if err := tmpl.Execute(f, data); err != nil {
			t.Fatalf("templating terraform file %v: %v", path, err)
		}
		if err := f.Close(); err != nil {
			t.Fatalf("closing file %v: %v", path, err)
		}
		log.Printf("Successfully created file %v", path)
	}
}

func normalizeAssets(t *testing.T, assets []google.Asset, offline bool) []google.Asset {
	t.Helper()
	ret := make([]google.Asset, len(assets))
	re := regexp.MustCompile(`/placeholder-[^/]+`)
	for i := range assets {
		// Get conformity by converting to/from json.
		bytes, err := json.Marshal(assets[i])
		if err != nil {
			t.Fatalf("marshaling: %v", err)
		}

		var asset google.Asset
		err = json.Unmarshal(bytes, &asset)
		if err != nil {
			t.Fatalf("marshaling: %v", err)
		}
		if !offline {
			// remove the ancestry as the value of that is dependent on project,
			// and is not important for the test.
			asset.Ancestry = ""
		}
		// Replace placeholder in names. This allows us to compare generated placeholders
		// (for example due to "unknown after apply") with the values in the expected
		// output files.
		asset.Name = re.ReplaceAllString(asset.Name, fmt.Sprintf("/placeholder-foobar"))
		ret[i] = asset
	}
	return ret
}
<commit_msg>Made creation of test files only get printed with verbose logging or on failures (#454)<commit_after>package test

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"testing"
	"text/template"
	"time"

	"github.com/GoogleCloudPlatform/terraform-validator/converters/google"
)

const (
	samplePolicyPath       = "../testdata/sample_policies"
	defaultAncestry        = "organization/12345/folder/67890"
	defaultOrganization    = "12345"
	defaultFolder          = "67890"
	defaultProject         = "foobar"
	defaultProviderVersion = "4.4.0"
)

var (
	data      *testData
	tfvBinary string
	tmpDir    = os.TempDir()
)

// testData represents the full dataset that is used for templating terraform
// configs. It contains Google API resources that are expected to be returned
// after converting the terraform plan.
type testData struct {
	// is not nil - Terraform 12 version used
	TFVersion string
	// provider "google"
	Provider map[string]string
	Project  map[string]string
	Time     map[string]string
	OrgID    string
	FolderID string
	Ancestry string
}

// init initializes the variables used for testing. As tests rely on
// environment variables, the parsing of those are only done once.
func init() {
	// don't raise errors in glog

	cwd, err := os.Getwd()
	if err != nil {
		log.Fatalf("cannot get current directory: %v", err)
	}
	tfvBinary = filepath.Join(cwd, "..", "bin", "terraform-validator")
	project, ok := os.LookupEnv("TEST_PROJECT")
	if !ok {
		log.Printf("Missing required env var TEST_PROJECT. Default (%s) will be used.", defaultProject)
		project = defaultProject
	}
	org, ok := os.LookupEnv("TEST_ORG_ID")
	if !ok {
		log.Printf("Missing required env var TEST_ORG_ID. Default (%s) will be used.", defaultOrganization)
		org = defaultOrganization
	}
	folder, ok := os.LookupEnv("TEST_FOLDER_ID")
	if !ok {
		log.Printf("Missing required env var TEST_FOLDER_ID. Default (%s) will be used.", defaultFolder)
		folder = defaultFolder
	}
	credentials, ok := os.LookupEnv("TEST_CREDENTIALS")
	if ok {
		// Make credentials path relative to repo root rather than
		// test/ dir if it is a relative path.
		if !filepath.IsAbs(credentials) {
			credentials = filepath.Join(cwd, "..", credentials)
		}
	} else {
		log.Printf("missing env var TEST_CREDENTIALS, will try to use Application Default Credentials")
	}
	ancestry, ok := os.LookupEnv("TEST_ANCESTRY")
	if !ok {
		log.Printf("Missing required env var TEST_ANCESTRY. Default (%s) will be used.", defaultAncestry)
		ancestry = defaultAncestry
	}
	providerVersion := defaultProviderVersion
	//As time is not information in terraform resource data, time is fixed for testing purposes
	fixedTime := time.Date(2021, time.April, 14, 15, 16, 17, 0, time.UTC)
	data = &testData{
		TFVersion: "0.12",
		Provider: map[string]string{
			"version":     providerVersion,
			"project":     project,
			"credentials": credentials,
		},
		Time: map[string]string{
			"RFC3339Nano": fixedTime.Format(time.RFC3339Nano),
		},
		Project: map[string]string{
			"Name":               "My Project Name",
			"ProjectId":          "my-project-id",
			"BillingAccountName": "012345-567890-ABCDEF",
			"Number":             "1234567890",
		},
		OrgID:    org,
		FolderID: folder,
		Ancestry: ancestry,
	}
}

func generateTestFiles(t *testing.T, sourceDir string, targetDir string, selector string) {
	funcMap := template.FuncMap{
		"pastLastSlash": func(s string) string {
			split := strings.Split(s, "/")
			return split[len(split)-1]
		},
	}
	tmpls, err := template.New("").Funcs(funcMap).
		ParseGlob(filepath.Join(sourceDir, selector))
	if err != nil {
		t.Fatalf("generateTestFiles: %v", err)
	}
	for _, tmpl := range tmpls.Templates() {
		if tmpl.Name() == "" {
			continue // Skip base template.
		}
		path := filepath.Join(targetDir, tmpl.Name())
		f, err := os.Create(path)
		if err != nil {
			t.Fatalf("creating terraform file %v: %v", path, err)
		}
		if err := tmpl.Execute(f, data); err != nil {
			t.Fatalf("templating terraform file %v: %v", path, err)
		}
		if err := f.Close(); err != nil {
			t.Fatalf("closing file %v: %v", path, err)
		}
		t.Logf("Successfully created file %v", path)
	}
}

func normalizeAssets(t *testing.T, assets []google.Asset, offline bool) []google.Asset {
	t.Helper()
	ret := make([]google.Asset, len(assets))
	re := regexp.MustCompile(`/placeholder-[^/]+`)
	for i := range assets {
		// Get conformity by converting to/from json.
		bytes, err := json.Marshal(assets[i])
		if err != nil {
			t.Fatalf("marshaling: %v", err)
		}

		var asset google.Asset
		err = json.Unmarshal(bytes, &asset)
		if err != nil {
			t.Fatalf("marshaling: %v", err)
		}
		if !offline {
			// remove the ancestry as the value of that is dependent on project,
			// and is not important for the test.
			asset.Ancestry = ""
		}
		// Replace placeholder in names. This allows us to compare generated placeholders
		// (for example due to "unknown after apply") with the values in the expected
		// output files.
		asset.Name = re.ReplaceAllString(asset.Name, fmt.Sprintf("/placeholder-foobar"))
		ret[i] = asset
	}
	return ret
}
<|endoftext|>


<commit_before>// © Copyright 2016 Jesse Allen. All rights reserved.
// Released under the MIT license found in the LICENSE file.

package handy_test

import (
	"bytes"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/jessecarl/handy"
)

func TestServeWithDelay(t *testing.T) {
	testCases := []struct {
		name      string
		method    string
		path      string
		wantPath  string
		wantDelay time.Duration
	}{
		{"no delay, GET",
			http.MethodGet,
			"/foo",
			"/foo",
			0,
		},
		{"no delay, no path, GET",
			http.MethodGet,
			"/",
			"/",
			0,
		},
		{"no delay, POST",
			http.MethodPost,
			"/bar/baz/",
			"/bar/baz/",
			0,
		},
		{"1s delay, PUT",
			http.MethodPut,
			"/foo/1s",
			"/foo",
			time.Second,
		},
		{"10ms delay, HEAD",
			http.MethodHead,
			"/10ms/",
			"/", // always have at least one slash
			time.Millisecond * 10,
		},
		{"300ms delay, GET",
			http.MethodGet,
			"/baz/bar/300ms/",
			"/baz/bar",
			time.Millisecond * 300,
		},
		{"1ms delay, GET",
			http.MethodGet,
			"/foo/bar/baz/1ms",
			"/foo/bar/baz",
			time.Millisecond,
		},
		{"100µs delay, GET",
			http.MethodGet,
			"/foo/bar/baz/100µs",
			"/foo/bar/baz",
			time.Microsecond * 100,
		},
		{"100µs delay 'us', GET",
			http.MethodGet,
			"/foo/bar/baz/100us",
			"/foo/bar/baz",
			time.Microsecond * 100,
		},
		{"10µs delay, GET",
			http.MethodGet,
			"/foo/bar/baz/10µs",
			"/foo/bar/baz",
			time.Microsecond * 10,
		},
		{"1µs delay, GET",
			http.MethodGet,
			"/foo/bar/baz/1µs",
			"/foo/bar/baz",
			time.Microsecond,
		},
	}

	for _, tc := range testCases {
		tc := tc
		t.Run(tc.name, func(t *testing.T) {
			var start time.Time
			s := httptest.NewServer(handy.ServeWithDelay(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				gotDelay := time.Since(start)
				if r.URL.Path != tc.wantPath {
					t.Fatalf("handler call to %q, expected %q", r.URL.Path, tc.wantPath)
				}
				delayError := (float64(gotDelay) - float64(tc.wantDelay)) / float64(tc.wantDelay)
				if delayError > 0.05 && delayError < -0.05 {
					t.Fatalf("handler delay took %v, expected %v, more than %v delay", gotDelay, tc.wantDelay, delayError)
				}
				w.Write([]byte("delayed handler"))
			})))

			req, err := http.NewRequest(tc.method, s.URL+tc.path, nil)
			if err != nil {
				t.Fatalf("constructing request: %+v", err)
			}
			start = time.Now()
			res, err := http.DefaultClient.Do(req)
			if err != nil {
				t.Fatalf("doing request: %+v", err)
			}
			defer res.Body.Close()
			body, err := ioutil.ReadAll(res.Body)
			if err != nil {
				t.Fatalf("reading response body: %+v", err)
			}
			if !bytes.Equal(body, []byte("delayed handler")) && tc.method != http.MethodHead {
				t.Fatalf("request to %q returned body %q, expected %q", tc.path, body, []byte("delayed handler"))
			}
		})
	}
}
<commit_msg>Adds test for empty get request to delay handler<commit_after>// © Copyright 2016 Jesse Allen. All rights reserved.
// Released under the MIT license found in the LICENSE file.

package handy_test

import (
	"bytes"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/jessecarl/handy"
)

func TestServeWithDelay(t *testing.T) {
	testCases := []struct {
		name      string
		method    string
		path      string
		wantPath  string
		wantDelay time.Duration
	}{
		{"no delay, GET",
			http.MethodGet,
			"/foo",
			"/foo",
			0,
		},
		{"no delay, no path, GET",
			http.MethodGet,
			"/",
			"/",
			0,
		},
		{"no delay, empty path, GET",
			http.MethodGet,
			"",
			"/",
			0,
		},
		{"no delay, POST",
			http.MethodPost,
			"/bar/baz/",
			"/bar/baz/",
			0,
		},
		{"1s delay, PUT",
			http.MethodPut,
			"/foo/1s",
			"/foo",
			time.Second,
		},
		{"10ms delay, HEAD",
			http.MethodHead,
			"/10ms/",
			"/", // always have at least one slash
			time.Millisecond * 10,
		},
		{"300ms delay, GET",
			http.MethodGet,
			"/baz/bar/300ms/",
			"/baz/bar",
			time.Millisecond * 300,
		},
		{"1ms delay, GET",
			http.MethodGet,
			"/foo/bar/baz/1ms",
			"/foo/bar/baz",
			time.Millisecond,
		},
		{"100µs delay, GET",
			http.MethodGet,
			"/foo/bar/baz/100µs",
			"/foo/bar/baz",
			time.Microsecond * 100,
		},
		{"100µs delay 'us', GET",
			http.MethodGet,
			"/foo/bar/baz/100us",
			"/foo/bar/baz",
			time.Microsecond * 100,
		},
		{"10µs delay, GET",
			http.MethodGet,
			"/foo/bar/baz/10µs",
			"/foo/bar/baz",
			time.Microsecond * 10,
		},
		{"1µs delay, GET",
			http.MethodGet,
			"/foo/bar/baz/1µs",
			"/foo/bar/baz",
			time.Microsecond,
		},
	}

	for _, tc := range testCases {
		tc := tc
		t.Run(tc.name, func(t *testing.T) {
			var start time.Time
			s := httptest.NewServer(handy.ServeWithDelay(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				gotDelay := time.Since(start)
				if r.URL.Path != tc.wantPath {
					t.Fatalf("handler call to %q, expected %q", r.URL.Path, tc.wantPath)
				}
				delayError := (float64(gotDelay) - float64(tc.wantDelay)) / float64(tc.wantDelay)
				if delayError > 0.05 && delayError < -0.05 {
					t.Fatalf("handler delay took %v, expected %v, more than %v delay", gotDelay, tc.wantDelay, delayError)
				}
				w.Write([]byte("delayed handler"))
			})))

			req, err := http.NewRequest(tc.method, s.URL+tc.path, nil)
			if err != nil {
				t.Fatalf("constructing request: %+v", err)
			}
			start = time.Now()
			res, err := http.DefaultClient.Do(req)
			if err != nil {
				t.Fatalf("doing request: %+v", err)
			}
			defer res.Body.Close()
			body, err := ioutil.ReadAll(res.Body)
			if err != nil {
				t.Fatalf("reading response body: %+v", err)
			}
			if !bytes.Equal(body, []byte("delayed handler")) && tc.method != http.MethodHead {
				t.Fatalf("request to %q returned body %q, expected %q", tc.path, body, []byte("delayed handler"))
			}
		})
	}
}
<|endoftext|>


<commit_before>package builder

import (
	"bytes"
	"log"
	"testing"

	dockercmd "github.com/docker/docker/builder/command"
	"github.com/docker/docker/builder/parser"
)

func TestReplaceValidCmd(t *testing.T) {
	tests := []struct {
		name           string
		cmd            string
		replaceArgs    string
		fileData       []byte
		edited         bool
		expectedOutput string
		expectedErr    error
	}{
		{
			name:           "from-replacement",
			cmd:            dockercmd.From,
			replaceArgs:    "other/image",
			fileData:       []byte(dockerFile),
			edited:         true,
			expectedOutput: expectedFROM,
			expectedErr:    nil,
		},
		{
			name:           "run-replacement",
			cmd:            dockercmd.Run,
			replaceArgs:    "This test kind-of-fails before string replacement so this string won't be used",
			fileData:       []byte(dockerFile),
			edited:         false,
			expectedOutput: "",
			expectedErr:    replaceCmdErr,
		},
		{
			name:           "invalid-dockerfile-cmd",
			cmd:            "blabla",
			replaceArgs:    "This test fails at start so this string won't be used",
			fileData:       []byte(dockerFile),
			edited:         false,
			expectedOutput: "",
			expectedErr:    invalidCmdErr,
		},
		{
			name:           "no-cmd-in-dockerfile",
			cmd:            dockercmd.Cmd,
			replaceArgs:    "runme.sh",
			fileData:       []byte(dockerFile),
			edited:         false,
			expectedOutput: "",
			expectedErr:    replaceCmdErr,
		},
		{
			name:           "trailing-slash",
			cmd:            dockercmd.From,
			replaceArgs:    "rhel",
			fileData:       []byte(trSlashFile),
			edited:         true,
			expectedOutput: expectedtrSlashFile,
			expectedErr:    nil,
		},
		{
			name:           "multiple trailing slashes plus plus",
			cmd:            dockercmd.From,
			replaceArgs:    "scratch",
			fileData:       []byte(trickierFile),
			edited:         true,
			expectedOutput: expectedTrickierFile,
			expectedErr:    nil,
		},
	}

	for _, test := range tests {
		out, err := replaceValidCmd(test.cmd, test.replaceArgs, test.fileData)
		if err != test.expectedErr {
			t.Errorf("%s: Unexpected error: Expected %v, got %v", test.name, test.expectedErr, err)
		}
		if out != test.expectedOutput {
			t.Errorf("%s: Unexpected output:\n\nExpected:\n%s\n(length: %d)\n\ngot:\n%s\n(length: %d)",
				test.name, test.expectedOutput, len(test.expectedOutput), out, len(out))
		}
	}

	// Re-use the tests above
	var buf *bytes.Buffer
	for _, test := range tests {
		buf = bytes.NewBuffer([]byte(test.fileData))
		original, err := parser.Parse(buf)
		if err != nil {
			log.Println(err)
		}
		repl, err := replaceValidCmd(test.cmd, test.replaceArgs, test.fileData)
		if err != nil {
			log.Println(err)
		}
		buf = bytes.NewBuffer([]byte(repl))
		edited, err := parser.Parse(buf)
		if err != nil {
			log.Println(err)
		}

		diff := cmpASTs(original, edited)

		// Note that these tests will probably fail for Dockerfiles where we have replaced
		// multiline command arguments
		if test.edited && diff != 1 {
			t.Errorf("%s: Edit mismatch, expected one edit, got %d", test.name, diff)
		} else if !test.edited && diff > 0 {
			t.Errorf("%s: Edit mismatch, expected no edit, got %d", test.name, diff)
		}
	}
}

// cmpASTs compares two Abstract Syntax Trees and returns the
// amount of differences between them
func cmpASTs(original *parser.Node, edited *parser.Node) int {
	index := 0
	if original.Value != edited.Value {
		index++
	}

	originalChildren := make([]*parser.Node, 0)
	for _, n := range original.Children {
		originalChildren = append(originalChildren, n)
	}
	editedChildren := make([]*parser.Node, 0)
	for _, n := range edited.Children {
		editedChildren = append(editedChildren, n)
	}
	for i := 0; i < len(editedChildren); i++ {
		index += cmpASTs(originalChildren[i], editedChildren[i])
	}

	if original.Next != nil && edited.Next != nil {
		index += cmpASTs(original.Next, edited.Next)
	} else if original.Next != edited.Next {
		index++
	}
	return index
}

func TestTraverseAST(t *testing.T) {
	tests := []struct {
		name     string
		cmd      string
		fileData []byte
		expected int
	}{
		{
			name:     "dockerFile",
			cmd:      dockercmd.Entrypoint,
			fileData: []byte(dockerFile),
			expected: 1,
		},
		{
			name:     "expectedFROM",
			cmd:      dockercmd.From,
			fileData: []byte(expectedFROM),
			expected: 2,
		},
		{
			name:     "trSlashFile",
			cmd:      dockercmd.Entrypoint,
			fileData: []byte(trSlashFile),
			expected: 0,
		},
		{
			name:     "expectedtrSlashFile",
			cmd:      dockercmd.Cmd,
			fileData: []byte(expectedtrSlashFile),
			expected: 1,
		},
	}

	var buf *bytes.Buffer
	for _, test := range tests {
		buf = bytes.NewBuffer([]byte(test.fileData))
		node, err := parser.Parse(buf)
		if err != nil {
			log.Println(err)
		}

		howMany := traverseAST(test.cmd, node)
		if howMany != test.expected {
			t.Errorf("Wrong result, expected %d, got %d", test.expected, howMany)
		}
	}
}

const (
	dockerFile = `
FROM openshift/origin-base
FROM candidate

RUN mkdir -p /var/lib/openshift

ADD bin/openshift        /usr/bin/openshift
RUN ln -s /usr/bin/openshift /usr/bin/osc && \

ENV HOME /root
WORKDIR /var/lib/openshift
ENTRYPOINT ["/usr/bin/openshift"]
`

	expectedFROM = `
FROM openshift/origin-base
FROM other/image

RUN mkdir -p /var/lib/openshift

ADD bin/openshift        /usr/bin/openshift
RUN ln -s /usr/bin/openshift /usr/bin/osc && \

ENV HOME /root
WORKDIR /var/lib/openshift
ENTRYPOINT ["/usr/bin/openshift"]
`

	trSlashFile = `
from \
centos
CMD "cat /etc/passwd"
`
	expectedtrSlashFile = `
from \
rhel
CMD "cat /etc/passwd"
`

	trickierFile = `
from centos \
rhel \
ubuntu

CMD ["executable","param1","param2"]
`

	expectedTrickierFile = `
from \
scratch

CMD ["executable","param1","param2"]
`
)
<commit_msg>Make tests for the Docker parser more robust<commit_after>package builder

import (
	"bytes"
	"log"
	"testing"

	dockercmd "github.com/docker/docker/builder/command"
	"github.com/docker/docker/builder/parser"
)

func TestReplaceValidCmd(t *testing.T) {
	tests := []struct {
		name           string
		cmd            string
		replaceArgs    string
		fileData       []byte
		expectedOutput string
		expectedDiffs  int
		expectedErr    error
	}{
		{
			name:           "from-replacement",
			cmd:            dockercmd.From,
			replaceArgs:    "other/image",
			fileData:       []byte(dockerFile),
			expectedOutput: expectedFROM,
			expectedDiffs:  1,
			expectedErr:    nil,
		},
		{
			name:           "run-replacement",
			cmd:            dockercmd.Run,
			replaceArgs:    "This test kind-of-fails before string replacement so this string won't be used",
			fileData:       []byte(dockerFile),
			expectedOutput: "",
			expectedErr:    replaceCmdErr,
		},
		{
			name:           "invalid-dockerfile-cmd",
			cmd:            "blabla",
			replaceArgs:    "This test fails at start so this string won't be used",
			fileData:       []byte(dockerFile),
			expectedOutput: "",
			expectedErr:    invalidCmdErr,
		},
		{
			name:           "no-cmd-in-dockerfile",
			cmd:            dockercmd.Cmd,
			replaceArgs:    "runme.sh",
			fileData:       []byte(dockerFile),
			expectedOutput: "",
			expectedErr:    replaceCmdErr,
		},
		{
			name:           "trailing-slash",
			cmd:            dockercmd.From,
			replaceArgs:    "rhel",
			fileData:       []byte(trSlashFile),
			expectedOutput: expectedtrSlashFile,
			expectedDiffs:  1,
			expectedErr:    nil,
		},
		{
			name:           "multiple trailing slashes plus plus",
			cmd:            dockercmd.From,
			replaceArgs:    "scratch",
			fileData:       []byte(trickierFile),
			expectedOutput: expectedTrickierFile,
			expectedDiffs:  1,
			expectedErr:    nil,
		},
	}

	for _, test := range tests {
		out, err := replaceValidCmd(test.cmd, test.replaceArgs, test.fileData)
		if err != test.expectedErr {
			t.Errorf("%s: Unexpected error: Expected %v, got %v", test.name, test.expectedErr, err)
		}
		if out != test.expectedOutput {
			t.Errorf("%s: Unexpected output:\n\nExpected:\n%s\n(length: %d)\n\ngot:\n%s\n(length: %d)",
				test.name, test.expectedOutput, len(test.expectedOutput), out, len(out))
		}
	}

	// Re-use the tests above
	var buf *bytes.Buffer
	for _, test := range tests {
		buf = bytes.NewBuffer([]byte(test.fileData))
		original, err := parser.Parse(buf)
		if err != nil {
			log.Println(err)
		}
		repl, err := replaceValidCmd(test.cmd, test.replaceArgs, test.fileData)
		if err != nil {
			log.Println(err)
		}
		buf = bytes.NewBuffer([]byte(repl))
		edited, err := parser.Parse(buf)
		if err != nil {
			log.Println(err)
		}

		diff := cmpASTs(original, edited)

		if diff != test.expectedDiffs {
			t.Errorf("%s: Edit mismatch, expected %d edit(s), got %d", test.name, test.expectedDiffs, diff)
		}
	}
}

// cmpASTs compares two Abstract Syntax Trees and returns the
// amount of differences between them
func cmpASTs(original *parser.Node, edited *parser.Node) int {
	index := 0
	if original.Value != edited.Value {
		index++
	}

	originalChildren := make([]*parser.Node, 0)
	for _, n := range original.Children {
		originalChildren = append(originalChildren, n)
	}
	editedChildren := make([]*parser.Node, 0)
	for _, n := range edited.Children {
		editedChildren = append(editedChildren, n)
	}
	for i := 0; i < len(editedChildren); i++ {
		index += cmpASTs(originalChildren[i], editedChildren[i])
	}

	if original.Next != nil && edited.Next != nil {
		index += cmpASTs(original.Next, edited.Next)
	} else if original.Next != edited.Next {
		index++
	}
	return index
}

func TestTraverseAST(t *testing.T) {
	tests := []struct {
		name     string
		cmd      string
		fileData []byte
		expected int
	}{
		{
			name:     "dockerFile",
			cmd:      dockercmd.Entrypoint,
			fileData: []byte(dockerFile),
			expected: 1,
		},
		{
			name:     "expectedFROM",
			cmd:      dockercmd.From,
			fileData: []byte(expectedFROM),
			expected: 2,
		},
		{
			name:     "trSlashFile",
			cmd:      dockercmd.Entrypoint,
			fileData: []byte(trSlashFile),
			expected: 0,
		},
		{
			name:     "expectedtrSlashFile",
			cmd:      dockercmd.Cmd,
			fileData: []byte(expectedtrSlashFile),
			expected: 1,
		},
	}

	var buf *bytes.Buffer
	for _, test := range tests {
		buf = bytes.NewBuffer([]byte(test.fileData))
		node, err := parser.Parse(buf)
		if err != nil {
			log.Println(err)
		}

		howMany := traverseAST(test.cmd, node)
		if howMany != test.expected {
			t.Errorf("Wrong result, expected %d, got %d", test.expected, howMany)
		}
	}
}

const (
	dockerFile = `
FROM openshift/origin-base
FROM candidate

RUN mkdir -p /var/lib/openshift

ADD bin/openshift        /usr/bin/openshift
RUN ln -s /usr/bin/openshift /usr/bin/osc && \

ENV HOME /root
WORKDIR /var/lib/openshift
ENTRYPOINT ["/usr/bin/openshift"]
`

	expectedFROM = `
FROM openshift/origin-base
FROM other/image

RUN mkdir -p /var/lib/openshift

ADD bin/openshift        /usr/bin/openshift
RUN ln -s /usr/bin/openshift /usr/bin/osc && \

ENV HOME /root
WORKDIR /var/lib/openshift
ENTRYPOINT ["/usr/bin/openshift"]
`

	trSlashFile = `
from \
centos
CMD "cat /etc/passwd"
`
	expectedtrSlashFile = `
from \
rhel
CMD "cat /etc/passwd"
`

	trickierFile = `
from centos \
rhel \
ubuntu

CMD ["executable","param1","param2"]
`

	expectedTrickierFile = `
from \
scratch

CMD ["executable","param1","param2"]
`
)
<|endoftext|>


<commit_before>package donut

import (
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"
	"time"

	"encoding/gob"
	"encoding/json"

	"github.com/minio-io/minio/pkg/encoding/erasure"
	"github.com/minio-io/minio/pkg/utils/split"
)

type donutDriver struct {
	buckets map[string]Bucket
	nodes   map[string]Node
}

// NewDonutDriver - instantiate new donut driver
func NewDonutDriver(root string) Donut {
	nodes := make(map[string]Node)
	nodes["localhost"] = localDirectoryNode{root: root}
	driver := donutDriver{
		buckets: make(map[string]Bucket),
		nodes:   nodes,
	}
	return driver
}

func (driver donutDriver) CreateBucket(bucketName string) error {
	if _, ok := driver.buckets[bucketName]; ok == false {
		bucketName = strings.TrimSpace(bucketName)
		if bucketName == "" {
			return errors.New("Cannot create bucket with no name")
		}
		// assign nodes
		// TODO assign other nodes
		nodes := make([]string, 16)
		for i := 0; i < 16; i++ {
			nodes[i] = "localhost"
		}
		bucket := bucketDriver{
			nodes: nodes,
		}
		driver.buckets[bucketName] = bucket
		return nil
	}
	return errors.New("Bucket exists")
}

func (driver donutDriver) ListBuckets() ([]string, error) {
	var buckets []string
	for bucket := range driver.buckets {
		buckets = append(buckets, bucket)
	}
	sort.Strings(buckets)
	return buckets, nil
}

func (driver donutDriver) GetObjectWriter(bucketName, objectName string) (ObjectWriter, error) {
	if bucket, ok := driver.buckets[bucketName]; ok == true {
		writers := make([]Writer, 16)
		nodes, err := bucket.GetNodes()
		if err != nil {
			return nil, err
		}
		for i, nodeID := range nodes {
			if node, ok := driver.nodes[nodeID]; ok == true {
				writer, _ := node.GetWriter(bucketName+":0:"+strconv.Itoa(i), objectName)
				writers[i] = writer
			}
		}
		return newErasureWriter(writers), nil
	}
	return nil, errors.New("Bucket not found")
}

func (driver donutDriver) GetObject(bucketName, objectName string) (io.ReadCloser, error) {
	r, w := io.Pipe()
	if bucket, ok := driver.buckets[bucketName]; ok == true {
		readers := make([]io.ReadCloser, 16)
		nodes, err := bucket.GetNodes()
		if err != nil {
			return nil, err
		}
		var metadata map[string]string
		for i, nodeID := range nodes {
			if node, ok := driver.nodes[nodeID]; ok == true {
				bucketID := bucketName + ":0:" + strconv.Itoa(i)
				reader, err := node.GetReader(bucketID, objectName)
				if err != nil {
					return nil, err
				}
				readers[i] = reader
				if metadata == nil {
					metadata, err = node.GetDonutMetadata(bucketID, objectName)
					if err != nil {
						return nil, err
					}
				}
			}
		}
		go erasureReader(readers, metadata, w)
		return r, nil
	}
	return nil, errors.New("Bucket not found")
}

func erasureReader(readers []io.ReadCloser, donutMetadata map[string]string, writer *io.PipeWriter) {
	totalChunks, _ := strconv.Atoi(donutMetadata["chunkCount"])
	totalLeft, _ := strconv.Atoi(donutMetadata["totalLength"])
	blockSize, _ := strconv.Atoi(donutMetadata["blockSize"])
	params, _ := erasure.ParseEncoderParams(8, 8, erasure.Cauchy)
	encoder := erasure.NewEncoder(params)
	for _, reader := range readers {
		defer reader.Close()
	}
	for i := 0; i < totalChunks; i++ {
		encodedBytes := make([][]byte, 16)
		for i, reader := range readers {
			var bytesArray []byte
			decoder := gob.NewDecoder(reader)
			err := decoder.Decode(&bytesArray)
			if err != nil {
				log.Println(err)
			}
			encodedBytes[i] = bytesArray
		}
		curBlockSize := totalLeft
		if blockSize < totalLeft {
			curBlockSize = blockSize
		}
		log.Println("decoding block size", curBlockSize)
		decodedData, err := encoder.Decode(encodedBytes, curBlockSize)
		if err != nil {
			writer.CloseWithError(err)
			return
		}
		io.Copy(writer, bytes.NewBuffer(decodedData))
		totalLeft = totalLeft - blockSize
	}
	writer.Close()
}

// erasure writer

type erasureWriter struct {
	writers       []Writer
	metadata      map[string]string
	donutMetadata map[string]string // not exposed
	erasureWriter *io.PipeWriter
	isClosed      <-chan bool
}

func newErasureWriter(writers []Writer) ObjectWriter {
	r, w := io.Pipe()
	isClosed := make(chan bool)
	writer := erasureWriter{
		writers:       writers,
		metadata:      make(map[string]string),
		erasureWriter: w,
		isClosed:      isClosed,
	}
	go erasureGoroutine(r, writer, isClosed)
	return writer
}

func erasureGoroutine(r *io.PipeReader, eWriter erasureWriter, isClosed chan<- bool) {
	chunks := split.Stream(r, 10*1024*1024)
	params, _ := erasure.ParseEncoderParams(8, 8, erasure.Cauchy)
	encoder := erasure.NewEncoder(params)
	chunkCount := 0
	totalLength := 0
	for chunk := range chunks {
		if chunk.Err == nil {
			totalLength = totalLength + len(chunk.Data)
			encodedBlocks, _ := encoder.Encode(chunk.Data)
			for blockIndex, block := range encodedBlocks {
				var byteBuffer bytes.Buffer
				gobEncoder := gob.NewEncoder(&byteBuffer)
				gobEncoder.Encode(block)
				io.Copy(eWriter.writers[blockIndex], &byteBuffer)
			}
		}
		chunkCount = chunkCount + 1
	}
	metadata := make(map[string]string)
	metadata["blockSize"] = strconv.Itoa(10 * 1024 * 1024)
	metadata["chunkCount"] = strconv.Itoa(chunkCount)
	metadata["created"] = time.Now().Format(time.RFC3339Nano)
	metadata["erasureK"] = "8"
	metadata["erasureM"] = "8"
	metadata["erasureTechnique"] = "Cauchy"
	metadata["totalLength"] = strconv.Itoa(totalLength)
	for _, nodeWriter := range eWriter.writers {
		if nodeWriter != nil {
			nodeWriter.SetMetadata(eWriter.metadata)
			nodeWriter.SetDonutMetadata(metadata)
			nodeWriter.Close()
		}
	}
	isClosed <- true
}

func (d erasureWriter) Write(data []byte) (int, error) {
	io.Copy(d.erasureWriter, bytes.NewBuffer(data))
	return len(data), nil
}

func (d erasureWriter) Close() error {
	d.erasureWriter.Close()
	<-d.isClosed
	return nil
}

func (d erasureWriter) CloseWithError(err error) error {
	for _, writer := range d.writers {
		if writer != nil {
			writer.CloseWithError(err)
		}
	}
	return nil
}

func (d erasureWriter) SetMetadata(metadata map[string]string) error {
	for k := range d.metadata {
		delete(d.metadata, k)
	}
	for k, v := range metadata {
		d.metadata[k] = v
	}
	return nil
}

func (d erasureWriter) GetMetadata() (map[string]string, error) {
	metadata := make(map[string]string)
	for k, v := range d.metadata {
		metadata[k] = v
	}
	return metadata, nil
}

type localDirectoryNode struct {
	root string
}

func (d localDirectoryNode) GetBuckets() ([]string, error) {
	return nil, errors.New("Not Implemented")
}

func (d localDirectoryNode) GetWriter(bucket, object string) (Writer, error) {
	objectPath := path.Join(d.root, bucket, object)
	err := os.MkdirAll(objectPath, 0700)
	if err != nil {
		return nil, err
	}
	return newDonutFileWriter(objectPath)
}

func (d localDirectoryNode) GetReader(bucket, object string) (io.ReadCloser, error) {
	return os.Open(path.Join(d.root, bucket, object, "data"))
}

func (d localDirectoryNode) GetMetadata(bucket, object string) (map[string]string, error) {
	return d.getMetadata(bucket, object, "metadata.json")
}
func (d localDirectoryNode) GetDonutMetadata(bucket, object string) (map[string]string, error) {
	return d.getMetadata(bucket, object, "donutMetadata.json")
}

func (d localDirectoryNode) getMetadata(bucket, object, fileName string) (map[string]string, error) {
	file, err := os.Open(path.Join(d.root, bucket, object, fileName))
	defer file.Close()
	if err != nil {
		return nil, err
	}
	metadata := make(map[string]string)
	decoder := json.NewDecoder(file)
	if err := decoder.Decode(&metadata); err != nil {
		return nil, err
	}
	return metadata, nil

}

func newDonutFileWriter(objectDir string) (Writer, error) {
	dataFile, err := os.OpenFile(path.Join(objectDir, "data"), os.O_WRONLY|os.O_CREATE, 0600)
	if err != nil {
		return nil, err
	}
	return donutFileWriter{
		root:          objectDir,
		file:          dataFile,
		metadata:      make(map[string]string),
		donutMetadata: make(map[string]string),
	}, nil
}

type donutFileWriter struct {
	root          string
	file          *os.File
	metadata      map[string]string
	donutMetadata map[string]string
	err           error
}

func (d donutFileWriter) Write(data []byte) (int, error) {
	return d.file.Write(data)
}

func (d donutFileWriter) Close() error {
	if d.err != nil {
		return d.err
	}

	d.file.Close()

	metadata, _ := json.Marshal(d.metadata)
	ioutil.WriteFile(path.Join(d.root, "metadata.json"), metadata, 0600)
	donutMetadata, _ := json.Marshal(d.donutMetadata)
	ioutil.WriteFile(path.Join(d.root, "donutMetadata.json"), donutMetadata, 0600)

	return nil
}

func (d donutFileWriter) CloseWithError(err error) error {
	if d.err != nil {
		d.err = err
	}
	d.file.Close()
	return nil
}

func (d donutFileWriter) SetMetadata(metadata map[string]string) error {
	for k := range d.metadata {
		delete(d.metadata, k)
	}
	for k, v := range metadata {
		d.metadata[k] = v
	}
	return nil
}

func (d donutFileWriter) GetMetadata() (map[string]string, error) {
	metadata := make(map[string]string)
	for k, v := range d.metadata {
		metadata[k] = v
	}
	return metadata, nil
}

func (d donutFileWriter) SetDonutMetadata(metadata map[string]string) error {
	for k := range d.donutMetadata {
		delete(d.donutMetadata, k)
	}
	for k, v := range metadata {
		d.donutMetadata[k] = v
	}
	return nil
}

func (d donutFileWriter) GetDonutMetadata() (map[string]string, error) {
	donutMetadata := make(map[string]string)
	for k, v := range d.donutMetadata {
		donutMetadata[k] = v
	}
	return donutMetadata, nil
}
<commit_msg>Removing gob from donut driver<commit_after>package donut

import (
	"bytes"
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/minio-io/minio/pkg/encoding/erasure"
	"github.com/minio-io/minio/pkg/utils/split"
)

type donutDriver struct {
	buckets map[string]Bucket
	nodes   map[string]Node
}

// NewDonutDriver - instantiate new donut driver
func NewDonutDriver(root string) Donut {
	nodes := make(map[string]Node)
	nodes["localhost"] = localDirectoryNode{root: root}
	driver := donutDriver{
		buckets: make(map[string]Bucket),
		nodes:   nodes,
	}
	return driver
}

func (driver donutDriver) CreateBucket(bucketName string) error {
	if _, ok := driver.buckets[bucketName]; ok == false {
		bucketName = strings.TrimSpace(bucketName)
		if bucketName == "" {
			return errors.New("Cannot create bucket with no name")
		}
		// assign nodes
		// TODO assign other nodes
		nodes := make([]string, 16)
		for i := 0; i < 16; i++ {
			nodes[i] = "localhost"
		}
		bucket := bucketDriver{
			nodes: nodes,
		}
		driver.buckets[bucketName] = bucket
		return nil
	}
	return errors.New("Bucket exists")
}

func (driver donutDriver) ListBuckets() ([]string, error) {
	var buckets []string
	for bucket := range driver.buckets {
		buckets = append(buckets, bucket)
	}
	sort.Strings(buckets)
	return buckets, nil
}

func (driver donutDriver) GetObjectWriter(bucketName, objectName string) (ObjectWriter, error) {
	if bucket, ok := driver.buckets[bucketName]; ok == true {
		writers := make([]Writer, 16)
		nodes, err := bucket.GetNodes()
		if err != nil {
			return nil, err
		}
		for i, nodeID := range nodes {
			if node, ok := driver.nodes[nodeID]; ok == true {
				writer, _ := node.GetWriter(bucketName+":0:"+strconv.Itoa(i), objectName)
				writers[i] = writer
			}
		}
		return newErasureWriter(writers), nil
	}
	return nil, errors.New("Bucket not found")
}

func (driver donutDriver) GetObject(bucketName, objectName string) (io.ReadCloser, error) {
	r, w := io.Pipe()
	if bucket, ok := driver.buckets[bucketName]; ok == true {
		readers := make([]io.ReadCloser, 16)
		nodes, err := bucket.GetNodes()
		if err != nil {
			return nil, err
		}
		var metadata map[string]string
		for i, nodeID := range nodes {
			if node, ok := driver.nodes[nodeID]; ok == true {
				bucketID := bucketName + ":0:" + strconv.Itoa(i)
				reader, err := node.GetReader(bucketID, objectName)
				if err != nil {
					return nil, err
				}
				readers[i] = reader
				if metadata == nil {
					metadata, err = node.GetDonutMetadata(bucketID, objectName)
					if err != nil {
						return nil, err
					}
				}
			}
		}
		go erasureReader(readers, metadata, w)
		return r, nil
	}
	return nil, errors.New("Bucket not found")
}

func erasureReader(readers []io.ReadCloser, donutMetadata map[string]string, writer *io.PipeWriter) {
	totalChunks, _ := strconv.Atoi(donutMetadata["chunkCount"])
	totalLeft, _ := strconv.Atoi(donutMetadata["totalLength"])
	blockSize, _ := strconv.Atoi(donutMetadata["blockSize"])
	params, _ := erasure.ParseEncoderParams(8, 8, erasure.Cauchy)
	encoder := erasure.NewEncoder(params)
	for _, reader := range readers {
		defer reader.Close()
	}
	for i := 0; i < totalChunks; i++ {
		encodedBytes := make([][]byte, 16)
		for i, reader := range readers {
			var bytesBuffer bytes.Buffer
			io.Copy(&bytesBuffer, reader)
			encodedBytes[i] = bytesBuffer.Bytes()
		}
		curBlockSize := totalLeft
		if blockSize < totalLeft {
			curBlockSize = blockSize
		}
		log.Println("decoding block size", curBlockSize)
		decodedData, err := encoder.Decode(encodedBytes, curBlockSize)
		if err != nil {
			writer.CloseWithError(err)
			return
		}
		io.Copy(writer, bytes.NewBuffer(decodedData))
		totalLeft = totalLeft - blockSize
	}
	writer.Close()
}

// erasure writer

type erasureWriter struct {
	writers       []Writer
	metadata      map[string]string
	donutMetadata map[string]string // not exposed
	erasureWriter *io.PipeWriter
	isClosed      <-chan bool
}

func newErasureWriter(writers []Writer) ObjectWriter {
	r, w := io.Pipe()
	isClosed := make(chan bool)
	writer := erasureWriter{
		writers:       writers,
		metadata:      make(map[string]string),
		erasureWriter: w,
		isClosed:      isClosed,
	}
	go erasureGoroutine(r, writer, isClosed)
	return writer
}

func erasureGoroutine(r *io.PipeReader, eWriter erasureWriter, isClosed chan<- bool) {
	chunks := split.Stream(r, 10*1024*1024)
	params, _ := erasure.ParseEncoderParams(8, 8, erasure.Cauchy)
	encoder := erasure.NewEncoder(params)
	chunkCount := 0
	totalLength := 0
	for chunk := range chunks {
		if chunk.Err == nil {
			totalLength = totalLength + len(chunk.Data)
			encodedBlocks, _ := encoder.Encode(chunk.Data)
			for blockIndex, block := range encodedBlocks {
				io.Copy(eWriter.writers[blockIndex], bytes.NewBuffer(block))
			}
		}
		chunkCount = chunkCount + 1
	}
	metadata := make(map[string]string)
	metadata["blockSize"] = strconv.Itoa(10 * 1024 * 1024)
	metadata["chunkCount"] = strconv.Itoa(chunkCount)
	metadata["created"] = time.Now().Format(time.RFC3339Nano)
	metadata["erasureK"] = "8"
	metadata["erasureM"] = "8"
	metadata["erasureTechnique"] = "Cauchy"
	metadata["totalLength"] = strconv.Itoa(totalLength)
	for _, nodeWriter := range eWriter.writers {
		if nodeWriter != nil {
			nodeWriter.SetMetadata(eWriter.metadata)
			nodeWriter.SetDonutMetadata(metadata)
			nodeWriter.Close()
		}
	}
	isClosed <- true
}

func (d erasureWriter) Write(data []byte) (int, error) {
	io.Copy(d.erasureWriter, bytes.NewBuffer(data))
	return len(data), nil
}

func (d erasureWriter) Close() error {
	d.erasureWriter.Close()
	<-d.isClosed
	return nil
}

func (d erasureWriter) CloseWithError(err error) error {
	for _, writer := range d.writers {
		if writer != nil {
			writer.CloseWithError(err)
		}
	}
	return nil
}

func (d erasureWriter) SetMetadata(metadata map[string]string) error {
	for k := range d.metadata {
		delete(d.metadata, k)
	}
	for k, v := range metadata {
		d.metadata[k] = v
	}
	return nil
}

func (d erasureWriter) GetMetadata() (map[string]string, error) {
	metadata := make(map[string]string)
	for k, v := range d.metadata {
		metadata[k] = v
	}
	return metadata, nil
}

type localDirectoryNode struct {
	root string
}

func (d localDirectoryNode) GetBuckets() ([]string, error) {
	return nil, errors.New("Not Implemented")
}

func (d localDirectoryNode) GetWriter(bucket, object string) (Writer, error) {
	objectPath := path.Join(d.root, bucket, object)
	err := os.MkdirAll(objectPath, 0700)
	if err != nil {
		return nil, err
	}
	return newDonutFileWriter(objectPath)
}

func (d localDirectoryNode) GetReader(bucket, object string) (io.ReadCloser, error) {
	return os.Open(path.Join(d.root, bucket, object, "data"))
}

func (d localDirectoryNode) GetMetadata(bucket, object string) (map[string]string, error) {
	return d.getMetadata(bucket, object, "metadata.json")
}
func (d localDirectoryNode) GetDonutMetadata(bucket, object string) (map[string]string, error) {
	return d.getMetadata(bucket, object, "donutMetadata.json")
}

func (d localDirectoryNode) getMetadata(bucket, object, fileName string) (map[string]string, error) {
	file, err := os.Open(path.Join(d.root, bucket, object, fileName))
	defer file.Close()
	if err != nil {
		return nil, err
	}
	metadata := make(map[string]string)
	decoder := json.NewDecoder(file)
	if err := decoder.Decode(&metadata); err != nil {
		return nil, err
	}
	return metadata, nil

}

func newDonutFileWriter(objectDir string) (Writer, error) {
	dataFile, err := os.OpenFile(path.Join(objectDir, "data"), os.O_WRONLY|os.O_CREATE, 0600)
	if err != nil {
		return nil, err
	}
	return donutFileWriter{
		root:          objectDir,
		file:          dataFile,
		metadata:      make(map[string]string),
		donutMetadata: make(map[string]string),
	}, nil
}

type donutFileWriter struct {
	root          string
	file          *os.File
	metadata      map[string]string
	donutMetadata map[string]string
	err           error
}

func (d donutFileWriter) Write(data []byte) (int, error) {
	return d.file.Write(data)
}

func (d donutFileWriter) Close() error {
	if d.err != nil {
		return d.err
	}

	d.file.Close()

	metadata, _ := json.Marshal(d.metadata)
	ioutil.WriteFile(path.Join(d.root, "metadata.json"), metadata, 0600)
	donutMetadata, _ := json.Marshal(d.donutMetadata)
	ioutil.WriteFile(path.Join(d.root, "donutMetadata.json"), donutMetadata, 0600)

	return nil
}

func (d donutFileWriter) CloseWithError(err error) error {
	if d.err != nil {
		d.err = err
	}
	d.file.Close()
	return nil
}

func (d donutFileWriter) SetMetadata(metadata map[string]string) error {
	for k := range d.metadata {
		delete(d.metadata, k)
	}
	for k, v := range metadata {
		d.metadata[k] = v
	}
	return nil
}

func (d donutFileWriter) GetMetadata() (map[string]string, error) {
	metadata := make(map[string]string)
	for k, v := range d.metadata {
		metadata[k] = v
	}
	return metadata, nil
}

func (d donutFileWriter) SetDonutMetadata(metadata map[string]string) error {
	for k := range d.donutMetadata {
		delete(d.donutMetadata, k)
	}
	for k, v := range metadata {
		d.donutMetadata[k] = v
	}
	return nil
}

func (d donutFileWriter) GetDonutMetadata() (map[string]string, error) {
	donutMetadata := make(map[string]string)
	for k, v := range d.donutMetadata {
		donutMetadata[k] = v
	}
	return donutMetadata, nil
}
<|endoftext|>


<commit_before>package test

import (
	"github.com/eaciit/toolkit"
	//. "github.com/frezadev/hdc/hive"
	. "github.com/eaciit/hdc/hive"
	//. "github.com/RyanCi/hdc/hive"
	"os"
	"testing"
)

var h *Hive
var e error

type Sample7 struct {
	Code        string `tag_name:"code"`
	Description string `tag_name:"description"`
	Total_emp   string `tag_name:"total_emp"`
	Salary      string `tag_name:"salary"`
}

type Students struct {
	Name    string
	Age     int
	Phone   string
	Address string
}

func killApp(code int) {
	if h != nil {
		h.Conn.Close()
	}
	os.Exit(code)
}

func fatalCheck(t *testing.T, what string, e error) {
	if e != nil {
		t.Fatalf("%s: %s", what, e.Error())
	}
}

func TestHiveConnect(t *testing.T) {
	h = HiveConfig("192.168.0.223:10000", "default", "hdfs", "", "")
}

/* Populate will exec query and immidiately return the value into object
Populate is suitable for short type query that return limited data,
Exec is suitable for long type query that return massive amount of data and require time to produce it

Ideally Populate should call Exec as well but already have predefined function on it receiving process
*/
func TestHivePopulate(t *testing.T) {
	q := "select * from sample_07 limit 5;"

	var result []toolkit.M

	h.Conn.Open()

	e := h.Populate(q, &result)
	fatalCheck(t, "Populate", e)

	if len(result) != 5 {
		t.Logf("Error want %d got %d", 5, len(result))
	}

	t.Logf("Result: \n%s", toolkit.JsonString(result))

	h.Conn.Close()
}

func TestHiveExec(t *testing.T) {
	i := 0
	q := "select * from sample_07 limit 5;"

	h.Conn.Open()

	e := h.Exec(q, func(x HiveResult) error {
		i++
		t.Logf("Receiving data: %s", toolkit.JsonString(x))
		return nil
	})

	if err != nil {
		fmt.Println(err)
	}
	h.Conn.Close()
	fmt.Println(retVal)
}

func TestLoadFileWithWorker(t *testing.T) {
	h.Conn.Open()

	var student students

	totalWorker := 10
	retVal, err := h.LoadFileWithWorker("/home/developer/contoh.txt", "students", "txt", &student, totalWorker)

	if err != nil {
		fmt.Println(err)
	}
	h.Conn.Close()
	fmt.Println(retVal)
}

func TestExecLine(t *testing.T) {
	h = HiveConfig("192.168.0.223:10000", "default", "hdfs", "", "")

	//func TestHiveExec(t *testing.T) {

	q := "select * from sample_07 limit 5;"
	// x := "select * from sample_07 limit 10;"
	DoSomething := func(res HiveResult) (e error) {
		toolkit.Serde(res, &res.ResultObj, "json")
		log.Printf("limit 5: %v", res.ResultObj)
		return
	}

	DoElse := func(res HiveResult) (e error) {
		tmp := toolkit.M{}
		toolkit.Serde(res, &res.ResultObj, "json")
		log.Printf("limit 10: %v", tmp)
		return
	}

	if e != nil {
		t.Fatalf("Error exec query: %s", e.Error())
	}

	if i < 5 {
		t.Fatalf("Error receive result. Expect %d got %d", 5, i)
	}

	h.Conn.Close()
}

func TestHiveExecMulti(t *testing.T) {
	h.Conn.Open()

	var ms1, ms2 []HiveResult
	q := "select * from sample_07 limit 5"

	e := h.Exec(q, func(x HiveResult) error {
		ms1 = append(ms1, x)
		return nil
	})

	fatalCheck(t, "HS1 exec", e)

	e = h.Exec(q, func(x HiveResult) error {
		ms2 = append(ms2, x)
		return nil
	})

	fatalCheck(t, "HS2 Exec", e)

	t.Logf("Value of HS1\n%s\n\nValue of HS2\n%s", toolkit.JsonString(ms1), toolkit.JsonString(ms2))

	h.Conn.Close()
}

func TestLoad(t *testing.T) {
	h.Conn.Open()

	var Student Students

	retVal, err := h.Load("students", "|", &Student)

	if err != nil {
		t.Log(err)
	}
	h.Conn.Close()
	t.Log(retVal)
}

//for now, this function works on simple csv file
func TestLoadFile(t *testing.T) {
	h.Conn.Open()

	var Student Students

	retVal, err := h.LoadFile("/home/developer/contoh.txt", "students", "txt", &Student)

	if err != nil {
		t.Log(err)
	}
	h.Conn.Close()
	t.Log(retVal)
}
<commit_msg>Modify hive func LoadFileWithWorker and main test file<commit_after>package test

import (
	"github.com/eaciit/toolkit"
	//. "github.com/frezadev/hdc/hive"
	. "github.com/masmeka/hdc/hive"
	//. "github.com/RyanCi/hdc/hive"
	"log"
	"os"
	"testing"
)

var h *Hive
var e error

type Sample7 struct {
	Code        string `tag_name:"code"`
	Description string `tag_name:"description"`
	Total_emp   string `tag_name:"total_emp"`
	Salary      string `tag_name:"salary"`
}

type Students struct {
	Name    string
	Age     int
	Phone   string
	Address string
}

func killApp(code int) {
	if h != nil {
		h.Conn.Close()
	}
	os.Exit(code)
}

func fatalCheck(t *testing.T, what string, e error) {
	if e != nil {
		t.Fatalf("%s: %s", what, e.Error())
	}
}

func TestHiveConnect(t *testing.T) {
	h = HiveConfig("192.168.0.223:10000", "default", "hdfs", "", "")
}

/* Populate will exec query and immidiately return the value into object
Populate is suitable for short type query that return limited data,
Exec is suitable for long type query that return massive amount of data and require time to produce it
Ideally Populate should call Exec as well but already have predefined function on it receiving process
*/
func TestHivePopulate(t *testing.T) {
	q := "select * from sample_07 limit 5;"

	var result []toolkit.M

	h.Conn.Open()

	e := h.Populate(q, &result)
	fatalCheck(t, "Populate", e)

	if len(result) != 5 {
		t.Logf("Error want %d got %d", 5, len(result))
	}

	t.Logf("Result: \n%s", toolkit.JsonString(result))

	h.Conn.Close()
}

func TestHiveExec(t *testing.T) {
	i := 0
	q := "select * from sample_07 limit 5;"

	h.Conn.Open()

	e := h.Exec(q, func(x HiveResult) error {
		i++
		t.Logf("Receiving data: %s", toolkit.JsonString(x))
		return nil
	})

	if e != nil {
		t.Fatalf("Error exec query: %s", e.Error())
	}

	if i < 5 {
		t.Fatalf("Error receive result. Expect %d got %d", 5, i)
	}

	h.Conn.Close()
}

func TestHiveExecMulti(t *testing.T) {
	h.Conn.Open()

	var ms1, ms2 []HiveResult
	q := "select * from sample_07 limit 5"

	e := h.Exec(q, func(x HiveResult) error {
		ms1 = append(ms1, x)
		return nil
	})

	fatalCheck(t, "HS1 exec", e)

	e = h.Exec(q, func(x HiveResult) error {
		ms2 = append(ms2, x)
		return nil
	})

	fatalCheck(t, "HS2 Exec", e)

	t.Logf("Value of HS1\n%s\n\nValue of HS2\n%s", toolkit.JsonString(ms1), toolkit.JsonString(ms2))

	h.Conn.Close()
}

func TestLoad(t *testing.T) {
	h.Conn.Open()

	var Student Students

	retVal, err := h.Load("students", "|", &Student)

	if err != nil {
		t.Log(err)
	}
	h.Conn.Close()
	t.Log(retVal)
}

//for now, this function works on simple csv file
func TestLoadFile(t *testing.T) {
	h.Conn.Open()

	var Student Students

	retVal, err := h.LoadFile("/home/developer/contoh.txt", "students", "txt", &Student)

	if err != nil {
		t.Log(err)
	}
	h.Conn.Close()
	t.Log(retVal)
}

func TestLoadFileWithWorker(t *testing.T) {
	h.Conn.Open()

	var student Students

	totalWorker := 10
	retVal, err := h.LoadFileWithWorker("/home/developer/contoh.txt", "students", "txt", &student, totalWorker)

	if err != nil {
		log.Println(err)
	}
	h.Conn.Close()
	log.Println(retVal)
}
<|endoftext|>


<commit_before>// +build darwin linux
// run

// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Test that maps don't go quadratic for NaNs and other values.

package main

import (
	"fmt"
	"math"
	"time"
)

// checkLinear asserts that the running time of f(n) is in O(n).
// tries is the initial number of iterations.
func checkLinear(typ string, tries int, f func(n int)) {
	// Depending on the machine and OS, this test might be too fast
	// to measure with accurate enough granularity. On failure,
	// make it run longer, hoping that the timing granularity
	// is eventually sufficient.

	timeF := func(n int) time.Duration {
		t1 := time.Now()
		f(n)
		return time.Since(t1)
	}

	t0 := time.Now()

	n := tries
	fails := 0
	for {
		t1 := timeF(n)
		t2 := timeF(2 * n)

		// should be 2x (linear); allow up to 3x
		if t2 < 3*t1 {
			if false {
				fmt.Println(typ, "\t", time.Since(t0))
			}
			return
		}
		fails++
		if fails == 6 {
			panic(fmt.Sprintf("%s: too slow: %d inserts: %v; %d inserts: %v\n",
				typ, n, t1, 2*n, t2))
		}
		if fails < 4 {
			n *= 2
		}
	}
}

type I interface {
	f()
}

type C int

func (C) f() {}

func main() {
	// NaNs. ~31ms on a 1.6GHz Zeon.
	checkLinear("NaN", 30000, func(n int) {
		m := map[float64]int{}
		nan := math.NaN()
		for i := 0; i < n; i++ {
			m[nan] = 1
		}
		if len(m) != n {
			panic("wrong size map after nan insertion")
		}
	})

	// ~6ms on a 1.6GHz Zeon.
	checkLinear("eface", 10000, func(n int) {
		m := map[interface{}]int{}
		for i := 0; i < n; i++ {
			m[i] = 1
		}
	})

	// ~7ms on a 1.6GHz Zeon.
	// Regression test for CL 119360043.
	checkLinear("iface", 10000, func(n int) {
		m := map[I]int{}
		for i := 0; i < n; i++ {
			m[C(i)] = 1
		}
	})

	// ~6ms on a 1.6GHz Zeon.
	checkLinear("int", 10000, func(n int) {
		m := map[int]int{}
		for i := 0; i < n; i++ {
			m[i] = 1
		}
	})

	// ~18ms on a 1.6GHz Zeon.
	checkLinear("string", 10000, func(n int) {
		m := map[string]int{}
		for i := 0; i < n; i++ {
			m[fmt.Sprint(i)] = 1
		}
	})

	// ~6ms on a 1.6GHz Zeon.
	checkLinear("float32", 10000, func(n int) {
		m := map[float32]int{}
		for i := 0; i < n; i++ {
			m[float32(i)] = 1
		}
	})

	// ~6ms on a 1.6GHz Zeon.
	checkLinear("float64", 10000, func(n int) {
		m := map[float64]int{}
		for i := 0; i < n; i++ {
			m[float64(i)] = 1
		}
	})

	// ~22ms on a 1.6GHz Zeon.
	checkLinear("complex64", 10000, func(n int) {
		m := map[complex64]int{}
		for i := 0; i < n; i++ {
			m[complex(float32(i), float32(i))] = 1
		}
	})

	// ~32ms on a 1.6GHz Zeon.
	checkLinear("complex128", 10000, func(n int) {
		m := map[complex128]int{}
		for i := 0; i < n; i++ {
			m[complex(float64(i), float64(i))] = 1
		}
	})
}
<commit_msg>runtime: add timing test for iterate/delete map idiom.<commit_after>// +build darwin linux
// run

// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Test that maps don't go quadratic for NaNs and other values.

package main

import (
	"fmt"
	"math"
	"time"
)

// checkLinear asserts that the running time of f(n) is in O(n).
// tries is the initial number of iterations.
func checkLinear(typ string, tries int, f func(n int)) {
	// Depending on the machine and OS, this test might be too fast
	// to measure with accurate enough granularity. On failure,
	// make it run longer, hoping that the timing granularity
	// is eventually sufficient.

	timeF := func(n int) time.Duration {
		t1 := time.Now()
		f(n)
		return time.Since(t1)
	}

	t0 := time.Now()

	n := tries
	fails := 0
	for {
		t1 := timeF(n)
		t2 := timeF(2 * n)

		// should be 2x (linear); allow up to 3x
		if t2 < 3*t1 {
			if false {
				fmt.Println(typ, "\t", time.Since(t0))
			}
			return
		}
		fails++
		if fails == 6 {
			panic(fmt.Sprintf("%s: too slow: %d inserts: %v; %d inserts: %v\n",
				typ, n, t1, 2*n, t2))
		}
		if fails < 4 {
			n *= 2
		}
	}
}

type I interface {
	f()
}

type C int

func (C) f() {}

func main() {
	// NaNs. ~31ms on a 1.6GHz Zeon.
	checkLinear("NaN", 30000, func(n int) {
		m := map[float64]int{}
		nan := math.NaN()
		for i := 0; i < n; i++ {
			m[nan] = 1
		}
		if len(m) != n {
			panic("wrong size map after nan insertion")
		}
	})

	// ~6ms on a 1.6GHz Zeon.
	checkLinear("eface", 10000, func(n int) {
		m := map[interface{}]int{}
		for i := 0; i < n; i++ {
			m[i] = 1
		}
	})

	// ~7ms on a 1.6GHz Zeon.
	// Regression test for CL 119360043.
	checkLinear("iface", 10000, func(n int) {
		m := map[I]int{}
		for i := 0; i < n; i++ {
			m[C(i)] = 1
		}
	})

	// ~6ms on a 1.6GHz Zeon.
	checkLinear("int", 10000, func(n int) {
		m := map[int]int{}
		for i := 0; i < n; i++ {
			m[i] = 1
		}
	})

	// ~18ms on a 1.6GHz Zeon.
	checkLinear("string", 10000, func(n int) {
		m := map[string]int{}
		for i := 0; i < n; i++ {
			m[fmt.Sprint(i)] = 1
		}
	})

	// ~6ms on a 1.6GHz Zeon.
	checkLinear("float32", 10000, func(n int) {
		m := map[float32]int{}
		for i := 0; i < n; i++ {
			m[float32(i)] = 1
		}
	})

	// ~6ms on a 1.6GHz Zeon.
	checkLinear("float64", 10000, func(n int) {
		m := map[float64]int{}
		for i := 0; i < n; i++ {
			m[float64(i)] = 1
		}
	})

	// ~22ms on a 1.6GHz Zeon.
	checkLinear("complex64", 10000, func(n int) {
		m := map[complex64]int{}
		for i := 0; i < n; i++ {
			m[complex(float32(i), float32(i))] = 1
		}
	})

	// ~32ms on a 1.6GHz Zeon.
	checkLinear("complex128", 10000, func(n int) {
		m := map[complex128]int{}
		for i := 0; i < n; i++ {
			m[complex(float64(i), float64(i))] = 1
		}
	})

	// ~70ms on a 1.6GHz Zeon.
	// The iterate/delete idiom currently takes expected
	// O(n lg n) time.  Fortunately, the checkLinear test
	// leaves enough wiggle room to include n lg n time
	// (it actually tests for O(n^log_2(3)).
	checkLinear("iterdelete", 10000, func(n int) {
		m := map[int]int{}
		for i := 0; i < n; i++ {
			m[i] = i
		}
		for i := 0; i < n; i++ {
			for k := range m {
				delete(m, k)
				break
			}
		}
	})
}
<|endoftext|>


<commit_before>package containers

import (
	"atlantis/crypto"
	"atlantis/supervisor/rpc/types"
	"fmt"
	"github.com/jigish/go-dockerclient"
	"log"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"sync"
)

var (
	dockerIdRegexp = regexp.MustCompile("^[A-Za-z0-9]+$")
	dockerLock     = sync.Mutex{}
	dockerClient   *docker.Client
)

type Container types.Container
type SSHCmd []string

func DockerInit() (err error) {
	dockerClient, err = docker.NewClient("unix:///var/run/docker.sock")
	return
}

func pretending() bool {
	return os.Getenv("SUPERVISOR_PRETEND") != ""
}

func (s SSHCmd) Execute() error {
	if pretending() {
		log.Printf("[pretend] ssh %s", strings.Join(s, " "))
		return nil
	}
	log.Printf("ssh %s", strings.Join(s, " "))
	cmd := exec.Command("ssh", s...)
	output, err := cmd.CombinedOutput()
	log.Printf("-> %s", output)
	if err != nil {
		log.Println("-> Error:", err)
	}
	return err
}

func removeExited() {
	if pretending() {
		return
	}
	dockerLock.Lock()
	defer dockerLock.Unlock()
	containers, err := dockerClient.ListContainers(docker.ListContainersOptions{All: true})
	if err != nil {
		log.Printf("[RemoveExited] could not list containers: %v", err)
		return
	}
	for _, cont := range containers {
		log.Printf("[RemoveExited] checking %s (%v) : %s", cont.ID, cont.Names, cont.Status)
		if !strings.HasPrefix(cont.Status, "Exit") {
			continue
		}
		log.Printf("[RemoveExited] remove %s (%v)", cont.ID, cont.Names)
		err := dockerClient.RemoveContainer(cont.ID)
		if err != nil {
			log.Printf("[RemoveExited] -> error: %v", err)
		} else {
			log.Printf("[RemoveExited] -> success")
		}
	}
}

func restartGhost() {
	if pretending() {
		return
	}
	dockerLock.Lock()
	defer dockerLock.Unlock()
	containers, err := dockerClient.ListContainers(docker.ListContainersOptions{All: true})
	if err != nil {
		log.Printf("[RestartGhost] could not list containers: %v", err)
		return
	}
	for _, cont := range containers {
		log.Printf("[RestartGhost] checking %s (%v) : %s", cont.ID, cont.Names, cont.Status)
		if !strings.HasPrefix(cont.Status, "Ghost") {
			continue
		}
		log.Printf("[RestartGhost] restart %s (%v)", cont.ID, cont.Names)
		err := dockerClient.RestartContainer(cont.ID, 0)
		if err != nil {
			log.Printf("[RestartGhost] -> error: %v", err)
		} else {
			log.Printf("[RestartGhost] -> success")
		}
	}
}

func (c *Container) dockerCfgs(repo string) (*docker.Config, *docker.HostConfig) {
	// get env cfg
	envs := []string{
		"ATLANTIS=true",
		fmt.Sprintf("CONTAINER_ID=%s", c.Id),
		fmt.Sprintf("CONTAINER_HOST=%s", c.Host),
		fmt.Sprintf("CONTAINER_ENV=%s", c.Env),
		fmt.Sprintf("HTTP_PORT=%d", c.PrimaryPort),
		fmt.Sprintf("SSHD_PORT=%d", c.SSHPort),
	}
	if c.Manifest.Deps != nil {
		for name, value := range c.Manifest.Deps {
			envs = append(envs, fmt.Sprintf("%s=%s", name, crypto.Decrypt([]byte(value))))
		}
	}

	// get port cfg
	exposedPorts := map[docker.Port]struct{}{}
	portBindings := map[docker.Port][]docker.PortBinding{}
	sPrimaryPort := fmt.Sprintf("%d", c.PrimaryPort)
	dPrimaryPort := docker.NewPort("tcp", sPrimaryPort)
	exposedPorts[dPrimaryPort] = struct{}{}
	portBindings[dPrimaryPort] = []docker.PortBinding{docker.PortBinding{
		HostIp:   "",
		HostPort: fmt.Sprintf("%d", sPrimaryPort),
	}}
	sSSHPort := fmt.Sprintf("%d", c.SSHPort)
	dSSHPort := docker.NewPort("tcp", sSSHPort)
	exposedPorts[dSSHPort] = struct{}{}
	portBindings[dSSHPort] = []docker.PortBinding{docker.PortBinding{
		HostIp:   "",
		HostPort: sSSHPort,
	}}
	for i, port := range c.SecondaryPorts {
		sPort := fmt.Sprintf("%d", port)
		dPort := docker.NewPort("tcp", sPort)
		exposedPorts[dPort] = struct{}{}
		portBindings[dPort] = []docker.PortBinding{docker.PortBinding{
			HostIp:   "",
			HostPort: sPort,
		}}
		envs = append(envs, fmt.Sprintf("SECONDARY_PORT%d=%d", i, port))
	}

	// setup actual cfg
	dCfg := &docker.Config{
		CpuShares:    int64(c.Manifest.CPUShares),
		Memory:       int64(c.Manifest.MemoryLimit) * int64(1024*1024), // this is in bytes
		MemorySwap:   int64(-1),                                        // -1 turns swap off
		ExposedPorts: exposedPorts,
		Env:          envs,
		Cmd:          []string{}, // images already specify run command
		Image:        fmt.Sprintf("%s/%s", RegistryHost, repo),
		Volumes: map[string]struct{}{
			fmt.Sprintf("/var/log/atlantis/containers/%s:/var/log/atlantis/syslog", c.Id): struct{}{},
		},
	}
	dHostCfg := &docker.HostConfig{
		PortBindings: portBindings,
		LxcConf:      []docker.KeyValuePair{},
	}
	return dCfg, dHostCfg
}

// Deploy the given app+sha with the dependencies defined in deps. This will spin up a new docker container.
func (c *Container) Deploy(host, app, sha, env string) error {
	c.Host = host
	c.App = app
	c.Sha = sha
	c.Env = env
	dRepo := fmt.Sprintf("apps/%s-%s", c.App, c.Sha)
	if pretending() {
	} else {
	}
	// Pull docker container
	if pretending() {
		log.Printf("[pretend] deploy %s with %s @ %s...", c.Id, c.App, c.Sha)
		log.Printf("[pretend] docker pull %s/%s", RegistryHost, dRepo)
		log.Printf("[pretend] docker run %s/%s", RegistryHost, dRepo)
		c.DockerId = fmt.Sprintf("pretend-docker-id-%d", c.PrimaryPort)
	} else {
		log.Printf("deploy %s with %s @ %s...", c.Id, c.App, c.Sha)
		dockerLock.Lock()
		err := dockerClient.PullImage(docker.PullImageOptions{Repository: dRepo, Registry: RegistryHost},
			os.Stdout)
		dockerLock.Unlock()
		if err != nil {
			return err
		}

		err = os.MkdirAll(fmt.Sprintf("/var/log/atlantis/containers/%s", c.Id), 0755)
		if err != nil {
			return err
		}

		// create docker container
		dCfg, dHostCfg := c.dockerCfgs(dRepo)
		dockerLock.Lock()
		dCont, err := dockerClient.CreateContainer(docker.CreateContainerOptions{Name: c.Id}, dCfg)
		dockerLock.Unlock()
		if err != nil {
			return err
		}
		c.DockerId = dCont.ID

		// start docker container
		dockerLock.Lock()
		err = dockerClient.StartContainer(c.DockerId, dHostCfg)
		dockerLock.Unlock()
		if err != nil {
			return err
		}
	}
	save() // save here because this is when we know the deployed container is actually alive
	return nil
}

// Teardown the container. This will kill the docker container but will not free the ports/containers
func (c *Container) teardown() {
	if pretending() {
		log.Printf("[pretend] teardown %s...", c.Id)
		return
	} else {
		log.Printf("teardown %s...", c.Id)
	}
	defer removeExited()
	dockerLock.Lock()
	err := dockerClient.KillContainer(c.DockerId)
	dockerLock.Unlock()
	if err != nil {
		log.Printf("failed to teardown %s: %v", c.Id, err)
	}
}

// This calls the Teardown(id string) method to ensure that the ports/containers are freed. That will in turn
// call c.teardown(id string)
func (c *Container) Teardown() {
	Teardown(c.Id)
}

func (c *Container) AuthorizeSSHUser(user, publicKey string) error {
	// copy file to container
	// rebuild authorize_keys
	return SSHCmd{"-p", fmt.Sprintf("%d", c.SSHPort), "-i", "/opt/atlantis/supervisor/master_id_rsa", "-o",
		"UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "root@localhost",
		fmt.Sprintf("echo \"%s\" >/root/.ssh/authorized_keys.d/%s.pub && rebuild_authorized_keys", publicKey,
			user)}.Execute()
}

func (c *Container) DeauthorizeSSHUser(user string) error {
	// delete file from container
	// rebuild authorize_keys
	return SSHCmd{"-p", fmt.Sprintf("%d", c.SSHPort), "-i", "/opt/atlantis/supervisor/master_id_rsa", "-o",
		"UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "root@localhost",
		fmt.Sprintf("rm /root/.ssh/authorized_keys.d/%s.pub && rebuild_authorized_keys",
			user)}.Execute()
}

func (c *Container) SetMaintenance(maint bool) error {
	if maint {
		// touch /etc/maint
		return SSHCmd{"-p", fmt.Sprintf("%d", c.SSHPort), "-i", "/opt/atlantis/supervisor/master_id_rsa", "-o",
			"UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "root@localhost",
			"touch /etc/maint"}.Execute()
	}
	// rm -f /etc/maint
	return SSHCmd{"-p", fmt.Sprintf("%d", c.SSHPort), "-i", "/opt/atlantis/supervisor/master_id_rsa", "-o",
		"UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "root@localhost",
		"rm -f /etc/maint"}.Execute()
}
<commit_msg>if restarting through api fails, try restarting daemon<commit_after>package containers

import (
	"atlantis/crypto"
	"atlantis/supervisor/rpc/types"
	"fmt"
	"github.com/jigish/go-dockerclient"
	"log"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"sync"
)

var (
	dockerIdRegexp = regexp.MustCompile("^[A-Za-z0-9]+$")
	dockerLock     = sync.Mutex{}
	dockerClient   *docker.Client
)

type Container types.Container
type SSHCmd []string

func DockerInit() (err error) {
	dockerClient, err = docker.NewClient("unix:///var/run/docker.sock")
	return
}

func pretending() bool {
	return os.Getenv("SUPERVISOR_PRETEND") != ""
}

func (s SSHCmd) Execute() error {
	if pretending() {
		log.Printf("[pretend] ssh %s", strings.Join(s, " "))
		return nil
	}
	log.Printf("ssh %s", strings.Join(s, " "))
	cmd := exec.Command("ssh", s...)
	output, err := cmd.CombinedOutput()
	log.Printf("-> %s", output)
	if err != nil {
		log.Println("-> Error:", err)
	}
	return err
}

func removeExited() {
	if pretending() {
		return
	}
	dockerLock.Lock()
	defer dockerLock.Unlock()
	containers, err := dockerClient.ListContainers(docker.ListContainersOptions{All: true})
	if err != nil {
		log.Printf("[RemoveExited] could not list containers: %v", err)
		return
	}
	for _, cont := range containers {
		log.Printf("[RemoveExited] checking %s (%v) : %s", cont.ID, cont.Names, cont.Status)
		if !strings.HasPrefix(cont.Status, "Exit") {
			continue
		}
		log.Printf("[RemoveExited] remove %s (%v)", cont.ID, cont.Names)
		err := dockerClient.RemoveContainer(cont.ID)
		if err != nil {
			log.Printf("[RemoveExited] -> error: %v", err)
		} else {
			log.Printf("[RemoveExited] -> success")
		}
	}
}

func restartGhost() {
	if pretending() {
		return
	}
	dockerLock.Lock()
	defer dockerLock.Unlock()
	containers, err := dockerClient.ListContainers(docker.ListContainersOptions{All: true})
	if err != nil {
		log.Printf("[RestartGhost] could not list containers: %v", err)
		return
	}
	hadError := false
	for _, cont := range containers {
		log.Printf("[RestartGhost] checking %s (%v) : %s", cont.ID, cont.Names, cont.Status)
		if !strings.HasPrefix(cont.Status, "Ghost") {
			continue
		}
		log.Printf("[RestartGhost] restart %s (%v)", cont.ID, cont.Names)
		err := dockerClient.RestartContainer(cont.ID, 0)
		if err != nil {
			hadError = true
			log.Printf("[RestartGhost] -> error: %v", err)
		} else {
			log.Printf("[RestartGhost] -> success")
		}
	}
	if hadError {
		log.Printf("[RestartGhost] attempting to force restart (restarting docker daemon)")
		cmd := exec.Command("sudo", "service docker restart")
		output, err := cmd.CombinedOutput()
		log.Printf("-> %s", output)
		if err != nil {
			log.Println("-> Error:", err)
		}
	}
}

func (c *Container) dockerCfgs(repo string) (*docker.Config, *docker.HostConfig) {
	// get env cfg
	envs := []string{
		"ATLANTIS=true",
		fmt.Sprintf("CONTAINER_ID=%s", c.Id),
		fmt.Sprintf("CONTAINER_HOST=%s", c.Host),
		fmt.Sprintf("CONTAINER_ENV=%s", c.Env),
		fmt.Sprintf("HTTP_PORT=%d", c.PrimaryPort),
		fmt.Sprintf("SSHD_PORT=%d", c.SSHPort),
	}
	if c.Manifest.Deps != nil {
		for name, value := range c.Manifest.Deps {
			envs = append(envs, fmt.Sprintf("%s=%s", name, crypto.Decrypt([]byte(value))))
		}
	}

	// get port cfg
	exposedPorts := map[docker.Port]struct{}{}
	portBindings := map[docker.Port][]docker.PortBinding{}
	sPrimaryPort := fmt.Sprintf("%d", c.PrimaryPort)
	dPrimaryPort := docker.NewPort("tcp", sPrimaryPort)
	exposedPorts[dPrimaryPort] = struct{}{}
	portBindings[dPrimaryPort] = []docker.PortBinding{docker.PortBinding{
		HostIp:   "",
		HostPort: fmt.Sprintf("%d", sPrimaryPort),
	}}
	sSSHPort := fmt.Sprintf("%d", c.SSHPort)
	dSSHPort := docker.NewPort("tcp", sSSHPort)
	exposedPorts[dSSHPort] = struct{}{}
	portBindings[dSSHPort] = []docker.PortBinding{docker.PortBinding{
		HostIp:   "",
		HostPort: sSSHPort,
	}}
	for i, port := range c.SecondaryPorts {
		sPort := fmt.Sprintf("%d", port)
		dPort := docker.NewPort("tcp", sPort)
		exposedPorts[dPort] = struct{}{}
		portBindings[dPort] = []docker.PortBinding{docker.PortBinding{
			HostIp:   "",
			HostPort: sPort,
		}}
		envs = append(envs, fmt.Sprintf("SECONDARY_PORT%d=%d", i, port))
	}

	// setup actual cfg
	dCfg := &docker.Config{
		CpuShares:    int64(c.Manifest.CPUShares),
		Memory:       int64(c.Manifest.MemoryLimit) * int64(1024*1024), // this is in bytes
		MemorySwap:   int64(-1),                                        // -1 turns swap off
		ExposedPorts: exposedPorts,
		Env:          envs,
		Cmd:          []string{}, // images already specify run command
		Image:        fmt.Sprintf("%s/%s", RegistryHost, repo),
		Volumes: map[string]struct{}{
			fmt.Sprintf("/var/log/atlantis/containers/%s:/var/log/atlantis/syslog", c.Id): struct{}{},
		},
	}
	dHostCfg := &docker.HostConfig{
		PortBindings: portBindings,
		LxcConf:      []docker.KeyValuePair{},
	}
	return dCfg, dHostCfg
}

// Deploy the given app+sha with the dependencies defined in deps. This will spin up a new docker container.
func (c *Container) Deploy(host, app, sha, env string) error {
	c.Host = host
	c.App = app
	c.Sha = sha
	c.Env = env
	dRepo := fmt.Sprintf("apps/%s-%s", c.App, c.Sha)
	if pretending() {
	} else {
	}
	// Pull docker container
	if pretending() {
		log.Printf("[pretend] deploy %s with %s @ %s...", c.Id, c.App, c.Sha)
		log.Printf("[pretend] docker pull %s/%s", RegistryHost, dRepo)
		log.Printf("[pretend] docker run %s/%s", RegistryHost, dRepo)
		c.DockerId = fmt.Sprintf("pretend-docker-id-%d", c.PrimaryPort)
	} else {
		log.Printf("deploy %s with %s @ %s...", c.Id, c.App, c.Sha)
		dockerLock.Lock()
		err := dockerClient.PullImage(docker.PullImageOptions{Repository: dRepo, Registry: RegistryHost},
			os.Stdout)
		dockerLock.Unlock()
		if err != nil {
			return err
		}

		err = os.MkdirAll(fmt.Sprintf("/var/log/atlantis/containers/%s", c.Id), 0755)
		if err != nil {
			return err
		}

		// create docker container
		dCfg, dHostCfg := c.dockerCfgs(dRepo)
		dockerLock.Lock()
		dCont, err := dockerClient.CreateContainer(docker.CreateContainerOptions{Name: c.Id}, dCfg)
		dockerLock.Unlock()
		if err != nil {
			return err
		}
		c.DockerId = dCont.ID

		// start docker container
		dockerLock.Lock()
		err = dockerClient.StartContainer(c.DockerId, dHostCfg)
		dockerLock.Unlock()
		if err != nil {
			return err
		}
	}
	save() // save here because this is when we know the deployed container is actually alive
	return nil
}

// Teardown the container. This will kill the docker container but will not free the ports/containers
func (c *Container) teardown() {
	if pretending() {
		log.Printf("[pretend] teardown %s...", c.Id)
		return
	} else {
		log.Printf("teardown %s...", c.Id)
	}
	defer removeExited()
	dockerLock.Lock()
	err := dockerClient.KillContainer(c.DockerId)
	dockerLock.Unlock()
	if err != nil {
		log.Printf("failed to teardown %s: %v", c.Id, err)
	}
}

// This calls the Teardown(id string) method to ensure that the ports/containers are freed. That will in turn
// call c.teardown(id string)
func (c *Container) Teardown() {
	Teardown(c.Id)
}

func (c *Container) AuthorizeSSHUser(user, publicKey string) error {
	// copy file to container
	// rebuild authorize_keys
	return SSHCmd{"-p", fmt.Sprintf("%d", c.SSHPort), "-i", "/opt/atlantis/supervisor/master_id_rsa", "-o",
		"UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "root@localhost",
		fmt.Sprintf("echo \"%s\" >/root/.ssh/authorized_keys.d/%s.pub && rebuild_authorized_keys", publicKey,
			user)}.Execute()
}

func (c *Container) DeauthorizeSSHUser(user string) error {
	// delete file from container
	// rebuild authorize_keys
	return SSHCmd{"-p", fmt.Sprintf("%d", c.SSHPort), "-i", "/opt/atlantis/supervisor/master_id_rsa", "-o",
		"UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "root@localhost",
		fmt.Sprintf("rm /root/.ssh/authorized_keys.d/%s.pub && rebuild_authorized_keys",
			user)}.Execute()
}

func (c *Container) SetMaintenance(maint bool) error {
	if maint {
		// touch /etc/maint
		return SSHCmd{"-p", fmt.Sprintf("%d", c.SSHPort), "-i", "/opt/atlantis/supervisor/master_id_rsa", "-o",
			"UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "root@localhost",
			"touch /etc/maint"}.Execute()
	}
	// rm -f /etc/maint
	return SSHCmd{"-p", fmt.Sprintf("%d", c.SSHPort), "-i", "/opt/atlantis/supervisor/master_id_rsa", "-o",
		"UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "root@localhost",
		"rm -f /etc/maint"}.Execute()
}
<|endoftext|>