<commit_before>/*
Copyright 2013 The Perkeep Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// This file adds the "tool" subcommand to devcam, to run pk against
// the dev server.

package main

import (
	"flag"
	"fmt"
	"path/filepath"

	"perkeep.org/pkg/cmdmain"
)

type toolCmd struct {
	// start of flag vars
	altkey bool
	// end of flag vars

	env *Env
}

func init() {
	cmdmain.RegisterMode("tool", func(flags *flag.FlagSet) cmdmain.CommandRunner {
		cmd := &toolCmd{
			env: NewCopyEnv(),
		}
		flags.BoolVar(&cmd.altkey, "altkey", false, "Use different gpg key and password from the server's.")
		return cmd
	})
}

func (c *toolCmd) Usage() {
	fmt.Fprintf(cmdmain.Stderr, "Usage: devcam tool [globalopts] <mode> [commandopts] [commandargs]\n")
}

func (c *toolCmd) Examples() []string {
	return []string{
		"sync --all",
	}
}

func (c *toolCmd) Describe() string {
	return "run pk in dev mode."
}

func (c *toolCmd) RunCommand(args []string) error {
	if !*noBuild {
		if err := build(filepath.Join("cmd", "pk")); err != nil {
			return fmt.Errorf("Could not build pk: %v", err)
		}
	}
	c.env.SetCamdevVars(c.altkey)
	// wipeCacheDir needs to be called after SetCamdevVars, because that is
	// where CAMLI_CACHE_DIR is defined.
	if *wipeCache {
		c.env.wipeCacheDir()
	}

	cmdBin := filepath.Join("bin", "pk")
	return runExec(cmdBin, args, c.env)
}
<commit_msg>dev/devcam: do not look for pk in ./bin dir<commit_after>/*
Copyright 2013 The Perkeep Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// This file adds the "tool" subcommand to devcam, to run pk against
// the dev server.

package main

import (
	"flag"
	"fmt"
	"path/filepath"

	"perkeep.org/internal/osutil"
	"perkeep.org/pkg/cmdmain"
)

type toolCmd struct {
	// start of flag vars
	altkey bool
	// end of flag vars

	env *Env
}

func init() {
	cmdmain.RegisterMode("tool", func(flags *flag.FlagSet) cmdmain.CommandRunner {
		cmd := &toolCmd{
			env: NewCopyEnv(),
		}
		flags.BoolVar(&cmd.altkey, "altkey", false, "Use different gpg key and password from the server's.")
		return cmd
	})
}

func (c *toolCmd) Usage() {
	fmt.Fprintf(cmdmain.Stderr, "Usage: devcam tool [globalopts] <mode> [commandopts] [commandargs]\n")
}

func (c *toolCmd) Examples() []string {
	return []string{
		"sync --all",
	}
}

func (c *toolCmd) Describe() string {
	return "run pk in dev mode."
}

func (c *toolCmd) RunCommand(args []string) error {
	if !*noBuild {
		if err := build(filepath.Join("cmd", "pk")); err != nil {
			return fmt.Errorf("Could not build pk: %v", err)
		}
	}
	c.env.SetCamdevVars(c.altkey)
	// wipeCacheDir needs to be called after SetCamdevVars, because that is
	// where CAMLI_CACHE_DIR is defined.
	if *wipeCache {
		c.env.wipeCacheDir()
	}

	cmdBin, err := osutil.LookPathGopath("pk")
	if err != nil {
		return err
	}
	return runExec(cmdBin, args, c.env)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/kch42/gomcmap/mcmap"
	"github.com/mattn/go-gtk/gdk"
	"github.com/mattn/go-gtk/gtk"
	"os"
	"strconv"
)

type biomeEditFrame struct {
	*gtk.Frame
	applyBtn                          *gtk.Button
	idInput, snowLineInput, nameInput *gtk.Entry
	colorInput                        *gtk.ColorButton
}

func newBiomeEditFrame() *biomeEditFrame {
	frm := &biomeEditFrame{
		Frame:         gtk.NewFrame("Edit Biome"),
		applyBtn:      gtk.NewButtonWithLabel("Apply"),
		idInput:       gtk.NewEntry(),
		snowLineInput: gtk.NewEntry(),
		nameInput:     gtk.NewEntry(),
		colorInput:    gtk.NewColorButton(),
	}

	frm.idInput.SetSizeRequest(40, -1)
	frm.snowLineInput.SetSizeRequest(40, -1)

	frm.idInput.Connect("changed", frm.unlockApply)
	frm.nameInput.Connect("changed", frm.unlockApply)
	frm.snowLineInput.Connect("changed", frm.unlockApply)
	frm.applyBtn.SetSensitive(false)

	vbox := gtk.NewVBox(false, 0)
	hbox := gtk.NewHBox(false, 0)

	hbox.PackStart(gtk.NewLabel("Color:"), false, false, 0)
	hbox.PackStart(frm.colorInput, false, false, 3)
	hbox.PackStart(gtk.NewLabel("ID:"), false, false, 0)
	hbox.PackStart(frm.idInput, false, false, 3)
	hbox.PackStart(gtk.NewLabel("Snowline:"), false, false, 0)
	hbox.PackStart(frm.snowLineInput, false, false, 3)
	hbox.PackStart(gtk.NewLabel("Name:"), false, false, 0)
	hbox.PackStart(frm.nameInput, true, true, 3)

	vbox.PackStart(hbox, false, false, 0)
	vbox.PackStart(frm.applyBtn, false, false, 3)
	frm.Add(vbox)

	return frm
}

func (frm *biomeEditFrame) setBiomeInfo(info BiomeInfo) {
	frm.colorInput.SetColor(gdk.NewColor(info.Color))
	frm.idInput.SetText(strconv.FormatInt(int64(info.ID), 10))
	frm.snowLineInput.SetText(strconv.FormatInt(int64(info.SnowLine), 10))
	frm.nameInput.SetText(info.Name)
}

func (frm *biomeEditFrame) getBiomeInfo() (BiomeInfo, bool) {
	id, err := strconv.ParseUint(frm.idInput.GetText(), 10, 8)
	if err != nil {
		return BiomeInfo{}, false
	}

	snow, err := strconv.ParseInt(frm.snowLineInput.GetText(), 10, 32)
	if err != nil {
		return BiomeInfo{}, false
	}
	if (snow > mcmap.ChunkSizeY) || (snow < 0) {
		snow = mcmap.ChunkSizeY
	}

	name := frm.nameInput.GetText()
	if name == "" {
		return BiomeInfo{}, false
	}

	col := frm.colorInput.GetColor()

	return BiomeInfo{
		ID:       mcmap.Biome(id),
		SnowLine: int(snow),
		Name:     name,
		Color:    fmt.Sprintf("#%02x%02x%02x", col.Red()<<8, col.Green()<<8, col.Blue()<<8),
	}, true
}

func (frm *biomeEditFrame) checkOK() bool {
	_, ok := frm.getBiomeInfo()
	return ok
}

func (frm *biomeEditFrame) unlockApply() {
	frm.applyBtn.SetSensitive(frm.checkOK())
}

type BiomeInfoEditor struct {
	*gtk.Dialog
	biomes []BiomeInfo
}

func NewBiomeInfoEditor(biomes []BiomeInfo) *BiomeInfoEditor {
	ed := &BiomeInfoEditor{
		Dialog: gtk.NewDialog(),
		biomes: biomes,
	}

	ed.SetModal(true)

	vbox := ed.GetVBox()

	btnHBox := gtk.NewHBox(true, 0)

	resetBtn := gtk.NewButtonWithLabel("Reset to defaults")
	resetBtn.Connect("clicked", ed.reset)
	loadBtn := gtk.NewButtonWithLabel("Load from file ...")
	loadBtn.Connect("clicked", ed.load)
	saveBtn := gtk.NewButtonWithLabel("Save to file ...")
	saveBtn.Connect("clicked", ed.save)

	btnHBox.PackStart(resetBtn, true, true, 3)
	btnHBox.PackStart(loadBtn, true, true, 3)
	btnHBox.PackStart(saveBtn, true, true, 3)
	vbox.PackStart(btnHBox, false, false, 3)

	editFrame := newBiomeEditFrame()
	vbox.PackStart(editFrame, false, false, 3)

	ed.AddButton("Cancel", gtk.RESPONSE_CANCEL)
	ed.AddButton("OK", gtk.RESPONSE_OK)
	ed.ShowAll()
	return ed
}

func (ed *BiomeInfoEditor) reset() {
	ed.biomes = ReadDefaultBiomes()
	// TODO: Update view
}

func mkBioFFilters() (*gtk.FileFilter, *gtk.FileFilter) {
	f1 := gtk.NewFileFilter()
	f1.AddPattern("*.biomes")
	f1.SetName("Biome Infos (.biomes)")

	f2 := gtk.NewFileFilter()
	f2.AddPattern("*")
	f2.SetName("All files")

	return f1, f2
}

func errdlg(msg string, params ...interface{}) {
	dlg := gtk.NewMessageDialog(nil, gtk.DIALOG_MODAL|gtk.DIALOG_DESTROY_WITH_PARENT, gtk.MESSAGE_ERROR, gtk.BUTTONS_OK, msg, params...)
	dlg.Run()
	dlg.Destroy()
}

func (ed *BiomeInfoEditor) load() {
	f1, f2 := mkBioFFilters()
	dlg := gtk.NewFileChooserDialog("Load", nil, gtk.FILE_CHOOSER_ACTION_OPEN, "OK", gtk.RESPONSE_OK, "Cancel", gtk.RESPONSE_CANCEL)
	dlg.AddFilter(f1)
	dlg.AddFilter(f2)
	defer dlg.Destroy()
	if dlg.Run() == gtk.RESPONSE_OK {
		path := dlg.GetFilename()

		f, err := os.Open(path)
		if err != nil {
			errdlg("Could not load biome infos %s:\n%s", path, err.Error())
			return
		}
		defer f.Close()

		infos, err := ReadBiomeInfos(f)
		if err != nil {
			errdlg("Could not load biome infos %s:\n%s", path, err.Error())
			return
		}

		ed.biomes = infos
		// TODO: Update view
	}
}

func (ed *BiomeInfoEditor) save() {
	// TODO
}

func (ed *BiomeInfoEditor) Biomes() []BiomeInfo { return ed.biomes }
<commit_msg>Saving biome infos implemented<commit_after>package main

import (
	"fmt"
	"github.com/kch42/gomcmap/mcmap"
	"github.com/mattn/go-gtk/gdk"
	"github.com/mattn/go-gtk/gtk"
	"os"
	"strconv"
)

type biomeEditFrame struct {
	*gtk.Frame
	applyBtn                          *gtk.Button
	idInput, snowLineInput, nameInput *gtk.Entry
	colorInput                        *gtk.ColorButton
}

func newBiomeEditFrame() *biomeEditFrame {
	frm := &biomeEditFrame{
		Frame:         gtk.NewFrame("Edit Biome"),
		applyBtn:      gtk.NewButtonWithLabel("Apply"),
		idInput:       gtk.NewEntry(),
		snowLineInput: gtk.NewEntry(),
		nameInput:     gtk.NewEntry(),
		colorInput:    gtk.NewColorButton(),
	}

	frm.idInput.SetSizeRequest(40, -1)
	frm.snowLineInput.SetSizeRequest(40, -1)

	frm.idInput.Connect("changed", frm.unlockApply)
	frm.nameInput.Connect("changed", frm.unlockApply)
	frm.snowLineInput.Connect("changed", frm.unlockApply)
	frm.applyBtn.SetSensitive(false)

	vbox := gtk.NewVBox(false, 0)
	hbox := gtk.NewHBox(false, 0)

	hbox.PackStart(gtk.NewLabel("Color:"), false, false, 0)
	hbox.PackStart(frm.colorInput, false, false, 3)
	hbox.PackStart(gtk.NewLabel("ID:"), false, false, 0)
	hbox.PackStart(frm.idInput, false, false, 3)
	hbox.PackStart(gtk.NewLabel("Snowline:"), false, false, 0)
	hbox.PackStart(frm.snowLineInput, false, false, 3)
	hbox.PackStart(gtk.NewLabel("Name:"), false, false, 0)
	hbox.PackStart(frm.nameInput, true, true, 3)

	vbox.PackStart(hbox, false, false, 0)
	vbox.PackStart(frm.applyBtn, false, false, 3)
	frm.Add(vbox)

	return frm
}

func (frm *biomeEditFrame) setBiomeInfo(info BiomeInfo) {
	frm.colorInput.SetColor(gdk.NewColor(info.Color))
	frm.idInput.SetText(strconv.FormatInt(int64(info.ID), 10))
	frm.snowLineInput.SetText(strconv.FormatInt(int64(info.SnowLine), 10))
	frm.nameInput.SetText(info.Name)
}

func (frm *biomeEditFrame) getBiomeInfo() (BiomeInfo, bool) {
	id, err := strconv.ParseUint(frm.idInput.GetText(), 10, 8)
	if err != nil {
		return BiomeInfo{}, false
	}

	snow, err := strconv.ParseInt(frm.snowLineInput.GetText(), 10, 32)
	if err != nil {
		return BiomeInfo{}, false
	}
	if (snow > mcmap.ChunkSizeY) || (snow < 0) {
		snow = mcmap.ChunkSizeY
	}

	name := frm.nameInput.GetText()
	if name == "" {
		return BiomeInfo{}, false
	}

	col := frm.colorInput.GetColor()

	return BiomeInfo{
		ID:       mcmap.Biome(id),
		SnowLine: int(snow),
		Name:     name,
		Color:    fmt.Sprintf("#%02x%02x%02x", col.Red()<<8, col.Green()<<8, col.Blue()<<8),
	}, true
}

func (frm *biomeEditFrame) checkOK() bool {
	_, ok := frm.getBiomeInfo()
	return ok
}

func (frm *biomeEditFrame) unlockApply() {
	frm.applyBtn.SetSensitive(frm.checkOK())
}

type BiomeInfoEditor struct {
	*gtk.Dialog
	biomes []BiomeInfo
}

func NewBiomeInfoEditor(biomes []BiomeInfo) *BiomeInfoEditor {
	ed := &BiomeInfoEditor{
		Dialog: gtk.NewDialog(),
		biomes: biomes,
	}

	ed.SetModal(true)

	vbox := ed.GetVBox()

	btnHBox := gtk.NewHBox(true, 0)

	resetBtn := gtk.NewButtonWithLabel("Reset to defaults")
	resetBtn.Connect("clicked", ed.reset)
	loadBtn := gtk.NewButtonWithLabel("Load from file ...")
	loadBtn.Connect("clicked", ed.load)
	saveBtn := gtk.NewButtonWithLabel("Save to file ...")
	saveBtn.Connect("clicked", ed.save)

	btnHBox.PackStart(resetBtn, true, true, 3)
	btnHBox.PackStart(loadBtn, true, true, 3)
	btnHBox.PackStart(saveBtn, true, true, 3)
	vbox.PackStart(btnHBox, false, false, 3)

	editFrame := newBiomeEditFrame()
	vbox.PackStart(editFrame, false, false, 3)

	ed.AddButton("Cancel", gtk.RESPONSE_CANCEL)
	ed.AddButton("OK", gtk.RESPONSE_OK)
	ed.ShowAll()
	return ed
}

func (ed *BiomeInfoEditor) reset() {
	ed.biomes = ReadDefaultBiomes()
	// TODO: Update view
}

func mkBioFFilters() (*gtk.FileFilter, *gtk.FileFilter) {
	f1 := gtk.NewFileFilter()
	f1.AddPattern("*.biomes")
	f1.SetName("Biome Infos (.biomes)")

	f2 := gtk.NewFileFilter()
	f2.AddPattern("*")
	f2.SetName("All files")

	return f1, f2
}

func errdlg(msg string, params ...interface{}) {
	dlg := gtk.NewMessageDialog(nil, gtk.DIALOG_MODAL|gtk.DIALOG_DESTROY_WITH_PARENT, gtk.MESSAGE_ERROR, gtk.BUTTONS_OK, msg, params...)
	dlg.Run()
	dlg.Destroy()
}

func (ed *BiomeInfoEditor) load() {
	dlg := gtk.NewFileChooserDialog("Load", nil, gtk.FILE_CHOOSER_ACTION_OPEN, "OK", gtk.RESPONSE_OK, "Cancel", gtk.RESPONSE_CANCEL)
	defer dlg.Destroy()
askFilename:
	if dlg.Run() == gtk.RESPONSE_OK {
		path := dlg.GetFilename()

		f, err := os.Open(path)
		if err != nil {
			errdlg("Could not load biome infos %s:\n%s", path, err.Error())
			goto askFilename
		}
		defer f.Close()

		infos, err := ReadBiomeInfos(f)
		if err != nil {
			errdlg("Could not load biome infos %s:\n%s", path, err.Error())
			goto askFilename
		}

		ed.biomes = infos
		// TODO: Update view
	}
}

func (ed *BiomeInfoEditor) save() {
	dlg := gtk.NewFileChooserDialog("Save", nil, gtk.FILE_CHOOSER_ACTION_SAVE, "OK", gtk.RESPONSE_OK, "Cancel", gtk.RESPONSE_CANCEL)
	defer dlg.Destroy()
askFilename:
	if dlg.Run() == gtk.RESPONSE_OK {
		path := dlg.GetFilename()

		if _, err := os.Stat(path); err == nil {
			qdlg := gtk.NewMessageDialog(nil, gtk.DIALOG_MODAL, gtk.MESSAGE_QUESTION, gtk.BUTTONS_YES_NO, "File %s already exists. Overwrite?", path)
			resp := qdlg.Run()
			qdlg.Destroy()

			if resp != gtk.RESPONSE_YES {
				goto askFilename
			}
		}

		f, err := os.Create(path)
		if err != nil {
			errdlg("Could not save biome infos %s:\n%s", path, err.Error())
			goto askFilename
		}
		defer f.Close()

		if err := WriteBiomeInfos(ed.biomes, f); err != nil {
			errdlg("Could not save biome infos %s:\n%s", path, err.Error())
			goto askFilename
		}
	}
}

func (ed *BiomeInfoEditor) Biomes() []BiomeInfo { return ed.biomes }
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"

	"encoding/base64"
	"encoding/json"
	"io"
	"net/url"

	"github.com/ae6rt/decap/web/k8stypes"
	"github.com/pborman/uuid"
	"golang.org/x/net/websocket"
)

type PodWatch struct {
	Object Object `json:"object"`
}

type Object struct {
	Meta   Metadata `json:"metadata"`
	Status Status   `json:"status"`
}

type Metadata struct {
	Name string `json:"name"`
}

type Status struct {
	Statuses []XContainerStatus `json:"containerStatuses"`
}

type XContainerStatus struct {
	Name  string `json:"name"`
	Ready bool   `json:"ready"`
	State State  `json:"state"`
}

type State struct {
	Terminated Terminated `json:"terminated"`
}

type Terminated struct {
	ContainerID string `json:"containerID"`
	ExitCode    int    `json:"exitCode"`
}

// TODO distinguish between pushes and branch creation.  Github has a header value that allows these to be differentitated.
// https://developer.github.com/webhooks/#delivery-headers
// https://gist.githubusercontent.com/ae6rt/53a25e726ac00b4cb535/raw/e3f412f6e7f408a56d0d691a1ec8b7658a495124/gh-create.json
// https://gist.githubusercontent.com/ae6rt/2be93f7d5edef8030b52/raw/29f591eb8ecc5555c55f1878b545613c1f9839b7/gh-push.json
type BuildEvent interface {
	Team() string
	Library() string
	ProjectKey() string
	Branches() []string
}

type DefaultDecap struct {
	MasterURL       string
	UserName        string
	Password        string
	AWSAccessKeyID  string
	AWSAccessSecret string
	AWSRegion       string
	Locker          Locker

	apiToken  string
	apiClient *http.Client
}

type RepoManagerCredential struct {
	User     string
	Password string
}

type StorageService interface {
	GetBuildsByProject(project Project, sinceUnixTime uint64, limit uint64) ([]Build, error)
	GetArtifacts(buildID string) ([]byte, error)
	GetConsoleLog(buildID string) ([]byte, error)
}

type Decap interface {
	LaunchBuild(buildEvent BuildEvent) error
	DeletePod(podName string) error
}

func NewDefaultDecap(apiServerURL, username, password, awsKey, awsSecret, awsRegion string, locker Locker) DefaultDecap {
	// todo when running in cluster, provide root certificate via /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
	apiClient := &http.Client{Transport: &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}}

	data, _ := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token")

	return DefaultDecap{
		MasterURL:       apiServerURL,
		apiToken:        string(data),
		UserName:        username,
		Password:        password,
		Locker:          locker,
		AWSAccessKeyID:  awsKey,
		AWSAccessSecret: awsSecret,
		AWSRegion:       awsRegion,
		apiClient:       apiClient,
	}
}

func (k8s DefaultDecap) LaunchBuild(buildEvent BuildEvent) error {
	projectKey := buildEvent.ProjectKey()

	projs := getProjects()

	for _, branch := range buildEvent.Branches() {
		key := k8s.Locker.Key(projectKey, branch)
		buildID := uuid.NewRandom().String()

		containers := make([]k8stypes.Container, 1+len(projs[projectKey].Sidecars))

		baseContainer := k8stypes.Container{
			Name:  "build-server",
			Image: projs[projectKey].Descriptor.Image,
			VolumeMounts: []k8stypes.VolumeMount{
				k8stypes.VolumeMount{
					Name:      "build-scripts",
					MountPath: "/home/decap/buildscripts",
				},
				k8stypes.VolumeMount{
					Name:      "decap-credentials",
					MountPath: "/etc/secrets",
				},
			},
			Env: []k8stypes.EnvVar{
				k8stypes.EnvVar{
					Name:  "BUILD_ID",
					Value: buildID,
				},
				k8stypes.EnvVar{
					Name:  "PROJECT_KEY",
					Value: projectKey,
				},
				k8stypes.EnvVar{
					Name:  "BRANCH_TO_BUILD",
					Value: branch,
				},
				k8stypes.EnvVar{
					Name:  "BUILD_LOCK_KEY",
					Value: key,
				},
				k8stypes.EnvVar{
					Name:  "AWS_ACCESS_KEY_ID",
					Value: k8s.AWSAccessKeyID,
				},
				k8stypes.EnvVar{
					Name:  "AWS_SECRET_ACCESS_KEY",
					Value: k8s.AWSAccessSecret,
				},
				k8stypes.EnvVar{
					Name:  "AWS_DEFAULT_REGION",
					Value: k8s.AWSRegion,
				},
			},
		}

		containers[0] = baseContainer
		for i, v := range projs[projectKey].Sidecars {
			var c k8stypes.Container
			err := json.Unmarshal([]byte(v), &c)
			if err != nil {
				Log.Println(err)
				continue
			}
			containers[i+1] = c
		}

		pod := k8stypes.Pod{
			TypeMeta: k8stypes.TypeMeta{
				Kind:       "Pod",
				APIVersion: "v1",
			},
			ObjectMeta: k8stypes.ObjectMeta{
				Name:      buildID,
				Namespace: "decap",
				Labels: map[string]string{
					"type":    "decap-build",
					"team":    buildEvent.Team(),
					"library": buildEvent.Library(),
					"branch":  branch,
				},
			},
			Spec: k8stypes.PodSpec{
				Volumes: []k8stypes.Volume{
					k8stypes.Volume{
						Name: "build-scripts",
						VolumeSource: k8stypes.VolumeSource{
							GitRepo: &k8stypes.GitRepoVolumeSource{
								Repository: *buildScriptsRepo,
								Revision:   *buildScriptsRepoBranch,
							},
						},
					},
					k8stypes.Volume{
						Name: "decap-credentials",
						VolumeSource: k8stypes.VolumeSource{
							Secret: &k8stypes.SecretVolumeSource{
								SecretName: "decap-credentials",
							},
						},
					},
				},
				Containers:    containers,
				RestartPolicy: "Never",
			},
		}

		podBytes, err := json.Marshal(&pod)
		if err != nil {
			Log.Println(err)
			continue
		}

		fullyQualifiedKey := "/buildlocks/" + key
		resp, err := k8s.Locker.Lock(fullyQualifiedKey, buildID)
		if err != nil {
			Log.Printf("Failed to acquire lock %s on build %s: %v\n", key, buildID, err)
			continue
		}

		if resp.Node.Value == buildID {
			Log.Printf("Acquired lock on build %s with key %s\n", buildID, key)
			if podError := k8s.CreatePod(podBytes); podError != nil {
				Log.Println(podError)
				if _, err := k8s.Locker.Unlock(fullyQualifiedKey, buildID); err != nil {
					Log.Println(err)
				} else {
					Log.Printf("Released lock on build %s with key %s because of pod creation error %v\n", buildID, key, podError)
				}
			}
			Log.Printf("Created pod=%s\n", buildID)
		} else {
			Log.Printf("Failed to acquire lock %s on build %s\n", key, buildID)
		}
	}
	return nil
}

func (base DefaultDecap) CreatePod(pod []byte) error {
	Log.Printf("spec pod:%+v\n", pod)

	req, err := http.NewRequest("POST", fmt.Sprintf("%s/api/v1/namespaces/decap/pods", base.MasterURL), bytes.NewReader(pod))
	if err != nil {
		Log.Println(err)
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	if base.apiToken != "" {
		req.Header.Set("Authorization", "Bearer "+base.apiToken)
	} else {
		req.SetBasicAuth(base.UserName, base.Password)
	}

	resp, err := base.apiClient.Do(req)
	if err != nil {
		return err
	}
	defer func() {
		resp.Body.Close()
	}()

	if resp.StatusCode != 201 {
		if data, err := ioutil.ReadAll(resp.Body); err != nil {
			Log.Printf("Error reading non-201 response body: %v\n", err)
			return err
		} else {
			Log.Printf("%s\n", string(data))
			return nil
		}
	}
	return nil
}

func (base DefaultDecap) DeletePod(podName string) error {
	req, err := http.NewRequest("DELETE", fmt.Sprintf("%s/api/v1/namespaces/decap/pods/%s", base.MasterURL, podName), nil)
	if err != nil {
		Log.Println(err)
		return err
	}
	if base.apiToken != "" {
		req.Header.Set("Authorization", "Bearer "+base.apiToken)
	} else {
		req.SetBasicAuth(base.UserName, base.Password)
	}

	resp, err := base.apiClient.Do(req)
	if err != nil {
		return err
	}
	defer func() {
		resp.Body.Close()
	}()

	if resp.StatusCode != 200 {
		if data, err := ioutil.ReadAll(resp.Body); err != nil {
			Log.Printf("Error reading non-200 response body: %v\n", err)
			return err
		} else {
			Log.Printf("%s\n", string(data))
			return nil
		}
	}
	return nil
}

func (decap DefaultDecap) Websock() {
	originURL, err := url.Parse(decap.MasterURL + "/api/v1/watch/namespaces/decap/pods?watch=true&labelSelector=type=decap-build")
	if err != nil {
		Log.Printf("Error parsing websocket origin URL.  Will be unable to reap exited pods.: %v\n", err)
		return
	}
	serviceURL, err := url.Parse("wss://" + originURL.Host + "/api/v1/watch/namespaces/decap/pods?watch=true&labelSelector=type=decap-build")
	if err != nil {
		Log.Printf("Error parsing websocket service URL.  Will be unable to reap exited pods.: %v\n", err)
		return
	}

	var hdrs http.Header
	if decap.apiToken != "" {
		hdrs = map[string][]string{"Authorization": []string{"Bearer " + decap.apiToken}}
	} else {
		hdrs = map[string][]string{"Authorization": []string{"Basic " + base64.StdEncoding.EncodeToString([]byte(decap.UserName+":"+decap.Password))}}
	}

	cfg := websocket.Config{
		Location:  serviceURL,
		Origin:    originURL,
		TlsConfig: &tls.Config{InsecureSkipVerify: true},
		Header:    hdrs,
		Version:   websocket.ProtocolVersionHybi13,
	}

	conn, err := websocket.DialConfig(&cfg)
	if err != nil {
		Log.Printf("Error opening websocket connection.  Will be unable to reap exited pods.: %v\n", err)
		return
	}
	Log.Print("Watching pods on websocket")

	var msg string
	for {
		err := websocket.Message.Receive(conn, &msg)
		if err != nil {
			if err == io.EOF {
				break
			}
			Log.Println("Couldn't receive msg " + err.Error())
		}
		var pod PodWatch
		if err := json.Unmarshal([]byte(msg), &pod); err != nil {
			Log.Println(err)
			continue
		}
		var deletePod bool
		for _, status := range pod.Object.Status.Statuses {
			if status.Name == "build-server" && status.State.Terminated.ContainerID != "" {
				deletePod = true
				break
			}
		}
		if deletePod {
			_, err := decap.Locker.Lock("/pods/"+pod.Object.Meta.Name, "anyvalue")
			if err == nil {
				if err := decap.DeletePod(pod.Object.Meta.Name); err != nil {
					Log.Print(err)
				} else {
					Log.Printf("Pod deleted: %s\n", pod.Object.Meta.Name)
				}
			}
		}
	}
}

func kubeSecret(file string, defaultValue string) string {
	if v, err := ioutil.ReadFile(file); err != nil {
		Log.Printf("Secret %s not found in the filesystem.  Using default.\n", file)
		return defaultValue
	} else {
		Log.Printf("Successfully read secret %s from the filesystem\n", file)
		return string(v)
	}
}
<commit_msg>fix typo<commit_after>package main

import (
	"bytes"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"

	"encoding/base64"
	"encoding/json"
	"io"
	"net/url"

	"github.com/ae6rt/decap/web/k8stypes"
	"github.com/pborman/uuid"
	"golang.org/x/net/websocket"
)

type PodWatch struct {
	Object Object `json:"object"`
}

type Object struct {
	Meta   Metadata `json:"metadata"`
	Status Status   `json:"status"`
}

type Metadata struct {
	Name string `json:"name"`
}

type Status struct {
	Statuses []XContainerStatus `json:"containerStatuses"`
}

type XContainerStatus struct {
	Name  string `json:"name"`
	Ready bool   `json:"ready"`
	State State  `json:"state"`
}

type State struct {
	Terminated Terminated `json:"terminated"`
}

type Terminated struct {
	ContainerID string `json:"containerID"`
	ExitCode    int    `json:"exitCode"`
}

// TODO distinguish between pushes and branch creation.  Github has a header value that allows these to be differentiated.
// https://developer.github.com/webhooks/#delivery-headers
// https://gist.githubusercontent.com/ae6rt/53a25e726ac00b4cb535/raw/e3f412f6e7f408a56d0d691a1ec8b7658a495124/gh-create.json
// https://gist.githubusercontent.com/ae6rt/2be93f7d5edef8030b52/raw/29f591eb8ecc5555c55f1878b545613c1f9839b7/gh-push.json
type BuildEvent interface {
	Team() string
	Library() string
	ProjectKey() string
	Branches() []string
}

type DefaultDecap struct {
	MasterURL       string
	UserName        string
	Password        string
	AWSAccessKeyID  string
	AWSAccessSecret string
	AWSRegion       string
	Locker          Locker

	apiToken  string
	apiClient *http.Client
}

type RepoManagerCredential struct {
	User     string
	Password string
}

type StorageService interface {
	GetBuildsByProject(project Project, sinceUnixTime uint64, limit uint64) ([]Build, error)
	GetArtifacts(buildID string) ([]byte, error)
	GetConsoleLog(buildID string) ([]byte, error)
}

type Decap interface {
	LaunchBuild(buildEvent BuildEvent) error
	DeletePod(podName string) error
}

func NewDefaultDecap(apiServerURL, username, password, awsKey, awsSecret, awsRegion string, locker Locker) DefaultDecap {
	// todo when running in cluster, provide root certificate via /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
	apiClient := &http.Client{Transport: &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}}

	data, _ := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token")

	return DefaultDecap{
		MasterURL:       apiServerURL,
		apiToken:        string(data),
		UserName:        username,
		Password:        password,
		Locker:          locker,
		AWSAccessKeyID:  awsKey,
		AWSAccessSecret: awsSecret,
		AWSRegion:       awsRegion,
		apiClient:       apiClient,
	}
}

func (k8s DefaultDecap) LaunchBuild(buildEvent BuildEvent) error {
	projectKey := buildEvent.ProjectKey()

	projs := getProjects()

	for _, branch := range buildEvent.Branches() {
		key := k8s.Locker.Key(projectKey, branch)
		buildID := uuid.NewRandom().String()

		containers := make([]k8stypes.Container, 1+len(projs[projectKey].Sidecars))

		baseContainer := k8stypes.Container{
			Name:  "build-server",
			Image: projs[projectKey].Descriptor.Image,
			VolumeMounts: []k8stypes.VolumeMount{
				k8stypes.VolumeMount{
					Name:      "build-scripts",
					MountPath: "/home/decap/buildscripts",
				},
				k8stypes.VolumeMount{
					Name:      "decap-credentials",
					MountPath: "/etc/secrets",
				},
			},
			Env: []k8stypes.EnvVar{
				k8stypes.EnvVar{
					Name:  "BUILD_ID",
					Value: buildID,
				},
				k8stypes.EnvVar{
					Name:  "PROJECT_KEY",
					Value: projectKey,
				},
				k8stypes.EnvVar{
					Name:  "BRANCH_TO_BUILD",
					Value: branch,
				},
				k8stypes.EnvVar{
					Name:  "BUILD_LOCK_KEY",
					Value: key,
				},
				k8stypes.EnvVar{
					Name:  "AWS_ACCESS_KEY_ID",
					Value: k8s.AWSAccessKeyID,
				},
				k8stypes.EnvVar{
					Name:  "AWS_SECRET_ACCESS_KEY",
					Value: k8s.AWSAccessSecret,
				},
				k8stypes.EnvVar{
					Name:  "AWS_DEFAULT_REGION",
					Value: k8s.AWSRegion,
				},
			},
		}

		containers[0] = baseContainer
		for i, v := range projs[projectKey].Sidecars {
			var c k8stypes.Container
			err := json.Unmarshal([]byte(v), &c)
			if err != nil {
				Log.Println(err)
				continue
			}
			containers[i+1] = c
		}

		pod := k8stypes.Pod{
			TypeMeta: k8stypes.TypeMeta{
				Kind:       "Pod",
				APIVersion: "v1",
			},
			ObjectMeta: k8stypes.ObjectMeta{
				Name:      buildID,
				Namespace: "decap",
				Labels: map[string]string{
					"type":    "decap-build",
					"team":    buildEvent.Team(),
					"library": buildEvent.Library(),
					"branch":  branch,
				},
			},
			Spec: k8stypes.PodSpec{
				Volumes: []k8stypes.Volume{
					k8stypes.Volume{
						Name: "build-scripts",
						VolumeSource: k8stypes.VolumeSource{
							GitRepo: &k8stypes.GitRepoVolumeSource{
								Repository: *buildScriptsRepo,
								Revision:   *buildScriptsRepoBranch,
							},
						},
					},
					k8stypes.Volume{
						Name: "decap-credentials",
						VolumeSource: k8stypes.VolumeSource{
							Secret: &k8stypes.SecretVolumeSource{
								SecretName: "decap-credentials",
							},
						},
					},
				},
				Containers:    containers,
				RestartPolicy: "Never",
			},
		}

		podBytes, err := json.Marshal(&pod)
		if err != nil {
			Log.Println(err)
			continue
		}

		fullyQualifiedKey := "/buildlocks/" + key
		resp, err := k8s.Locker.Lock(fullyQualifiedKey, buildID)
		if err != nil {
			Log.Printf("Failed to acquire lock %s on build %s: %v\n", key, buildID, err)
			continue
		}

		if resp.Node.Value == buildID {
			Log.Printf("Acquired lock on build %s with key %s\n", buildID, key)
			if podError := k8s.CreatePod(podBytes); podError != nil {
				Log.Println(podError)
				if _, err := k8s.Locker.Unlock(fullyQualifiedKey, buildID); err != nil {
					Log.Println(err)
				} else {
					Log.Printf("Released lock on build %s with key %s because of pod creation error %v\n", buildID, key, podError)
				}
			}
			Log.Printf("Created pod=%s\n", buildID)
		} else {
			Log.Printf("Failed to acquire lock %s on build %s\n", key, buildID)
		}
	}
	return nil
}

func (base DefaultDecap) CreatePod(pod []byte) error {
	Log.Printf("spec pod:%+v\n", pod)

	req, err := http.NewRequest("POST", fmt.Sprintf("%s/api/v1/namespaces/decap/pods", base.MasterURL), bytes.NewReader(pod))
	if err != nil {
		Log.Println(err)
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	if base.apiToken != "" {
		req.Header.Set("Authorization", "Bearer "+base.apiToken)
	} else {
		req.SetBasicAuth(base.UserName, base.Password)
	}

	resp, err := base.apiClient.Do(req)
	if err != nil {
		return err
	}
	defer func() {
		resp.Body.Close()
	}()

	if resp.StatusCode != 201 {
		if data, err := ioutil.ReadAll(resp.Body); err != nil {
			Log.Printf("Error reading non-201 response body: %v\n", err)
			return err
		} else {
			Log.Printf("%s\n", string(data))
			return nil
		}
	}
	return nil
}

func (base DefaultDecap) DeletePod(podName string) error {
	req, err := http.NewRequest("DELETE", fmt.Sprintf("%s/api/v1/namespaces/decap/pods/%s", base.MasterURL, podName), nil)
	if err != nil {
		Log.Println(err)
		return err
	}
	if base.apiToken != "" {
		req.Header.Set("Authorization", "Bearer "+base.apiToken)
	} else {
		req.SetBasicAuth(base.UserName, base.Password)
	}

	resp, err := base.apiClient.Do(req)
	if err != nil {
		return err
	}
	defer func() {
		resp.Body.Close()
	}()

	if resp.StatusCode != 200 {
		if data, err := ioutil.ReadAll(resp.Body); err != nil {
			Log.Printf("Error reading non-200 response body: %v\n", err)
			return err
		} else {
			Log.Printf("%s\n", string(data))
			return nil
		}
	}
	return nil
}

func (decap DefaultDecap) Websock() {
	originURL, err := url.Parse(decap.MasterURL + "/api/v1/watch/namespaces/decap/pods?watch=true&labelSelector=type=decap-build")
	if err != nil {
		Log.Printf("Error parsing websocket origin URL.  Will be unable to reap exited pods.: %v\n", err)
		return
	}
	serviceURL, err := url.Parse("wss://" + originURL.Host + "/api/v1/watch/namespaces/decap/pods?watch=true&labelSelector=type=decap-build")
	if err != nil {
		Log.Printf("Error parsing websocket service URL.  Will be unable to reap exited pods.: %v\n", err)
		return
	}

	var hdrs http.Header
	if decap.apiToken != "" {
		hdrs = map[string][]string{"Authorization": []string{"Bearer " + decap.apiToken}}
	} else {
		hdrs = map[string][]string{"Authorization": []string{"Basic " + base64.StdEncoding.EncodeToString([]byte(decap.UserName+":"+decap.Password))}}
	}

	cfg := websocket.Config{
		Location:  serviceURL,
		Origin:    originURL,
		TlsConfig: &tls.Config{InsecureSkipVerify: true},
		Header:    hdrs,
		Version:   websocket.ProtocolVersionHybi13,
	}

	conn, err := websocket.DialConfig(&cfg)
	if err != nil {
		Log.Printf("Error opening websocket connection.  Will be unable to reap exited pods.: %v\n", err)
		return
	}
	Log.Print("Watching pods on websocket")

	var msg string
	for {
		err := websocket.Message.Receive(conn, &msg)
		if err != nil {
			if err == io.EOF {
				break
			}
			Log.Println("Couldn't receive msg " + err.Error())
		}
		var pod PodWatch
		if err := json.Unmarshal([]byte(msg), &pod); err != nil {
			Log.Println(err)
			continue
		}
		var deletePod bool
		for _, status := range pod.Object.Status.Statuses {
			if status.Name == "build-server" && status.State.Terminated.ContainerID != "" {
				deletePod = true
				break
			}
		}
		if deletePod {
			_, err := decap.Locker.Lock("/pods/"+pod.Object.Meta.Name, "anyvalue")
			if err == nil {
				if err := decap.DeletePod(pod.Object.Meta.Name); err != nil {
					Log.Print(err)
				} else {
					Log.Printf("Pod deleted: %s\n", pod.Object.Meta.Name)
				}
			}
		}
	}
}

func kubeSecret(file string, defaultValue string) string {
	if v, err := ioutil.ReadFile(file); err != nil {
		Log.Printf("Secret %s not found in the filesystem.  Using default.\n", file)
		return defaultValue
	} else {
		Log.Printf("Successfully read secret %s from the filesystem\n", file)
		return string(v)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//
// VMDK Docker driver sanity tests.
//

package main

import (
	"flag"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/docker/engine-api/client"
	"github.com/docker/engine-api/types"
	"github.com/docker/engine-api/types/container"
	"github.com/docker/engine-api/types/filters"
	"github.com/docker/engine-api/types/strslice"
	"github.com/vmware/docker-vmdk-plugin/config"
	"golang.org/x/net/context"
	"testing"
	"time"
)

const (
	defaultMountLocation = "/mnt/testvol"
	// tests are often run under regular account and have no access to /var/log
	defaultTestLogPath = "/tmp/test-docker-volume-vsphere.log"
)

var (
	// flag vars - see init() for help
	endPoint1        string
	endPoint2        string
	volumeName       string
	removeContainers bool
)

// prepares the environment. Kind of "main"-ish code for tests.
// Parses flags and inits logs and mount ref counters (the latter waits on Docker
// actuallu replyimg). As any other init(), it is called somewhere during init phase
// so do not expect ALL inits from other tests (if any) to compete by now.
func init() {
	waitSec := flag.Int("t", defaultSleepIntervalSec, "Reconnect polling time, in seconds")
	logLevel := flag.String("log_level", "debug", "Logging Level")
	logFile := flag.String("log_file", config.DefaultLogPath, "Log file path")
	configFile := flag.String("config", config.DefaultConfigPath, "Configuration file path")

	flag.StringVar(&endPoint1, "H1", dockerUSocket, "Endpoint (Host1) to connect to")
	flag.StringVar(&endPoint2, "H2", dockerUSocket, "Endpoint (Host2) to connect to")
	flag.StringVar(&volumeName, "v", "TestVol", "Volume name to use in sanity tests")
	flag.BoolVar(&removeContainers, "rm", true, "rm container after run")
	flag.StringVar(&driverName, "d", "vmdk", "Driver name. We refcount volumes on this driver")
	flag.Parse()

	usingConfigFileDefaults := logInit(logLevel, logFile, configFile)

	reconnectSleepInterval = time.Duration(*waitSec) * time.Second
	defaultHeaders = map[string]string{"User-Agent": "engine-api-client-1.0"}

	log.WithFields(log.Fields{
		"driver":          driverName,
		"waitSec":         *waitSec,
		"log_level":       *logLevel,
		"log_file":        *logFile,
		"conf_file":       *configFile,
		"using_conf_file_defaults": usingConfigFileDefaults,
	}).Info("VMDK plugin tests started ")
}

// returns in-container mount point for a volume
func getMountpoint(vol string) string {
	return defaultMountLocation + "/" + vol
}

// runs a command in a container , with volume mounted
// returns completion code.
// exits (t.Fatal() or create/start/wait errors
func runContainerCmd(t *testing.T, client *client.Client, volumeName string,
	image string, cmd *strslice.StrSlice, addr string) int {

	mountPoint := getMountpoint(volumeName)
	bind := volumeName + ":" + mountPoint
	t.Logf("Running cmd=%v with vol=%s on client %s", cmd, volumeName, addr)

	r, err := client.ContainerCreate(
		&container.Config{Image: image, Cmd: *cmd,
			Volumes: map[string]struct{}{mountPoint: {}}},
		&container.HostConfig{Binds: []string{bind}}, nil, "")
	if err != nil {
		t.Fatalf("\tContainer create failed: %v", err)
	}

	err = client.ContainerStart(r.ID)
	if err != nil {
		t.Fatalf("\tContainer start failed: id=%s, err %v", r.ID, err)
	}

	code, err := client.ContainerWait(context.Background(), r.ID)
	if err != nil {
		t.Fatalf("\tContainer wait failed: id=%s, err %v", r.ID, err)
	}

	if removeContainers == false {
		t.Logf("\tSkipping container removal, id=%s (removeContainers == false)",
			r.ID)
		return code
	}

	err = client.ContainerRemove(types.ContainerRemoveOptions{
		ContainerID:   r.ID,
		RemoveVolumes: true,
		Force:         true,
	})
	if err != nil {
		t.Fatalf("\nContainer removal failed: %v", err)
	}

	return code
}

// Checks that we can touch a file in one container and then stat it
// in another container, using the same (vmdk-based) volume
//
// goes over 'cases' and runs commands, then checks expected return code
func checkTouch(t *testing.T, c *client.Client, vol string,
	file string, addr string) {

	cases := []struct {
		image    string             // Container image to use
		cmd      *strslice.StrSlice // Command to run under busybox
		expected int                // expected results
	}{
		{"busybox", &strslice.StrSlice{"touch", getMountpoint(vol) + "/" + file}, 0},
		{"busybox", &strslice.StrSlice{"stat", getMountpoint(vol) + "/" + file}, 0},
	}

	for _, i := range cases {
		code := runContainerCmd(t, c, vol, i.image, i.cmd, addr)
		if code != i.expected {
			t.Errorf("Expected  %d, got %d (cmd: %v)", i.expected, code, i.cmd)
		}
	}
}

// returns nil for NOT_FOUND and  if volume exists
// still fails the test if driver for this volume is not vmdk
func volumeVmdkExists(t *testing.T, c *client.Client, vol string) *types.Volume {
	reply, err := c.VolumeList(filters.Args{})
	if err != nil {
		t.Fatalf("Failed to enumerate  volumes: %v", err)
	}

	for _, v := range reply.Volumes {
		//	t.Log(v.Name, v.Driver, v.Mountpoint)
		if v.Name == vol {
			return v
		}
	}
	return nil
}

// Sanity test for VMDK volumes
// - check we can attach/detach correct volume (we use 'touch' and 'stat' to validate
// - check volumes are correctly created and deleted.
// - check we see it properly from another docker VM (-H2 flag)
func TestSanity(t *testing.T) {

	fmt.Printf("Running tests on  %s (may take a while)...\n", endPoint1)
	clients := []struct {
		endPoint string
		client   *client.Client
	}{
		{endPoint1, new(client.Client)},
		{endPoint2, new(client.Client)},
	}

	for idx, elem := range clients {
		c, err := client.NewClient(elem.endPoint, apiVersion, nil, defaultHeaders)
		if err != nil {
			t.Fatalf("Failed to connect to %s, err: %v", elem.endPoint, err)
		}
		t.Logf("Successfully connected to %s", elem.endPoint)
		clients[idx].client = c
	}

	c := clients[0].client // this is the endpoint we use as master
	t.Logf("Creating vol=%s on client %s.", volumeName, clients[0].endPoint)
	_, err := c.VolumeCreate(
		types.VolumeCreateRequest{
			Name:   volumeName,
			Driver: driverName,
			DriverOpts: map[string]string{
				"size":   "1gb",
				"policy": "good",
			},
		})
	if err != nil {
		t.Fatal(err)
	}

	checkTouch(t, c, volumeName, "file_to_touch", clients[0].endPoint)

	for _, elem := range clients {
		v := volumeVmdkExists(t, elem.client, volumeName)
		if v == nil {
			t.Fatalf("Volume=%s is missing on %s after create",
				volumeName, elem.endPoint)
		}
		if v.Driver != driverName {
			t.Fatalf("wrong driver (%s) for volume %s", v.Driver, v.Name)
		}
	}

	err = c.VolumeRemove(volumeName)
	if err != nil {
		t.Fatalf("Failed to delete volume, err: %v", err)
	}

	for _, elem := range clients {
		if volumeVmdkExists(t, elem.client, volumeName) != nil {
			t.Errorf("Volume=%s is still present on %s after removal",
				volumeName, elem.endPoint)
		}
	}
}
<commit_msg>Minor code clean up via gotfmt<commit_after>// Copyright 2016 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//
// VMDK Docker driver sanity tests.
//

package main

import (
	"flag"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/docker/engine-api/client"
	"github.com/docker/engine-api/types"
	"github.com/docker/engine-api/types/container"
	"github.com/docker/engine-api/types/filters"
	"github.com/docker/engine-api/types/strslice"
	"github.com/vmware/docker-vmdk-plugin/config"
	"golang.org/x/net/context"
	"testing"
	"time"
)

const (
	defaultMountLocation = "/mnt/testvol"
	// tests are often run under regular account and have no access to /var/log
	defaultTestLogPath = "/tmp/test-docker-volume-vsphere.log"
)

var (
	// flag vars - see init() for help
	endPoint1        string
	endPoint2        string
	volumeName       string
	removeContainers bool
)

// prepares the environment. Kind of "main"-ish code for tests.
// Parses flags and inits logs and mount ref counters (the latter waits on Docker
// actuallu replyimg). As any other init(), it is called somewhere during init phase
// so do not expect ALL inits from other tests (if any) to compete by now.
func init() {
	waitSec := flag.Int("t", defaultSleepIntervalSec, "Reconnect polling time, in seconds")
	logLevel := flag.String("log_level", "debug", "Logging Level")
	logFile := flag.String("log_file", config.DefaultLogPath, "Log file path")
	configFile := flag.String("config", config.DefaultConfigPath, "Configuration file path")

	flag.StringVar(&endPoint1, "H1", dockerUSocket, "Endpoint (Host1) to connect to")
	flag.StringVar(&endPoint2, "H2", dockerUSocket, "Endpoint (Host2) to connect to")
	flag.StringVar(&volumeName, "v", "TestVol", "Volume name to use in sanity tests")
	flag.BoolVar(&removeContainers, "rm", true, "rm container after run")
	flag.StringVar(&driverName, "d", "vmdk", "Driver name. We refcount volumes on this driver")
	flag.Parse()

	usingConfigFileDefaults := logInit(logLevel, logFile, configFile)

	reconnectSleepInterval = time.Duration(*waitSec) * time.Second
	defaultHeaders = map[string]string{"User-Agent": "engine-api-client-1.0"}

	log.WithFields(log.Fields{
		"driver":                   driverName,
		"waitSec":                  *waitSec,
		"log_level":                *logLevel,
		"log_file":                 *logFile,
		"conf_file":                *configFile,
		"using_conf_file_defaults": usingConfigFileDefaults,
	}).Info("VMDK plugin tests started ")
}

// returns in-container mount point for a volume
func getMountpoint(vol string) string {
	return defaultMountLocation + "/" + vol
}

// runs a command in a container , with volume mounted
// returns completion code.
// exits (t.Fatal() or create/start/wait errors
func runContainerCmd(t *testing.T, client *client.Client, volumeName string,
	image string, cmd *strslice.StrSlice, addr string) int {

	mountPoint := getMountpoint(volumeName)
	bind := volumeName + ":" + mountPoint
	t.Logf("Running cmd=%v with vol=%s on client %s", cmd, volumeName, addr)

	r, err := client.ContainerCreate(
		&container.Config{Image: image, Cmd: *cmd,
			Volumes: map[string]struct{}{mountPoint: {}}},
		&container.HostConfig{Binds: []string{bind}}, nil, "")
	if err != nil {
		t.Fatalf("\tContainer create failed: %v", err)
	}

	err = client.ContainerStart(r.ID)
	if err != nil {
		t.Fatalf("\tContainer start failed: id=%s, err %v", r.ID, err)
	}

	code, err := client.ContainerWait(context.Background(), r.ID)
	if err != nil {
		t.Fatalf("\tContainer wait failed: id=%s, err %v", r.ID, err)
	}

	if removeContainers == false {
		t.Logf("\tSkipping container removal, id=%s (removeContainers == false)",
			r.ID)
		return code
	}

	err = client.ContainerRemove(types.ContainerRemoveOptions{
		ContainerID:   r.ID,
		RemoveVolumes: true,
		Force:         true,
	})
	if err != nil {
		t.Fatalf("\nContainer removal failed: %v", err)
	}

	return code
}

// Checks that we can touch a file in one container and then stat it
// in another container, using the same (vmdk-based) volume
//
// goes over 'cases' and runs commands, then checks expected return code
func checkTouch(t *testing.T, c *client.Client, vol string,
	file string, addr string) {

	cases := []struct {
		image    string             // Container image to use
		cmd      *strslice.StrSlice // Command to run under busybox
		expected int                // expected results
	}{
		{"busybox", &strslice.StrSlice{"touch", getMountpoint(vol) + "/" + file}, 0},
		{"busybox", &strslice.StrSlice{"stat", getMountpoint(vol) + "/" + file}, 0},
	}

	for _, i := range cases {
		code := runContainerCmd(t, c, vol, i.image, i.cmd, addr)
		if code != i.expected {
			t.Errorf("Expected  %d, got %d (cmd: %v)", i.expected, code, i.cmd)
		}
	}
}

// returns nil for NOT_FOUND and  if volume exists
// still fails the test if driver for this volume is not vmdk
func volumeVmdkExists(t *testing.T, c *client.Client, vol string) *types.Volume {
	reply, err := c.VolumeList(filters.Args{})
	if err != nil {
		t.Fatalf("Failed to enumerate  volumes: %v", err)
	}

	for _, v := range reply.Volumes {
		//	t.Log(v.Name, v.Driver, v.Mountpoint)
		if v.Name == vol {
			return v
		}
	}
	return nil
}

// Sanity test for VMDK volumes
// - check we can attach/detach correct volume (we use 'touch' and 'stat' to validate
// - check volumes are correctly created and deleted.
// - check we see it properly from another docker VM (-H2 flag)
func TestSanity(t *testing.T) {

	fmt.Printf("Running tests on  %s (may take a while)...\n", endPoint1)
	clients := []struct {
		endPoint string
		client   *client.Client
	}{
		{endPoint1, new(client.Client)},
		{endPoint2, new(client.Client)},
	}

	for idx, elem := range clients {
		c, err := client.NewClient(elem.endPoint, apiVersion, nil, defaultHeaders)
		if err != nil {
			t.Fatalf("Failed to connect to %s, err: %v", elem.endPoint, err)
		}
		t.Logf("Successfully connected to %s", elem.endPoint)
		clients[idx].client = c
	}

	c := clients[0].client // this is the endpoint we use as master
	t.Logf("Creating vol=%s on client %s.", volumeName, clients[0].endPoint)
	_, err := c.VolumeCreate(
		types.VolumeCreateRequest{
			Name:   volumeName,
			Driver: driverName,
			DriverOpts: map[string]string{
				"size":   "1gb",
				"policy": "good",
			},
		})
	if err != nil {
		t.Fatal(err)
	}

	checkTouch(t, c, volumeName, "file_to_touch", clients[0].endPoint)

	for _, elem := range clients {
		v := volumeVmdkExists(t, elem.client, volumeName)
		if v == nil {
			t.Fatalf("Volume=%s is missing on %s after create",
				volumeName, elem.endPoint)
		}
		if v.Driver != driverName {
			t.Fatalf("wrong driver (%s) for volume %s", v.Driver, v.Name)
		}
	}

	err = c.VolumeRemove(volumeName)
	if err != nil {
		t.Fatalf("Failed to delete volume, err: %v", err)
	}

	for _, elem := range clients {
		if volumeVmdkExists(t, elem.client, volumeName) != nil {
			t.Errorf("Volume=%s is still present on %s after removal",
				volumeName, elem.endPoint)
		}
	}
}
<|endoftext|>


<commit_before>package goadb

import (
	"fmt"
	"io"
	"strings"

	"github.com/zach-klippenstein/goadb/util"
	"github.com/zach-klippenstein/goadb/wire"
)

// DeviceClient communicates with a specific Android device.
type DeviceClient struct {
	config     ClientConfig
	descriptor DeviceDescriptor
}

func NewDeviceClient(config ClientConfig, descriptor DeviceDescriptor) *DeviceClient {
	return &DeviceClient{
		config:     config.sanitized(),
		descriptor: descriptor,
	}
}

func (c *DeviceClient) String() string {
	return c.descriptor.String()
}

// get-product is documented, but not implemented in the server.
// TODO(z): Make getProduct exported if get-product is ever implemented in adb.
func (c *DeviceClient) getProduct() (string, error) {
	attr, err := c.getAttribute("get-product")
	return attr, wrapClientError(err, c, "GetProduct")
}

func (c *DeviceClient) GetSerial() (string, error) {
	attr, err := c.getAttribute("get-serialno")
	return attr, wrapClientError(err, c, "GetSerial")
}

func (c *DeviceClient) GetDevicePath() (string, error) {
	attr, err := c.getAttribute("get-devpath")
	return attr, wrapClientError(err, c, "GetDevicePath")
}

func (c *DeviceClient) GetState() (string, error) {
	attr, err := c.getAttribute("get-state")
	return attr, wrapClientError(err, c, "GetState")
}

/*
RunCommand runs the specified commands on a shell on the device.

From the Android docs:
	Run 'command arg1 arg2 ...' in a shell on the device, and return
	its output and error streams. Note that arguments must be separated
	by spaces. If an argument contains a space, it must be quoted with
	double-quotes. Arguments cannot contain double quotes or things
	will go very wrong.

	Note that this is the non-interactive version of "adb shell"
Source: https://android.googlesource.com/platform/system/core/+/master/adb/SERVICES.TXT

This method quotes the arguments for you, and will return an error if any of them
contain double quotes.
*/
func (c *DeviceClient) RunCommand(cmd string, args ...string) (string, error) {
	cmd, err := prepareCommandLine(cmd, args...)
	if err != nil {
		return "", wrapClientError(err, c, "RunCommand")
	}

	conn, err := c.dialDevice()
	if err != nil {
		return "", wrapClientError(err, c, "RunCommand")
	}
	defer conn.Close()

	req := fmt.Sprintf("shell:%s", cmd)

	// Shell responses are special, they don't include a length header.
	// We read until the stream is closed.
	// So, we can't use conn.RoundTripSingleResponse.
	if err = conn.SendMessage([]byte(req)); err != nil {
		return "", wrapClientError(err, c, "RunCommand")
	}
	if err = wire.ReadStatusFailureAsError(conn, req); err != nil {
		return "", wrapClientError(err, c, "RunCommand")
	}

	resp, err := conn.ReadUntilEof()
	return string(resp), wrapClientError(err, c, "RunCommand")
}

/*
Remount, from the docs,
	Ask adbd to remount the device's filesystem in read-write mode,
	instead of read-only. This is usually necessary before performing
	an "adb sync" or "adb push" request.
	This request may not succeed on certain builds which do not allow
	that.
Source: https://android.googlesource.com/platform/system/core/+/master/adb/SERVICES.TXT
*/
func (c *DeviceClient) Remount() (string, error) {
	conn, err := c.dialDevice()
	if err != nil {
		return "", wrapClientError(err, c, "Remount")
	}
	defer conn.Close()

	resp, err := conn.RoundTripSingleResponse([]byte("remount"))
	return string(resp), wrapClientError(err, c, "Remount")
}

func (c *DeviceClient) ListDirEntries(path string) (*DirEntries, error) {
	conn, err := c.getSyncConn()
	if err != nil {
		return nil, wrapClientError(err, c, "ListDirEntries(%s)", path)
	}

	entries, err := listDirEntries(conn, path)
	return entries, wrapClientError(err, c, "ListDirEntries(%s)", path)
}

func (c *DeviceClient) Stat(path string) (*DirEntry, error) {
	conn, err := c.getSyncConn()
	if err != nil {
		return nil, wrapClientError(err, c, "Stat(%s)", path)
	}

	entry, err := stat(conn, path)
	return entry, wrapClientError(err, c, "Stat(%s)", path)
}

func (c *DeviceClient) OpenRead(path string) (io.ReadCloser, error) {
	conn, err := c.getSyncConn()
	if err != nil {
		return nil, wrapClientError(err, c, "OpenRead(%s)", path)
	}

	reader, err := receiveFile(conn, path)
	return reader, wrapClientError(err, c, "OpenRead(%s)", path)
}

// getAttribute returns the first message returned by the server by running
// <host-prefix>:<attr>, where host-prefix is determined from the DeviceDescriptor.
func (c *DeviceClient) getAttribute(attr string) (string, error) {
	resp, err := roundTripSingleResponse(c.config.Dialer,
		fmt.Sprintf("%s:%s", c.descriptor.getHostPrefix(), attr))
	if err != nil {
		return "", err
	}
	return string(resp), nil
}

func (c *DeviceClient) getSyncConn() (*wire.SyncConn, error) {
	conn, err := c.dialDevice()
	if err != nil {
		return nil, err
	}

	// Switch the connection to sync mode.
	if err := wire.SendMessageString(conn, "sync:"); err != nil {
		return nil, err
	}
	if err := wire.ReadStatusFailureAsError(conn, "sync"); err != nil {
		return nil, err
	}

	return conn.NewSyncConn(), nil
}

// dialDevice switches the connection to communicate directly with the device
// by requesting the transport defined by the DeviceDescriptor.
func (c *DeviceClient) dialDevice() (*wire.Conn, error) {
	conn, err := c.config.Dialer.Dial()
	if err != nil {
		return nil, err
	}

	req := fmt.Sprintf("host:%s", c.descriptor.getTransportDescriptor())
	if err = wire.SendMessageString(conn, req); err != nil {
		conn.Close()
		return nil, util.WrapErrf(err, "error connecting to device '%s'", c.descriptor)
	}

	if err = wire.ReadStatusFailureAsError(conn, req); err != nil {
		conn.Close()
		return nil, err
	}

	return conn, nil
}

// prepareCommandLine validates the command and argument strings, quotes
// arguments if required, and joins them into a valid adb command string.
func prepareCommandLine(cmd string, args ...string) (string, error) {
	if isBlank(cmd) {
		return "", util.AssertionErrorf("command cannot be empty")
	}

	for i, arg := range args {
		if strings.ContainsRune(arg, '"') {
			return "", util.Errorf(util.ParseError, "arg at index %d contains an invalid double quote: %s", i, arg)
		}
		if containsWhitespace(arg) {
			args[i] = fmt.Sprintf("\"%s\"", arg)
		}
	}

	// Prepend the command to the args array.
	if len(args) > 0 {
		cmd = fmt.Sprintf("%s %s", cmd, strings.Join(args, " "))
	}

	return cmd, nil
}
<commit_msg>Fixed DeviceClient not closing the connection on Stat.<commit_after>package goadb

import (
	"fmt"
	"io"
	"strings"

	"github.com/zach-klippenstein/goadb/util"
	"github.com/zach-klippenstein/goadb/wire"
)

// DeviceClient communicates with a specific Android device.
type DeviceClient struct {
	config     ClientConfig
	descriptor DeviceDescriptor
}

func NewDeviceClient(config ClientConfig, descriptor DeviceDescriptor) *DeviceClient {
	return &DeviceClient{
		config:     config.sanitized(),
		descriptor: descriptor,
	}
}

func (c *DeviceClient) String() string {
	return c.descriptor.String()
}

// get-product is documented, but not implemented in the server.
// TODO(z): Make getProduct exported if get-product is ever implemented in adb.
func (c *DeviceClient) getProduct() (string, error) {
	attr, err := c.getAttribute("get-product")
	return attr, wrapClientError(err, c, "GetProduct")
}

func (c *DeviceClient) GetSerial() (string, error) {
	attr, err := c.getAttribute("get-serialno")
	return attr, wrapClientError(err, c, "GetSerial")
}

func (c *DeviceClient) GetDevicePath() (string, error) {
	attr, err := c.getAttribute("get-devpath")
	return attr, wrapClientError(err, c, "GetDevicePath")
}

func (c *DeviceClient) GetState() (string, error) {
	attr, err := c.getAttribute("get-state")
	return attr, wrapClientError(err, c, "GetState")
}

/*
RunCommand runs the specified commands on a shell on the device.

From the Android docs:
	Run 'command arg1 arg2 ...' in a shell on the device, and return
	its output and error streams. Note that arguments must be separated
	by spaces. If an argument contains a space, it must be quoted with
	double-quotes. Arguments cannot contain double quotes or things
	will go very wrong.

	Note that this is the non-interactive version of "adb shell"
Source: https://android.googlesource.com/platform/system/core/+/master/adb/SERVICES.TXT

This method quotes the arguments for you, and will return an error if any of them
contain double quotes.
*/
func (c *DeviceClient) RunCommand(cmd string, args ...string) (string, error) {
	cmd, err := prepareCommandLine(cmd, args...)
	if err != nil {
		return "", wrapClientError(err, c, "RunCommand")
	}

	conn, err := c.dialDevice()
	if err != nil {
		return "", wrapClientError(err, c, "RunCommand")
	}
	defer conn.Close()

	req := fmt.Sprintf("shell:%s", cmd)

	// Shell responses are special, they don't include a length header.
	// We read until the stream is closed.
	// So, we can't use conn.RoundTripSingleResponse.
	if err = conn.SendMessage([]byte(req)); err != nil {
		return "", wrapClientError(err, c, "RunCommand")
	}
	if err = wire.ReadStatusFailureAsError(conn, req); err != nil {
		return "", wrapClientError(err, c, "RunCommand")
	}

	resp, err := conn.ReadUntilEof()
	return string(resp), wrapClientError(err, c, "RunCommand")
}

/*
Remount, from the docs,
	Ask adbd to remount the device's filesystem in read-write mode,
	instead of read-only. This is usually necessary before performing
	an "adb sync" or "adb push" request.
	This request may not succeed on certain builds which do not allow
	that.
Source: https://android.googlesource.com/platform/system/core/+/master/adb/SERVICES.TXT
*/
func (c *DeviceClient) Remount() (string, error) {
	conn, err := c.dialDevice()
	if err != nil {
		return "", wrapClientError(err, c, "Remount")
	}
	defer conn.Close()

	resp, err := conn.RoundTripSingleResponse([]byte("remount"))
	return string(resp), wrapClientError(err, c, "Remount")
}

func (c *DeviceClient) ListDirEntries(path string) (*DirEntries, error) {
	conn, err := c.getSyncConn()
	if err != nil {
		return nil, wrapClientError(err, c, "ListDirEntries(%s)", path)
	}

	entries, err := listDirEntries(conn, path)
	return entries, wrapClientError(err, c, "ListDirEntries(%s)", path)
}

func (c *DeviceClient) Stat(path string) (*DirEntry, error) {
	conn, err := c.getSyncConn()
	if err != nil {
		return nil, wrapClientError(err, c, "Stat(%s)", path)
	}
	defer conn.Close()

	entry, err := stat(conn, path)
	return entry, wrapClientError(err, c, "Stat(%s)", path)
}

func (c *DeviceClient) OpenRead(path string) (io.ReadCloser, error) {
	conn, err := c.getSyncConn()
	if err != nil {
		return nil, wrapClientError(err, c, "OpenRead(%s)", path)
	}

	reader, err := receiveFile(conn, path)
	return reader, wrapClientError(err, c, "OpenRead(%s)", path)
}

// getAttribute returns the first message returned by the server by running
// <host-prefix>:<attr>, where host-prefix is determined from the DeviceDescriptor.
func (c *DeviceClient) getAttribute(attr string) (string, error) {
	resp, err := roundTripSingleResponse(c.config.Dialer,
		fmt.Sprintf("%s:%s", c.descriptor.getHostPrefix(), attr))
	if err != nil {
		return "", err
	}
	return string(resp), nil
}

func (c *DeviceClient) getSyncConn() (*wire.SyncConn, error) {
	conn, err := c.dialDevice()
	if err != nil {
		return nil, err
	}

	// Switch the connection to sync mode.
	if err := wire.SendMessageString(conn, "sync:"); err != nil {
		return nil, err
	}
	if err := wire.ReadStatusFailureAsError(conn, "sync"); err != nil {
		return nil, err
	}

	return conn.NewSyncConn(), nil
}

// dialDevice switches the connection to communicate directly with the device
// by requesting the transport defined by the DeviceDescriptor.
func (c *DeviceClient) dialDevice() (*wire.Conn, error) {
	conn, err := c.config.Dialer.Dial()
	if err != nil {
		return nil, err
	}

	req := fmt.Sprintf("host:%s", c.descriptor.getTransportDescriptor())
	if err = wire.SendMessageString(conn, req); err != nil {
		conn.Close()
		return nil, util.WrapErrf(err, "error connecting to device '%s'", c.descriptor)
	}

	if err = wire.ReadStatusFailureAsError(conn, req); err != nil {
		conn.Close()
		return nil, err
	}

	return conn, nil
}

// prepareCommandLine validates the command and argument strings, quotes
// arguments if required, and joins them into a valid adb command string.
func prepareCommandLine(cmd string, args ...string) (string, error) {
	if isBlank(cmd) {
		return "", util.AssertionErrorf("command cannot be empty")
	}

	for i, arg := range args {
		if strings.ContainsRune(arg, '"') {
			return "", util.Errorf(util.ParseError, "arg at index %d contains an invalid double quote: %s", i, arg)
		}
		if containsWhitespace(arg) {
			args[i] = fmt.Sprintf("\"%s\"", arg)
		}
	}

	// Prepend the command to the args array.
	if len(args) > 0 {
		cmd = fmt.Sprintf("%s %s", cmd, strings.Join(args, " "))
	}

	return cmd, nil
}
<|endoftext|>


<commit_before>package sched

import (
    "log"
    "net"
    "time"
    "sync"
    "strings"
    "container/list"
    "container/heap"
)

type Sched struct {
    timer            *time.Timer
    grabQueue        *list.List
    jobQueue         *list.List
    entryPoint       string
    JobLocker        *sync.Mutex
    Funcs            map[string]*FuncStat
    store            Storer
    pq               map[string]PriorityQueue
}

type Counter uint

func (c *Counter) Incr() {
    *c = *c + 1
}

func (c *Counter) Decr() {
    *c = *c - 1
}

type FuncStat struct {
    Worker     Counter `json:"worker_count"`
    Job        Counter `json:"job_count"`
    Processing Counter `json:"processing"`
}

func NewSched(entryPoint string, store Storer) *Sched {
    sched := new(Sched)
    sched.timer = time.NewTimer(1 * time.Hour)
    sched.grabQueue = list.New()
    sched.jobQueue = list.New()
    sched.entryPoint = entryPoint
    sched.JobLocker = new(sync.Mutex)
    sched.Funcs = make(map[string]*FuncStat)
    sched.store = store
    sched.pq = make(map[string]PriorityQueue)
    return sched
}

func (sched *Sched) Serve() {
    parts := strings.SplitN(sched.entryPoint, "://", 2)
    if parts[0] == "unix" {
        sockCheck(parts[1])
    }
    sched.checkJobQueue()
    go sched.handle()
    listen, err := net.Listen(parts[0], parts[1])
    if err != nil {
        log.Fatal(err)
    }
    defer listen.Close()
    log.Printf("huabot-sched started on %s\n", sched.entryPoint)
    for {
        conn, err := listen.Accept()
        if err != nil {
            log.Fatal(err)
        }
        sched.HandleConnection(conn)
    }
}

func (sched *Sched) Notify() {
    sched.timer.Reset(time.Millisecond)
}

func (sched *Sched) DieWorker(worker *Worker) {
    defer sched.Notify()
    sched.removeGrabQueue(worker)
    worker.Close()
}

func (sched *Sched) HandleConnection(conn net.Conn) {
    c := Conn{Conn: conn}
    payload, err := c.Receive()
    if err != nil {
        return
    }
    switch payload[0] {
    case TYPE_CLIENT:
        client := NewClient(sched, c)
        go client.Handle()
        break
    case TYPE_WORKER:
        worker := NewWorker(sched, c)
        go worker.Handle()
        break
    default:
        log.Printf("Unsupport client %d\n", payload[0])
        c.Close()
        break
    }
}

func (sched *Sched) Done(jobId int64) {
    defer sched.Notify()
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    removeListJob(sched.jobQueue, jobId)
    job, err := sched.store.Get(jobId)
    if err == nil {
        sched.store.Delete(jobId)
        sched.DecrStatJob(job)
        sched.DecrStatProc(job)
    }
    return
}

func (sched *Sched) isDoJob(job Job) bool {
    now := time.Now()
    current := int64(now.Unix())
    ret := false
    for e := sched.jobQueue.Front(); e != nil; e = e.Next() {
        chk := e.Value.(Job)
        runAt := chk.RunAt
        if runAt < chk.SchedAt {
            runAt = chk.SchedAt
        }
        if chk.Timeout > 0 && runAt + chk.Timeout < current {
            newJob, _ := sched.store.Get(chk.Id)
            if newJob.Status == JOB_STATUS_PROC {
                sched.DecrStatProc(newJob)
                newJob.Status = JOB_STATUS_READY
                sched.store.Save(newJob)
                pq := sched.pq[newJob.Func]
                item := &Item{
                    value: newJob.Id,
                    priority: newJob.SchedAt,
                }
                heap.Push(&pq, item)
            }
            sched.jobQueue.Remove(e)
            continue
        }
        if chk.Id == job.Id {
            old := e.Value.(Job)
            runAt := old.RunAt
            if runAt < old.SchedAt {
                runAt = old.SchedAt
            }
            if old.Timeout > 0 && runAt + old.Timeout < current {
                ret = false
            } else {
                ret = true
            }
        }
    }
    return ret
}

func (sched *Sched) SubmitJob(worker *Worker, job Job) {
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    if job.Name == "" {
        sched.store.Delete(job.Id)
        return
    }
    if sched.isDoJob(job) {
        return
    }
    if !worker.alive {
        return
    }
    if err := worker.HandleDo(job); err != nil {
        worker.alive = false
        go sched.DieWorker(worker)
        return
    }
    now := time.Now()
    current := int64(now.Unix())
    job.Status = JOB_STATUS_PROC
    job.RunAt = current
    sched.store.Save(job)
    sched.IncrStatProc(job)
    sched.jobQueue.PushBack(job)
    sched.removeGrabQueue(worker)
}

func (sched *Sched) handle() {
    var current time.Time
    var timestamp int64
    var schedJob Job
    for {
        if sched.grabQueue.Len() == 0 {
            sched.timer.Reset(time.Minute)
            current =<-sched.timer.C
            continue
        }

        maybeItem := make(map[string]*Item)
        for Func, _ := range sched.Funcs {
            pq, ok := sched.pq[Func]
            if !ok || pq.Len() == 0 {
                continue
            }

            item := heap.Pop(&pq).(*Item)

            maybeItem[Func] = item

        }

        if len(maybeItem) == 0 {
            sched.timer.Reset(time.Minute)
            current =<-sched.timer.C
        }

        var lessItem *Item
        var lessFunc string

        for Func, item := range maybeItem {
            if lessItem == nil {
                lessItem = item
                lessFunc = Func
                continue
            }
            if lessItem.priority > item.priority {
                lessItem = item
                lessFunc = Func
            }
        }

        for Func, item := range maybeItem {
            if Func == lessFunc {
                continue
            }
            pq := sched.pq[Func]
            heap.Push(&pq, item)
        }

        schedJob, err := sched.store.Get(lessItem.value)

        if err != nil {
            continue
        }

        pq := sched.pq[schedJob.Func]
        timestamp = int64(time.Now().Unix())

        if schedJob.SchedAt > timestamp {
            sched.timer.Reset(time.Second * time.Duration(schedJob.SchedAt - timestamp))
            current =<-sched.timer.C
            timestamp = int64(current.Unix())
            if schedJob.SchedAt > timestamp {
                heap.Push(&pq, lessItem)
                continue
            }
        }

        isSubmited := false
        for e := sched.grabQueue.Front(); e != nil; e = e.Next() {
            worker := e.Value.(*Worker)
            for _, Func := range worker.Funcs {
                if schedJob.Func == Func {
                    sched.SubmitJob(worker, schedJob)
                    isSubmited = true
                    break
                }
            }
            if isSubmited {
                break
            }
        }

        if !isSubmited {
            sched.DecrStatFunc(schedJob.Func)
        }
    }
}

func (sched *Sched) Fail(jobId int64) {
    defer sched.Notify()
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    removeListJob(sched.jobQueue, jobId)
    job, _ := sched.store.Get(jobId)
    sched.DecrStatProc(job)
    job.Status = JOB_STATUS_READY
    sched.store.Save(job)
    pq := sched.pq[job.Func]
    item := &Item{
        value: job.Id,
        priority: job.SchedAt,
    }
    heap.Push(&pq, item)
    return
}

func (sched *Sched) IncrStatFunc(Func string) {
    stat, ok := sched.Funcs[Func]
    if !ok {
        stat = new(FuncStat)
        sched.Funcs[Func] = stat
    }
    stat.Worker.Incr()
}

func (sched *Sched) DecrStatFunc(Func string) {
    stat, ok := sched.Funcs[Func]
    if ok {
        stat.Worker.Decr()
    }
}

func (sched *Sched) IncrStatJob(job Job) {
    stat, ok := sched.Funcs[job.Func]
    if !ok {
        stat = new(FuncStat)
        sched.Funcs[job.Func] = stat
    }
    stat.Job.Incr()
}

func (sched *Sched) DecrStatJob(job Job) {
    stat, ok := sched.Funcs[job.Func]
    if ok {
        stat.Job.Decr()
    }
}

func (sched *Sched) IncrStatProc(job Job) {
    stat, ok := sched.Funcs[job.Func]
    if !ok {
        stat = new(FuncStat)
        sched.Funcs[job.Func] = stat
    }
    if job.Status == JOB_STATUS_PROC {
        stat.Processing.Incr()
    }
}

func (sched *Sched) DecrStatProc(job Job) {
    stat, ok := sched.Funcs[job.Func]
    if ok && job.Status == JOB_STATUS_PROC {
        stat.Processing.Decr()
    }
}

func (sched *Sched) SchedLater(jobId int64, delay int64) {
    defer sched.Notify()
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    removeListJob(sched.jobQueue, jobId)
    job, _ := sched.store.Get(jobId)
    sched.DecrStatProc(job)
    job.Status = JOB_STATUS_READY
    var now = time.Now()
    job.SchedAt = int64(now.Unix()) + delay
    sched.store.Save(job)
    pq := sched.pq[job.Func]
    item := &Item{
        value: job.Id,
        priority: job.SchedAt,
    }
    heap.Push(&pq, item)
    return
}

func (sched *Sched) removeGrabQueue(worker *Worker) {
    for e := sched.grabQueue.Front(); e != nil; e = e.Next() {
        if e.Value.(*Worker) == worker {
            sched.grabQueue.Remove(e)
        }
    }
}

func (sched *Sched) checkJobQueue() {
    updateQueue := make([]Job, 0)
    removeQueue := make([]Job, 0)
    var now = time.Now()
    current := int64(now.Unix())

    iter := sched.store.NewIterator(nil, nil)
    for iter.Next() {
        job := iter.Value()
        if job.Name == "" {
            removeQueue = append(removeQueue, job)
            continue
        }
        sched.IncrStatJob(job)
        if job.Status != JOB_STATUS_PROC {
            pq, ok := sched.pq[job.Func]
            if !ok {
                pq = make(PriorityQueue, 0)
                sched.pq[job.Func] = pq
                heap.Init(&pq)
            }
            item := &Item{
                value: job.Id,
                priority: job.SchedAt,
            }
            heap.Push(&pq, item)
            continue
        }
        runAt := job.RunAt
        if runAt < job.SchedAt {
            runAt = job.SchedAt
        }
        if runAt + job.Timeout < current {
            updateQueue = append(updateQueue, job)
        } else {
            sched.jobQueue.PushBack(job)
            sched.IncrStatProc(job)
        }
    }

    iter.Close()

    for _, job := range updateQueue {
        job.Status = JOB_STATUS_READY
        sched.store.Save(job)
    }

    for _, job := range removeQueue {
        sched.store.Delete(job.Id)
    }
}

func (sched *Sched) Close() {
}
<commit_msg>remove no used code<commit_after>package sched

import (
    "log"
    "net"
    "time"
    "sync"
    "strings"
    "container/list"
    "container/heap"
)

type Sched struct {
    timer            *time.Timer
    grabQueue        *list.List
    jobQueue         *list.List
    entryPoint       string
    JobLocker        *sync.Mutex
    Funcs            map[string]*FuncStat
    store            Storer
    pq               map[string]PriorityQueue
}

type Counter uint

func (c *Counter) Incr() {
    *c = *c + 1
}

func (c *Counter) Decr() {
    *c = *c - 1
}

type FuncStat struct {
    Worker     Counter `json:"worker_count"`
    Job        Counter `json:"job_count"`
    Processing Counter `json:"processing"`
}

func NewSched(entryPoint string, store Storer) *Sched {
    sched := new(Sched)
    sched.timer = time.NewTimer(1 * time.Hour)
    sched.grabQueue = list.New()
    sched.jobQueue = list.New()
    sched.entryPoint = entryPoint
    sched.JobLocker = new(sync.Mutex)
    sched.Funcs = make(map[string]*FuncStat)
    sched.store = store
    sched.pq = make(map[string]PriorityQueue)
    return sched
}

func (sched *Sched) Serve() {
    parts := strings.SplitN(sched.entryPoint, "://", 2)
    if parts[0] == "unix" {
        sockCheck(parts[1])
    }
    sched.checkJobQueue()
    go sched.handle()
    listen, err := net.Listen(parts[0], parts[1])
    if err != nil {
        log.Fatal(err)
    }
    defer listen.Close()
    log.Printf("huabot-sched started on %s\n", sched.entryPoint)
    for {
        conn, err := listen.Accept()
        if err != nil {
            log.Fatal(err)
        }
        sched.HandleConnection(conn)
    }
}

func (sched *Sched) Notify() {
    sched.timer.Reset(time.Millisecond)
}

func (sched *Sched) DieWorker(worker *Worker) {
    defer sched.Notify()
    sched.removeGrabQueue(worker)
    worker.Close()
}

func (sched *Sched) HandleConnection(conn net.Conn) {
    c := Conn{Conn: conn}
    payload, err := c.Receive()
    if err != nil {
        return
    }
    switch payload[0] {
    case TYPE_CLIENT:
        client := NewClient(sched, c)
        go client.Handle()
        break
    case TYPE_WORKER:
        worker := NewWorker(sched, c)
        go worker.Handle()
        break
    default:
        log.Printf("Unsupport client %d\n", payload[0])
        c.Close()
        break
    }
}

func (sched *Sched) Done(jobId int64) {
    defer sched.Notify()
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    removeListJob(sched.jobQueue, jobId)
    job, err := sched.store.Get(jobId)
    if err == nil {
        sched.store.Delete(jobId)
        sched.DecrStatJob(job)
        sched.DecrStatProc(job)
    }
    return
}

func (sched *Sched) isDoJob(job Job) bool {
    now := time.Now()
    current := int64(now.Unix())
    ret := false
    for e := sched.jobQueue.Front(); e != nil; e = e.Next() {
        chk := e.Value.(Job)
        runAt := chk.RunAt
        if runAt < chk.SchedAt {
            runAt = chk.SchedAt
        }
        if chk.Timeout > 0 && runAt + chk.Timeout < current {
            newJob, _ := sched.store.Get(chk.Id)
            if newJob.Status == JOB_STATUS_PROC {
                sched.DecrStatProc(newJob)
                newJob.Status = JOB_STATUS_READY
                sched.store.Save(newJob)
                pq := sched.pq[newJob.Func]
                item := &Item{
                    value: newJob.Id,
                    priority: newJob.SchedAt,
                }
                heap.Push(&pq, item)
            }
            sched.jobQueue.Remove(e)
            continue
        }
        if chk.Id == job.Id {
            old := e.Value.(Job)
            runAt := old.RunAt
            if runAt < old.SchedAt {
                runAt = old.SchedAt
            }
            if old.Timeout > 0 && runAt + old.Timeout < current {
                ret = false
            } else {
                ret = true
            }
        }
    }
    return ret
}

func (sched *Sched) SubmitJob(worker *Worker, job Job) {
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    if job.Name == "" {
        sched.store.Delete(job.Id)
        return
    }
    if sched.isDoJob(job) {
        return
    }
    if !worker.alive {
        return
    }
    if err := worker.HandleDo(job); err != nil {
        worker.alive = false
        go sched.DieWorker(worker)
        return
    }
    now := time.Now()
    current := int64(now.Unix())
    job.Status = JOB_STATUS_PROC
    job.RunAt = current
    sched.store.Save(job)
    sched.IncrStatProc(job)
    sched.jobQueue.PushBack(job)
    sched.removeGrabQueue(worker)
}

func (sched *Sched) handle() {
    var current time.Time
    var timestamp int64
    for {
        if sched.grabQueue.Len() == 0 {
            sched.timer.Reset(time.Minute)
            current =<-sched.timer.C
            continue
        }

        maybeItem := make(map[string]*Item)
        for Func, _ := range sched.Funcs {
            pq, ok := sched.pq[Func]
            if !ok || pq.Len() == 0 {
                continue
            }

            item := heap.Pop(&pq).(*Item)

            maybeItem[Func] = item

        }

        if len(maybeItem) == 0 {
            sched.timer.Reset(time.Minute)
            current =<-sched.timer.C
        }

        var lessItem *Item
        var lessFunc string

        for Func, item := range maybeItem {
            if lessItem == nil {
                lessItem = item
                lessFunc = Func
                continue
            }
            if lessItem.priority > item.priority {
                lessItem = item
                lessFunc = Func
            }
        }

        for Func, item := range maybeItem {
            if Func == lessFunc {
                continue
            }
            pq := sched.pq[Func]
            heap.Push(&pq, item)
        }

        schedJob, err := sched.store.Get(lessItem.value)

        if err != nil {
            continue
        }

        pq := sched.pq[schedJob.Func]
        timestamp = int64(time.Now().Unix())

        if schedJob.SchedAt > timestamp {
            sched.timer.Reset(time.Second * time.Duration(schedJob.SchedAt - timestamp))
            current =<-sched.timer.C
            timestamp = int64(current.Unix())
            if schedJob.SchedAt > timestamp {
                heap.Push(&pq, lessItem)
                continue
            }
        }

        isSubmited := false
        for e := sched.grabQueue.Front(); e != nil; e = e.Next() {
            worker := e.Value.(*Worker)
            for _, Func := range worker.Funcs {
                if schedJob.Func == Func {
                    sched.SubmitJob(worker, schedJob)
                    isSubmited = true
                    break
                }
            }
            if isSubmited {
                break
            }
        }

        if !isSubmited {
            sched.DecrStatFunc(schedJob.Func)
        }
    }
}

func (sched *Sched) Fail(jobId int64) {
    defer sched.Notify()
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    removeListJob(sched.jobQueue, jobId)
    job, _ := sched.store.Get(jobId)
    sched.DecrStatProc(job)
    job.Status = JOB_STATUS_READY
    sched.store.Save(job)
    pq := sched.pq[job.Func]
    item := &Item{
        value: job.Id,
        priority: job.SchedAt,
    }
    heap.Push(&pq, item)
    return
}

func (sched *Sched) IncrStatFunc(Func string) {
    stat, ok := sched.Funcs[Func]
    if !ok {
        stat = new(FuncStat)
        sched.Funcs[Func] = stat
    }
    stat.Worker.Incr()
}

func (sched *Sched) DecrStatFunc(Func string) {
    stat, ok := sched.Funcs[Func]
    if ok {
        stat.Worker.Decr()
    }
}

func (sched *Sched) IncrStatJob(job Job) {
    stat, ok := sched.Funcs[job.Func]
    if !ok {
        stat = new(FuncStat)
        sched.Funcs[job.Func] = stat
    }
    stat.Job.Incr()
}

func (sched *Sched) DecrStatJob(job Job) {
    stat, ok := sched.Funcs[job.Func]
    if ok {
        stat.Job.Decr()
    }
}

func (sched *Sched) IncrStatProc(job Job) {
    stat, ok := sched.Funcs[job.Func]
    if !ok {
        stat = new(FuncStat)
        sched.Funcs[job.Func] = stat
    }
    if job.Status == JOB_STATUS_PROC {
        stat.Processing.Incr()
    }
}

func (sched *Sched) DecrStatProc(job Job) {
    stat, ok := sched.Funcs[job.Func]
    if ok && job.Status == JOB_STATUS_PROC {
        stat.Processing.Decr()
    }
}

func (sched *Sched) SchedLater(jobId int64, delay int64) {
    defer sched.Notify()
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    removeListJob(sched.jobQueue, jobId)
    job, _ := sched.store.Get(jobId)
    sched.DecrStatProc(job)
    job.Status = JOB_STATUS_READY
    var now = time.Now()
    job.SchedAt = int64(now.Unix()) + delay
    sched.store.Save(job)
    pq := sched.pq[job.Func]
    item := &Item{
        value: job.Id,
        priority: job.SchedAt,
    }
    heap.Push(&pq, item)
    return
}

func (sched *Sched) removeGrabQueue(worker *Worker) {
    for e := sched.grabQueue.Front(); e != nil; e = e.Next() {
        if e.Value.(*Worker) == worker {
            sched.grabQueue.Remove(e)
        }
    }
}

func (sched *Sched) checkJobQueue() {
    updateQueue := make([]Job, 0)
    removeQueue := make([]Job, 0)
    var now = time.Now()
    current := int64(now.Unix())

    iter := sched.store.NewIterator(nil, nil)
    for iter.Next() {
        job := iter.Value()
        if job.Name == "" {
            removeQueue = append(removeQueue, job)
            continue
        }
        sched.IncrStatJob(job)
        if job.Status != JOB_STATUS_PROC {
            pq, ok := sched.pq[job.Func]
            if !ok {
                pq = make(PriorityQueue, 0)
                sched.pq[job.Func] = pq
                heap.Init(&pq)
            }
            item := &Item{
                value: job.Id,
                priority: job.SchedAt,
            }
            heap.Push(&pq, item)
            continue
        }
        runAt := job.RunAt
        if runAt < job.SchedAt {
            runAt = job.SchedAt
        }
        if runAt + job.Timeout < current {
            updateQueue = append(updateQueue, job)
        } else {
            sched.jobQueue.PushBack(job)
            sched.IncrStatProc(job)
        }
    }

    iter.Close()

    for _, job := range updateQueue {
        job.Status = JOB_STATUS_READY
        sched.store.Save(job)
    }

    for _, job := range removeQueue {
        sched.store.Delete(job.Id)
    }
}

func (sched *Sched) Close() {
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package atomic

func generalCAS64(addr *uint64, old uint64, new uint64) bool

var GeneralCAS64 = generalCAS64
<commit_msg>sync: fix linux/arm build<commit_after>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package atomic

var GeneralCAS64 = generalCAS64
<|endoftext|>