<commit_before>// Copyright (c) 2014-2019 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package messagebus

import (
	"fmt"
	"reflect"
	"strconv"
	"sync"
)

// internal constants
const (
	defaultQueueSize = 1000 // if specific size is not specified
)

// Message - message to put into a queue
type Message struct {
	Command    string   // type of packed data
	Parameters [][]byte // array of parameters
}

// Queue - a 1:1 queue
type Queue struct {
	c    chan Message
	size int
	used bool
}

// BroadcastQueue - a 1:M queue
// out is synchronous, so messages to routines not waiting are dropped
type BroadcastQueue struct {
	in               chan Message
	out              []chan Message
	defaultSize      int
	deliveredMessage map[string]Message // cache for the delivered message
	sync.RWMutex
}

// the exported message queues and their sizes
// any item with a size option will be allocated that size
// absent then default size is used
type busses struct {
	Broadcast  *BroadcastQueue `size:"1000"` // to broadcast to other nodes
	Connector  *Queue          `size:"50"`   // to control connector
	Announce   *Queue          `size:"50"`   // to control the announcer
	Blockstore *Queue          `size:"50"`   // to sequentially store blocks
	TestQueue  *Queue          `size:"50"`   // for testing use
}

// Bus - all available message queues
var Bus busses

// initialise all queues with preset size
func init() {

	// this will be a struct type
	busType := reflect.TypeOf(Bus)

	// get write access by using pointer + Elem()
	busValue := reflect.ValueOf(&Bus).Elem()

	// scan each field
	for i := 0; i < busType.NumField(); i += 1 {

		fieldInfo := busType.Field(i)

		sizeTag := fieldInfo.Tag.Get("size")

		queueSize := defaultQueueSize

		// if size specified and valid positive integer override default
		if len(sizeTag) > 0 {
			s, err := strconv.Atoi(sizeTag)
			if nil == err && s > 0 {
				queueSize = s
			} else {
				m := fmt.Sprintf("queue: %v  has invalid size: %q", fieldInfo, sizeTag)
				panic(m)
			}
		}

		switch qt := busValue.Field(i).Type(); qt {

		case reflect.TypeOf((*BroadcastQueue)(nil)):
			q := &BroadcastQueue{
				in:               make(chan Message, queueSize),
				out:              make([]chan Message, 0, 10),
				defaultSize:      queueSize,
				deliveredMessage: make(map[string]Message),
			}
			go q.multicast()

			newQueue := reflect.ValueOf(q)
			busValue.Field(i).Set(newQueue)

		case reflect.TypeOf((*Queue)(nil)):
			q := &Queue{
				c:    make(chan Message, queueSize),
				size: queueSize,
				used: false,
			}
			newQueue := reflect.ValueOf(q)
			busValue.Field(i).Set(newQueue)
		default:
			panic(fmt.Sprintf("queue type: %q is not handled", qt))
		}
	}
}

// Send - send a message to a 1:1 queue
// but only if listener is connected
func (queue *Queue) Send(command string, parameters ...[]byte) {
	queue.c <- Message{
		Command:    command,
		Parameters: parameters,
	}
}

// Chan - channel to read from 1:1 queue
// can only be called once
func (queue *Queue) Chan() <-chan Message {
	if queue.used {
		panic("cannot get a second receive channel from a 1:1 queue")
	}
	queue.used = true
	return queue.c
}

// Release - give the channel back
func (queue *Queue) Release() {
	queue.used = false
	close(queue.c)
	queue.c = make(chan Message, queue.size)
}

// Send - send a message to a 1:M queue
func (queue *BroadcastQueue) Send(command string, parameters ...[]byte) {
	m := Message{
		Command:    command,
		Parameters: parameters,
	}

	if queue.isCached(m) {
		// Drop the message that already cached
		return
	}

	queue.in <- m

	// Ignore caching rpc/peer command
	if "rpc" == command || "peer" == command {
		return
	}
	queue.cache(m)
}

// Chan - get a new channel to read from a 1:M queue
// each call gets a distinct channel
func (queue *BroadcastQueue) Chan(size int) <-chan Message {
	if size < 0 {
		size = queue.defaultSize
	}
	c := make(chan Message, size)
	queue.out = append(queue.out, c)
	return c
}

// Release - release the incomming and outgoing queue
func (queue *BroadcastQueue) Release() {
	// close all channel
	close(queue.in)
	for _, o := range queue.out {
		close(o)
	}

	// give them back
	queue.in = make(chan Message, queue.defaultSize)
	queue.out = make([]chan Message, 0, 10)
}

// DropCache - drop the items from cache
func (queue *BroadcastQueue) DropCache(msgs ...Message) {

	if nil == msgs {
		queue.clearCache()
		return
	}

clean_cache:
	for _, m := range msgs {

		if !queue.isCached(m) {
			continue clean_cache
		}

		queue.Lock()
		delete(queue.deliveredMessage, m.packHex())
		queue.Unlock()
	}
}

func (queue *BroadcastQueue) clearCache() {
	queue.Lock()
	defer queue.Unlock()
	queue.deliveredMessage = map[string]Message{}
}

func (queue *BroadcastQueue) cache(m Message) {
	queue.Lock()
	defer queue.Unlock()
	queue.deliveredMessage[m.packHex()] = m
}

func (queue *BroadcastQueue) isCached(m Message) bool {
	queue.RLock()
	defer queue.RUnlock()
	_, ok := queue.deliveredMessage[m.packHex()]
	return ok
}

// background processing for the 1:M queue
//
// if an outgoing queue is full, determine whether it is block message
// drop lower priority message for taking slot for new one
func (queue *BroadcastQueue) multicast() {
loop:
	for {
		data, ok := <-queue.in
		if !ok {
			// invalid data
			continue loop
		}
		for _, out := range queue.out {
			select {
			case out <- data:
			default:
				// the outgoing queue is full
				if "block" == data.Command {
					// drop lower priority item for taking more slot
					ci := make([]Message, 0)

					found := false
				append_ci:
					for len(out) > 0 {
						m := <-out
						if !found && "block" != m.Command {
							found = true
							continue append_ci
						} else {
							ci = append(ci, m)
						}
					}

					if !found {
						// couldn't find any existing message type block in queue
						// just drop cache
						queue.DropCache(data)
					} else {
						// reput item to the queue
						for _, m := range ci {
							out <- m
						}
						out <- data
					}

				} else {
					// drop existing cache item
					queue.DropCache(data)
				}
			}
		}
	}
}

// pack Message
func (m Message) pack() []byte {
	s := make([]byte, 0)
	s = append(s, []byte(m.Command)...)

	for _, a := range m.Parameters {
		s = append(s, a...)
	}
	return s
}

// pack Message in hex string
func (m Message) packHex() string {
	return fmt.Sprintf("%x", m.pack())
}
<commit_msg>[messagebus] limit duplication checker<commit_after>// Copyright (c) 2014-2019 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package messagebus

import (
	"container/list"
	"fmt"
	"reflect"
	"strconv"
	"sync"

	"golang.org/x/crypto/sha3"
)

// for select the default queue size
const Default = -1

// internal constants
const (
	defaultQueueSize = 1000 // if specific size is not specified
)

// Message - message to put into a queue
type Message struct {
	Command    string   // type of packed data
	Parameters [][]byte // array of parameters
}

// Queue - a 1:1 queue
type Queue struct {
	c    chan Message
	size int
	used bool
}

// to cache messages
type signature [32]byte

// BroadcastQueue - a 1:M queue
// out is synchronous, so messages to routines not waiting are dropped
type BroadcastQueue struct {
	sync.RWMutex

	out         []chan Message
	defaultSize int
	cache       map[signature]struct{}
	index       list.List
}

// the exported message queues and their sizes
// any item with a size option will be allocated that size
// absent then default size is used
type busses struct {
	Broadcast  *BroadcastQueue `size:"1000"` // to broadcast to other nodes
	Connector  *Queue          `size:"50"`   // to control connector
	Announce   *Queue          `size:"50"`   // to control the announcer
	Blockstore *Queue          `size:"50"`   // to sequentially store blocks
	TestQueue  *Queue          `size:"50"`   // for testing use
}

// Bus - all available message queues
var Bus busses

// initialise all queues with preset size
func init() {

	// this will be a struct type
	busType := reflect.TypeOf(Bus)

	// get write access by using pointer + Elem()
	busValue := reflect.ValueOf(&Bus).Elem()

	// scan each field
	for i := 0; i < busType.NumField(); i += 1 {

		fieldInfo := busType.Field(i)

		sizeTag := fieldInfo.Tag.Get("size")

		queueSize := defaultQueueSize

		// if size specified and valid positive integer override default
		if len(sizeTag) > 0 {
			s, err := strconv.Atoi(sizeTag)
			if nil == err && s > 0 {
				queueSize = s
			} else {
				m := fmt.Sprintf("queue: %v  has invalid size: %q", fieldInfo, sizeTag)
				panic(m)
			}
		}

		switch qt := busValue.Field(i).Type(); qt {

		case reflect.TypeOf((*BroadcastQueue)(nil)):
			q := &BroadcastQueue{
				out:         make([]chan Message, 0, 10),
				defaultSize: queueSize,
				cache:       make(map[signature]struct{}),
			}

			newQueue := reflect.ValueOf(q)
			busValue.Field(i).Set(newQueue)

		case reflect.TypeOf((*Queue)(nil)):
			q := &Queue{
				c:    make(chan Message, queueSize),
				size: queueSize,
				used: false,
			}
			newQueue := reflect.ValueOf(q)
			busValue.Field(i).Set(newQueue)
		default:
			panic(fmt.Sprintf("queue type: %q is not handled", qt))
		}
	}
}

// Send - send a message to a 1:1 queue
// but only if listener is connected
func (queue *Queue) Send(command string, parameters ...[]byte) {
	queue.c <- Message{
		Command:    command,
		Parameters: parameters,
	}
}

// Chan - channel to read from 1:1 queue
// can only be called once
func (queue *Queue) Chan() <-chan Message {
	if queue.used {
		panic("cannot get a second receive channel from a 1:1 queue")
	}
	queue.used = true
	return queue.c
}

// Release - give the channel back
func (queue *Queue) Release() {
	queue.used = false
	close(queue.c)
	queue.c = make(chan Message, queue.size)
}

// Send - send a message to a 1:M queue
func (queue *BroadcastQueue) Send(command string, parameters ...[]byte) {
	m := Message{
		Command:    command,
		Parameters: parameters,
	}

	h := sha3.New256()
	h.Write([]byte(command))
	for _, p := range parameters {
		h.Write(p)
	}
	var sum signature
	copy(sum[:], h.Sum([]byte{}))

	queue.Lock()
	if _, ok := queue.cache[sum]; ok {
		queue.Unlock()
		return
	}
	queue.cache[sum] = struct{}{}
	queue.index.PushBack(sum)

	if queue.index.Len() > 100 {
		e := queue.index.Front()
		s := e.Value.(signature)
		delete(queue.cache, s)
		queue.index.Remove(e)
	}
	queue.Unlock()

	for _, out := range queue.out {

		// check for more that one free entry
		if len(out) < cap(out)-1 {
			out <- m
		} else if "block" == command {
			// allow block messages to use the last free entry
			select {
			case out <- m:
			default:
			}
		}
	}
}

// Chan - get a new channel to read from a 1:M queue
// each call gets a distinct channel
func (queue *BroadcastQueue) Chan(size int) <-chan Message {
	if size < 0 {
		size = queue.defaultSize
	}
	c := make(chan Message, size)
	queue.out = append(queue.out, c)
	return c
}

// Release - release the incoming and outgoing queue
func (queue *BroadcastQueue) Release() {

	for _, o := range queue.out {
		close(o)
	}

	// empty the list
	queue.out = make([]chan Message, 0, 10)
}
<|endoftext|>


<commit_before>package middleware

import (
	"net/http"

	"github.com/PuerkitoBio/goquery"
)

// ExtractAllLinks simply extracts all the <a href="<urL>">  in a page
// provided that they aren't silly like a hash. It also expands relative
// links automagically.
func ExtractAllLinks(resp *http.Response) []string {
	var links []string

	tempURL := *resp.Request.URL
	tempURL.Fragment = ""
	tempURL.Path = ""
	tempURL.RawQuery = ""
	uri := tempURL.String()

	doc, err := goquery.NewDocumentFromReader(resp.Body)
	resp.Body.Close()
	if err != nil {
		return links
	}

	doc.Find("a").Each(func(i int, s *goquery.Selection) {
		href, exists := s.Attr("href")

		if exists && href != "" {

			if href[0] == '#' {
				return
			}
			if href[0] == '/' {
				href = uri + href
			}
			links = append(links, href)
		}
	})

	return links
}
<commit_msg>Added double slash href handling to ExtractAllLinks Parse function.<commit_after>package middleware

import (
	"net/http"
	"strings"

	"github.com/PuerkitoBio/goquery"
)

// ExtractAllLinks simply extracts all the <a href="<urL>">  in a page
// provided that they aren't silly like a hash. It also expands relative
// links automagically.
func ExtractAllLinks(resp *http.Response) []string {
	var links []string

	tempURL := *resp.Request.URL
	tempURL.Fragment = ""
	tempURL.Path = ""
	tempURL.RawQuery = ""
	uri := tempURL.String()

	doc, err := goquery.NewDocumentFromReader(resp.Body)
	resp.Body.Close()
	if err != nil {
		return links
	}

	doc.Find("a").Each(func(i int, s *goquery.Selection) {
		href, exists := s.Attr("href")

		if exists && href != "" {

			if href[0] == '#' {
				return
			}
			if strings.HasPrefix(href, "//") {
				href = "http:" + href
			}
			if strings.HasPrefix(href, "/") {
				href = uri + href
			}
			links = append(links, href)
		}
	})

	return links
}
<|endoftext|>


<commit_before>package main

import (
    "log"
    "github.com/fsouza/go-dockerclient"
    "github.com/spf13/viper"
    "io"
    "fmt"
    "bufio"
    "strings"
    "time"
    "bytes"
)

var hostConfig   *docker.HostConfig

//AWSDOCKER is a dockerobj. It is identified by an image id
type AWSDOCKER struct {
    id             string
    taskArn        string
    timeout        time.Duration
    docker_client  *docker.Client
    eventsCh       chan *docker.APIEvents
    taskDefinition *string
}

func getDockerHostConfig() *docker.HostConfig {
    if hostConfig != nil {
        return hostConfig
    }
    dockerKey := func(key string) string {
        return "dockerobj.docker.hostConfig." + key
    }
    getInt64 := func(key string) int64 {
        defer func() {
            if err := recover(); err != nil {
                log.Printf("load dockerobj.docker.hostConfig.%s failed, error: %v\n", key, err)
            }
        }()
        n := viper.GetInt(dockerKey(key))
        return int64(n)
    }

    var logConfig docker.LogConfig
    err := viper.UnmarshalKey(dockerKey("LogConfig"), &logConfig)
    if err != nil {
        log.Printf("load docker HostConfig.LogConfig failed, error: %s\n", err.Error())
    }
    networkMode := viper.GetString(dockerKey("NetworkMode"))
    if networkMode == "" {
        networkMode = "host"
    }
    log.Printf("docker container hostconfig NetworkMode: %s\n", networkMode)

    hostConfig = &docker.HostConfig{
        CapAdd:  viper.GetStringSlice(dockerKey("CapAdd")),
        CapDrop: viper.GetStringSlice(dockerKey("CapDrop")),

        DNS:         viper.GetStringSlice(dockerKey("Dns")),
        DNSSearch:   viper.GetStringSlice(dockerKey("DnsSearch")),
        ExtraHosts:  viper.GetStringSlice(dockerKey("ExtraHosts")),
        NetworkMode: networkMode,
        IpcMode:     viper.GetString(dockerKey("IpcMode")),
        PidMode:     viper.GetString(dockerKey("PidMode")),
        UTSMode:     viper.GetString(dockerKey("UTSMode")),
        LogConfig:   logConfig,

        ReadonlyRootfs:   viper.GetBool(dockerKey("ReadonlyRootfs")),
        SecurityOpt:      viper.GetStringSlice(dockerKey("SecurityOpt")),
        CgroupParent:     viper.GetString(dockerKey("CgroupParent")),
        Memory:           getInt64("Memory"),
        MemorySwap:       getInt64("MemorySwap"),
        MemorySwappiness: getInt64("MemorySwappiness"),
        OOMKillDisable:   viper.GetBool(dockerKey("OomKillDisable")),
        CPUShares:        getInt64("CpuShares"),
        CPUSet:           viper.GetString(dockerKey("Cpuset")),
        CPUSetCPUs:       viper.GetString(dockerKey("CpusetCPUs")),
        CPUSetMEMs:       viper.GetString(dockerKey("CpusetMEMs")),
        CPUQuota:         getInt64("CpuQuota"),
        CPUPeriod:        getInt64("CpuPeriod"),
        BlkioWeight:      getInt64("BlkioWeight"),
    }

    return hostConfig
}

func (dockerobj *AWSDOCKER) createContainer(imageID string, args []string, env []string, attachStdout bool) (string, error) {
    docker_config := docker.Config{Cmd: args, Image: imageID, Env: env, AttachStdout: attachStdout, AttachStderr: attachStdout}
    copts := docker.CreateContainerOptions{Name: imageID, Config: &docker_config, HostConfig: getDockerHostConfig()}
    log.Printf("Create container for image id: %s\n", imageID)
    container, err := dockerobj.docker_client.CreateContainer(copts)
    if err != nil {
        return "", err
    }
    log.Printf("Created container id: %s\n", container.ID)
    return container.ID, err
}

func (dockerobj *AWSDOCKER) deployImage(id string, args []string, env []string, reader io.Reader) error {
    outputbuf := bytes.NewBuffer(nil)
    opts := docker.BuildImageOptions{
        Name:         id,
        Pull:         false,
        InputStream:  reader,
        OutputStream: outputbuf,
    }

    if err := dockerobj.docker_client.BuildImage(opts); err != nil {
        log.Printf("Error building images: %s", err)
        log.Printf("Image Output:\n********************\n%s\n********************", outputbuf.String())
        return err
    }

    log.Printf("Created image: %s", id)

    return nil
}

//Deploy use the reader containing targz to create a docker image
//for docker inputbuf is tar reader ready for use by docker.Client
//the stream from end docker_client to peer could directly be this tar stream
//talk to docker daemon using docker Client and build the image
func (dockerobj *AWSDOCKER) Deploy(id string, args []string, env []string, reader io.Reader) error {
    if err := dockerobj.deployImage(id, args, env, reader); err != nil {
        return err
    }
    return nil
}

type BuildSpecFactory func() (io.Reader, error)

func (dockerobj *AWSDOCKER) stopInternal(id string, timeout uint, dontkill bool, dontremove bool) error {

    err := dockerobj.docker_client.StopContainer(id, timeout)
    if err != nil {
        log.Printf("Stop container %s(%s)", id, err)
    } else {
        log.Printf("Stopped container %s", id)
    }
    if !dontkill {
        err = dockerobj.docker_client.KillContainer(docker.KillContainerOptions{ID: id})
        if err != nil {
            log.Printf("Kill container %s (%s)", id, err)
        } else {
            log.Printf("Killed container %s", id)
        }
    }
    if !dontremove {
        err = dockerobj.docker_client.RemoveContainer(docker.RemoveContainerOptions{ID: id, Force: true})
        if err != nil {
            log.Printf("Remove container %s (%s)", id, err)
        } else {
            log.Printf("Removed container %s", id)
        }
    }
    return err
}

//Start starts a container using a previously created docker image
func (dockerobj *AWSDOCKER) Start(imageID string, args []string, env []string, builder BuildSpecFactory) error {

    containerID := strings.Replace(imageID, ":", "_", -1)
    attachStdout := viper.GetBool("dockerobj.docker.attachStdout")

    //stop,force remove if necessary
    log.Printf("Cleanup container %s", containerID)
    dockerobj.stopInternal(containerID, 0, false, false)

    log.Printf("Start container %s", containerID)
    containerID, err := dockerobj.createContainer(imageID, args, env, attachStdout)
    if err != nil {
        //if image not found try to create image and retry
        if err == docker.ErrNoSuchImage {
            if builder != nil {
                log.Printf("start-could not find image ...attempt to recreate image %s", err)

                reader, err1 := builder()
                if err1 != nil {
                    log.Printf("Error creating image builder: %s", err1)
                }

                if err1 = dockerobj.deployImage(imageID, args, env, reader); err1 != nil {
                    return err1
                }

                log.Printf("start-recreated image successfully")
                if containerID, err1 = dockerobj.createContainer(imageID, args, env, attachStdout); err1 != nil {
                    log.Printf("start-could not recreate container post recreate image: %s", err1)
                    return err1
                }
            } else {
                log.Printf("start-could not find image: %s", err)
                return err
            }
        } else {
            log.Printf("start-could not recreate container: %s", err)
            return err
        }
    }

    if attachStdout {
        // Launch a few go-threads to manage output streams from the container.
        // They will be automatically destroyed when the container exits
        attached := make(chan struct{})
        r, w := io.Pipe()

        go func() {
            // AttachToContainer will fire off a message on the "attached" channel once the
            // attachment completes, and then block until the container is terminated.
            // The returned error is not used outside the scope of this function. Assign the
            // error to a local variable to prevent clobbering the function variable 'err'.
            err := dockerobj.docker_client.AttachToContainer(docker.AttachToContainerOptions{
                Container:    containerID,
                OutputStream: w,
                ErrorStream:  w,
                Logs:         true,
                Stdout:       true,
                Stderr:       true,
                Stream:       true,
                Success:      attached,
            })

            // If we get here, the container has terminated.  Send a signal on the pipe
            // so that downstream may clean up appropriately
            _ = w.CloseWithError(err)
        }()

        go func() {
            // Block here until the attachment completes or we timeout
            select {
            case <-attached:
                // successful attach
            case <-time.After(10 * time.Second):
                log.Printf("Timeout while attaching to IO channel in container %s", containerID)
                return
            }

            // Acknowledge the attachment?  This was included in the gist I followed
            // (http://bit.ly/2jBrCtM).  Not sure it's actually needed but it doesn't
            // appear to hurt anything.
            attached <- struct{}{}

            // Establish a buffer for our IO channel so that we may do readline-style
            // ingestion of the IO, one log entry per line
            is := bufio.NewReader(r)

            for {
                // Loop forever dumping lines of text into the containerLogger
                // until the pipe is closed
                line, err2 := is.ReadString('\n')
                if err2 != nil {
                    switch err2 {
                    case io.EOF:
                        log.Printf("Container %s has closed its IO channel", containerID)
                    default:
                        log.Printf("Error reading container output: %s", err2)
                    }

                    return
                }

                log.Printf(line)
            }
        }()
    }

    // start container with HostConfig was deprecated since v1.10 and removed in v1.2
    err = dockerobj.docker_client.StartContainer(containerID, nil)
    if err != nil {
        log.Printf("start-could not start container: %s", err)
        return err
    }

    log.Printf("Started container %s", containerID)
    return nil
}

//Stop stops a running chaincode
func (dockerobj *AWSDOCKER) Stop(id string, timeout uint, dontkill bool, dontremove bool) error {

    id = strings.Replace(id, ":", "_", -1)
    err := dockerobj.stopInternal(id, timeout, dontkill, dontremove)

    return err
}

//Destroy destroys an image
func (dockerobj *AWSDOCKER) Destroy(id string, force bool, noprune bool) error {
    id = strings.Replace(id, ":", "_", -1)

    err := dockerobj.docker_client.RemoveImageExtended(id, docker.RemoveImageOptions{Force: force, NoPrune: noprune})

    if err != nil {
        log.Printf("error while destroying image: %s", err)
    } else {
        log.Printf("Destroyed image %s", id)
    }

    return err
}

func (executable AWSDOCKER) execute(handler MessageHandler) {
    handler.initialize()
    if handler.receive() {
        executable.executableTimeoutHelper(handler)
    }
}

func (executable *AWSDOCKER) executableTimeoutHelper(handler MessageHandler) {
    ch := make(chan error)
    go func() {
        ch <- executable.executionHelper(handler.body(), handler.id())
    }()
    select {
    case err := <-ch:
        if err != nil {
            log.Printf("E: %s %s", *executable.taskDefinition, err.Error())
            handler.failure(err)
        } else {
            log.Printf("I: %s finished successfully", *executable.taskDefinition)
            handler.success()
        }
    case <-time.After(executable.timeout):
        err := fmt.Errorf("%s timed out after %f seconds", *executable.taskDefinition, executable.timeout.Seconds())
        log.Println(err)
        handler.failure(err)
    }
}

func (executable *AWSDOCKER) executionHelper(messageBody *string, messageID *string) error {
    var err error

    args := make([]string, 1)
    env := make([]string, 1)
    err = executable.Start("pipeline-agisoft", args, env, nil)
    if err != nil {
        return err
    }
    err = executable.monitorDocker()
    if err != nil {
        return err
    }
    return nil
}

func (executable *AWSDOCKER) monitorDocker() error {
    executable.addListener()
    // Monitor docker events for sibling Projector task
    status, err := executable.listenForDie()
    if err != nil {
        return err
    }

    if status == "0" {
        // status is die
        log.Printf("[INFO] Execution completed successfully")
        executable.success()
        return nil
    }
    // non-zero exit
    log.Printf("[ERROR] Execution completed with non-zero exit status")
    err = fmt.Errorf("%s died with non-zero exit status (exit code %s)", *executable.taskDefinition, status)
    executable.failure()
    return err

}

func (executable *AWSDOCKER) listenForDie() (exitCode string, err error) {
    log.Printf("[INFO] Monitoring Docker events.")
    log.Printf("[DEBUG] %+v\n", executable)
    duration := getTimeout()
    timeout := time.After(duration)
    defer executable.removeListener()
    for {
        select {
        case msg := <-executable.eventsCh:
            if msg != nil {
                matched := msg.Actor.Attributes["com.amazonaws.ecs.task-arn"] == executable.taskArn
                if matched {
                    log.Printf("[DEBUG] %+v\n", msg)
                    switch msg.Action {
                    case "die":
                        log.Printf("[INFO] Container die event")
                        return msg.Actor.Attributes["exitCode"], nil
                    }
                }
            }
        case <-timeout:
            log.Printf("[INFO] Instance timeout reached.")
            err := fmt.Errorf("Docker container %s timed out after %f seconds", *executable.taskDefinition, duration.Seconds())
            return "timeout", err
        }
    }
}

func (dockerobj *AWSDOCKER) connect(dockerEndpointPath string) {
    log.Printf("[INFO] Connecting to Docker API.")
    endpoint := dockerEndpointPath
    client, err := docker.NewClient(endpoint)
    if err != nil {
        panic(err)
    }
    dockerobj.docker_client = client
    dockerobj.eventsCh = make(chan *docker.APIEvents)
}

func (dockerobj *AWSDOCKER) addListener() {
    err := dockerobj.docker_client.AddEventListener(dockerobj.eventsCh)
    if err != nil {
        log.Fatal(err)
    }
}

func (dockerobj *AWSDOCKER) removeListener() {
    err := dockerobj.docker_client.RemoveEventListener(dockerobj.eventsCh)
    if err != nil {
        log.Fatal(err)
    }
}

func (executable *AWSDOCKER) success() {}
func (executable *AWSDOCKER) failure() {}<commit_msg>INFRA-1778 working version that has been tested with blaines/whalesay:latest<commit_after>package main

import (
    "log"
    "github.com/fsouza/go-dockerclient"
    "github.com/spf13/viper"
    "io"
    "fmt"
    "bufio"
    "strings"
    "time"
    "bytes"
)

var hostConfig   *docker.HostConfig

//AWSDOCKER is a dockerobj. It is identified by an image id
type AWSDOCKER struct {
    id             string
    taskArn        string
    timeout        time.Duration
    docker_client  *docker.Client
    eventsCh       chan *docker.APIEvents
    taskDefinition *string
}

func getDockerHostConfig() *docker.HostConfig {
    if hostConfig != nil {
        return hostConfig
    }
    dockerKey := func(key string) string {
        return "dockerobj.docker.hostConfig." + key
    }
    getInt64 := func(key string) int64 {
        defer func() {
            if err := recover(); err != nil {
                log.Printf("load dockerobj.docker.hostConfig.%s failed, error: %v\n", key, err)
            }
        }()
        n := viper.GetInt(dockerKey(key))
        return int64(n)
    }

    var logConfig docker.LogConfig
    err := viper.UnmarshalKey(dockerKey("LogConfig"), &logConfig)
    if err != nil {
        log.Printf("load docker HostConfig.LogConfig failed, error: %s\n", err.Error())
    }
    networkMode := viper.GetString(dockerKey("NetworkMode"))
    if networkMode == "" {
        networkMode = "host"
    }
    log.Printf("docker container hostconfig NetworkMode: %s\n", networkMode)

    hostConfig = &docker.HostConfig{
        CapAdd:  viper.GetStringSlice(dockerKey("CapAdd")),
        CapDrop: viper.GetStringSlice(dockerKey("CapDrop")),

        DNS:         viper.GetStringSlice(dockerKey("Dns")),
        DNSSearch:   viper.GetStringSlice(dockerKey("DnsSearch")),
        ExtraHosts:  viper.GetStringSlice(dockerKey("ExtraHosts")),
        NetworkMode: networkMode,
        IpcMode:     viper.GetString(dockerKey("IpcMode")),
        PidMode:     viper.GetString(dockerKey("PidMode")),
        UTSMode:     viper.GetString(dockerKey("UTSMode")),
        LogConfig:   logConfig,

        ReadonlyRootfs:   viper.GetBool(dockerKey("ReadonlyRootfs")),
        SecurityOpt:      viper.GetStringSlice(dockerKey("SecurityOpt")),
        CgroupParent:     viper.GetString(dockerKey("CgroupParent")),
        Memory:           getInt64("Memory"),
        MemorySwap:       getInt64("MemorySwap"),
        MemorySwappiness: getInt64("MemorySwappiness"),
        OOMKillDisable:   viper.GetBool(dockerKey("OomKillDisable")),
        CPUShares:        getInt64("CpuShares"),
        CPUSet:           viper.GetString(dockerKey("Cpuset")),
        CPUSetCPUs:       viper.GetString(dockerKey("CpusetCPUs")),
        CPUSetMEMs:       viper.GetString(dockerKey("CpusetMEMs")),
        CPUQuota:         getInt64("CpuQuota"),
        CPUPeriod:        getInt64("CpuPeriod"),
        BlkioWeight:      getInt64("BlkioWeight"),
    }

    return hostConfig
}

func (dockerobj *AWSDOCKER) createContainer(imageID string, args []string, env []string, attachStdout bool) (string, error) {
    docker_config := docker.Config{Cmd:[]string{"malaka", ">", "temp"}, Image: imageID, AttachStdout: attachStdout, AttachStderr: attachStdout}
    copts := docker.CreateContainerOptions{Name: "whalesay-test", Config: &docker_config, HostConfig: getDockerHostConfig()}
    log.Printf("Create container for image id: %s\n", imageID)
    container, err := dockerobj.docker_client.CreateContainer(copts)
    if err != nil {
        return "", err
    }
    log.Printf("Created container id: %s\n", container.ID)
    return container.ID, err
}

func (dockerobj *AWSDOCKER) deployImage(id string, args []string, env []string, reader io.Reader) error {
    outputbuf := bytes.NewBuffer(nil)
    opts := docker.BuildImageOptions{
        Name:         id,
        Pull:         false,
        InputStream:  reader,
        OutputStream: outputbuf,
    }

    if err := dockerobj.docker_client.BuildImage(opts); err != nil {
        log.Printf("Error building images: %s", err)
        log.Printf("Image Output:\n********************\n%s\n********************", outputbuf.String())
        return err
    }

    log.Printf("Created image: %s", id)

    return nil
}

//Deploy use the reader containing targz to create a docker image
//for docker inputbuf is tar reader ready for use by docker.Client
//the stream from end docker_client to peer could directly be this tar stream
//talk to docker daemon using docker Client and build the image
func (dockerobj *AWSDOCKER) Deploy(id string, args []string, env []string, reader io.Reader) error {
    if err := dockerobj.deployImage(id, args, env, reader); err != nil {
        return err
    }
    return nil
}

type BuildSpecFactory func() (io.Reader, error)

func (dockerobj *AWSDOCKER) stopInternal(id string, timeout uint, dontkill bool, dontremove bool) error {

    err := dockerobj.docker_client.StopContainer(id, timeout)
    if err != nil {
        log.Printf("Stop container %s(%s)", id, err)
    } else {
        log.Printf("Stopped container %s", id)
    }
    if !dontkill {
        err = dockerobj.docker_client.KillContainer(docker.KillContainerOptions{ID: id})
        if err != nil {
            log.Printf("Kill container %s (%s)", id, err)
        } else {
            log.Printf("Killed container %s", id)
        }
    }
    if !dontremove {
        err = dockerobj.docker_client.RemoveContainer(docker.RemoveContainerOptions{ID: id, Force: true})
        if err != nil {
            log.Printf("Remove container %s (%s)", id, err)
        } else {
            log.Printf("Removed container %s", id)
        }
    }
    return err
}

//Start starts a container using a previously created docker image
func (dockerobj *AWSDOCKER) Start(imageID string, args []string, env []string, builder BuildSpecFactory) error {

    containerID := strings.Replace(imageID, ":", "_", -1)
    attachStdout := true

    //stop,force remove if necessary
    log.Printf("Cleanup image id %s", containerID)

    //dockerobj.stopInternal(containerID, 0, false, false)

    log.Printf("Start container %s", containerID)
    containerID, err := dockerobj.createContainer(imageID, args, env, attachStdout)
    if err != nil {
        //if image not found try to create image and retry
        if err == docker.ErrNoSuchImage {
            if builder != nil {
                log.Printf("start-could not find image ...attempt to recreate image %s", err)

                reader, err1 := builder()
                if err1 != nil {
                    log.Printf("Error creating image builder: %s", err1)
                }

                if err1 = dockerobj.deployImage(imageID, args, env, reader); err1 != nil {
                    return err1
                }

                log.Printf("start-recreated image successfully")
                if containerID, err1 = dockerobj.createContainer(imageID, args, env, attachStdout); err1 != nil {
                    log.Printf("start-could not recreate container post recreate image: %s", err1)
                    return err1
                }
            } else {
                log.Printf("start-could not find image: %s", err)
                return err
            }
        } else {
            log.Printf("start-could not recreate container: %s", err)
            return err
        }
    }

    if attachStdout {
        // Launch a few go-threads to manage output streams from the container.
        // They will be automatically destroyed when the container exits
        attached := make(chan struct{})
        r, w := io.Pipe()

        go func() {
            // AttachToContainer will fire off a message on the "attached" channel once the
            // attachment completes, and then block until the container is terminated.
            // The returned error is not used outside the scope of this function. Assign the
            // error to a local variable to prevent clobbering the function variable 'err'.
            err := dockerobj.docker_client.AttachToContainer(docker.AttachToContainerOptions{
                Container:    containerID,
                OutputStream: w,
                ErrorStream:  w,
                Logs:         true,
                Stdout:       true,
                Stderr:       true,
                Stream:       true,
                Success:      attached,
            })

            // If we get here, the container has terminated.  Send a signal on the pipe
            // so that downstream may clean up appropriately
            _ = w.CloseWithError(err)
        }()

        go func() {
            // Block here until the attachment completes or we timeout
            select {
            case <-attached:
                // successful attach
            case <-time.After(10 * time.Second):
                log.Printf("Timeout while attaching to IO channel in container %s", containerID)
                return
            }

            // Acknowledge the attachment?  This was included in the gist I followed
            // (http://bit.ly/2jBrCtM).  Not sure it's actually needed but it doesn't
            // appear to hurt anything.
            attached <- struct{}{}

            // Establish a buffer for our IO channel so that we may do readline-style
            // ingestion of the IO, one log entry per line
            is := bufio.NewReader(r)

            for {
                // Loop forever dumping lines of text into the containerLogger
                // until the pipe is closed
                line, err2 := is.ReadString('\n')
                if err2 != nil {
                    switch err2 {
                    case io.EOF:
                        log.Printf("Container %s has closed its IO channel", containerID)
                    default:
                        log.Printf("Error reading container output: %s", err2)
                    }

                    return
                }

                log.Printf(line)
            }
        }()
    }

    // start container with HostConfig was deprecated since v1.10 and removed in v1.2
    err = dockerobj.docker_client.StartContainer(containerID, nil)
    if err != nil {
        log.Printf("start-could not start container: %s", err)
        return err
    }

    log.Printf("Started container %s", containerID)
    return nil
}

//Stop stops a running chaincode
func (dockerobj *AWSDOCKER) Stop(id string, timeout uint, dontkill bool, dontremove bool) error {

    id = strings.Replace(id, ":", "_", -1)
    err := dockerobj.stopInternal(id, timeout, dontkill, dontremove)

    return err
}

//Destroy destroys an image
func (dockerobj *AWSDOCKER) Destroy(id string, force bool, noprune bool) error {
    id = strings.Replace(id, ":", "_", -1)

    err := dockerobj.docker_client.RemoveImageExtended(id, docker.RemoveImageOptions{Force: force, NoPrune: noprune})

    if err != nil {
        log.Printf("error while destroying image: %s", err)
    } else {
        log.Printf("Destroyed image %s", id)
    }

    return err
}

func (executable AWSDOCKER) execute(handler MessageHandler) {
    handler.initialize()
    if handler.receive() {
        executable.executableTimeoutHelper(handler)
    }
}

func (executable *AWSDOCKER) executableTimeoutHelper(handler MessageHandler) {
    ch := make(chan error)
    go func() {
        ch <- executable.executionHelper(handler.body(), handler.id())
    }()
    select {
    case err := <-ch:
        if err != nil {
            log.Printf("XX: %s %s", *executable.taskDefinition, err.Error())
            handler.failure(err)
        } else {
            log.Printf("I: %s finished successfully", *executable.taskDefinition)
            handler.success()
        }
    case <-time.After(executable.timeout):
        err := fmt.Errorf("%s timed out after %f seconds", *executable.taskDefinition, executable.timeout.Seconds())
        log.Println(err)
        handler.failure(err)
    }
}

func (executable *AWSDOCKER) executionHelper(messageBody *string, messageID *string) error {
    var err error

    args := make([]string, 1)
    env := make([]string, 1)
    env = append(env, *messageBody)
    err = executable.Start("94bbd31594fa", args, env, nil)
    if err != nil {
        return err
    }
    err = executable.monitorDocker()
    if err != nil {
        return err
    }
    return nil
}

func (executable *AWSDOCKER) monitorDocker() error {
    executable.addListener()
    // Monitor docker events for sibling Projector task
    status, err := executable.listenForDie()
    if err != nil {
        return err
    }

    if status == "0" {
        // status is die
        log.Printf("[INFO] Execution completed successfully")
        executable.success()
        return nil
    }
    // non-zero exit
    log.Printf("[ERROR] Execution completed with non-zero exit status")
    err = fmt.Errorf("%s died with non-zero exit status (exit code %s)", *executable.taskDefinition, status)
    executable.failure()
    return err

}

func (executable *AWSDOCKER) listenForDie() (exitCode string, err error) {
    log.Printf("[INFO] Monitoring Docker events.")
    log.Printf("[DEBUG] %+v\n", executable)
    duration := getTimeout()
    timeout := time.After(duration)
    defer executable.removeListener()
    for {
        select {
        case msg := <-executable.eventsCh:
            if msg != nil {
                matched := msg.Actor.Attributes["com.amazonaws.ecs.task-arn"] == executable.taskArn
                if matched {
                    log.Printf("[DEBUG] %+v\n", msg)
                    switch msg.Action {
                    case "die":
                        log.Printf("[INFO] Container die event")
                        return msg.Actor.Attributes["exitCode"], nil
                    }
                }
            }
        case <-timeout:
            log.Printf("[INFO] Instance timeout reached.")
            err := fmt.Errorf("Docker container %s timed out after %f seconds", *executable.taskDefinition, duration.Seconds())
            return "timeout", err
        }
    }
}

func (dockerobj *AWSDOCKER) connect(dockerEndpointPath string) {
    log.Printf("[INFO] Connecting to Docker API.")
    endpoint := dockerEndpointPath
    client, err := docker.NewClient(endpoint)
    if err != nil {
        panic(err)
    }
    dockerobj.docker_client = client
    dockerobj.eventsCh = make(chan *docker.APIEvents)
}

func (dockerobj *AWSDOCKER) addListener() {
    err := dockerobj.docker_client.AddEventListener(dockerobj.eventsCh)
    if err != nil {
        log.Fatal(err)
    }
}

func (dockerobj *AWSDOCKER) removeListener() {
    err := dockerobj.docker_client.RemoveEventListener(dockerobj.eventsCh)
    if err != nil {
        log.Fatal(err)
    }
}

func (executable *AWSDOCKER) success() {}
func (executable *AWSDOCKER) failure() {}<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/api/types/versions"
	"github.com/docker/docker/cli"
	"github.com/docker/docker/cli/command"
	"github.com/docker/docker/cli/command/commands"
	cliflags "github.com/docker/docker/cli/flags"
	"github.com/docker/docker/cliconfig"
	"github.com/docker/docker/dockerversion"
	"github.com/docker/docker/pkg/term"
	"github.com/docker/docker/utils"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
)

func newDockerCommand(dockerCli *command.DockerCli) *cobra.Command {
	opts := cliflags.NewClientOptions()
	var flags *pflag.FlagSet

	cmd := &cobra.Command{
		Use:              "docker [OPTIONS] COMMAND [ARG...]",
		Short:            "A self-sufficient runtime for containers",
		SilenceUsage:     true,
		SilenceErrors:    true,
		TraverseChildren: true,
		Args:             noArgs,
		RunE: func(cmd *cobra.Command, args []string) error {
			if opts.Version {
				showVersion()
				return nil
			}
			return dockerCli.ShowHelp(cmd, args)
		},
		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
			// daemon command is special, we redirect directly to another binary
			if cmd.Name() == "daemon" {
				return nil
			}
			// flags must be the top-level command flags, not cmd.Flags()
			opts.Common.SetDefaultOptions(flags)
			dockerPreRun(opts)
			return dockerCli.Initialize(opts)
		},
	}
	cli.SetupRootCommand(cmd)

	cmd.SetHelpFunc(func(ccmd *cobra.Command, args []string) {
		if dockerCli.Client() == nil {
			// flags must be the top-level command flags, not cmd.Flags()
			opts.Common.SetDefaultOptions(flags)
			dockerPreRun(opts)
			dockerCli.Initialize(opts)
		}

		hideUnsupportedFeatures(ccmd, dockerCli.Client().ClientVersion(), dockerCli.HasExperimental())

		if err := ccmd.Help(); err != nil {
			ccmd.Println(err)
		}
	})

	flags = cmd.Flags()
	flags.BoolVarP(&opts.Version, "version", "v", false, "Print version information and quit")
	flags.StringVar(&opts.ConfigDir, "config", cliconfig.ConfigDir(), "Location of client config files")
	opts.Common.InstallFlags(flags)

	cmd.SetOutput(dockerCli.Out())
	cmd.AddCommand(newDaemonCommand())
	commands.AddCommands(cmd, dockerCli)

	return cmd
}

func noArgs(cmd *cobra.Command, args []string) error {
	if len(args) == 0 {
		return nil
	}
	return fmt.Errorf(
		"docker: '%s' is not a docker command.\nSee 'docker --help'.", args[0])
}

func main() {
	// Set terminal emulation based on platform as required.
	stdin, stdout, stderr := term.StdStreams()
	logrus.SetOutput(stderr)

	dockerCli := command.NewDockerCli(stdin, stdout, stderr)
	cmd := newDockerCommand(dockerCli)

	if err := cmd.Execute(); err != nil {
		if sterr, ok := err.(cli.StatusError); ok {
			if sterr.Status != "" {
				fmt.Fprintln(stderr, sterr.Status)
			}
			// StatusError should only be used for errors, and all errors should
			// have a non-zero exit status, so never exit with 0
			if sterr.StatusCode == 0 {
				os.Exit(1)
			}
			os.Exit(sterr.StatusCode)
		}
		fmt.Fprintln(stderr, err)
		os.Exit(1)
	}
}

func showVersion() {
	fmt.Printf("Docker version %s, build %s\n", dockerversion.Version, dockerversion.GitCommit)
}

func dockerPreRun(opts *cliflags.ClientOptions) {
	cliflags.SetLogLevel(opts.Common.LogLevel)

	if opts.ConfigDir != "" {
		cliconfig.SetConfigDir(opts.ConfigDir)
	}

	if opts.Common.Debug {
		utils.EnableDebug()
	}
}

func hideUnsupportedFeatures(cmd *cobra.Command, clientVersion string, hasExperimental bool) {
	cmd.Flags().VisitAll(func(f *pflag.Flag) {
		// hide experimental flags
		if !hasExperimental {
			if _, ok := f.Annotations["experimental"]; ok {
				f.Hidden = true
			}
		}

		// hide flags not supported by the server
		if flagVersion, ok := f.Annotations["version"]; ok && len(flagVersion) == 1 && versions.LessThan(clientVersion, flagVersion[0]) {
			f.Hidden = true
		}

	})

	for _, subcmd := range cmd.Commands() {
		// hide experimental subcommands
		if !hasExperimental {
			if _, ok := subcmd.Tags["experimental"]; ok {
				subcmd.Hidden = true
			}
		}

		// hide subcommands not supported by the server
		if subcmdVersion, ok := subcmd.Tags["version"]; ok && versions.LessThan(clientVersion, subcmdVersion) {
			subcmd.Hidden = true
		}
	}
}
<commit_msg>fix a few golint errors<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/api/types/versions"
	"github.com/docker/docker/cli"
	"github.com/docker/docker/cli/command"
	"github.com/docker/docker/cli/command/commands"
	cliflags "github.com/docker/docker/cli/flags"
	"github.com/docker/docker/cliconfig"
	"github.com/docker/docker/dockerversion"
	"github.com/docker/docker/pkg/term"
	"github.com/docker/docker/utils"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
)

func newDockerCommand(dockerCli *command.DockerCli) *cobra.Command {
	opts := cliflags.NewClientOptions()
	var flags *pflag.FlagSet

	cmd := &cobra.Command{
		Use:              "docker [OPTIONS] COMMAND [ARG...]",
		Short:            "A self-sufficient runtime for containers",
		SilenceUsage:     true,
		SilenceErrors:    true,
		TraverseChildren: true,
		Args:             noArgs,
		RunE: func(cmd *cobra.Command, args []string) error {
			if opts.Version {
				showVersion()
				return nil
			}
			return dockerCli.ShowHelp(cmd, args)
		},
		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
			// daemon command is special, we redirect directly to another binary
			if cmd.Name() == "daemon" {
				return nil
			}
			// flags must be the top-level command flags, not cmd.Flags()
			opts.Common.SetDefaultOptions(flags)
			dockerPreRun(opts)
			return dockerCli.Initialize(opts)
		},
	}
	cli.SetupRootCommand(cmd)

	cmd.SetHelpFunc(func(ccmd *cobra.Command, args []string) {
		if dockerCli.Client() == nil {
			// flags must be the top-level command flags, not cmd.Flags()
			opts.Common.SetDefaultOptions(flags)
			dockerPreRun(opts)
			dockerCli.Initialize(opts)
		}

		hideUnsupportedFeatures(ccmd, dockerCli.Client().ClientVersion(), dockerCli.HasExperimental())

		if err := ccmd.Help(); err != nil {
			ccmd.Println(err)
		}
	})

	flags = cmd.Flags()
	flags.BoolVarP(&opts.Version, "version", "v", false, "Print version information and quit")
	flags.StringVar(&opts.ConfigDir, "config", cliconfig.ConfigDir(), "Location of client config files")
	opts.Common.InstallFlags(flags)

	cmd.SetOutput(dockerCli.Out())
	cmd.AddCommand(newDaemonCommand())
	commands.AddCommands(cmd, dockerCli)

	return cmd
}

func noArgs(cmd *cobra.Command, args []string) error {
	if len(args) == 0 {
		return nil
	}
	return fmt.Errorf(
		"docker: '%s' is not a docker command.\nSee 'docker --help'", args[0])
}

func main() {
	// Set terminal emulation based on platform as required.
	stdin, stdout, stderr := term.StdStreams()
	logrus.SetOutput(stderr)

	dockerCli := command.NewDockerCli(stdin, stdout, stderr)
	cmd := newDockerCommand(dockerCli)

	if err := cmd.Execute(); err != nil {
		if sterr, ok := err.(cli.StatusError); ok {
			if sterr.Status != "" {
				fmt.Fprintln(stderr, sterr.Status)
			}
			// StatusError should only be used for errors, and all errors should
			// have a non-zero exit status, so never exit with 0
			if sterr.StatusCode == 0 {
				os.Exit(1)
			}
			os.Exit(sterr.StatusCode)
		}
		fmt.Fprintln(stderr, err)
		os.Exit(1)
	}
}

func showVersion() {
	fmt.Printf("Docker version %s, build %s\n", dockerversion.Version, dockerversion.GitCommit)
}

func dockerPreRun(opts *cliflags.ClientOptions) {
	cliflags.SetLogLevel(opts.Common.LogLevel)

	if opts.ConfigDir != "" {
		cliconfig.SetConfigDir(opts.ConfigDir)
	}

	if opts.Common.Debug {
		utils.EnableDebug()
	}
}

func hideUnsupportedFeatures(cmd *cobra.Command, clientVersion string, hasExperimental bool) {
	cmd.Flags().VisitAll(func(f *pflag.Flag) {
		// hide experimental flags
		if !hasExperimental {
			if _, ok := f.Annotations["experimental"]; ok {
				f.Hidden = true
			}
		}

		// hide flags not supported by the server
		if flagVersion, ok := f.Annotations["version"]; ok && len(flagVersion) == 1 && versions.LessThan(clientVersion, flagVersion[0]) {
			f.Hidden = true
		}

	})

	for _, subcmd := range cmd.Commands() {
		// hide experimental subcommands
		if !hasExperimental {
			if _, ok := subcmd.Tags["experimental"]; ok {
				subcmd.Hidden = true
			}
		}

		// hide subcommands not supported by the server
		if subcmdVersion, ok := subcmd.Tags["version"]; ok && versions.LessThan(clientVersion, subcmdVersion) {
			subcmd.Hidden = true
		}
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Mute Communications Ltd.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package nymaddr implements nym address generation and decoding.
package nymaddr

import (
	"bytes"
	"crypto"
	"crypto/hmac"
	"crypto/rand"
	"encoding/asn1"
	"errors"
	"io"

	"github.com/davecgh/go-spew/spew"
	"github.com/mutecomm/mute/mix/mixaddr"
	"github.com/mutecomm/mute/util/times"
	"github.com/ronperry/cryptoedge/lioness"
	"golang.org/x/crypto/curve25519"
)

var (
	// ErrNoMix is returned if no mix could be found.
	ErrNoMix = errors.New("nymaddr: No mix found")
	// ErrNoKey is returned if a private key was unavailable.
	ErrNoKey = errors.New("nymaddr: Private key not found")
	// ErrExpired is returned if a nymaddr has already expired.
	ErrExpired = errors.New("nymaddr: Expired")
	// ErrHMAC is returned if a nym-header HMAC verification failed.
	ErrHMAC = errors.New("nymaddr: HMAC mismatch")
	// ErrBadKey is returned if a header key is not reproduceable.
	ErrBadKey = errors.New("nymaddr: Bad key in header")
)

// Rand is the random source of this package.
var Rand = rand.Reader

const (
	// KeySize is the size in bytes of a key for curve25519.
	KeySize = 32
)

// KeyFunc is a function that returns a private key for a public key, or nil.
type KeyFunc func(*[KeySize]byte) *[KeySize]byte

// Address contains a NymAddress.
type Address struct {
	MixAddress  []byte // The address of the mix which will handle messages for this Nym
	Expire      int64  // The time when the address should not be used anymore
	SingleUse   bool   // If the address may be used more than once
	TokenPubKey []byte // The token receive key of the mix
	MixPubKey   []byte // Public key of the mix
	AddressKey  []byte // Random, single use key used for this address
	PrivateData []byte // Encrypted private part
}

// AddressPrivate is the private/encrypted part of a nymaddress.
type AddressPrivate struct {
	System         int32  // The system number. should be 0
	Address        []byte // The final address to deliver to
	Expire         int64  // The time when the address should not be used anymore
	SingleUse      bool   // If the address may be used more than once
	MinDelay       int32  // Minimum delay in the mix
	MaxDelay       int32  // Maximum delay in the mix
	Nonce          []byte // Random data, size KeySize
	ReceiverPubKey []byte // The pubkey of the receiver
	EncNym         []byte // The encrypted Nym
	HMACHead       []byte // The HMAC of the header (nonce, receiverpubkey, encnym)
}

// RelayHeader is the header of a relayed (post-mix) message.
type RelayHeader struct {
	SenderKey      []byte // Pubkey of the sender (mix), per message
	Nonce          []byte // Random data, size KeySize
	ReceiverPubKey []byte // The pubkey of the receiver
	EncNym         []byte // The encrypted Nym
	HMACHead       []byte // The HMAC of the header (nonce, receiverpubkey, encnym)
}

// AddressTemplate contains parameters for address creation.
type AddressTemplate struct {
	Secret []byte // The local secret for address creation, must be random and long-lived

	System        int32               // The system number. should be 0
	MixCandidates mixaddr.AddressList // A list of mixes

	Expire    int64 // The time when the address should not be used anymore
	SingleUse bool  // If the address may be used more than once
	MinDelay  int32 // Minimum delay in the mix
	MaxDelay  int32 // Maximum delay in the mix
}

func genKeyRandom() (pub, priv *[KeySize]byte, err error) {
	privateKey, err := genNonce()
	if err != nil {
		return nil, nil, err
	}
	publicKey := new([KeySize]byte)
	curve25519.ScalarBaseMult(publicKey, privateKey)
	return publicKey, privateKey, nil
}

func genNonce() (nonce *[KeySize]byte, err error) {
	nonce = new([KeySize]byte)
	if _, err = io.ReadFull(Rand, nonce[:]); err != nil {
		return nil, err
	}
	return nonce, nil
}

// ParseAddress parses an address.
func ParseAddress(address []byte) (*Address, error) {
	a := new(Address)
	_, err := asn1.Unmarshal(address, a)
	if err != nil {
		return nil, err
	}
	if a.Expire < times.Now() {
		return nil, ErrExpired
	}
	return a, nil
}

// GetMixData decrypts the private portion of a nymaddress.
func (ad *Address) GetMixData(keysLookup KeyFunc) (*AddressPrivate, error) {
	pubkey := new([KeySize]byte)
	copy(pubkey[:], ad.MixPubKey)
	privkey := keysLookup(pubkey)
	if privkey == nil {
		return nil, ErrNoKey
	}
	sharedSecret := new([KeySize]byte)
	addrKey := new([KeySize]byte)
	copy(addrKey[:], ad.AddressKey)
	curve25519.ScalarMult(sharedSecret, privkey, addrKey)
	cr, err := lioness.New(sharedSecret[:]) // saves some bytes and is safe against tagging
	if err != nil {
		return nil, err
	}
	privmarshal, err := cr.Decrypt(ad.PrivateData)
	if err != nil {
		return nil, err
	}
	ap := new(AddressPrivate)
	_, err = asn1.Unmarshal(privmarshal, ap)
	if err != nil {
		return nil, err
	}
	return ap, nil
}

// GetUnique returns a unique value of the nymaddress IF the nymaddress is
// single use, nil otherwise.
func (ap AddressPrivate) GetUnique() []byte {
	if ap.SingleUse {
		return ap.ReceiverPubKey
	}
	return nil
}

// GetHeader returns the header for a relay message and a secret for encryption.
func (ap AddressPrivate) GetHeader() (header, secret []byte, err error) {
	pubkey, privkey, err := genKeyRandom()
	if err != nil {
		return nil, nil, err
	}
	rh := RelayHeader{
		SenderKey:      pubkey[:],
		Nonce:          ap.Nonce,
		ReceiverPubKey: ap.ReceiverPubKey,
		EncNym:         ap.EncNym,
		HMACHead:       ap.HMACHead,
	}
	d, err := asn1.Marshal(rh)
	if err != nil {
		return nil, nil, err
	}
	sharedSecret := new([KeySize]byte)
	receiverPubKey := new([KeySize]byte)
	copy(receiverPubKey[:], ap.ReceiverPubKey)
	curve25519.ScalarMult(sharedSecret, privkey, receiverPubKey)
	return d, sharedSecret[:], nil
}

// GetPrivate gets the shared secret from a header.
func (tmp AddressTemplate) GetPrivate(header, MailboxAddress []byte) (nym, secret []byte, err error) {
	rh := new(RelayHeader)
	_, err = asn1.Unmarshal(header, rh)
	if err != nil {
		return nil, nil, err
	}
	symkey := tmp.genSymKey(rh.Nonce, rh.ReceiverPubKey, MailboxAddress)
	HMACHead := calcHmac(symkey, rh.Nonce, rh.ReceiverPubKey, rh.EncNym)
	if !hmac.Equal(rh.HMACHead, HMACHead) {
		return nil, nil, ErrHMAC
	}
	nym = decryptNym(symkey, rh.EncNym)
	recPub, recPriv := tmp.genDetermKeys(rh.Nonce, nym)
	if !bytes.Equal(recPub[:], rh.ReceiverPubKey) {
		return nil, nil, ErrBadKey
	}
	sharedSecret := new([KeySize]byte)
	senderPubKey := new([KeySize]byte)
	copy(senderPubKey[:], rh.SenderKey)
	curve25519.ScalarMult(sharedSecret, recPriv, senderPubKey)
	return nym, sharedSecret[:], nil
}

// NewAddress generates a new nymaddress for nym/address from AddressTemplate.
// Only the first KeySize bytes of Nym are used, so use a hash of the true nym
// here.
func (tmp AddressTemplate) NewAddress(MailboxAddress, Nym []byte) ([]byte, error) {
	xnym := make([]byte, KeySize)
	copy(xnym, Nym)
	Nym = xnym
	mixAddress := tmp.MixCandidates.Expire(tmp.Expire - 1).Rand() // Select a random mix
	if mixAddress == nil {
		spew.Dump(tmp)
		return nil, ErrNoMix
	}
	pubKeyRand, privKeyRand, err := genKeyRandom()
	if err != nil {
		return nil, err
	}
	nonce, err := genNonce()
	if err != nil {
		return nil, err
	}

	nymaddr := new(Address)
	nymaddr.MixAddress = []byte(mixAddress.Address)
	nymaddr.MixPubKey = mixAddress.Pubkey
	nymaddr.TokenPubKey = mixAddress.TokenKey
	nymaddr.Expire = tmp.Expire
	nymaddr.SingleUse = tmp.SingleUse
	nymaddr.AddressKey = pubKeyRand[:]

	nymprivate := new(AddressPrivate)
	nymprivate.System = tmp.System
	nymprivate.Expire = tmp.Expire
	nymprivate.Address = MailboxAddress
	nymprivate.SingleUse = tmp.SingleUse
	nymprivate.MinDelay = tmp.MinDelay
	nymprivate.MaxDelay = tmp.MaxDelay
	nymprivate.Nonce = nonce[:]

	// Calculate public key that the mix will encrypt TO
	recPub, _ := tmp.genDetermKeys(nymprivate.Nonce, Nym)
	nymprivate.ReceiverPubKey = recPub[:]

	// Calculate symmetric key
	symkey := tmp.genSymKey(nymprivate.Nonce, nymprivate.ReceiverPubKey, nymprivate.Address)

	nymprivate.EncNym = encryptNym(symkey, Nym)

	nymprivate.HMACHead = calcHmac(symkey, nymprivate.Nonce, nymprivate.ReceiverPubKey, nymprivate.EncNym)

	privmarshal, err := asn1.Marshal(*nymprivate)
	if err != nil {
		return nil, err
	}
	mixPubKey := new([KeySize]byte)
	sharedSecret := new([KeySize]byte)
	copy(mixPubKey[:], nymaddr.MixPubKey)
	curve25519.ScalarMult(sharedSecret, privKeyRand, mixPubKey)
	cr, err := lioness.New(sharedSecret[:]) // saves some bytes and is safe against tagging
	if err != nil {
		return nil, err
	}
	privmarshalEnc, err := cr.Encrypt(privmarshal)
	if err != nil {
		return nil, err
	}
	nymaddr.PrivateData = privmarshalEnc
	return asn1.Marshal(*nymaddr)
}

func calcHmac(key []byte, data ...[]byte) []byte {
	h := hmac.New(crypto.SHA256.New, key)
	for _, e := range data {
		h.Write(e)
	}
	x := h.Sum(make([]byte, 0))
	return x[:]
}

func (tmp AddressTemplate) genDetermKeys(nonce, nym []byte) (pub, priv *[KeySize]byte) {
	pub, priv = new([KeySize]byte), new([KeySize]byte)
	privkeyA := calcHmac(tmp.Secret, nonce, nym)
	copy(priv[:], privkeyA)
	curve25519.ScalarBaseMult(pub, priv)
	return pub, priv
}

func (tmp AddressTemplate) genSymKey(nonce, receiverpubkey, deliveryaddress []byte) []byte {
	return calcHmac(tmp.Secret, nonce, receiverpubkey, deliveryaddress)
}
<commit_msg>Revert "update"<commit_after>// Copyright (c) 2015 Mute Communications Ltd.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package nymaddr implements nym address generation and decoding.
package nymaddr

import (
	"bytes"
	"crypto"
	"crypto/hmac"
	"crypto/rand"
	"encoding/asn1"
	"errors"
	"io"

	"github.com/mutecomm/mute/mix/mixaddr"
	"github.com/mutecomm/mute/util/times"
	"github.com/ronperry/cryptoedge/lioness"
	"golang.org/x/crypto/curve25519"
)

var (
	// ErrNoMix is returned if no mix could be found.
	ErrNoMix = errors.New("nymaddr: No mix found")
	// ErrNoKey is returned if a private key was unavailable.
	ErrNoKey = errors.New("nymaddr: Private key not found")
	// ErrExpired is returned if a nymaddr has already expired.
	ErrExpired = errors.New("nymaddr: Expired")
	// ErrHMAC is returned if a nym-header HMAC verification failed.
	ErrHMAC = errors.New("nymaddr: HMAC mismatch")
	// ErrBadKey is returned if a header key is not reproduceable.
	ErrBadKey = errors.New("nymaddr: Bad key in header")
)

// Rand is the random source of this package.
var Rand = rand.Reader

const (
	// KeySize is the size in bytes of a key for curve25519.
	KeySize = 32
)

// KeyFunc is a function that returns a private key for a public key, or nil.
type KeyFunc func(*[KeySize]byte) *[KeySize]byte

// Address contains a NymAddress.
type Address struct {
	MixAddress  []byte // The address of the mix which will handle messages for this Nym
	Expire      int64  // The time when the address should not be used anymore
	SingleUse   bool   // If the address may be used more than once
	TokenPubKey []byte // The token receive key of the mix
	MixPubKey   []byte // Public key of the mix
	AddressKey  []byte // Random, single use key used for this address
	PrivateData []byte // Encrypted private part
}

// AddressPrivate is the private/encrypted part of a nymaddress.
type AddressPrivate struct {
	System         int32  // The system number. should be 0
	Address        []byte // The final address to deliver to
	Expire         int64  // The time when the address should not be used anymore
	SingleUse      bool   // If the address may be used more than once
	MinDelay       int32  // Minimum delay in the mix
	MaxDelay       int32  // Maximum delay in the mix
	Nonce          []byte // Random data, size KeySize
	ReceiverPubKey []byte // The pubkey of the receiver
	EncNym         []byte // The encrypted Nym
	HMACHead       []byte // The HMAC of the header (nonce, receiverpubkey, encnym)
}

// RelayHeader is the header of a relayed (post-mix) message.
type RelayHeader struct {
	SenderKey      []byte // Pubkey of the sender (mix), per message
	Nonce          []byte // Random data, size KeySize
	ReceiverPubKey []byte // The pubkey of the receiver
	EncNym         []byte // The encrypted Nym
	HMACHead       []byte // The HMAC of the header (nonce, receiverpubkey, encnym)
}

// AddressTemplate contains parameters for address creation.
type AddressTemplate struct {
	Secret []byte // The local secret for address creation, must be random and long-lived

	System        int32               // The system number. should be 0
	MixCandidates mixaddr.AddressList // A list of mixes

	Expire    int64 // The time when the address should not be used anymore
	SingleUse bool  // If the address may be used more than once
	MinDelay  int32 // Minimum delay in the mix
	MaxDelay  int32 // Maximum delay in the mix
}

func genKeyRandom() (pub, priv *[KeySize]byte, err error) {
	privateKey, err := genNonce()
	if err != nil {
		return nil, nil, err
	}
	publicKey := new([KeySize]byte)
	curve25519.ScalarBaseMult(publicKey, privateKey)
	return publicKey, privateKey, nil
}

func genNonce() (nonce *[KeySize]byte, err error) {
	nonce = new([KeySize]byte)
	if _, err = io.ReadFull(Rand, nonce[:]); err != nil {
		return nil, err
	}
	return nonce, nil
}

// ParseAddress parses an address.
func ParseAddress(address []byte) (*Address, error) {
	a := new(Address)
	_, err := asn1.Unmarshal(address, a)
	if err != nil {
		return nil, err
	}
	if a.Expire < times.Now() {
		return nil, ErrExpired
	}
	return a, nil
}

// GetMixData decrypts the private portion of a nymaddress.
func (ad *Address) GetMixData(keysLookup KeyFunc) (*AddressPrivate, error) {
	pubkey := new([KeySize]byte)
	copy(pubkey[:], ad.MixPubKey)
	privkey := keysLookup(pubkey)
	if privkey == nil {
		return nil, ErrNoKey
	}
	sharedSecret := new([KeySize]byte)
	addrKey := new([KeySize]byte)
	copy(addrKey[:], ad.AddressKey)
	curve25519.ScalarMult(sharedSecret, privkey, addrKey)
	cr, err := lioness.New(sharedSecret[:]) // saves some bytes and is safe against tagging
	if err != nil {
		return nil, err
	}
	privmarshal, err := cr.Decrypt(ad.PrivateData)
	if err != nil {
		return nil, err
	}
	ap := new(AddressPrivate)
	_, err = asn1.Unmarshal(privmarshal, ap)
	if err != nil {
		return nil, err
	}
	return ap, nil
}

// GetUnique returns a unique value of the nymaddress IF the nymaddress is
// single use, nil otherwise.
func (ap AddressPrivate) GetUnique() []byte {
	if ap.SingleUse {
		return ap.ReceiverPubKey
	}
	return nil
}

// GetHeader returns the header for a relay message and a secret for encryption.
func (ap AddressPrivate) GetHeader() (header, secret []byte, err error) {
	pubkey, privkey, err := genKeyRandom()
	if err != nil {
		return nil, nil, err
	}
	rh := RelayHeader{
		SenderKey:      pubkey[:],
		Nonce:          ap.Nonce,
		ReceiverPubKey: ap.ReceiverPubKey,
		EncNym:         ap.EncNym,
		HMACHead:       ap.HMACHead,
	}
	d, err := asn1.Marshal(rh)
	if err != nil {
		return nil, nil, err
	}
	sharedSecret := new([KeySize]byte)
	receiverPubKey := new([KeySize]byte)
	copy(receiverPubKey[:], ap.ReceiverPubKey)
	curve25519.ScalarMult(sharedSecret, privkey, receiverPubKey)
	return d, sharedSecret[:], nil
}

// GetPrivate gets the shared secret from a header.
func (tmp AddressTemplate) GetPrivate(header, MailboxAddress []byte) (nym, secret []byte, err error) {
	rh := new(RelayHeader)
	_, err = asn1.Unmarshal(header, rh)
	if err != nil {
		return nil, nil, err
	}
	symkey := tmp.genSymKey(rh.Nonce, rh.ReceiverPubKey, MailboxAddress)
	HMACHead := calcHmac(symkey, rh.Nonce, rh.ReceiverPubKey, rh.EncNym)
	if !hmac.Equal(rh.HMACHead, HMACHead) {
		return nil, nil, ErrHMAC
	}
	nym = decryptNym(symkey, rh.EncNym)
	recPub, recPriv := tmp.genDetermKeys(rh.Nonce, nym)
	if !bytes.Equal(recPub[:], rh.ReceiverPubKey) {
		return nil, nil, ErrBadKey
	}
	sharedSecret := new([KeySize]byte)
	senderPubKey := new([KeySize]byte)
	copy(senderPubKey[:], rh.SenderKey)
	curve25519.ScalarMult(sharedSecret, recPriv, senderPubKey)
	return nym, sharedSecret[:], nil
}

// NewAddress generates a new nymaddress for nym/address from AddressTemplate.
// Only the first KeySize bytes of Nym are used, so use a hash of the true nym
// here.
func (tmp AddressTemplate) NewAddress(MailboxAddress, Nym []byte) ([]byte, error) {
	xnym := make([]byte, KeySize)
	copy(xnym, Nym)
	Nym = xnym
	mixAddress := tmp.MixCandidates.Expire(tmp.Expire).Rand() // Select a random mix
	if mixAddress == nil {
		return nil, ErrNoMix
	}
	pubKeyRand, privKeyRand, err := genKeyRandom()
	if err != nil {
		return nil, err
	}
	nonce, err := genNonce()
	if err != nil {
		return nil, err
	}

	nymaddr := new(Address)
	nymaddr.MixAddress = []byte(mixAddress.Address)
	nymaddr.MixPubKey = mixAddress.Pubkey
	nymaddr.TokenPubKey = mixAddress.TokenKey
	nymaddr.Expire = tmp.Expire
	nymaddr.SingleUse = tmp.SingleUse
	nymaddr.AddressKey = pubKeyRand[:]

	nymprivate := new(AddressPrivate)
	nymprivate.System = tmp.System
	nymprivate.Expire = tmp.Expire
	nymprivate.Address = MailboxAddress
	nymprivate.SingleUse = tmp.SingleUse
	nymprivate.MinDelay = tmp.MinDelay
	nymprivate.MaxDelay = tmp.MaxDelay
	nymprivate.Nonce = nonce[:]

	// Calculate public key that the mix will encrypt TO
	recPub, _ := tmp.genDetermKeys(nymprivate.Nonce, Nym)
	nymprivate.ReceiverPubKey = recPub[:]

	// Calculate symmetric key
	symkey := tmp.genSymKey(nymprivate.Nonce, nymprivate.ReceiverPubKey, nymprivate.Address)

	nymprivate.EncNym = encryptNym(symkey, Nym)

	nymprivate.HMACHead = calcHmac(symkey, nymprivate.Nonce, nymprivate.ReceiverPubKey, nymprivate.EncNym)

	privmarshal, err := asn1.Marshal(*nymprivate)
	if err != nil {
		return nil, err
	}
	mixPubKey := new([KeySize]byte)
	sharedSecret := new([KeySize]byte)
	copy(mixPubKey[:], nymaddr.MixPubKey)
	curve25519.ScalarMult(sharedSecret, privKeyRand, mixPubKey)
	cr, err := lioness.New(sharedSecret[:]) // saves some bytes and is safe against tagging
	if err != nil {
		return nil, err
	}
	privmarshalEnc, err := cr.Encrypt(privmarshal)
	if err != nil {
		return nil, err
	}
	nymaddr.PrivateData = privmarshalEnc
	return asn1.Marshal(*nymaddr)
}

func calcHmac(key []byte, data ...[]byte) []byte {
	h := hmac.New(crypto.SHA256.New, key)
	for _, e := range data {
		h.Write(e)
	}
	x := h.Sum(make([]byte, 0))
	return x[:]
}

func (tmp AddressTemplate) genDetermKeys(nonce, nym []byte) (pub, priv *[KeySize]byte) {
	pub, priv = new([KeySize]byte), new([KeySize]byte)
	privkeyA := calcHmac(tmp.Secret, nonce, nym)
	copy(priv[:], privkeyA)
	curve25519.ScalarBaseMult(pub, priv)
	return pub, priv
}

func (tmp AddressTemplate) genSymKey(nonce, receiverpubkey, deliveryaddress []byte) []byte {
	return calcHmac(tmp.Secret, nonce, receiverpubkey, deliveryaddress)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"

	"github.com/calavera/dkvolume"

	"github.com/noahdesu/go-ceph/rados"
	"github.com/noahdesu/go-ceph/rbd"
)

type volume struct {
	name        string
	device		string
	connections int
}

type cephDriver struct {
	root       	string
	pool		string
	volumes    	map[string]*volume
	m          	sync.Mutex
}

func newCephDriver(root, pool string) cephDriver {
	d := cephDriver{
		root:   root,
		pool:	pool,
		volumes: map[string]*volume{},
	}
	return d
}

func checkFs(device string) (string, error) {
	log.Printf("blkid %s", device)
 	out, err := exec.Command("blkid", device).Output()
 	if out != nil {
	 	sout := strings.TrimRight(string(out), "\n")
		return sout, err
 	}
 	return "", err
}

func ensureFs(name string) dkvolume.Response {
	log.Printf("ensureFs(%q)", name)
	device, err := rbdMap(name)
	if err != nil {
		return dkvolume.Response{Err: err.Error()}
	}
	defer rbdUnmap(*device)

	t, err := checkFs(*device)
	log.Printf("t => %q", t)
	if !strings.HasSuffix(t, "TYPE=\"xfs\"") {
		log.Printf("Formatting %s", *device)
		out, _, err := sh(fmt.Sprintf("mkfs.xfs -f %s", *device))
		if err != nil {
			return dkvolume.Response{Err: err.Error()}
		}
		log.Printf(string(out))
	}
	return dkvolume.Response{}
}

func rbdExists(name string) (bool, error) {
	conn, err := rados.NewConn()
	if err != nil {
		log.Fatal(err)
		return false, err
	}
	conn.ReadDefaultConfigFile()
	conn.Connect()
	defer conn.Shutdown()

	ioContext, err := conn.OpenIOContext("rbd")
	if err != nil {
		log.Fatal(err)
		return false, err
	}
	defer ioContext.Destroy()

	volumes, err := rbd.GetImageNames(ioContext)
	if err != nil {
		log.Fatal(err)
		return false, err
	}
	for _, volumeName := range volumes {
		if name == volumeName {
			return true, nil
		}
	}
	return false, nil
}

func (d cephDriver) Create(r dkvolume.Request) dkvolume.Response {
	log.Printf("Create(%q)", r.Name)
	d.m.Lock()
	defer d.m.Unlock()
	name := r.Name
	mountpoint := d.mountpoint(name)

	if _, ok := d.volumes[mountpoint]; ok {
		return dkvolume.Response{}
	}

	exists, err := rbdExists(name)
	if err != nil {
		return dkvolume.Response{Err: err.Error()}
	}

	if !exists {
		if out, _, err := sh(fmt.Sprintf("rbd create --size 512 %s", name)); err != nil {
			log.Print(string(out))
			return dkvolume.Response{Err: err.Error()}
		}
	}

	return ensureFs(name)
}

func (d cephDriver) Remove(r dkvolume.Request) dkvolume.Response {
	log.Printf("Remove(%q)", r.Name)
	d.m.Lock()
	defer d.m.Unlock()
	name := r.Name
	mountpoint := d.mountpoint(name)

	if volume, ok := d.volumes[mountpoint]; ok {
		if volume.connections <= 1 {
			cmd := fmt.Sprintf("rbd rm %s", name)
			if out, _, err := sh(cmd); err != nil {
				log.Print(string(out))
				return dkvolume.Response{Err: err.Error()}
			}
			delete(d.volumes, mountpoint)
		}
	}
	return dkvolume.Response{}
}

func (d cephDriver) Path(r dkvolume.Request) dkvolume.Response {
	return dkvolume.Response{Mountpoint: d.mountpoint(r.Name)}
}

func (d cephDriver) Mount(r dkvolume.Request) dkvolume.Response {
	log.Printf("Mount(%q)", r.Name)
	d.m.Lock()
	defer d.m.Unlock()
	mountpoint := d.mountpoint(r.Name)
	log.Printf("mountpoint(%q) => %q", r.Name, mountpoint)

	vol, ok := d.volumes[mountpoint]
	if ok && vol.connections > 0 {
		vol.connections++
		return dkvolume.Response{Mountpoint: mountpoint}
	}

	fi, err := os.Lstat(mountpoint)

	if os.IsNotExist(err) {
		if err := os.MkdirAll(mountpoint, 0755); err != nil {
			return dkvolume.Response{Err: err.Error()}
		}
	} else if err != nil {
		return dkvolume.Response{Err: err.Error()}
	}

	if fi != nil && !fi.IsDir() {
		return dkvolume.Response{Err: fmt.Sprintf("%v already exist and it's not a directory", mountpoint)}
	}

	device, err := d.mountVolume(r.Name, mountpoint);
	if err != nil {
		return dkvolume.Response{Err: err.Error()}
	}

	d.volumes[mountpoint] = &volume{name: r.Name, device: *device, connections: 1}

	return dkvolume.Response{Mountpoint: mountpoint}
}

func (d cephDriver) Unmount(r dkvolume.Request) dkvolume.Response {
	log.Printf("Unmount(%q)\n", r.Name)
	d.m.Lock()
	defer d.m.Unlock()
	mountpoint := d.mountpoint(r.Name)
	log.Printf("mountpoint(%q) => %s\n", r.Name, mountpoint)

	if volume, ok := d.volumes[mountpoint]; ok {
		if volume.connections == 1 {
			if err := d.unmountVolume(mountpoint, volume.device); err != nil {
				return dkvolume.Response{Err: err.Error()}
			}
		}
		volume.connections--
	} else {
		return dkvolume.Response{Err: fmt.Sprintf("Unable to find volume mounted on %s", mountpoint)}
	}

	return dkvolume.Response{}
}

func (d *cephDriver) mountpoint(name string) string {
	return filepath.Join(d.root, name)
}

func execCommand(name string, arg... string) ([]byte, []byte, error) {
 	cmd := exec.Command(name, arg...)
	stdout, err := cmd.StdoutPipe(); if err != nil {
		return nil, nil, err
	}
 	stderr, err := cmd.StderrPipe()
	if err != nil {
		return nil, nil, err
	}
	if err := cmd.Start(); err != nil {
		return nil, nil, err
	}
	sout, err := ioutil.ReadAll(stdout); if err != nil {
		return nil, nil, err
	}
	serr, err := ioutil.ReadAll(stderr); if err != nil {
		return nil, nil, err
	}
	if err := cmd.Wait(); err != nil {
		return nil, nil, err
	}
	return sout, serr, nil
}

func sh(s string) ([]byte, []byte, error) {
	log.Print(s)
	return execCommand("sh", "-c", s)
}

func rbdMap(name string) (*string, error) {
	log.Printf("rbdMap(%q)", name)
	out, _, err := sh(fmt.Sprintf("rbd map %s", name))
	if err != nil {
		log.Print(string(out))
		return nil, err
	}
	out1, _, err := sh(fmt.Sprintf("rbd showmapped |grep %s| awk 'NR==1 {print $5}'", name))
	if err != nil {
		log.Print(string(out1))
		return nil, err
	}

	device := strings.TrimRight(string(out1), "\n")
	log.Printf("rbd map %s => %q", name, device)
	return &device, nil
}

func rbdUnmap(device string) error {
	log.Printf("rbdUnmap(%q)", device)
	if out, _, err := sh(fmt.Sprintf("rbd unmap %s", device)); err != nil {
		log.Print(string(out))
		return err
	}
	return nil
}

func (d *cephDriver) mountVolume(name, target string) (*string, error) {
	log.Printf("mountVolume(%q, %q)", name, target)
	device, err := rbdMap(name)
	if err != nil {
		log.Fatal(err)
		return nil, err
	}
	log.Printf("rbdMap(%q) => %q", name, *device)
	if out, _, err := sh(fmt.Sprintf("mount %s %s", *device, target)); err != nil {
		log.Print(string(out))
		return nil, err
	}
	return device, nil
}

func (d *cephDriver) unmountVolume(target, device string) error {
	log.Printf("unmountVolume(%q, %q)", target, device)
	cmd := fmt.Sprintf("umount %s", target)
	if out, _, err := sh(cmd); err != nil {
		log.Print(string(out))
		return err
	}
	return rbdUnmap(device)
}
<commit_msg>Update driver.go<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"

	"github.com/calavera/dkvolume"

	"github.com/noahdesu/go-ceph/rados"
	"github.com/noahdesu/go-ceph/rbd"
)

type volume struct {
	name        string
	device		string
	connections int
}

type cephDriver struct {
	root       	string
	pool		string
	volumes    	map[string]*volume
	m          	sync.Mutex
}

func newCephDriver(root, pool string) cephDriver {
	d := cephDriver{
		root:   root,
		pool:	pool,
		volumes: map[string]*volume{},
	}
	return d
}

func checkFs(device string) (string, error) {
	log.Printf("blkid %s", device)
 	out, err := exec.Command("blkid", device).Output()
 	if out != nil {
	 	sout := strings.TrimRight(string(out), "\n")
		return sout, err
 	}
 	return "", err
}

func ensureFs(name string) dkvolume.Response {
	log.Printf("ensureFs(%q)", name)
	device, err := rbdMap(name)
	if err != nil {
		return dkvolume.Response{Err: err.Error()}
	}
	defer rbdUnmap(*device)

	t, err := checkFs(*device)
	log.Printf("t => %q", t)
	if !strings.ContainsAny(t, "TYPE=\"xfs\"") {
		log.Printf("Formatting %s", *device)
		out, _, err := sh(fmt.Sprintf("mkfs.xfs -f %s", *device))
		if err != nil {
			return dkvolume.Response{Err: err.Error()}
		}
		log.Printf(string(out))
	}
	return dkvolume.Response{}
}

func rbdExists(name string) (bool, error) {
	conn, err := rados.NewConn()
	if err != nil {
		log.Fatal(err)
		return false, err
	}
	conn.ReadDefaultConfigFile()
	conn.Connect()
	defer conn.Shutdown()

	ioContext, err := conn.OpenIOContext("rbd")
	if err != nil {
		log.Fatal(err)
		return false, err
	}
	defer ioContext.Destroy()

	volumes, err := rbd.GetImageNames(ioContext)
	if err != nil {
		log.Fatal(err)
		return false, err
	}
	for _, volumeName := range volumes {
		if name == volumeName {
			return true, nil
		}
	}
	return false, nil
}

func (d cephDriver) Create(r dkvolume.Request) dkvolume.Response {
	log.Printf("Create(%q)", r.Name)
	d.m.Lock()
	defer d.m.Unlock()
	name := r.Name
	mountpoint := d.mountpoint(name)

	if _, ok := d.volumes[mountpoint]; ok {
		return dkvolume.Response{}
	}

	exists, err := rbdExists(name)
	if err != nil {
		return dkvolume.Response{Err: err.Error()}
	}

	if !exists {
		if out, _, err := sh(fmt.Sprintf("rbd create --size 512 %s", name)); err != nil {
			log.Print(string(out))
			return dkvolume.Response{Err: err.Error()}
		}
	}

	return ensureFs(name)
}

func (d cephDriver) Remove(r dkvolume.Request) dkvolume.Response {
	log.Printf("Remove(%q)", r.Name)
	d.m.Lock()
	defer d.m.Unlock()
	name := r.Name
	mountpoint := d.mountpoint(name)

	if volume, ok := d.volumes[mountpoint]; ok {
		if volume.connections <= 1 {
			cmd := fmt.Sprintf("rbd rm %s", name)
			if out, _, err := sh(cmd); err != nil {
				log.Print(string(out))
				return dkvolume.Response{Err: err.Error()}
			}
			delete(d.volumes, mountpoint)
		}
	}
	return dkvolume.Response{}
}

func (d cephDriver) Path(r dkvolume.Request) dkvolume.Response {
	return dkvolume.Response{Mountpoint: d.mountpoint(r.Name)}
}

func (d cephDriver) Mount(r dkvolume.Request) dkvolume.Response {
	log.Printf("Mount(%q)", r.Name)
	d.m.Lock()
	defer d.m.Unlock()
	mountpoint := d.mountpoint(r.Name)
	log.Printf("mountpoint(%q) => %q", r.Name, mountpoint)

	vol, ok := d.volumes[mountpoint]
	if ok && vol.connections > 0 {
		vol.connections++
		return dkvolume.Response{Mountpoint: mountpoint}
	}

	fi, err := os.Lstat(mountpoint)

	if os.IsNotExist(err) {
		if err := os.MkdirAll(mountpoint, 0755); err != nil {
			return dkvolume.Response{Err: err.Error()}
		}
	} else if err != nil {
		return dkvolume.Response{Err: err.Error()}
	}

	if fi != nil && !fi.IsDir() {
		return dkvolume.Response{Err: fmt.Sprintf("%v already exist and it's not a directory", mountpoint)}
	}

	device, err := d.mountVolume(r.Name, mountpoint);
	if err != nil {
		return dkvolume.Response{Err: err.Error()}
	}

	d.volumes[mountpoint] = &volume{name: r.Name, device: *device, connections: 1}

	return dkvolume.Response{Mountpoint: mountpoint}
}

func (d cephDriver) Unmount(r dkvolume.Request) dkvolume.Response {
	log.Printf("Unmount(%q)\n", r.Name)
	d.m.Lock()
	defer d.m.Unlock()
	mountpoint := d.mountpoint(r.Name)
	log.Printf("mountpoint(%q) => %s\n", r.Name, mountpoint)

	if volume, ok := d.volumes[mountpoint]; ok {
		if volume.connections == 1 {
			if err := d.unmountVolume(mountpoint, volume.device); err != nil {
				return dkvolume.Response{Err: err.Error()}
			}
		}
		volume.connections--
	} else {
		return dkvolume.Response{Err: fmt.Sprintf("Unable to find volume mounted on %s", mountpoint)}
	}

	return dkvolume.Response{}
}

func (d *cephDriver) mountpoint(name string) string {
	return filepath.Join(d.root, name)
}

func execCommand(name string, arg... string) ([]byte, []byte, error) {
 	cmd := exec.Command(name, arg...)
	stdout, err := cmd.StdoutPipe(); if err != nil {
		return nil, nil, err
	}
 	stderr, err := cmd.StderrPipe()
	if err != nil {
		return nil, nil, err
	}
	if err := cmd.Start(); err != nil {
		return nil, nil, err
	}
	sout, err := ioutil.ReadAll(stdout); if err != nil {
		return nil, nil, err
	}
	serr, err := ioutil.ReadAll(stderr); if err != nil {
		return nil, nil, err
	}
	if err := cmd.Wait(); err != nil {
		return nil, nil, err
	}
	return sout, serr, nil
}

func sh(s string) ([]byte, []byte, error) {
	log.Print(s)
	return execCommand("sh", "-c", s)
}

func rbdMap(name string) (*string, error) {
	log.Printf("rbdMap(%q)", name)
	out, _, err := sh(fmt.Sprintf("rbd map %s", name))
	if err != nil {
		log.Print(string(out))
		return nil, err
	}
	out1, _, err := sh(fmt.Sprintf("rbd showmapped |grep %s| awk 'NR==1 {print $5}'", name))
	if err != nil {
		log.Print(string(out1))
		return nil, err
	}

	device := strings.TrimRight(string(out1), "\n")
	log.Printf("rbd map %s => %q", name, device)
	return &device, nil
}

func rbdUnmap(device string) error {
	log.Printf("rbdUnmap(%q)", device)
	if out, _, err := sh(fmt.Sprintf("rbd unmap %s", device)); err != nil {
		log.Print(string(out))
		return err
	}
	return nil
}

func (d *cephDriver) mountVolume(name, target string) (*string, error) {
	log.Printf("mountVolume(%q, %q)", name, target)
	device, err := rbdMap(name)
	if err != nil {
		log.Fatal(err)
		return nil, err
	}
	log.Printf("rbdMap(%q) => %q", name, *device)
	if out, _, err := sh(fmt.Sprintf("mount %s %s", *device, target)); err != nil {
		log.Print(string(out))
		return nil, err
	}
	return device, nil
}

func (d *cephDriver) unmountVolume(target, device string) error {
	log.Printf("unmountVolume(%q, %q)", target, device)
	cmd := fmt.Sprintf("umount %s", target)
	if out, _, err := sh(cmd); err != nil {
		log.Print(string(out))
		return err
	}
	return rbdUnmap(device)
}
<|endoftext|>