<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/dotcloud/docker"
	dclient "github.com/fsouza/go-dockerclient"
	"github.com/globocom/config"
	"github.com/globocom/docker-cluster/cluster"
	"github.com/globocom/docker-cluster/storage"
	"github.com/globocom/tsuru/action"
	"github.com/globocom/tsuru/fs"
	"github.com/globocom/tsuru/log"
	"github.com/globocom/tsuru/provision"
	"io"
	"labix.org/v2/mgo/bson"
	"strings"
	"sync"
)

var (
	dCluster *cluster.Cluster
	cmutext  sync.Mutex
	fsystem  fs.Fs
)

func dockerCluster() *cluster.Cluster {
	cmutext.Lock()
	defer cmutext.Unlock()
	if dCluster == nil {
		servers, _ := config.GetList("docker:servers")
		if len(servers) < 1 {
			log.Fatal(`Tsuru is misconfigured. Setting "docker:servers" is mandatory`)
		}
		nodes := []cluster.Node{}
		for index, server := range servers {
			node := cluster.Node{
				ID:      fmt.Sprintf("server%d", index),
				Address: server,
			}
			nodes = append(nodes, node)
		}
		if segregate, _ := config.GetBool("docker:segregate"); segregate {
			var scheduler segregatedScheduler
			dCluster, _ = cluster.New(&scheduler, nodes...)
		} else {
			dCluster, _ = cluster.New(nil, nodes...)
		}
		if redisServer, err := config.GetString("docker:scheduler:redis-server"); err == nil {
			if password, err := config.GetString("docker:scheduler:redis-password"); err == nil {
				dCluster.SetStorage(storage.AuthenticatedRedis(redisServer, password))
			} else {
				dCluster.SetStorage(storage.Redis(redisServer))
			}
		}
	}
	return dCluster
}

func filesystem() fs.Fs {
	if fsystem == nil {
		fsystem = fs.OsFs{}
	}
	return fsystem
}

// runCmd executes commands and log the given stdout and stderror.
func runCmd(cmd string, args ...string) (string, error) {
	out := bytes.Buffer{}
	err := executor().Execute(cmd, args, nil, &out, &out)
	log.Printf("running the cmd: %s with the args: %s", cmd, args)
	if err != nil {
		return "", &cmdError{cmd: cmd, args: args, err: err, out: out.String()}
	}
	return out.String(), nil
}

func getPort() (string, error) {
	return config.GetString("docker:run-cmd:port")
}

type container struct {
	ID       string `bson:"_id"`
	AppName  string
	Type     string
	IP       string
	Port     string
	HostPort string
	Status   string
	Version  string
	Image    string
}

func (c *container) getAddress() string {
	hostAddr, err := config.Get("docker:host-address")
	if err != nil {
		log.Printf("Failed to obtain container address: %s", err.Error())
		return ""
	}
	return fmt.Sprintf("http://%s:%s", hostAddr, c.HostPort)
}

// newContainer creates a new container in Docker and stores it in the database.
func newContainer(app provision.App, imageId string, cmds []string) (container, error) {
	cont := container{
		AppName: app.GetName(),
		Type:    app.GetPlatform(),
	}
	port, err := getPort()
	if err != nil {
		log.Printf("error on getting port for container %s - %s", cont.AppName, port)
		return container{}, err
	}
	config := docker.Config{
		Image:        imageId,
		Cmd:          cmds,
		PortSpecs:    []string{port},
		AttachStdin:  false,
		AttachStdout: false,
		AttachStderr: false,
	}
	c, err := dockerCluster().CreateContainer(&config)
	if err != nil {
		log.Printf("error on creating container in docker %s - %s", cont.AppName, err.Error())
		return container{}, err
	}
	cont.ID = c.ID
	cont.Port = port
	return cont, nil
}

// hostPort returns the host port mapped for the container.
func (c *container) hostPort() (string, error) {
	if c.Port == "" {
		return "", errors.New("Container does not contain any mapped port")
	}
	dockerContainer, err := dockerCluster().InspectContainer(c.ID)
	if err != nil {
		return "", err
	}
	if dockerContainer.NetworkSettings != nil {
		mappedPorts := dockerContainer.NetworkSettings.PortMapping
		if port, ok := mappedPorts["Tcp"][c.Port]; ok {
			return port, nil
		}
	}
	return "", fmt.Errorf("Container port %s is not mapped to any host port", c.Port)
}

// ip returns the ip for the container.
func (c *container) ip() (string, error) {
	dockerContainer, err := dockerCluster().InspectContainer(c.ID)
	if err != nil {
		return "", err
	}
	if dockerContainer.NetworkSettings == nil {
		msg := "Error when getting container information. NetworkSettings is missing."
		log.Print(msg)
		return "", errors.New(msg)
	}
	instanceIP := dockerContainer.NetworkSettings.IPAddress
	if instanceIP == "" {
		msg := "error: Can't get ipaddress..."
		log.Print(msg)
		return "", errors.New(msg)
	}
	log.Printf("Instance IpAddress: %s", instanceIP)
	return instanceIP, nil
}

func (c *container) setStatus(status string) error {
	c.Status = status
	coll := collection()
	defer coll.Database.Session.Close()
	return coll.UpdateId(c.ID, c)
}

func (c *container) setImage(imageId string) error {
	c.Image = imageId
	coll := collection()
	defer coll.Database.Session.Close()
	return coll.UpdateId(c.ID, c)
}

func deploy(app provision.App, version string, w io.Writer) (string, error) {
	commands, err := deployCmds(app, version)
	if err != nil {
		return "", err
	}
	imageId := getImage(app)
	actions := []*action.Action{&createContainer, &startContainer, &insertContainer}
	pipeline := action.NewPipeline(actions...)
	err = pipeline.Execute(app, imageId, commands)
	if err != nil {
		log.Printf("error on execute deploy pipeline for app %s - %s", app.GetName(), err.Error())
		return "", err
	}
	c := pipeline.Result().(container)
	for {
		result, err := c.stopped()
		if err != nil {
			log.Printf("error on stopped for container %s - %s", c.ID, err.Error())
			return "", err
		}
		if result {
			break
		}
	}
	err = c.logs(w)
	if err != nil {
		log.Printf("error on get logs for container %s - %s", c.ID, err.Error())
		return "", err
	}
	imageId, err = c.commit()
	if err != nil {
		log.Printf("error on commit container %s - %s", c.ID, err.Error())
		return "", err
	}
	c.remove()
	return imageId, nil
}

func start(app provision.App, imageId string, w io.Writer) (*container, error) {
	commands, err := runCmds()
	if err != nil {
		return nil, err
	}
	actions := []*action.Action{&createContainer, &startContainer, &setIp, &setHostPort, &insertContainer, &addRoute}
	pipeline := action.NewPipeline(actions...)
	err = pipeline.Execute(app, imageId, commands)
	if err != nil {
		return nil, err
	}
	c := pipeline.Result().(container)
	err = c.setImage(imageId)
	if err != nil {
		return nil, err
	}
	err = c.setStatus("running")
	if err != nil {
		return nil, err
	}
	return &c, nil
}

// remove removes a docker container.
func (c *container) remove() error {
	address := c.getAddress()
	log.Printf("Removing container %s from docker", c.ID)
	err := dockerCluster().RemoveContainer(c.ID)
	if err != nil {
		log.Printf("Failed to remove container from docker: %s", err)
	}
	runCmd("ssh-keygen", "-R", c.IP)
	log.Printf("Removing container %s from database", c.ID)
	coll := collection()
	defer coll.Database.Session.Close()
	if err := coll.RemoveId(c.ID); err != nil {
		log.Printf("Failed to remove container from database: %s", err)
	}
	r, err := getRouter()
	if err != nil {
		log.Printf("Failed to obtain router: %s", err)
	}
	if err := r.RemoveRoute(c.AppName, address); err != nil {
		log.Printf("Failed to remove route: %s", err)
	}
	return nil
}

func (c *container) ssh(stdout, stderr io.Writer, cmd string, args ...string) error {
	stderr = &filter{w: stderr, content: []byte("unable to resolve host")}
	user, err := config.GetString("docker:ssh:user")
	if err != nil {
		return err
	}
	sshArgs := []string{c.IP, "-l", user, "-o", "StrictHostKeyChecking no"}
	if keyFile, err := config.GetString("docker:ssh:private-key"); err == nil {
		sshArgs = append(sshArgs, "-i", keyFile)
	}
	sshArgs = append(sshArgs, "--", cmd)
	sshArgs = append(sshArgs, args...)
	return executor().Execute("ssh", sshArgs, nil, stdout, stderr)
}

// commit commits an image in docker based in the container
// and returns the image repository.
func (c *container) commit() (string, error) {
	log.Printf("commiting container %s", c.ID)
	repository := buildImageName(c.AppName)
	opts := dclient.CommitContainerOptions{Container: c.ID, Repository: repository}
	image, err := dockerCluster().CommitContainer(opts)
	if err != nil {
		log.Printf("Could not commit docker image: %s", err.Error())
		return "", err
	}
	log.Printf("image %s generated from container %s", image.ID, c.ID)
	replicateImage(repository)
	return repository, nil
}

// stopped returns true if the container is stopped.
func (c *container) stopped() (bool, error) {
	dockerContainer, err := dockerCluster().InspectContainer(c.ID)
	if err != nil {
		log.Printf("error on get log for container %s: %s", c.ID, err)
		return false, err
	}
	return !dockerContainer.State.Running, nil
}

// stop stops the container.
func (c *container) stop() error {
	err := dockerCluster().StopContainer(c.ID, 10)
	if err != nil {
		log.Printf("error on stop container %s: %s", c.ID, err)
	}
	return err
}

// logs returns logs for the container.
func (c *container) logs(w io.Writer) error {
	opts := dclient.AttachToContainerOptions{
		Container:    c.ID,
		Logs:         true,
		Stdout:       true,
		OutputStream: w,
	}
	err := dockerCluster().AttachToContainer(opts)
	if err != nil {
		return err
	}
	opts = dclient.AttachToContainerOptions{
		Container:    c.ID,
		Logs:         true,
		Stderr:       true,
		OutputStream: w,
	}
	return dockerCluster().AttachToContainer(opts)
}

func getContainer(id string) (*container, error) {
	var c container
	coll := collection()
	defer coll.Database.Session.Close()
	err := coll.Find(bson.M{"_id": id}).One(&c)
	if err != nil {
		return nil, err
	}
	return &c, nil
}

func listAppContainers(appName string) ([]container, error) {
	var containers []container
	err := collection().Find(bson.M{"appname": appName}).All(&containers)
	return containers, err
}

// getImage returns the image name or id from an app.
func getImage(app provision.App) string {
	var c container
	collection().Find(bson.M{"appname": app.GetName()}).One(&c)
	if c.Image != "" {
		return c.Image
	}
	repoNamespace, err := config.GetString("docker:repository-namespace")
	if err != nil {
		return ""
	}
	return fmt.Sprintf("%s/%s", repoNamespace, app.GetPlatform())
}

// removeImage removes an image from docker registry
func removeImage(imageId string) error {
	return dockerCluster().RemoveImage(imageId)
}

type cmdError struct {
	cmd  string
	args []string
	err  error
	out  string
}

func (e *cmdError) Error() string {
	command := e.cmd + " " + strings.Join(e.args, " ")
	return fmt.Sprintf("Failed to run command %q (%s): %s.", command, e.err, e.out)
}

// replicateImage replicates the given image through all nodes in the cluster.
func replicateImage(name string) error {
	var buf bytes.Buffer
	if registry, err := config.GetString("docker:registry"); err == nil {
		if !strings.HasPrefix(name, registry) {
			name = registry + "/" + name
		}
		pushOpts := dclient.PushImageOptions{Name: name}
		err := dockerCluster().PushImage(pushOpts, &buf)
		if err != nil {
			log.Printf("[docker] Failed to push image %q (%s): %s", name, err, buf.String())
			return err
		}
		buf.Reset()
		pullOpts := dclient.PullImageOptions{Repository: name, Registry: registry}
		err = dockerCluster().PullImage(pullOpts, &buf)
		if err != nil {
			log.Printf("[docker] Failed to replicate image %q through nodes (%s): %s", name, err, buf.String())
			return err
		}
	}
	return nil
}

func buildImageName(appName string) string {
	parts := make([]string, 0, 3)
	registry, _ := config.GetString("docker:registry")
	if registry != "" {
		parts = append(parts, registry)
	}
	repoNamespace, _ := config.GetString("docker:repository-namespace")
	parts = append(parts, repoNamespace, appName)
	return strings.Join(parts, "/")
}
<commit_msg>provision/docker: add a loop to try to push multiple times<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/dotcloud/docker"
	dclient "github.com/fsouza/go-dockerclient"
	"github.com/globocom/config"
	"github.com/globocom/docker-cluster/cluster"
	"github.com/globocom/docker-cluster/storage"
	"github.com/globocom/tsuru/action"
	"github.com/globocom/tsuru/fs"
	"github.com/globocom/tsuru/log"
	"github.com/globocom/tsuru/provision"
	"io"
	"labix.org/v2/mgo/bson"
	"strings"
	"sync"
)

var (
	dCluster *cluster.Cluster
	cmutext  sync.Mutex
	fsystem  fs.Fs
)

const pushTries = 5

func dockerCluster() *cluster.Cluster {
	cmutext.Lock()
	defer cmutext.Unlock()
	if dCluster == nil {
		servers, _ := config.GetList("docker:servers")
		if len(servers) < 1 {
			log.Fatal(`Tsuru is misconfigured. Setting "docker:servers" is mandatory`)
		}
		nodes := []cluster.Node{}
		for index, server := range servers {
			node := cluster.Node{
				ID:      fmt.Sprintf("server%d", index),
				Address: server,
			}
			nodes = append(nodes, node)
		}
		if segregate, _ := config.GetBool("docker:segregate"); segregate {
			var scheduler segregatedScheduler
			dCluster, _ = cluster.New(&scheduler, nodes...)
		} else {
			dCluster, _ = cluster.New(nil, nodes...)
		}
		if redisServer, err := config.GetString("docker:scheduler:redis-server"); err == nil {
			if password, err := config.GetString("docker:scheduler:redis-password"); err == nil {
				dCluster.SetStorage(storage.AuthenticatedRedis(redisServer, password))
			} else {
				dCluster.SetStorage(storage.Redis(redisServer))
			}
		}
	}
	return dCluster
}

func filesystem() fs.Fs {
	if fsystem == nil {
		fsystem = fs.OsFs{}
	}
	return fsystem
}

// runCmd executes commands and log the given stdout and stderror.
func runCmd(cmd string, args ...string) (string, error) {
	out := bytes.Buffer{}
	err := executor().Execute(cmd, args, nil, &out, &out)
	log.Printf("running the cmd: %s with the args: %s", cmd, args)
	if err != nil {
		return "", &cmdError{cmd: cmd, args: args, err: err, out: out.String()}
	}
	return out.String(), nil
}

func getPort() (string, error) {
	return config.GetString("docker:run-cmd:port")
}

type container struct {
	ID       string `bson:"_id"`
	AppName  string
	Type     string
	IP       string
	Port     string
	HostPort string
	Status   string
	Version  string
	Image    string
}

func (c *container) getAddress() string {
	hostAddr, err := config.Get("docker:host-address")
	if err != nil {
		log.Printf("Failed to obtain container address: %s", err.Error())
		return ""
	}
	return fmt.Sprintf("http://%s:%s", hostAddr, c.HostPort)
}

// newContainer creates a new container in Docker and stores it in the database.
func newContainer(app provision.App, imageId string, cmds []string) (container, error) {
	cont := container{
		AppName: app.GetName(),
		Type:    app.GetPlatform(),
	}
	port, err := getPort()
	if err != nil {
		log.Printf("error on getting port for container %s - %s", cont.AppName, port)
		return container{}, err
	}
	config := docker.Config{
		Image:        imageId,
		Cmd:          cmds,
		PortSpecs:    []string{port},
		AttachStdin:  false,
		AttachStdout: false,
		AttachStderr: false,
	}
	c, err := dockerCluster().CreateContainer(&config)
	if err != nil {
		log.Printf("error on creating container in docker %s - %s", cont.AppName, err.Error())
		return container{}, err
	}
	cont.ID = c.ID
	cont.Port = port
	return cont, nil
}

// hostPort returns the host port mapped for the container.
func (c *container) hostPort() (string, error) {
	if c.Port == "" {
		return "", errors.New("Container does not contain any mapped port")
	}
	dockerContainer, err := dockerCluster().InspectContainer(c.ID)
	if err != nil {
		return "", err
	}
	if dockerContainer.NetworkSettings != nil {
		mappedPorts := dockerContainer.NetworkSettings.PortMapping
		if port, ok := mappedPorts["Tcp"][c.Port]; ok {
			return port, nil
		}
	}
	return "", fmt.Errorf("Container port %s is not mapped to any host port", c.Port)
}

// ip returns the ip for the container.
func (c *container) ip() (string, error) {
	dockerContainer, err := dockerCluster().InspectContainer(c.ID)
	if err != nil {
		return "", err
	}
	if dockerContainer.NetworkSettings == nil {
		msg := "Error when getting container information. NetworkSettings is missing."
		log.Print(msg)
		return "", errors.New(msg)
	}
	instanceIP := dockerContainer.NetworkSettings.IPAddress
	if instanceIP == "" {
		msg := "error: Can't get ipaddress..."
		log.Print(msg)
		return "", errors.New(msg)
	}
	log.Printf("Instance IpAddress: %s", instanceIP)
	return instanceIP, nil
}

func (c *container) setStatus(status string) error {
	c.Status = status
	coll := collection()
	defer coll.Database.Session.Close()
	return coll.UpdateId(c.ID, c)
}

func (c *container) setImage(imageId string) error {
	c.Image = imageId
	coll := collection()
	defer coll.Database.Session.Close()
	return coll.UpdateId(c.ID, c)
}

func deploy(app provision.App, version string, w io.Writer) (string, error) {
	commands, err := deployCmds(app, version)
	if err != nil {
		return "", err
	}
	imageId := getImage(app)
	actions := []*action.Action{&createContainer, &startContainer, &insertContainer}
	pipeline := action.NewPipeline(actions...)
	err = pipeline.Execute(app, imageId, commands)
	if err != nil {
		log.Printf("error on execute deploy pipeline for app %s - %s", app.GetName(), err.Error())
		return "", err
	}
	c := pipeline.Result().(container)
	for {
		result, err := c.stopped()
		if err != nil {
			log.Printf("error on stopped for container %s - %s", c.ID, err.Error())
			return "", err
		}
		if result {
			break
		}
	}
	err = c.logs(w)
	if err != nil {
		log.Printf("error on get logs for container %s - %s", c.ID, err.Error())
		return "", err
	}
	imageId, err = c.commit()
	if err != nil {
		log.Printf("error on commit container %s - %s", c.ID, err.Error())
		return "", err
	}
	c.remove()
	return imageId, nil
}

func start(app provision.App, imageId string, w io.Writer) (*container, error) {
	commands, err := runCmds()
	if err != nil {
		return nil, err
	}
	actions := []*action.Action{&createContainer, &startContainer, &setIp, &setHostPort, &insertContainer, &addRoute}
	pipeline := action.NewPipeline(actions...)
	err = pipeline.Execute(app, imageId, commands)
	if err != nil {
		return nil, err
	}
	c := pipeline.Result().(container)
	err = c.setImage(imageId)
	if err != nil {
		return nil, err
	}
	err = c.setStatus("running")
	if err != nil {
		return nil, err
	}
	return &c, nil
}

// remove removes a docker container.
func (c *container) remove() error {
	address := c.getAddress()
	log.Printf("Removing container %s from docker", c.ID)
	err := dockerCluster().RemoveContainer(c.ID)
	if err != nil {
		log.Printf("Failed to remove container from docker: %s", err)
	}
	runCmd("ssh-keygen", "-R", c.IP)
	log.Printf("Removing container %s from database", c.ID)
	coll := collection()
	defer coll.Database.Session.Close()
	if err := coll.RemoveId(c.ID); err != nil {
		log.Printf("Failed to remove container from database: %s", err)
	}
	r, err := getRouter()
	if err != nil {
		log.Printf("Failed to obtain router: %s", err)
	}
	if err := r.RemoveRoute(c.AppName, address); err != nil {
		log.Printf("Failed to remove route: %s", err)
	}
	return nil
}

func (c *container) ssh(stdout, stderr io.Writer, cmd string, args ...string) error {
	stderr = &filter{w: stderr, content: []byte("unable to resolve host")}
	user, err := config.GetString("docker:ssh:user")
	if err != nil {
		return err
	}
	sshArgs := []string{c.IP, "-l", user, "-o", "StrictHostKeyChecking no"}
	if keyFile, err := config.GetString("docker:ssh:private-key"); err == nil {
		sshArgs = append(sshArgs, "-i", keyFile)
	}
	sshArgs = append(sshArgs, "--", cmd)
	sshArgs = append(sshArgs, args...)
	return executor().Execute("ssh", sshArgs, nil, stdout, stderr)
}

// commit commits an image in docker based in the container
// and returns the image repository.
func (c *container) commit() (string, error) {
	log.Printf("commiting container %s", c.ID)
	repository := buildImageName(c.AppName)
	opts := dclient.CommitContainerOptions{Container: c.ID, Repository: repository}
	image, err := dockerCluster().CommitContainer(opts)
	if err != nil {
		log.Printf("Could not commit docker image: %s", err.Error())
		return "", err
	}
	log.Printf("image %s generated from container %s", image.ID, c.ID)
	replicateImage(repository)
	return repository, nil
}

// stopped returns true if the container is stopped.
func (c *container) stopped() (bool, error) {
	dockerContainer, err := dockerCluster().InspectContainer(c.ID)
	if err != nil {
		log.Printf("error on get log for container %s: %s", c.ID, err)
		return false, err
	}
	return !dockerContainer.State.Running, nil
}

// stop stops the container.
func (c *container) stop() error {
	err := dockerCluster().StopContainer(c.ID, 10)
	if err != nil {
		log.Printf("error on stop container %s: %s", c.ID, err)
	}
	return err
}

// logs returns logs for the container.
func (c *container) logs(w io.Writer) error {
	opts := dclient.AttachToContainerOptions{
		Container:    c.ID,
		Logs:         true,
		Stdout:       true,
		OutputStream: w,
	}
	err := dockerCluster().AttachToContainer(opts)
	if err != nil {
		return err
	}
	opts = dclient.AttachToContainerOptions{
		Container:    c.ID,
		Logs:         true,
		Stderr:       true,
		OutputStream: w,
	}
	return dockerCluster().AttachToContainer(opts)
}

func getContainer(id string) (*container, error) {
	var c container
	coll := collection()
	defer coll.Database.Session.Close()
	err := coll.Find(bson.M{"_id": id}).One(&c)
	if err != nil {
		return nil, err
	}
	return &c, nil
}

func listAppContainers(appName string) ([]container, error) {
	var containers []container
	err := collection().Find(bson.M{"appname": appName}).All(&containers)
	return containers, err
}

// getImage returns the image name or id from an app.
func getImage(app provision.App) string {
	var c container
	collection().Find(bson.M{"appname": app.GetName()}).One(&c)
	if c.Image != "" {
		return c.Image
	}
	repoNamespace, err := config.GetString("docker:repository-namespace")
	if err != nil {
		return ""
	}
	return fmt.Sprintf("%s/%s", repoNamespace, app.GetPlatform())
}

// removeImage removes an image from docker registry
func removeImage(imageId string) error {
	return dockerCluster().RemoveImage(imageId)
}

type cmdError struct {
	cmd  string
	args []string
	err  error
	out  string
}

func (e *cmdError) Error() string {
	command := e.cmd + " " + strings.Join(e.args, " ")
	return fmt.Sprintf("Failed to run command %q (%s): %s.", command, e.err, e.out)
}

// replicateImage replicates the given image through all nodes in the cluster.
func replicateImage(name string) error {
	var buf bytes.Buffer
	if registry, err := config.GetString("docker:registry"); err == nil {
		if !strings.HasPrefix(name, registry) {
			name = registry + "/" + name
		}
		pushOpts := dclient.PushImageOptions{Name: name}
		for i := 0; i < pushMaxTry; i++ {
			err := dockerCluster().PushImage(pushOpts, &buf)
			if err == nil {
				buf.Reset()
				break
			}
			log.Printf("[docker] Failed to push image %q (%s): %s", name, err, buf.String())
			buf.Reset()
		}
		pullOpts := dclient.PullImageOptions{Repository: name}
		err = dockerCluster().PullImage(pullOpts, &buf)
		if err != nil {
			log.Printf("[docker] Failed to replicate image %q through nodes (%s): %s", name, err, buf.String())
			return err
		}
	}
	return nil
}

func buildImageName(appName string) string {
	parts := make([]string, 0, 3)
	registry, _ := config.GetString("docker:registry")
	if registry != "" {
		parts = append(parts, registry)
	}
	repoNamespace, _ := config.GetString("docker:repository-namespace")
	parts = append(parts, repoNamespace, appName)
	return strings.Join(parts, "/")
}
<|endoftext|>


<commit_before>
// Copyright (c) James Percent and Unlock contributors.
// All rights reserved.
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//    1. Redistributions of source code must retain the above copyright notice,
//       this list of conditions and the following disclaimer.
//
//    2. Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in the
//       documentation and/or other materials provided with the distribution.
//
//    3. Neither the name of Unlock nor the names of its contributors may be used
//       to endorse or promote products derived from this software without
//       specific prior written permission.

// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package main

import (
    "npl/bu/edu/unzip"
    "net/http"
    "fmt"
    "io/ioutil"
    "flag"
    "os"
    "os/exec"
    "log"
    "path/filepath"
    "io"
)

func runCommand(command string, errorMsg string, failOnError bool) bool {
    result := true
    cmd := exec.Command("cmd", "/C", command) // note the mixing of ` and "; my editor `\` are not friends
    if err := cmd.Run(); err != nil {
        if failOnError == true {
            log.Fatalln(`FATAL: `+errorMsg+`; command = `+command, err)
        } else {
            log.Println(`Non-fatal `+errorMsg+`; command = `+command, err)
        }
        result = false
    }
    return result
}

func install(command string, packageName string, failOnError bool) {
    log.Print(`Installing `+packageName+`: `)
    log.Println("command = "+ command)
    result := runCommand(command, `Failed to install `+packageName, failOnError)
    if result {
        fmt.Println(`Success`)
    } else {
        fmt.Println(`Failure`)
    }
}

func chdirFailOnError(directory string, errorMsg string) {
    if err := os.Chdir(directory); err != nil {
        log.Fatalln(`install-win.chdirFailOnError: ERROR: Change directory to `+directory+` failed: `+errorMsg, err)
    }
}

func unzipExpand(fileName string) {
    u := &unzip.Unzip{fileName, ``, nil}
    if err := u.Expand(); err != nil {
        log.Fatalln(`Failed to expand `+fileName, err)
    }
}

func downloadAndWriteFile(url string, fileName string) string {
    fullPath := filepath.Join(getDownloadDirectory(), fileName)

	isFileExist,_ := checkFileExists(fullPath)

    if isFileExist == false {
        log.Println("Downloading file "+fileName+" from URL = "+url)
        resp, err := http.Get(url)
        if err != nil {
            log.Fatalln(err)
        }

        defer resp.Body.Close()
        body, err1 := ioutil.ReadAll(resp.Body)
        if err1 != nil {
            log.Fatalln(err)
        }

        if err = ioutil.WriteFile(fullPath, body, 0744); err != nil {
            log.Fatalln(err)
        }
    }

    return fullPath
}

func checkFileExists(path string) (bool, error) {
    _, err := os.Stat(path)
    if err == nil { return true, nil }
    if os.IsNotExist(err) { return false, nil }
    return false, err
}

func getDownloadDirectory() string {
    path := getWorkingDirectoryAbsolutePath()

    if *repoPath != `` {
		path = filepath.Join(*repoPath, `package`)
	}

    return path
}

func getWorkingDirectoryAbsolutePath() string {
    cwd, err := filepath.Abs(``)
	log.Println(`Current working directory = `, cwd)
    if err != nil {
        log.Fatalln(err)
    }
    return cwd
}

func installZippedPythonPackage(pythonPath string, baseUrl string, fileName string, packageName string, packageDirectory string) {
    log.Println(`Downloading `+packageName+`... `)
    downloadedFile := downloadAndWriteFile(baseUrl+fileName, fileName)

	unzipExpand(downloadedFile)
    chdirFailOnError(packageDirectory, `Failed to install `+packageName)
    log.Println("CWD = "+getWorkingDirectoryAbsolutePath())
    command := pythonPath+` setup.py install`
    install(command, packageName, true)
    os.Chdir(`..`)
}

func installPython(baseUrl string, pythonPathEnvVar string, pythonInstallerName string, pythonBasePath string, pythonPackageName string) {
    //cwd := getWorkingDirectoryAbsolutePath()
    log.Println(`Downloading `+pythonPackageName+`...`)
    downloadedFile := downloadAndWriteFile(baseUrl+pythonInstallerName, pythonInstallerName)
    log.Println(`Installing `+pythonPackageName)
//    output, err := exec.Command("cmd", "/C", "msiexec /i ", cwd+"\\"+pythonInstallerName,`TARGETDIR=`+pythonBasePath,`/qb`, `ALLUSERS=0`).CombinedOutput()
    //output, err := exec.Command("cmd", "/C", cwd+"\\"+pythonInstallerName).CombinedOutput()
    output, err := exec.Command("cmd", "/C", downloadedFile).CombinedOutput()
    if len(output) > 0 {
        log.Printf("%s\n", output)
    }

    if err != nil {
        log.Fatalln(`FATAL: failed to install python `, err)
    }

    if err := os.Setenv(`PYTHONPATH`, pythonPathEnvVar); err != nil {
        log.Println(`Could not properly set the PYTHONPATH env variable; on some systems this can cause problems during virtual env creation`)
    }
    log.Println(`PYTHON PATH = `+os.Getenv(`PYTHONPATH`))
}

func installEasyInstall(baseUrl string, pythonPath string) {
    downloadAndWriteFile(baseUrl+`distribute_setup-py`, `distribute_setup.py`)
    install(pythonPath+` distribute_setup.py`, `easy_install`, true)
}

func installPip(easyInstallPath string) {
    install(easyInstallPath+` pip`, `pip`, true)
}

func installVirtualenv(pipPath string) {
    install(pipPath+` install virtualenv`, `virtualenv`, true)
}

func createVirtualenv(unlockDirectory string, virtualenvPath string, envName string) {
    errorMsg := `Failed to create virtual environment`
    var cwd = getWorkingDirectoryAbsolutePath()
    chdirFailOnError(unlockDirectory, errorMsg)
    command := virtualenvPath+` --system-site-packages `+envName //python27`
    runCommand(command, errorMsg, true)
    os.Chdir(cwd)
}

func installPyglet12alpha(pythonPath string, baseUrl string, fileName string, packageName string, packageDirectory string) {
    installZippedPythonPackage(pythonPath, baseUrl, fileName, packageName, packageDirectory)
}

func installNumPy(baseUrl string, numpyPath string) {
    /*downloadAndWriteFile(baseUrl+numpyPath, numpyPath)//numpy-MKL-1.7.1.win32-py2.7.exe)
    var cwd = getWorkingDirectoryAbsolutePath()
    install(cwd+"\\"+numpyPath, `numpy`, true)*/

    downloadAndInstallBinPackage(baseUrl, numpyPath, `numpy`)
}

func downloadAndInstallBinPackage(baseUrl string, fileName string, packageName string) {
    downloadedFile := downloadAndWriteFile(baseUrl + fileName, fileName)
    install(downloadedFile, packageName, true)
}

func installPySerial26(pythonPath string, baseUrl string, fileName string, packageName string, packageDirectory string) {
    installZippedPythonPackage(pythonPath, baseUrl, fileName, packageName, packageDirectory);
}

func installAvbin(baseUrl string, avbin string) {
    /*downloadAndWriteFile(baseUrl+avbin, avbin)
    var cwd = getWorkingDirectoryAbsolutePath()
    install(cwd+"\\"+avbin, `avbin`, true)*/

    downloadAndInstallBinPackage(baseUrl, avbin, `avbin`)

    // XXX - last minute hack
    data, err1 := ioutil.ReadFile(`C:\Windows\System32\avbin.dll`)
    if err1 != nil {
        log.Fatalln(err1)
    }

    if err := ioutil.WriteFile(`C:\Windows\SysWOW64\avbin.dll`, data, 0744); err != nil {
        log.Println(err)
    }

}

func installScons(pythonPath string, baseUrl string, fileName string, packageName string, packageDirectory string) {
    installZippedPythonPackage(pythonPath, baseUrl, fileName, packageName, packageDirectory);
}

func installUnlock(pythonPath string, baseUrl string, fileName string, packageName string, packageDirectory string) {
    installZippedPythonPackage(pythonPath, baseUrl, fileName, packageName, packageDirectory)
}

var confFile = flag.String("conf", "", "Qualified file name of Unlock installation configuration file")
var devOption = flag.Bool("dev", false, "Setup development env")
var repoPath = flag.String("repo", "", "Path to project's git repo")

func createConf() UnlockInstallConf {
    if *confFile == `` {
        return UnlockInstallConf {`C:\Unlock`, `http://jpercent.org/unlock/`, `C:\Python33;C:\Python33\Lib;C:\Python33\DLLs`, `python-3.3.2.msi`,
            `Python-3.3.2`, `C:\Python33`, `C:\Python33\python.exe`, `numpy-MKL-1.7.1.win32-py3.3.exe`,
            `C:\Python33\Scripts\easy_install.exe`, `C:\Python33\Scripts\pip.exe`,
            `C:\Python33\Scripts\virtualenv.exe`, `python33`,
            `C:\Python33\Lib\site-packages\numpy`, `C:\Unlock\python33\Lib\site-packages`,
            `C:\Unlock\python33\Scripts\python.exe`, `C:\Unlock\python33\Scripts\pip.exe`,
            `pyglet-1.2alpha-p3.zip`, `pyglet-1.2alpha`, `pyglet-1.2alpha1`, `AVbin10-win32.exe`,
            `pyserial-2.6.zip`, `pyserial-2.6`, `pyserial-2.6`, `unlock-0.3.7-win32.zip`, `unlock`, `unlock-0.3.7`,
            `scons-2.3.0.zip`, `scons`, `scons-2.3.0`,
            `unlock.exe`, `vcredist_2010_x86.exe`, `pyaudio-0.2.7.py33.exe`, `pywin32-218.win32-py3.3.exe`}
    } else {
        return ParseConf(*confFile)
    }
}

func numpyHack(pythonPath string, from string, to string) {
    var copydir = "import shutil\n"
    copydir += "import os\n"
    copydir += `shutil.copytree('`+from+`','`+to+`')`+"\n"
    fmt.Println(copydir)
    command := pythonPath+` -c '`+copydir+`'`
    runCommand(command, "numpyHack Failed", false)
}

func installUnlockRunner(baseUrl string, unlockDirectory string, unlockexe string) {
    var cwd = getWorkingDirectoryAbsolutePath()
    chdirFailOnError(unlockDirectory, ` ERROR: Failed to install unlock.exe: couldn't change dir `)
    downloadAndWriteFile(baseUrl+unlockexe, unlockexe)
    os.Chdir(cwd)
}

func main() {

    flag.Parse()
    logf, err := os.OpenFile(`unlock-install.log`, os.O_WRONLY|os.O_APPEND|os.O_CREATE,0640)
    if err != nil {
        log.Fatalln(err)
    }

    log.SetOutput(io.MultiWriter(logf, os.Stdout))
    log.Printf("conf file = "+*confFile)

    var conf = createConf()

    installPython(conf.BaseUrl, conf.PythonPathEnvVar, conf.PythonInstallerName, conf.PythonBasePath, conf.PythonPackageName)
    downloadAndInstallBinPackage(conf.BaseUrl, conf.VCRedistPackageName, `vcredist`)
	installNumPy(conf.BaseUrl, conf.NumpyPackageName)
    //installEasyInstall(conf.BaseUrl, conf.PythonPath)
    //installPip(conf.EasyInstallPath)
    //installVirtualenv(conf.PipPath)
    installAvbin(conf.BaseUrl, conf.Avbin)

    if err := os.MkdirAll(conf.UnlockDirectory, 0755); err != nil {
        log.Fatalln(`Failed to create `+conf.UnlockDirectory, err)
    }
    //createVirtualenv(conf.UnlockDirectory, conf.VirtualenvPath, conf.EnvName)
    // XXX - this is a hack for numpy.  on my machine the virtual env does the right thing, but on other machines it does not.
    //       I found this solution on stackoverflow; its not the best as it does register numpy with pip, but it does work for
    //       now.
    //numpyHack(conf.EnvPythonPath, conf.NumpyHack, conf.NumpyHack1)

    installPyglet12alpha(conf.PythonPath, conf.BaseUrl, conf.PygletZipName, conf.PygletPackageName, conf.PygletDirectory)
    installPySerial26(conf.PythonPath, conf.BaseUrl, conf.PyserialZipName, conf.PyserialPackageName, conf.PyserialDirectory)
    downloadAndInstallBinPackage(conf.BaseUrl, conf.PyAudioPackageName, `pyaudio`)
    downloadAndInstallBinPackage(conf.BaseUrl, conf.PyWinPackageName, `pywin`)

	// Skip install unlock software for development option
	if *devOption == false {
        installUnlock(conf.PythonPath, conf.BaseUrl, conf.UnlockZipName, conf.UnlockPackageName, conf.UnlockPackageDirectory)
        //installScons(conf.PythonPath, conf.BaseUrl, conf.SconsZipName, conf.SconsPackageName, conf.SconsPackageDirectory)
        //installUnlockRunner(conf.BaseUrl, conf.UnlockDirectory, conf.Unlockexe)
	}
}
<commit_msg>Working on file checksum.<commit_after>
// Copyright (c) James Percent and Unlock contributors.
// All rights reserved.
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//    1. Redistributions of source code must retain the above copyright notice,
//       this list of conditions and the following disclaimer.
//
//    2. Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in the
//       documentation and/or other materials provided with the distribution.
//
//    3. Neither the name of Unlock nor the names of its contributors may be used
//       to endorse or promote products derived from this software without
//       specific prior written permission.

// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package main

import (
    "npl/bu/edu/unzip"
    "net/http"
    "fmt"
    "io/ioutil"
    "flag"
    "os"
    "os/exec"
    "log"
    "path/filepath"
    "io"
    "bytes"
    "crypto/sha1"
)

func runCommand(command string, errorMsg string, failOnError bool) bool {
    result := true
    cmd := exec.Command("cmd", "/C", command) // note the mixing of ` and "; my editor `\` are not friends
    if err := cmd.Run(); err != nil {
        if failOnError == true {
            log.Fatalln(`FATAL: `+errorMsg+`; command = `+command, err)
        } else {
            log.Println(`Non-fatal `+errorMsg+`; command = `+command, err)
        }
        result = false
    }
    return result
}

func install(command string, packageName string, failOnError bool) {
    log.Print(`Installing `+packageName+`: `)
    log.Println("command = "+ command)
    result := runCommand(command, `Failed to install `+packageName, failOnError)
    if result {
        fmt.Println(`Success`)
    } else {
        fmt.Println(`Failure`)
    }
}

func chdirFailOnError(directory string, errorMsg string) {
    if err := os.Chdir(directory); err != nil {
        log.Fatalln(`install-win.chdirFailOnError: ERROR: Change directory to `+directory+` failed: `+errorMsg, err)
    }
}

func unzipExpand(fileName string) {
    u := &unzip.Unzip{fileName, ``, nil}
    if err := u.Expand(); err != nil {
        log.Fatalln(`Failed to expand `+fileName, err)
    }
}

func downloadAndWriteFile(fileUrl string, fileName string) string {
    return downloadAndWriteFileWithChecksum(fileUrl, fileName, true)
}

func downloadAndWriteFileWithChecksum(fileUrl string, fileName string, shouldCheckSum bool) string {
    fullPath := filepath.Join(getDownloadDirectory(), fileName)

	//isFileExist,_ := checkFileExists(fullPath)
    isFileGood := false
    if (shouldCheckSum) {
        isFileGood = !checkSum(fullPath, fileUrl+".sha1")
    }

    if /*!isFileExist ||*/ !isFileGood {
        log.Println("Downloading file "+fileName+" from URL = "+fileUrl)
        resp, err := http.Get(fileUrl)
        if err != nil {
            log.Fatalln(err)
        }

        defer resp.Body.Close()
        body, err1 := ioutil.ReadAll(resp.Body)
        if err1 != nil {
            log.Fatalln(err)
        }

        if err = ioutil.WriteFile(fullPath, body, 0744); err != nil {
            log.Fatalln(err)
        }
    }

    return fullPath
}

func checkSum(filePath string, checksumFileUrl string) bool {
    computedHash := computeChecksum(filePath)
    downloadedHash := downloadChecksum(checksumFileUrl)

    return bytes.Compare(computedHash, downloadedHash) == 0
}

func computeChecksum(filePath string) []byte {
    content,err := ioutil.ReadFile(filePath)
    if err != nil { panic(err) }

    s1 := sha1.New()
    s1.Write([]byte(content))
    hashed := s1.Sum(nil)

    log.Println(`Computed checksum: `, hashed)

    return hashed
}

func downloadChecksum(checksumFileUrl string) []byte {
    checksumFile := downloadAndWriteFileWithChecksum(checksumFileUrl, `checksum`, false)

    content,err := ioutil.ReadFile(checksumFile)
    if err != nil { panic(err) }

    log.Println(`Downloaded checksum: `, content)

    return content
}

func checkFileExists(path string) (bool, error) {
    _, err := os.Stat(path)
    if err == nil { return true, nil }
    if os.IsNotExist(err) { return false, nil }
    return false, err
}

func getDownloadDirectory() string {
    path := getWorkingDirectoryAbsolutePath()

    if *repoPath != `` {
		path = filepath.Join(*repoPath, `package`)
	}

    return path
}

func getWorkingDirectoryAbsolutePath() string {
    cwd, err := filepath.Abs(``)
	log.Println(`Current working directory = `, cwd)
    if err != nil {
        log.Fatalln(err)
    }
    return cwd
}

func installZippedPythonPackage(pythonPath string, baseUrl string, fileName string, packageName string, packageDirectory string) {
    log.Println(`Downloading `+packageName+`... `)
    downloadedFile := downloadAndWriteFile(baseUrl+fileName, fileName)

	unzipExpand(downloadedFile)
    chdirFailOnError(packageDirectory, `Failed to install `+packageName)
    log.Println("CWD = "+getWorkingDirectoryAbsolutePath())
    command := pythonPath+` setup.py install`
    install(command, packageName, true)
    os.Chdir(`..`)
}

func installPython(baseUrl string, pythonPathEnvVar string, pythonInstallerName string, pythonBasePath string, pythonPackageName string) {
    //cwd := getWorkingDirectoryAbsolutePath()
    log.Println(`Downloading `+pythonPackageName+`...`)
    downloadedFile := downloadAndWriteFile(baseUrl+pythonInstallerName, pythonInstallerName)
    log.Println(`Installing `+pythonPackageName)
//    output, err := exec.Command("cmd", "/C", "msiexec /i ", cwd+"\\"+pythonInstallerName,`TARGETDIR=`+pythonBasePath,`/qb`, `ALLUSERS=0`).CombinedOutput()
    //output, err := exec.Command("cmd", "/C", cwd+"\\"+pythonInstallerName).CombinedOutput()
    output, err := exec.Command("cmd", "/C", downloadedFile).CombinedOutput()
    if len(output) > 0 {
        log.Printf("%s\n", output)
    }

    if err != nil {
        log.Fatalln(`FATAL: failed to install python `, err)
    }

    if err := os.Setenv(`PYTHONPATH`, pythonPathEnvVar); err != nil {
        log.Println(`Could not properly set the PYTHONPATH env variable; on some systems this can cause problems during virtual env creation`)
    }
    log.Println(`PYTHON PATH = `+os.Getenv(`PYTHONPATH`))
}

func installEasyInstall(baseUrl string, pythonPath string) {
    downloadAndWriteFile(baseUrl+`distribute_setup-py`, `distribute_setup.py`)
    install(pythonPath+` distribute_setup.py`, `easy_install`, true)
}

func installPip(easyInstallPath string) {
    install(easyInstallPath+` pip`, `pip`, true)
}

func installVirtualenv(pipPath string) {
    install(pipPath+` install virtualenv`, `virtualenv`, true)
}

func createVirtualenv(unlockDirectory string, virtualenvPath string, envName string) {
    errorMsg := `Failed to create virtual environment`
    var cwd = getWorkingDirectoryAbsolutePath()
    chdirFailOnError(unlockDirectory, errorMsg)
    command := virtualenvPath+` --system-site-packages `+envName //python27`
    runCommand(command, errorMsg, true)
    os.Chdir(cwd)
}

func installPyglet12alpha(pythonPath string, baseUrl string, fileName string, packageName string, packageDirectory string) {
    installZippedPythonPackage(pythonPath, baseUrl, fileName, packageName, packageDirectory)
}

func installNumPy(baseUrl string, numpyPath string) {
    /*downloadAndWriteFile(baseUrl+numpyPath, numpyPath)//numpy-MKL-1.7.1.win32-py2.7.exe)
    var cwd = getWorkingDirectoryAbsolutePath()
    install(cwd+"\\"+numpyPath, `numpy`, true)*/

    downloadAndInstallBinPackage(baseUrl, numpyPath, `numpy`)
}

func downloadAndInstallBinPackage(baseUrl string, fileName string, packageName string) {
    downloadedFile := downloadAndWriteFile(baseUrl + fileName, fileName)
    install(downloadedFile, packageName, true)
}

func installPySerial26(pythonPath string, baseUrl string, fileName string, packageName string, packageDirectory string) {
    installZippedPythonPackage(pythonPath, baseUrl, fileName, packageName, packageDirectory);
}

func installAvbin(baseUrl string, avbin string) {
    /*downloadAndWriteFile(baseUrl+avbin, avbin)
    var cwd = getWorkingDirectoryAbsolutePath()
    install(cwd+"\\"+avbin, `avbin`, true)*/

    downloadAndInstallBinPackage(baseUrl, avbin, `avbin`)

    // XXX - last minute hack
    data, err1 := ioutil.ReadFile(`C:\Windows\System32\avbin.dll`)
    if err1 != nil {
        log.Fatalln(err1)
    }

    if err := ioutil.WriteFile(`C:\Windows\SysWOW64\avbin.dll`, data, 0744); err != nil {
        log.Println(err)
    }

}

func installScons(pythonPath string, baseUrl string, fileName string, packageName string, packageDirectory string) {
    installZippedPythonPackage(pythonPath, baseUrl, fileName, packageName, packageDirectory);
}

func installUnlock(pythonPath string, baseUrl string, fileName string, packageName string, packageDirectory string) {
    installZippedPythonPackage(pythonPath, baseUrl, fileName, packageName, packageDirectory)
}

var confFile = flag.String("conf", "", "Qualified file name of Unlock installation configuration file")
var devOption = flag.Bool("dev", false, "Setup development env")
var repoPath = flag.String("repo", "", "Path to project's git repo")

func createConf() UnlockInstallConf {
    if *confFile == `` {
        return UnlockInstallConf {`C:\Unlock`, `http://jpercent.org/unlock/`, `C:\Python33;C:\Python33\Lib;C:\Python33\DLLs`, `python-3.3.2.msi`,
            `Python-3.3.2`, `C:\Python33`, `C:\Python33\python.exe`, `numpy-MKL-1.7.1.win32-py3.3.exe`,
            `C:\Python33\Scripts\easy_install.exe`, `C:\Python33\Scripts\pip.exe`,
            `C:\Python33\Scripts\virtualenv.exe`, `python33`,
            `C:\Python33\Lib\site-packages\numpy`, `C:\Unlock\python33\Lib\site-packages`,
            `C:\Unlock\python33\Scripts\python.exe`, `C:\Unlock\python33\Scripts\pip.exe`,
            `pyglet-1.2alpha-p3.zip`, `pyglet-1.2alpha`, `pyglet-1.2alpha1`, `AVbin10-win32.exe`,
            `pyserial-2.6.zip`, `pyserial-2.6`, `pyserial-2.6`, `unlock-0.3.7-win32.zip`, `unlock`, `unlock-0.3.7`,
            `scons-2.3.0.zip`, `scons`, `scons-2.3.0`,
            `unlock.exe`, `vcredist_2010_x86.exe`, `pyaudio-0.2.7.py33.exe`, `pywin32-218.win32-py3.3.exe`}
    } else {
        return ParseConf(*confFile)
    }
}

func numpyHack(pythonPath string, from string, to string) {
    var copydir = "import shutil\n"
    copydir += "import os\n"
    copydir += `shutil.copytree('`+from+`','`+to+`')`+"\n"
    fmt.Println(copydir)
    command := pythonPath+` -c '`+copydir+`'`
    runCommand(command, "numpyHack Failed", false)
}

func installUnlockRunner(baseUrl string, unlockDirectory string, unlockexe string) {
    var cwd = getWorkingDirectoryAbsolutePath()
    chdirFailOnError(unlockDirectory, ` ERROR: Failed to install unlock.exe: couldn't change dir `)
    downloadAndWriteFile(baseUrl+unlockexe, unlockexe)
    os.Chdir(cwd)
}

func main() {

    flag.Parse()
    logf, err := os.OpenFile(`unlock-install.log`, os.O_WRONLY|os.O_APPEND|os.O_CREATE,0640)
    if err != nil {
        log.Fatalln(err)
    }

    log.SetOutput(io.MultiWriter(logf, os.Stdout))
    log.Printf("conf file = "+*confFile)

    var conf = createConf()

    installPython(conf.BaseUrl, conf.PythonPathEnvVar, conf.PythonInstallerName, conf.PythonBasePath, conf.PythonPackageName)
    downloadAndInstallBinPackage(conf.BaseUrl, conf.VCRedistPackageName, `vcredist`)
	installNumPy(conf.BaseUrl, conf.NumpyPackageName)
    //installEasyInstall(conf.BaseUrl, conf.PythonPath)
    //installPip(conf.EasyInstallPath)
    //installVirtualenv(conf.PipPath)
    installAvbin(conf.BaseUrl, conf.Avbin)

    if err := os.MkdirAll(conf.UnlockDirectory, 0755); err != nil {
        log.Fatalln(`Failed to create `+conf.UnlockDirectory, err)
    }
    //createVirtualenv(conf.UnlockDirectory, conf.VirtualenvPath, conf.EnvName)
    // XXX - this is a hack for numpy.  on my machine the virtual env does the right thing, but on other machines it does not.
    //       I found this solution on stackoverflow; its not the best as it does register numpy with pip, but it does work for
    //       now.
    //numpyHack(conf.EnvPythonPath, conf.NumpyHack, conf.NumpyHack1)

    installPyglet12alpha(conf.PythonPath, conf.BaseUrl, conf.PygletZipName, conf.PygletPackageName, conf.PygletDirectory)
    installPySerial26(conf.PythonPath, conf.BaseUrl, conf.PyserialZipName, conf.PyserialPackageName, conf.PyserialDirectory)
    downloadAndInstallBinPackage(conf.BaseUrl, conf.PyAudioPackageName, `pyaudio`)
    downloadAndInstallBinPackage(conf.BaseUrl, conf.PyWinPackageName, `pywin`)

	// Skip install unlock software for development option
	if *devOption == false {
        installUnlock(conf.PythonPath, conf.BaseUrl, conf.UnlockZipName, conf.UnlockPackageName, conf.UnlockPackageDirectory)
        //installScons(conf.PythonPath, conf.BaseUrl, conf.SconsZipName, conf.SconsPackageName, conf.SconsPackageDirectory)
        //installUnlockRunner(conf.BaseUrl, conf.UnlockDirectory, conf.Unlockexe)
	}
}
<|endoftext|>


<commit_before>package fp_reg

import (
	"net"
	"net/http"
	"path"

	"appengine"
	"appengine/urlfetch"
)

const BASE = "https://fp-facilitator.org/reg/"

func robotsTxtHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	w.Write([]byte("User-agent: *\nDisallow:\n"))
}

func ipHandler(w http.ResponseWriter, r *http.Request) {
	remoteAddr := r.RemoteAddr
	if net.ParseIP(remoteAddr).To4() == nil {
		remoteAddr = "[" + remoteAddr + "]"
	}
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	w.Write([]byte(remoteAddr))
}

func regHandler(w http.ResponseWriter, r *http.Request) {
	dir, blob := path.Split(path.Clean(r.URL.Path))
	if dir != "/reg/" {
		http.NotFound(w, r)
		return
	}
	c := appengine.NewContext(r)
	client := urlfetch.Client(c)
	_, err := client.Get(BASE + blob)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Write([]byte("Thanks."))
}

func init() {
	http.HandleFunc("/robots.txt", robotsTxtHandler)
	http.HandleFunc("/ip", ipHandler)
	http.HandleFunc("/reg/", regHandler)
}
<commit_msg>Shorter.<commit_after>package fp_reg

import (
	"net"
	"net/http"
	"path"

	"appengine"
	"appengine/urlfetch"
)

const BASE = "https://fp-facilitator.org/reg/"

func robotsTxtHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	w.Write([]byte("User-agent: *\nDisallow:\n"))
}

func ipHandler(w http.ResponseWriter, r *http.Request) {
	remoteAddr := r.RemoteAddr
	if net.ParseIP(remoteAddr).To4() == nil {
		remoteAddr = "[" + remoteAddr + "]"
	}
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	w.Write([]byte(remoteAddr))
}

func regHandler(w http.ResponseWriter, r *http.Request) {
	dir, blob := path.Split(path.Clean(r.URL.Path))
	if dir != "/reg/" {
		http.NotFound(w, r)
		return
	}
	client := urlfetch.Client(appengine.NewContext(r))
	resp, err := client.Get(BASE + blob)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Write([]byte("Thanks."))
}

func init() {
	http.HandleFunc("/robots.txt", robotsTxtHandler)
	http.HandleFunc("/ip", ipHandler)
	http.HandleFunc("/reg/", regHandler)
}
<|endoftext|>


<commit_before>package archive

import (
	"archive/tar"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"

	"golang.org/x/sys/unix"

	"github.com/lxc/lxd/lxd/apparmor"
	"github.com/lxc/lxd/lxd/sys"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/ioprogress"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/subprocess"
)

// ExtractWithFds runs extractor process under specifc AppArmor profile.
// The allowedCmds argument specify commands which are allowed to run by apparmor.
// The cmd argument is automatically added to allowedCmds slice.
func ExtractWithFds(cmd string, args []string, allowedCmds []string, stdin io.ReadCloser, sysOS *sys.OS, output *os.File) error {
	outputPath := output.Name()

	allowedCmds = append(allowedCmds, cmd)
	allowedCmdPaths := []string{}
	for _, c := range allowedCmds {
		cmdPath, err := exec.LookPath(c)
		if err != nil {
			return fmt.Errorf("Failed to start extract: Failed to find executable: %w", err)
		}
		allowedCmdPaths = append(allowedCmdPaths, cmdPath)
	}

	err := apparmor.ArchiveLoad(sysOS, outputPath, allowedCmdPaths)
	if err != nil {
		return fmt.Errorf("Failed to start extract: Failed to load profile: %w", err)
	}
	defer apparmor.ArchiveDelete(sysOS, outputPath)
	defer apparmor.ArchiveUnload(sysOS, outputPath)

	p, err := subprocess.NewProcessWithFds(cmd, args, stdin, output, nil)
	if err != nil {
		return fmt.Errorf("Failed to start extract: Failed to creating subprocess: %w", err)
	}

	p.SetApparmor(apparmor.ArchiveProfileName(outputPath))

	err = p.Start()
	if err != nil {
		return fmt.Errorf("Failed to start extract: Failed running: tar: %w", err)
	}

	p.Wait(context.Background())
	return nil
}

// CompressedTarReader returns a tar reader from the supplied (optionally compressed) tarball stream.
// The unpacker arguments are those returned by DetectCompressionFile().
// The returned cancelFunc should be called when finished with reader to clean up any resources used.
// This can be done before reading to the end of the tarball if desired.
func CompressedTarReader(ctx context.Context, r io.ReadSeeker, unpacker []string, sysOS *sys.OS, outputPath string) (*tar.Reader, context.CancelFunc, error) {
	ctx, cancelFunc := context.WithCancel(ctx)

	r.Seek(0, 0)
	var tr *tar.Reader

	if len(unpacker) > 0 {
		cmdPath, err := exec.LookPath(unpacker[0])
		if err != nil {
			return nil, cancelFunc, fmt.Errorf("Failed to start unpack: Failed to find executable: %w", err)
		}

		err = apparmor.ArchiveLoad(sysOS, outputPath, []string{cmdPath})
		if err != nil {
			return nil, cancelFunc, fmt.Errorf("Failed to start unpack: Failed to load profile: %w", err)
		}

		pipeReader, pipeWriter := io.Pipe()
		p, err := subprocess.NewProcessWithFds(unpacker[0], unpacker[1:], ioutil.NopCloser(r), pipeWriter, nil)
		if err != nil {
			return nil, cancelFunc, fmt.Errorf("Failed to start unpack: Failed to creating subprocess: %w", err)
		}

		p.SetApparmor(apparmor.ArchiveProfileName(outputPath))
		err = p.Start()
		if err != nil {
			return nil, cancelFunc, fmt.Errorf("Failed to start unpack: Failed running: %s: %w", unpacker[0], err)
		}

		ctxCancelFunc := cancelFunc

		// Now that unpacker process has started, wrap context cancel function with one that waits for
		// the unpacker process to complete.
		cancelFunc = func() {
			ctxCancelFunc()
			pipeWriter.Close()
			p.Wait(ctx)
			apparmor.ArchiveUnload(sysOS, outputPath)
			apparmor.ArchiveDelete(sysOS, outputPath)
		}

		tr = tar.NewReader(pipeReader)
	} else {
		tr = tar.NewReader(r)
	}

	return tr, cancelFunc, nil
}

// Unpack extracts image from archive.
func Unpack(file string, path string, blockBackend bool, sysOS *sys.OS, tracker *ioprogress.ProgressTracker) error {
	extractArgs, extension, unpacker, err := shared.DetectCompression(file)
	if err != nil {
		return err
	}

	command := ""
	args := []string{}
	var allowedCmds []string
	var reader io.Reader
	if strings.HasPrefix(extension, ".tar") {
		command = "tar"
		if sysOS.RunningInUserNS {
			// We can't create char/block devices so avoid extracting them.
			args = append(args, "--wildcards")
			args = append(args, "--exclude=dev/*")
			args = append(args, "--exclude=./dev/*")
			args = append(args, "--exclude=rootfs/dev/*")
			args = append(args, "--exclude=rootfs/./dev/*")
		}
		args = append(args, "--restrict", "--force-local")
		args = append(args, "-C", path, "--numeric-owner", "--xattrs-include=*")
		args = append(args, extractArgs...)
		args = append(args, "-")

		f, err := os.Open(file)
		if err != nil {
			return err
		}
		defer f.Close()

		reader = f

		// Attach the ProgressTracker if supplied.
		if tracker != nil {
			fsinfo, err := f.Stat()
			if err != nil {
				return err
			}

			tracker.Length = fsinfo.Size()
			reader = &ioprogress.ProgressReader{
				ReadCloser: f,
				Tracker:    tracker,
			}
		}

		// Allow supplementary commands for the unpacker to use.
		if len(unpacker) > 0 {
			allowedCmds = append(allowedCmds, unpacker[0])
		}
	} else if strings.HasPrefix(extension, ".squashfs") {
		// unsquashfs does not support reading from stdin,
		// so ProgressTracker is not possible.
		command = "unsquashfs"
		args = append(args, "-f", "-d", path, "-n")

		// Limit unsquashfs chunk size to 10% of memory and up to 256MB (default)
		// When running on a low memory system, also disable multi-processing
		mem, err := shared.DeviceTotalMemory()
		mem = mem / 1024 / 1024 / 10
		if err == nil && mem < 256 {
			args = append(args, "-da", fmt.Sprintf("%d", mem), "-fr", fmt.Sprintf("%d", mem), "-p", "1")
		}

		args = append(args, file)
	} else {
		return fmt.Errorf("Unsupported image format: %s", extension)
	}

	outputDir, err := os.OpenFile(path, os.O_RDONLY, 0)
	if err != nil {
		return fmt.Errorf("Error opening directory: %w", err)
	}
	defer outputDir.Close()

	var readCloser io.ReadCloser
	if reader != nil {
		readCloser = ioutil.NopCloser(reader)
	}

	err = ExtractWithFds(command, args, allowedCmds, readCloser, sysOS, outputDir)
	if err != nil {
		// We can't create char/block devices in unpriv containers so ignore related errors.
		if sysOS.RunningInUserNS && command == "unsquashfs" {
			runError, ok := err.(shared.RunError)
			if !ok || runError.Stderr == "" {
				return err
			}

			// Confirm that all errors are related to character or block devices.
			found := false
			for _, line := range strings.Split(runError.Stderr, "\n") {
				line = strings.TrimSpace(line)
				if line == "" {
					continue
				}

				if !strings.Contains(line, "failed to create block device") {
					continue
				}

				if !strings.Contains(line, "failed to create character device") {
					continue
				}

				// We found an actual error.
				found = true
			}

			if !found {
				// All good, assume everything unpacked.
				return nil
			}
		}

		// Check if we ran out of space
		fs := unix.Statfs_t{}

		err1 := unix.Statfs(path, &fs)
		if err1 != nil {
			return err1
		}

		// Check if we're running out of space
		if int64(fs.Bfree) < 10 {
			if blockBackend {
				return fmt.Errorf("Unable to unpack image, run out of disk space (consider increasing your pool's volume.size)")
			}

			return fmt.Errorf("Unable to unpack image, run out of disk space")
		}

		logger.Debugf("Unpacking failed")
		logger.Debugf(err.Error())
		return fmt.Errorf("Unpack failed, %s.", err)
	}

	return nil
}
<commit_msg>lxd/archive: Improve error and logging in Unpack<commit_after>package archive

import (
	"archive/tar"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"

	"golang.org/x/sys/unix"
	log "gopkg.in/inconshreveable/log15.v2"

	"github.com/lxc/lxd/lxd/apparmor"
	"github.com/lxc/lxd/lxd/sys"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/ioprogress"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/subprocess"
)

// ExtractWithFds runs extractor process under specifc AppArmor profile.
// The allowedCmds argument specify commands which are allowed to run by apparmor.
// The cmd argument is automatically added to allowedCmds slice.
func ExtractWithFds(cmd string, args []string, allowedCmds []string, stdin io.ReadCloser, sysOS *sys.OS, output *os.File) error {
	outputPath := output.Name()

	allowedCmds = append(allowedCmds, cmd)
	allowedCmdPaths := []string{}
	for _, c := range allowedCmds {
		cmdPath, err := exec.LookPath(c)
		if err != nil {
			return fmt.Errorf("Failed to start extract: Failed to find executable: %w", err)
		}
		allowedCmdPaths = append(allowedCmdPaths, cmdPath)
	}

	err := apparmor.ArchiveLoad(sysOS, outputPath, allowedCmdPaths)
	if err != nil {
		return fmt.Errorf("Failed to start extract: Failed to load profile: %w", err)
	}
	defer apparmor.ArchiveDelete(sysOS, outputPath)
	defer apparmor.ArchiveUnload(sysOS, outputPath)

	p, err := subprocess.NewProcessWithFds(cmd, args, stdin, output, nil)
	if err != nil {
		return fmt.Errorf("Failed to start extract: Failed to creating subprocess: %w", err)
	}

	p.SetApparmor(apparmor.ArchiveProfileName(outputPath))

	err = p.Start()
	if err != nil {
		return fmt.Errorf("Failed to start extract: Failed running: tar: %w", err)
	}

	p.Wait(context.Background())
	return nil
}

// CompressedTarReader returns a tar reader from the supplied (optionally compressed) tarball stream.
// The unpacker arguments are those returned by DetectCompressionFile().
// The returned cancelFunc should be called when finished with reader to clean up any resources used.
// This can be done before reading to the end of the tarball if desired.
func CompressedTarReader(ctx context.Context, r io.ReadSeeker, unpacker []string, sysOS *sys.OS, outputPath string) (*tar.Reader, context.CancelFunc, error) {
	ctx, cancelFunc := context.WithCancel(ctx)

	r.Seek(0, 0)
	var tr *tar.Reader

	if len(unpacker) > 0 {
		cmdPath, err := exec.LookPath(unpacker[0])
		if err != nil {
			return nil, cancelFunc, fmt.Errorf("Failed to start unpack: Failed to find executable: %w", err)
		}

		err = apparmor.ArchiveLoad(sysOS, outputPath, []string{cmdPath})
		if err != nil {
			return nil, cancelFunc, fmt.Errorf("Failed to start unpack: Failed to load profile: %w", err)
		}

		pipeReader, pipeWriter := io.Pipe()
		p, err := subprocess.NewProcessWithFds(unpacker[0], unpacker[1:], ioutil.NopCloser(r), pipeWriter, nil)
		if err != nil {
			return nil, cancelFunc, fmt.Errorf("Failed to start unpack: Failed to creating subprocess: %w", err)
		}

		p.SetApparmor(apparmor.ArchiveProfileName(outputPath))
		err = p.Start()
		if err != nil {
			return nil, cancelFunc, fmt.Errorf("Failed to start unpack: Failed running: %s: %w", unpacker[0], err)
		}

		ctxCancelFunc := cancelFunc

		// Now that unpacker process has started, wrap context cancel function with one that waits for
		// the unpacker process to complete.
		cancelFunc = func() {
			ctxCancelFunc()
			pipeWriter.Close()
			p.Wait(ctx)
			apparmor.ArchiveUnload(sysOS, outputPath)
			apparmor.ArchiveDelete(sysOS, outputPath)
		}

		tr = tar.NewReader(pipeReader)
	} else {
		tr = tar.NewReader(r)
	}

	return tr, cancelFunc, nil
}

// Unpack extracts image from archive.
func Unpack(file string, path string, blockBackend bool, sysOS *sys.OS, tracker *ioprogress.ProgressTracker) error {
	extractArgs, extension, unpacker, err := shared.DetectCompression(file)
	if err != nil {
		return err
	}

	command := ""
	args := []string{}
	var allowedCmds []string
	var reader io.Reader
	if strings.HasPrefix(extension, ".tar") {
		command = "tar"
		if sysOS.RunningInUserNS {
			// We can't create char/block devices so avoid extracting them.
			args = append(args, "--wildcards")
			args = append(args, "--exclude=dev/*")
			args = append(args, "--exclude=./dev/*")
			args = append(args, "--exclude=rootfs/dev/*")
			args = append(args, "--exclude=rootfs/./dev/*")
		}
		args = append(args, "--restrict", "--force-local")
		args = append(args, "-C", path, "--numeric-owner", "--xattrs-include=*")
		args = append(args, extractArgs...)
		args = append(args, "-")

		f, err := os.Open(file)
		if err != nil {
			return err
		}
		defer f.Close()

		reader = f

		// Attach the ProgressTracker if supplied.
		if tracker != nil {
			fsinfo, err := f.Stat()
			if err != nil {
				return err
			}

			tracker.Length = fsinfo.Size()
			reader = &ioprogress.ProgressReader{
				ReadCloser: f,
				Tracker:    tracker,
			}
		}

		// Allow supplementary commands for the unpacker to use.
		if len(unpacker) > 0 {
			allowedCmds = append(allowedCmds, unpacker[0])
		}
	} else if strings.HasPrefix(extension, ".squashfs") {
		// unsquashfs does not support reading from stdin,
		// so ProgressTracker is not possible.
		command = "unsquashfs"
		args = append(args, "-f", "-d", path, "-n")

		// Limit unsquashfs chunk size to 10% of memory and up to 256MB (default)
		// When running on a low memory system, also disable multi-processing
		mem, err := shared.DeviceTotalMemory()
		mem = mem / 1024 / 1024 / 10
		if err == nil && mem < 256 {
			args = append(args, "-da", fmt.Sprintf("%d", mem), "-fr", fmt.Sprintf("%d", mem), "-p", "1")
		}

		args = append(args, file)
	} else {
		return fmt.Errorf("Unsupported image format: %s", extension)
	}

	outputDir, err := os.OpenFile(path, os.O_RDONLY, 0)
	if err != nil {
		return fmt.Errorf("Error opening directory: %w", err)
	}
	defer outputDir.Close()

	var readCloser io.ReadCloser
	if reader != nil {
		readCloser = ioutil.NopCloser(reader)
	}

	err = ExtractWithFds(command, args, allowedCmds, readCloser, sysOS, outputDir)
	if err != nil {
		// We can't create char/block devices in unpriv containers so ignore related errors.
		if sysOS.RunningInUserNS && command == "unsquashfs" {
			runError, ok := err.(shared.RunError)
			if !ok || runError.Stderr == "" {
				return err
			}

			// Confirm that all errors are related to character or block devices.
			found := false
			for _, line := range strings.Split(runError.Stderr, "\n") {
				line = strings.TrimSpace(line)
				if line == "" {
					continue
				}

				if !strings.Contains(line, "failed to create block device") {
					continue
				}

				if !strings.Contains(line, "failed to create character device") {
					continue
				}

				// We found an actual error.
				found = true
			}

			if !found {
				// All good, assume everything unpacked.
				return nil
			}
		}

		// Check if we ran out of space
		fs := unix.Statfs_t{}

		err1 := unix.Statfs(path, &fs)
		if err1 != nil {
			return err1
		}

		// Check if we're running out of space
		if int64(fs.Bfree) < 10 {
			if blockBackend {
				return fmt.Errorf("Unable to unpack image, run out of disk space (consider increasing your pool's volume.size)")
			}

			return fmt.Errorf("Unable to unpack image, run out of disk space")
		}

		logger.Warn("Unpack failed", log.Ctx{"file": file, "allowedCmds": allowedCmds, "extension": extension, "path": path, "err": err})
		return fmt.Errorf("Unpack failed: %w", err)
	}

	return nil
}
<|endoftext|>


<commit_before>//go:build linux && cgo && !agent
// +build linux,cgo,!agent

package db

import (
	"fmt"

	"github.com/lxc/lxd/lxd/db/query"
	"github.com/lxc/lxd/shared/api"
	"github.com/pkg/errors"
)

// Code generation directives.
//
//go:generate -command mapper lxd-generate db mapper -t certificates.mapper.go
//go:generate mapper reset
//
//go:generate mapper stmt -p db -e certificate objects
//go:generate mapper stmt -p db -e certificate objects-by-Fingerprint
//go:generate mapper stmt -p db -e certificate projects-ref
//go:generate mapper stmt -p db -e certificate projects-ref-by-Fingerprint
//go:generate mapper stmt -p db -e certificate id
//go:generate mapper stmt -p db -e certificate create struct=Certificate
//go:generate mapper stmt -p db -e certificate create-projects-ref
//go:generate mapper stmt -p db -e certificate delete-by-Fingerprint
//go:generate mapper stmt -p db -e certificate delete-by-Name-and-Type
//go:generate mapper stmt -p db -e certificate update struct=Certificate
//
//go:generate mapper method -p db -e certificate GetMany
//go:generate mapper method -p db -e certificate GetOne
//go:generate mapper method -p db -e certificate ID struct=Certificate
//go:generate mapper method -p db -e certificate Exists struct=Certificate
//go:generate mapper method -p db -e certificate Create struct=Certificate
//go:generate mapper method -p db -e certificate ProjectsRef
//go:generate mapper method -p db -e certificate DeleteOne-by-Fingerprint
//go:generate mapper method -p db -e certificate DeleteMany-by-Name-and-Type
//go:generate mapper method -p db -e certificate Update struct=Certificate

// CertificateType indicates the type of the certificate.
type CertificateType int

// CertificateTypeClient indicates a client certificate type.
const CertificateTypeClient = CertificateType(1)

// CertificateTypeServer indicates a server certificate type.
const CertificateTypeServer = CertificateType(2)

// CertificateTypeMetrics indicates a metrics certificate type.
const CertificateTypeMetrics = CertificateType(3)

// CertificateAPITypeToDBType converts an API type to the equivalent DB type.
func CertificateAPITypeToDBType(apiType string) (CertificateType, error) {
	switch apiType {
	case api.CertificateTypeClient:
		return CertificateTypeClient, nil
	case api.CertificateTypeServer:
		return CertificateTypeServer, nil
	case api.CertificateTypeMetrics:
		return CertificateTypeMetrics, nil
	}

	return -1, fmt.Errorf("Invalid certificate type")
}

// Certificate is here to pass the certificates content from the database around.
type Certificate struct {
	ID          int
	Fingerprint string `db:"primary=yes"`
	Type        CertificateType
	Name        string
	Certificate string
	Restricted  bool
	Projects    []string
}

// ToAPIType returns the API equivalent type.
func (cert *Certificate) ToAPIType() string {
	switch cert.Type {
	case CertificateTypeClient:
		return api.CertificateTypeClient
	case CertificateTypeServer:
		return api.CertificateTypeServer
	case CertificateTypeMetrics:
		return api.CertificateTypeMetrics
	}

	return api.CertificateTypeUnknown
}

// ToAPI converts the database Certificate struct to an api.Certificate entry.
func (cert *Certificate) ToAPI() api.Certificate {
	resp := api.Certificate{}
	resp.Fingerprint = cert.Fingerprint
	resp.Certificate = cert.Certificate
	resp.Name = cert.Name
	resp.Restricted = cert.Restricted
	resp.Projects = cert.Projects
	resp.Type = cert.ToAPIType()

	return resp
}

// UpdateCertificateProjects updates the list of projects on a certificate.
func (c *ClusterTx) UpdateCertificateProjects(id int, projects []string) error {
	// Clear all projects from the restrictions.
	q := "DELETE FROM certificates_projects WHERE certificate_id=?"
	_, err := c.tx.Exec(q, id)
	if err != nil {
		return err
	}

	// Add the new restrictions.
	for _, name := range projects {
		projID, err := c.GetProjectID(name)
		if err != nil {
			return err
		}

		q := "INSERT INTO certificates_projects (certificate_id, project_id) VALUES (?, ?)"
		_, err = c.tx.Exec(q, id, projID)
		if err != nil {
			return err
		}
	}

	return nil
}

// CertificateFilter specifies potential query parameter fields.
type CertificateFilter struct {
	Fingerprint *string
	Name        *string
	Type        *CertificateType
}

// GetCertificate gets an CertBaseInfo object from the database.
// The argument fingerprint will be queried with a LIKE query, means you can
// pass a shortform and will get the full fingerprint.
// There can never be more than one certificate with a given fingerprint, as it is
// enforced by a UNIQUE constraint in the schema.
func (c *Cluster) GetCertificate(fingerprintPrefix string) (*Certificate, error) {
	var err error
	var cert *Certificate
	objects := []Certificate{}
	err = c.Transaction(func(tx *ClusterTx) error {
		sql := `
SELECT certificates.id, certificates.fingerprint, certificates.type, certificates.name, certificates.certificate, certificates.restricted
FROM certificates
WHERE certificates.fingerprint LIKE ?
ORDER BY certificates.fingerprint
		`
		stmt, err := tx.prepare(sql)
		if err != nil {
			return err
		}
		dest := func(i int) []interface{} {
			objects = append(objects, Certificate{})
			return []interface{}{
				&objects[i].ID,
				&objects[i].Fingerprint,
				&objects[i].Type,
				&objects[i].Name,
				&objects[i].Certificate,
				&objects[i].Restricted,
			}
		}

		err = query.SelectObjects(stmt, dest, fingerprintPrefix+"%")
		if err != nil {
			return errors.Wrap(err, "Failed to fetch certificates")
		}

		if len(objects) > 1 {
			return fmt.Errorf("More than one certificate matches")
		}

		if len(objects) == 0 {
			return ErrNoSuchObject
		}

		cert, err = tx.GetCertificate(objects[0].Fingerprint)
		return err
	})
	if err != nil {
		return nil, err
	}

	return cert, nil
}

// CreateCertificate stores a CertInfo object in the db, it will ignore the ID
// field from the CertInfo.
func (c *Cluster) CreateCertificate(cert Certificate) (int64, error) {
	var id int64
	var err error
	err = c.Transaction(func(tx *ClusterTx) error {
		id, err = tx.CreateCertificate(cert)
		return err
	})
	return id, err
}

// DeleteCertificate deletes a certificate from the db.
func (c *Cluster) DeleteCertificate(fingerprint string) error {
	err := c.Transaction(func(tx *ClusterTx) error {
		return tx.DeleteCertificate(fingerprint)
	})
	return err
}

// UpdateCertificate updates a certificate in the db.
func (c *Cluster) UpdateCertificate(fingerprint string, cert Certificate) error {
	err := c.Transaction(func(tx *ClusterTx) error {
		return tx.UpdateCertificate(fingerprint, cert)
	})
	return err
}

// GetCertificates returns all available local certificates.
func (n *NodeTx) GetCertificates() ([]Certificate, error) {
	dbCerts := []struct {
		fingerprint string
		certType    CertificateType
		name        string
		certificate string
	}{}
	dest := func(i int) []interface{} {
		dbCerts = append(dbCerts, struct {
			fingerprint string
			certType    CertificateType
			name        string
			certificate string
		}{})
		return []interface{}{&dbCerts[i].fingerprint, &dbCerts[i].certType, &dbCerts[i].name, &dbCerts[i].certificate}
	}

	stmt, err := n.tx.Prepare("SELECT fingerprint, type, name, certificate FROM certificates")
	if err != nil {
		return nil, err
	}
	defer stmt.Close()

	err = query.SelectObjects(stmt, dest)
	if err != nil {
		return nil, err
	}

	certs := make([]Certificate, 0, len(dbCerts))
	for _, dbCert := range dbCerts {
		certs = append(certs, Certificate{
			Fingerprint: dbCert.fingerprint,
			Type:        dbCert.certType,
			Name:        dbCert.name,
			Certificate: dbCert.certificate,
		})
	}

	return certs, nil
}

// ReplaceCertificates removes all existing certificates from the local certificates table and replaces them with
// the ones provided.
func (n *NodeTx) ReplaceCertificates(certs []Certificate) error {
	_, err := n.tx.Exec("DELETE FROM certificates")
	if err != nil {
		return err
	}

	stmt, err := n.tx.Prepare("INSERT INTO certificates (fingerprint, type, name, certificate) VALUES(?,?,?,?)")
	if err != nil {
		return err
	}
	defer stmt.Close()

	for _, cert := range certs {
		_, err = stmt.Exec(cert.Fingerprint, cert.Type, cert.Name, cert.Certificate)
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>lxd/db/certificates: remove UpdateCertificateProjects<commit_after>//go:build linux && cgo && !agent
// +build linux,cgo,!agent

package db

import (
	"fmt"

	"github.com/lxc/lxd/lxd/db/query"
	"github.com/lxc/lxd/shared/api"
	"github.com/pkg/errors"
)

// Code generation directives.
//
//go:generate -command mapper lxd-generate db mapper -t certificates.mapper.go
//go:generate mapper reset
//
//go:generate mapper stmt -p db -e certificate objects
//go:generate mapper stmt -p db -e certificate objects-by-Fingerprint
//go:generate mapper stmt -p db -e certificate projects-ref
//go:generate mapper stmt -p db -e certificate projects-ref-by-Fingerprint
//go:generate mapper stmt -p db -e certificate id
//go:generate mapper stmt -p db -e certificate create struct=Certificate
//go:generate mapper stmt -p db -e certificate create-projects-ref
//go:generate mapper stmt -p db -e certificate delete-by-Fingerprint
//go:generate mapper stmt -p db -e certificate delete-by-Name-and-Type
//go:generate mapper stmt -p db -e certificate update struct=Certificate
//
//go:generate mapper method -p db -e certificate GetMany
//go:generate mapper method -p db -e certificate GetOne
//go:generate mapper method -p db -e certificate ID struct=Certificate
//go:generate mapper method -p db -e certificate Exists struct=Certificate
//go:generate mapper method -p db -e certificate Create struct=Certificate
//go:generate mapper method -p db -e certificate ProjectsRef
//go:generate mapper method -p db -e certificate DeleteOne-by-Fingerprint
//go:generate mapper method -p db -e certificate DeleteMany-by-Name-and-Type
//go:generate mapper method -p db -e certificate Update struct=Certificate

// CertificateType indicates the type of the certificate.
type CertificateType int

// CertificateTypeClient indicates a client certificate type.
const CertificateTypeClient = CertificateType(1)

// CertificateTypeServer indicates a server certificate type.
const CertificateTypeServer = CertificateType(2)

// CertificateTypeMetrics indicates a metrics certificate type.
const CertificateTypeMetrics = CertificateType(3)

// CertificateAPITypeToDBType converts an API type to the equivalent DB type.
func CertificateAPITypeToDBType(apiType string) (CertificateType, error) {
	switch apiType {
	case api.CertificateTypeClient:
		return CertificateTypeClient, nil
	case api.CertificateTypeServer:
		return CertificateTypeServer, nil
	case api.CertificateTypeMetrics:
		return CertificateTypeMetrics, nil
	}

	return -1, fmt.Errorf("Invalid certificate type")
}

// Certificate is here to pass the certificates content from the database around.
type Certificate struct {
	ID          int
	Fingerprint string `db:"primary=yes"`
	Type        CertificateType
	Name        string
	Certificate string
	Restricted  bool
	Projects    []string
}

// ToAPIType returns the API equivalent type.
func (cert *Certificate) ToAPIType() string {
	switch cert.Type {
	case CertificateTypeClient:
		return api.CertificateTypeClient
	case CertificateTypeServer:
		return api.CertificateTypeServer
	case CertificateTypeMetrics:
		return api.CertificateTypeMetrics
	}

	return api.CertificateTypeUnknown
}

// ToAPI converts the database Certificate struct to an api.Certificate entry.
func (cert *Certificate) ToAPI() api.Certificate {
	resp := api.Certificate{}
	resp.Fingerprint = cert.Fingerprint
	resp.Certificate = cert.Certificate
	resp.Name = cert.Name
	resp.Restricted = cert.Restricted
	resp.Projects = cert.Projects
	resp.Type = cert.ToAPIType()

	return resp
}

// CertificateFilter specifies potential query parameter fields.
type CertificateFilter struct {
	Fingerprint *string
	Name        *string
	Type        *CertificateType
}

// GetCertificate gets an CertBaseInfo object from the database.
// The argument fingerprint will be queried with a LIKE query, means you can
// pass a shortform and will get the full fingerprint.
// There can never be more than one certificate with a given fingerprint, as it is
// enforced by a UNIQUE constraint in the schema.
func (c *Cluster) GetCertificate(fingerprintPrefix string) (*Certificate, error) {
	var err error
	var cert *Certificate
	objects := []Certificate{}
	err = c.Transaction(func(tx *ClusterTx) error {
		sql := `
SELECT certificates.id, certificates.fingerprint, certificates.type, certificates.name, certificates.certificate, certificates.restricted
FROM certificates
WHERE certificates.fingerprint LIKE ?
ORDER BY certificates.fingerprint
		`
		stmt, err := tx.prepare(sql)
		if err != nil {
			return err
		}
		dest := func(i int) []interface{} {
			objects = append(objects, Certificate{})
			return []interface{}{
				&objects[i].ID,
				&objects[i].Fingerprint,
				&objects[i].Type,
				&objects[i].Name,
				&objects[i].Certificate,
				&objects[i].Restricted,
			}
		}

		err = query.SelectObjects(stmt, dest, fingerprintPrefix+"%")
		if err != nil {
			return errors.Wrap(err, "Failed to fetch certificates")
		}

		if len(objects) > 1 {
			return fmt.Errorf("More than one certificate matches")
		}

		if len(objects) == 0 {
			return ErrNoSuchObject
		}

		cert, err = tx.GetCertificate(objects[0].Fingerprint)
		return err
	})
	if err != nil {
		return nil, err
	}

	return cert, nil
}

// CreateCertificate stores a CertInfo object in the db, it will ignore the ID
// field from the CertInfo.
func (c *Cluster) CreateCertificate(cert Certificate) (int64, error) {
	var id int64
	var err error
	err = c.Transaction(func(tx *ClusterTx) error {
		id, err = tx.CreateCertificate(cert)
		return err
	})
	return id, err
}

// DeleteCertificate deletes a certificate from the db.
func (c *Cluster) DeleteCertificate(fingerprint string) error {
	err := c.Transaction(func(tx *ClusterTx) error {
		return tx.DeleteCertificate(fingerprint)
	})
	return err
}

// UpdateCertificate updates a certificate in the db.
func (c *Cluster) UpdateCertificate(fingerprint string, cert Certificate) error {
	err := c.Transaction(func(tx *ClusterTx) error {
		return tx.UpdateCertificate(fingerprint, cert)
	})
	return err
}

// GetCertificates returns all available local certificates.
func (n *NodeTx) GetCertificates() ([]Certificate, error) {
	dbCerts := []struct {
		fingerprint string
		certType    CertificateType
		name        string
		certificate string
	}{}
	dest := func(i int) []interface{} {
		dbCerts = append(dbCerts, struct {
			fingerprint string
			certType    CertificateType
			name        string
			certificate string
		}{})
		return []interface{}{&dbCerts[i].fingerprint, &dbCerts[i].certType, &dbCerts[i].name, &dbCerts[i].certificate}
	}

	stmt, err := n.tx.Prepare("SELECT fingerprint, type, name, certificate FROM certificates")
	if err != nil {
		return nil, err
	}
	defer stmt.Close()

	err = query.SelectObjects(stmt, dest)
	if err != nil {
		return nil, err
	}

	certs := make([]Certificate, 0, len(dbCerts))
	for _, dbCert := range dbCerts {
		certs = append(certs, Certificate{
			Fingerprint: dbCert.fingerprint,
			Type:        dbCert.certType,
			Name:        dbCert.name,
			Certificate: dbCert.certificate,
		})
	}

	return certs, nil
}

// ReplaceCertificates removes all existing certificates from the local certificates table and replaces them with
// the ones provided.
func (n *NodeTx) ReplaceCertificates(certs []Certificate) error {
	_, err := n.tx.Exec("DELETE FROM certificates")
	if err != nil {
		return err
	}

	stmt, err := n.tx.Prepare("INSERT INTO certificates (fingerprint, type, name, certificate) VALUES(?,?,?,?)")
	if err != nil {
		return err
	}
	defer stmt.Close()

	for _, cert := range certs {
		_, err = stmt.Exec(cert.Fingerprint, cert.Type, cert.Name, cert.Certificate)
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>