<commit_before>package compose

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"text/tabwriter"

	yaml "gopkg.in/yaml.v2"
)

type Command []string

type Commands map[string]Command

type VariableFile struct {
	File string
}

// Service ...
type Service struct {
	Abstract    bool
	Parent      string
	Path        string
	Commands    Commands
	Environment Environment
	variables   map[string]VariableFile
}

type Environment []string

// Compose ... composed infrastructure
type Compose struct {
	Version      string
	projectDir   string
	Services     map[string]Service
	Environments map[string]Environment

	DryRun bool
}

type execResult struct {
	environmentID string
	serviceID     string
	commandID     string
	command       Command
	execError     error
}

// Exec ...
func (c *Compose) Exec(args []string) error {
	var execResults []execResult
	//serviceCmdAlias := args[0]

	// cmds, present := c.Commands[serviceCmdAlias]
	// var err error
	// if present {
	// 	for _, cmd := range cmds {
	// 		res := c.execServiceCmd(cmd)
	// 		execResults = append(execResults, res)
	// 		err = res.execError
	// 		if res.execError != nil {
	// 			break
	// 		}
	// 	}
	// } else {
	res := c.execServiceCmds(args)
	execResults = append(execResults, res)
	err := res.execError
	//}

	dumpExecResults(execResults)

	return err
}

// List ... List all available command
func (c *Compose) List(args []string) error {
	const padding = 8
	w := tabwriter.NewWriter(os.Stdout, 0, 0, padding, ' ', 0)
	fmt.Fprintln(w, "SERVICE\tCOMMAND\tSUB-COMMAND\t")

	var srvKeys []string
	for k, srv := range c.Services {
		if !srv.Abstract {
			srvKeys = append(srvKeys, k)
		}
	}
	sort.Strings(srvKeys)
	for _, srv := range srvKeys {
		service := c.Services[srv]

		commands := Commands{}

		for cmdKey, cmd := range service.Commands {
			commands[cmdKey] = cmd
		}

		dumpCommand(w, srv, commands)
	}

	w.Flush()

	return nil
}

func (c *Compose) findServiceCommand(service Service) {

}

func dumpCommand(w *tabwriter.Writer, serviceName string, commands Commands) {
	// sort command
	var keys []string
	for k := range commands {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	for _, cmd := range keys {
		subCommands := commands[cmd]

		commandList := ellipsis(40, strings.Join(subCommands, " | "))

		fmt.Fprintf(w, "%s\t%s\t%s\t\n", serviceName, cmd, commandList)
	}
}

func ellipsis(length int, text string) string {
	r := []rune(text)
	if len(r) > length {
		return string(r[0:length]) + "..."
	}
	return text
}

func dumpExecResults(execResults []execResult) {
	fmt.Println("Execution summary")
	fmt.Println()
	const padding = 4
	w := tabwriter.NewWriter(os.Stdout, 0, 0, padding, ' ', 0)
	fmt.Fprintln(w, "ENVIRONMENT\tSERVICE\tCOMMAND\tSTATUS\t")
	//	fmt.Fprintln(w, "\t\t\t\t\t\t")
	for _, res := range execResults {
		status := "Success"
		if res.execError != nil {
			status = "Error"
		}
		fmt.Fprintf(w, "%s\t%s\t%s\t%s\t\n", res.environmentID,
			res.serviceID, res.commandID, status)
	}
	w.Flush()
	fmt.Println()
}

func (c *Compose) execServiceCmd(args string) execResult {
	return c.execServiceCmds(strings.Fields(args))
}

func (c *Compose) execServiceCmds(args []string) execResult {
	result := execResult{}
	//	fmt.Println("Exec args:" + strings.Join(args, " "))

	var env Environment

	// check if environment is defined
	envID := args[0]
	envConf, present := c.Environments[envID]
	if present {
		env = envConf
		args = args[1:]
		result.environmentID = envID
	}

	serviceName := args[0]
	result.serviceID = serviceName
	service, present := c.Services[serviceName]
	if !present {
		result.execError = errors.New("Invalid service name")
		return result
	}

	servicePath := filepath.Join(c.projectDir, service.Path)
	err := os.Chdir(servicePath)
	if err != nil {
		result.execError = err
		return result
	}

	command := args[1]
	commandArgs := args[2:]

	// Merge service environment
	env = appendEnv(service.Environment, env)

	// search if a command is defined
	commandList, present := service.Commands[command]
	if present {
		result.commandID = command
		for _, commands := range commandList {
			commandsSplit := strings.Fields(commands)
			err = c.executeCommand(commandsSplit[0], commandsSplit[1:], servicePath, env, service)
			if err != nil {
				result.execError = err
				return result
			}
		}
		return result
	}

	// Execute command in service directory
	result.commandID = "-"
	result.execError = c.executeCommand(command, commandArgs, servicePath, env, service)
	return result
}

func (c *Compose) executeCommand(name string, args []string, dir string, env Environment, service Service) error {
	if c.DryRun {
		//		fmt.Println("Plan to Execute ")
		fmt.Println("Exec : " + name + " " + strings.Join(args, " "))
		fmt.Println("Dir  : " + dir)
		fmt.Println("Env  : " + strings.Join(env, " "))
		fmt.Println("")
		return nil
	}

	// create variables files

	cmd := exec.Command(name, args...)
	cmd.Dir = dir
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin

	fullEnv := appendEnv(env, os.Environ())
	cmd.Env = fullEnv

	err := cmd.Run()

	fmt.Println("State: " + cmd.ProcessState.String())

	if err != nil {
		return errors.New("Execute command error. " + err.Error())
	}

	return nil
}

// Load ...
func (c *Compose) Load(file string, projectDir string) error {
	validComposeFile, err := findComposeFile(file, projectDir)
	if err != nil {
		return err
	}

	err = c.loadCompose(validComposeFile)

	c.init()

	return err
}

func (c *Compose) init() {
	for _, service := range c.Services {
		if service.Parent != "" {
			parentService := c.Services[service.Parent]
			for cmdKey, cmd := range parentService.Commands {
				if service.Commands == nil {
					service.Commands = make(Commands)
				}
				service.Commands[cmdKey] = cmd
			}
		}
	}
}

func (c *Compose) loadCompose(composeFile string) error {
	source, err := ioutil.ReadFile(composeFile)
	if err != nil {
		return err
	}

	composeStr := string(source)

	os.Setenv("branch.first", "prod")
	os.Setenv("branch.last", "prod")

	composeParsed := os.ExpandEnv(composeStr)

	err = yaml.Unmarshal([]byte(composeParsed), &c)
	if err != nil {
		return err
	}

	absFileName, _ := filepath.Abs(composeFile)
	c.projectDir = filepath.Dir(absFileName)

	return nil
}

func findComposeFile(file string, projectDir string) (string, error) {
	if projectDir != "" {
		err := os.Chdir(projectDir)
		if err != nil {
			return "", err
		}
	}

	_, err := os.Stat(file)
	if err != nil {

		// if not root path find in parent
		absProjectDir, _ := filepath.Abs(projectDir)
		parentDir := filepath.Dir(absProjectDir)

		if absProjectDir == "/" {
			return "", errors.New("Compose file not found")
		}

		return findComposeFile(file, parentDir)
	}

	return file, nil
}
<commit_msg>create variables file<commit_after>package compose

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"text/tabwriter"

	yaml "gopkg.in/yaml.v2"
)

type Command []string

type Commands map[string]Command

type VariableFile struct {
	File        string
	Environment Environment
}

// Service ...
type Service struct {
	Abstract    bool
	Parent      string
	Path        string
	Commands    Commands
	Environment Environment
	Variables   map[string]VariableFile
}

type Environment []string

// Compose ... composed infrastructure
type Compose struct {
	Version      string
	projectDir   string
	Services     map[string]Service
	Environments map[string]Environment

	DryRun bool
}

type execResult struct {
	environmentID string
	serviceID     string
	commandID     string
	command       Command
	execError     error
}

// Exec ...
func (c *Compose) Exec(args []string) error {
	var execResults []execResult
	//serviceCmdAlias := args[0]

	// cmds, present := c.Commands[serviceCmdAlias]
	// var err error
	// if present {
	// 	for _, cmd := range cmds {
	// 		res := c.execServiceCmd(cmd)
	// 		execResults = append(execResults, res)
	// 		err = res.execError
	// 		if res.execError != nil {
	// 			break
	// 		}
	// 	}
	// } else {
	res := c.execServiceCmds(args)
	execResults = append(execResults, res)
	err := res.execError
	//}

	dumpExecResults(execResults)

	return err
}

// List ... List all available command
func (c *Compose) List(args []string) error {
	const padding = 8
	w := tabwriter.NewWriter(os.Stdout, 0, 0, padding, ' ', 0)
	fmt.Fprintln(w, "SERVICE\tCOMMAND\tSUB-COMMAND\t")

	var srvKeys []string
	for k, srv := range c.Services {
		if !srv.Abstract {
			srvKeys = append(srvKeys, k)
		}
	}
	sort.Strings(srvKeys)
	for _, srv := range srvKeys {
		service := c.Services[srv]

		commands := Commands{}

		for cmdKey, cmd := range service.Commands {
			commands[cmdKey] = cmd
		}

		dumpCommand(w, srv, commands)
	}

	w.Flush()

	return nil
}

func (c *Compose) findServiceCommand(service Service) {

}

func dumpCommand(w *tabwriter.Writer, serviceName string, commands Commands) {
	// sort command
	var keys []string
	for k := range commands {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	for _, cmd := range keys {
		subCommands := commands[cmd]

		commandList := ellipsis(40, strings.Join(subCommands, " | "))

		fmt.Fprintf(w, "%s\t%s\t%s\t\n", serviceName, cmd, commandList)
	}
}

func ellipsis(length int, text string) string {
	r := []rune(text)
	if len(r) > length {
		return string(r[0:length]) + "..."
	}
	return text
}

func dumpExecResults(execResults []execResult) {
	fmt.Println("Execution summary")
	fmt.Println()
	const padding = 4
	w := tabwriter.NewWriter(os.Stdout, 0, 0, padding, ' ', 0)
	fmt.Fprintln(w, "ENVIRONMENT\tSERVICE\tCOMMAND\tSTATUS\t")
	//	fmt.Fprintln(w, "\t\t\t\t\t\t")
	for _, res := range execResults {
		status := "Success"
		if res.execError != nil {
			status = "Error"
		}
		fmt.Fprintf(w, "%s\t%s\t%s\t%s\t\n", res.environmentID,
			res.serviceID, res.commandID, status)
	}
	w.Flush()
	fmt.Println()
}

func (c *Compose) execServiceCmd(args string) execResult {
	return c.execServiceCmds(strings.Fields(args))
}

func (c *Compose) execServiceCmds(args []string) execResult {
	result := execResult{}
	//	fmt.Println("Exec args:" + strings.Join(args, " "))

	var env Environment

	// check if environment is defined
	envID := args[0]
	envConf, present := c.Environments[envID]
	if present {
		env = envConf
		args = args[1:]
		result.environmentID = envID
	}

	serviceName := args[0]
	result.serviceID = serviceName
	service, present := c.Services[serviceName]
	if !present {
		result.execError = errors.New("Invalid service name")
		return result
	}

	servicePath := filepath.Join(c.projectDir, service.Path)
	err := os.Chdir(servicePath)
	if err != nil {
		result.execError = err
		return result
	}

	command := args[1]
	commandArgs := args[2:]

	// Merge service environment
	env = appendEnv(service.Environment, env)

	// search if a command is defined
	commandList, present := service.Commands[command]
	if present {
		result.commandID = command
		for _, commands := range commandList {
			commandsSplit := strings.Fields(commands)
			err = c.executeCommand(commandsSplit[0], commandsSplit[1:], servicePath, env, service)
			if err != nil {
				result.execError = err
				return result
			}
		}
		return result
	}

	// Execute command in service directory
	result.commandID = "-"
	result.execError = c.executeCommand(command, commandArgs, servicePath, env, service)
	return result
}

func (c *Compose) executeCommand(name string, args []string, dir string, env Environment, service Service) error {

	for _, variableFile := range service.Variables {
		fmt.Println("Var file  : " + variableFile.File)

		outputVars := ""
		for _, variable := range variableFile.Environment {
			outputVars += variable + "\n"
		}

		ioutil.WriteFile(variableFile.File, []byte(outputVars), 0644)
	}

	if c.DryRun {
		//		fmt.Println("Plan to Execute ")
		fmt.Println("Exec : " + name + " " + strings.Join(args, " "))
		fmt.Println("Dir  : " + dir)
		fmt.Println("Env  : " + strings.Join(env, " "))
		fmt.Println("")
		return nil
	}

	// create variables files

	cmd := exec.Command(name, args...)
	cmd.Dir = dir
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin

	fullEnv := appendEnv(env, os.Environ())
	cmd.Env = fullEnv

	err := cmd.Run()

	fmt.Println("State: " + cmd.ProcessState.String())

	if err != nil {
		return errors.New("Execute command error. " + err.Error())
	}

	return nil
}

// Load ...
func (c *Compose) Load(file string, projectDir string) error {
	validComposeFile, err := findComposeFile(file, projectDir)
	if err != nil {
		return err
	}

	err = c.loadCompose(validComposeFile)

	c.init()

	return err
}

func (c *Compose) init() {
	services := make(map[string]Service)
	for serviceKey, service := range c.Services {
		if service.Parent != "" {
			parentService := c.Services[service.Parent]
			for cmdKey, cmd := range parentService.Commands {
				if service.Commands == nil {
					service.Commands = make(Commands)
				}
				service.Commands[cmdKey] = cmd
			}

			for varKey, variable := range parentService.Variables {
				if service.Variables == nil {
					service.Variables = make(map[string]VariableFile)
				}
				service.Variables[varKey] = variable
			}
		}
		services[serviceKey] = service
	}

	c.Services = services
}

func (c *Compose) loadCompose(composeFile string) error {
	source, err := ioutil.ReadFile(composeFile)
	if err != nil {
		return err
	}

	composeStr := string(source)

	os.Setenv("branch.first", "prod")
	os.Setenv("branch.last", "prod")

	composeParsed := os.ExpandEnv(composeStr)

	err = yaml.Unmarshal([]byte(composeParsed), &c)
	if err != nil {
		return err
	}

	absFileName, _ := filepath.Abs(composeFile)
	c.projectDir = filepath.Dir(absFileName)

	return nil
}

func findComposeFile(file string, projectDir string) (string, error) {
	if projectDir != "" {
		err := os.Chdir(projectDir)
		if err != nil {
			return "", err
		}
	}

	_, err := os.Stat(file)
	if err != nil {

		// if not root path find in parent
		absProjectDir, _ := filepath.Abs(projectDir)
		parentDir := filepath.Dir(absProjectDir)

		if absProjectDir == "/" {
			return "", errors.New("Compose file not found")
		}

		return findComposeFile(file, parentDir)
	}

	return file, nil
}
<|endoftext|>


<commit_before>package systemtests_test

import (
	"io/ioutil"
	"os"
	"sync"
	"time"

	"github.com/apoydence/talaria/broker"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("SingleConnectionSingleBroker", func() {

	var (
		session *gexec.Session
		client  *broker.Client
		URL     string
	)

	BeforeEach(func() {
		var err error
		tmpDir, err = ioutil.TempDir("/tmp", "systemtalaria")
		Expect(err).ToNot(HaveOccurred())
		URL, session = startTalaria(tmpDir)
		client = startClient(URL)
	})

	AfterEach(func() {
		session.Kill()
		session.Wait("10s", "100ms")

		Expect(os.RemoveAll(tmpDir)).To(Succeed())
		client.Close()
	})

	It("Writes and reads from a single file", func(done Done) {
		defer close(done)
		fileId, err := client.FetchFile("some-file")
		Expect(err).ToNot(HaveOccurred())
		for i := byte(0); i < 100; i++ {
			_, err = client.WriteToFile(fileId, []byte{i})
			Expect(err).ToNot(HaveOccurred())
		}

		for i := 0; i < 100; i++ {
			data, _, err := client.ReadFromFile(fileId)
			Expect(err).ToNot(HaveOccurred())
			Expect(data).To(HaveLen(1))
			Expect(data[0]).To(BeEquivalentTo(i))
		}
	}, 5)

	It("Writes and reads from a single file at the same time", func(done Done) {
		defer close(done)
		clientW := startClient(URL)
		clientR := startClient(URL)
		fileIdW, err := clientW.FetchFile("some-file")
		Expect(err).ToNot(HaveOccurred())
		fileIdR, err := clientR.FetchFile("some-file")
		Expect(err).ToNot(HaveOccurred())

		var wg sync.WaitGroup
		defer wg.Wait()
		wg.Add(1)
		go func() {
			defer GinkgoRecover()
			defer wg.Done()
			for i := 0; i < 10; i++ {
				_, err = clientW.WriteToFile(fileIdW, []byte{byte(i)})
				Expect(err).ToNot(HaveOccurred())
				time.Sleep(time.Millisecond)
			}
		}()

		var result []byte
		for len(result) < 10 {
			data, _, err := clientR.ReadFromFile(fileIdR)
			Expect(err).ToNot(HaveOccurred())
			result = append(result, data...)
		}

		for i := 0; i < 10; i++ {
			Expect(result[i]).To(BeEquivalentTo(i))
		}
	}, 5)

	It("inits a file", func(done Done) {
		defer close(done)

		fileId, err := client.FetchFile("some-file")
		Expect(err).ToNot(HaveOccurred())
		expectedData := []byte("some-data")
		client.InitWriteIndex(fileId, 1000, expectedData)
		data, _, err := client.ReadFromFile(fileId)

		Expect(err).ToNot(HaveOccurred())
		Expect(data).To(Equal(expectedData))
	}, 3)

})
<commit_msg>Adjusts system test to include check for initialized index<commit_after>package systemtests_test

import (
	"io/ioutil"
	"os"
	"sync"
	"time"

	"github.com/apoydence/talaria/broker"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("SingleConnectionSingleBroker", func() {

	var (
		session *gexec.Session
		client  *broker.Client
		URL     string
	)

	BeforeEach(func() {
		var err error
		tmpDir, err = ioutil.TempDir("/tmp", "systemtalaria")
		Expect(err).ToNot(HaveOccurred())
		URL, session = startTalaria(tmpDir)
		client = startClient(URL)
	})

	AfterEach(func() {
		session.Kill()
		session.Wait("10s", "100ms")

		Expect(os.RemoveAll(tmpDir)).To(Succeed())
		client.Close()
	})

	It("Writes and reads from a single file", func(done Done) {
		defer close(done)
		fileId, err := client.FetchFile("some-file")
		Expect(err).ToNot(HaveOccurred())
		for i := byte(0); i < 100; i++ {
			_, err = client.WriteToFile(fileId, []byte{i})
			Expect(err).ToNot(HaveOccurred())
		}

		for i := 0; i < 100; i++ {
			data, _, err := client.ReadFromFile(fileId)
			Expect(err).ToNot(HaveOccurred())
			Expect(data).To(HaveLen(1))
			Expect(data[0]).To(BeEquivalentTo(i))
		}
	}, 5)

	It("Writes and reads from a single file at the same time", func(done Done) {
		defer close(done)
		clientW := startClient(URL)
		clientR := startClient(URL)
		fileIdW, err := clientW.FetchFile("some-file")
		Expect(err).ToNot(HaveOccurred())
		fileIdR, err := clientR.FetchFile("some-file")
		Expect(err).ToNot(HaveOccurred())

		var wg sync.WaitGroup
		defer wg.Wait()
		wg.Add(1)
		go func() {
			defer GinkgoRecover()
			defer wg.Done()
			for i := 0; i < 10; i++ {
				_, err = clientW.WriteToFile(fileIdW, []byte{byte(i)})
				Expect(err).ToNot(HaveOccurred())
				time.Sleep(time.Millisecond)
			}
		}()

		var result []byte
		for len(result) < 10 {
			data, _, err := clientR.ReadFromFile(fileIdR)
			Expect(err).ToNot(HaveOccurred())
			result = append(result, data...)
		}

		for i := 0; i < 10; i++ {
			Expect(result[i]).To(BeEquivalentTo(i))
		}
	}, 5)

	It("inits a file", func(done Done) {
		defer close(done)

		fileId, err := client.FetchFile("some-file")
		Expect(err).ToNot(HaveOccurred())
		expectedData := []byte("some-data")
		client.InitWriteIndex(fileId, 1000, expectedData)
		data, index, err := client.ReadFromFile(fileId)

		Expect(err).ToNot(HaveOccurred())
		Expect(data).To(Equal(expectedData))
		Expect(index).To(BeEquivalentTo(1001))
	}, 3)

})
<|endoftext|>


<commit_before>/*!
 * Copyright 2014 Docker, Inc.
 * Licensed under the Apache License, Version 2.0
 * github.com/docker/docker/LICENSE
 *
 * github.com/docker/docker/api/client/commands.go
 * github.com/docker/docker/pkg/archive/archive.go
 * github.com/docker/docker/pkg/fileutils/fileutils.go
 */

package api

import (
	"archive/tar"
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"strings"

	log "github.com/sirupsen/logrus"
)

const (
	DOCKERFILE   = "Dockerfile"
	DOCKERIGNORE = ".dockerignore"
)

func (client *DockerClient) BuildImage(path, tag string) error {
	v := url.Values{}
	v.Set("rm", "1")
	if tag != "" {
		v.Set("t", tag)
	}

	uri := fmt.Sprintf("/v%s/build?%s", API_VERSION, v.Encode())

	dockerfile := path

	fi, err := os.Lstat(dockerfile)
	if err != nil {
		return err
	}

	fm := fi.Mode()
	if fm.IsDir() {
		dockerfile = filepath.Join(dockerfile, DOCKERFILE)
		if _, err := os.Stat(dockerfile); os.IsNotExist(err) {
			return fmt.Errorf("No Dockerfile found in %s", path)
		}
	}

	var (
		root     = filepath.Dir(dockerfile)
		filename = filepath.Base(dockerfile)
	)

	ignore, err := ioutil.ReadFile(filepath.Join(root, DOCKERIGNORE))
	if err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("Error reading .dockerignore: %s", err)
	}

	var excludes []string
	for _, pattern := range strings.Split(string(ignore), "\n") {
		pattern = strings.TrimSpace(pattern)
		if pattern == "" {
			continue
		}
		pattern = filepath.Clean(pattern)
		ok, err := filepath.Match(pattern, filename)
		if err != nil {
			return fmt.Errorf("Bad .dockerignore pattern: %s, error: %s", pattern, err)
		}
		if ok {
			return fmt.Errorf("Dockerfile was excluded by .dockerignore pattern %s", pattern)
		}
		excludes = append(excludes, pattern)
	}

	if filename != DOCKERFILE {
		excludes = append(excludes, DOCKERFILE)
	}

	fmt.Fprintf(client.out, "Sending build context to Docker daemon\n")
	if log.GetLevel() < log.InfoLevel {
		fmt.Fprintf(client.out, "---> ")
	}

	pipeReader, pipeWriter := io.Pipe()

	go func() {
		var (
			files int64 = 0
			total int64 = 0
		)

		bufWriter := bufio.NewWriterSize(pipeWriter, 32*1024)
		tarWriter := tar.NewWriter(bufWriter)
		tmpWriter := bufio.NewWriterSize(nil, 32*1024)
		defer tmpWriter.Reset(nil)

		filepath.Walk(filepath.Join(root, "."), func(filePath string, f os.FileInfo, err error) error {
			if err != nil {
				log.Debugf("Can't stat file %s, error: %s", filePath, err)
				return nil
			}

			relFilePath, err := filepath.Rel(root, filePath)
			if err != nil || (relFilePath == "." && f.IsDir()) {
				return nil
			}

			skip, err := func() (bool, error) { // Excluding
				for _, exclude := range excludes {
					matched, err := filepath.Match(exclude, relFilePath)
					if err != nil {
						log.Errorf("Error matching: %s, pattern: %s", relFilePath, exclude)
						return false, err
					}
					if matched {
						if filepath.Clean(relFilePath) == "." {
							log.Errorf("Can't exclude whole path, excluding pattern: %s", exclude)
							continue
						}
						return true, nil
					}
				}
				return false, nil
			}()
			if err != nil {
				log.Debugf("Error matching: %s, %s", relFilePath, err)
				return err
			}
			if skip {
				log.WithField("", " Skipped").Debugf("---> %s", relFilePath)
				if f.IsDir() {
					return filepath.SkipDir
				}
				return nil
			}

			var size int64

			if err := func() error { // Adding a file to tar
				fi, err := os.Lstat(filePath)
				if err != nil {
					log.Errorf("Can't get file info: %s, error: %s", filePath, err)
					return err
				}

				size = fi.Size()

				link := ""
				if (fi.Mode() & os.ModeSymlink) != 0 {
					if link, err = os.Readlink(filePath); err != nil {
						log.Errorf("Can't read link to tar: %s, error: %s", filePath, err)
						return err
					}
				}

				hdr, err := tar.FileInfoHeader(fi, link)
				if err != nil {
					log.Errorf("Can't get file info header to tar: %s, error: %s", filePath, err)
					return err
				}

				name := relFilePath
				if fi.IsDir() && !strings.HasSuffix(name, "/") {
					name = name + "/"
				}
				hdr.Name = name

				if name == filename {
					hdr.Name = DOCKERFILE
				}

				if err := tarWriter.WriteHeader(hdr); err != nil {
					log.Errorf("Can't write tar header, error: %s", err)
					return err
				}

				if hdr.Typeflag == tar.TypeReg {
					file, err := os.Open(filePath)
					if err != nil {
						log.Errorf("Can't open file: %s, error: %s", filePath, err)
						return err
					}

					tmpWriter.Reset(tarWriter)
					defer tmpWriter.Reset(nil)
					_, err = io.Copy(tmpWriter, file)
					file.Close()
					if err != nil {
						log.Errorf("Can't write file to tar: %s, error: %s", filePath, err)
						return err
					}
					err = tmpWriter.Flush()
					if err != nil {
						log.Errorf("Can't flush file to tar, error: %s", err)
						return err
					}
				}

				return nil
			}(); err != nil {
				log.Debugf("Can't add file %s to tar, error: %s", filePath, err)
			}

			files++
			total += size

			if log.GetLevel() < log.InfoLevel {
				fmt.Fprintf(client.out, ".")
			}
			log.WithFields(log.Fields{
				"": fmt.Sprintf(" %7.2f KB", float64(size)/1000),
			}).Infof("---> %s", relFilePath)

			return nil
		})

		if err := tarWriter.Close(); err != nil {
			log.Debugf("Can't close tar writer: %s", err)
		}

		bufWriter.Flush()
		if err := pipeWriter.Close(); err != nil {
			log.Debugf("Can't close pipe writer: %s", err)
		}

		if log.GetLevel() < log.InfoLevel {
			fmt.Fprintf(client.out, "\n")
		}
		fmt.Fprintf(client.out, "---> Sent %d file(s), %.2f KB\n", files, float64(total)/1000)
	}()

	headers := map[string]string{}
	headers["Content-type"] = "application/tar"

	return client.doStreamRequest("POST", uri, pipeReader, headers)
}
<commit_msg>Follow docker/docker@6d78013<commit_after>/*!
 * Copyright 2014 Docker, Inc.
 * Licensed under the Apache License, Version 2.0
 * github.com/docker/docker/LICENSE
 *
 * github.com/docker/docker/api/client/commands.go
 * github.com/docker/docker/pkg/archive/archive.go
 * github.com/docker/docker/pkg/fileutils/fileutils.go
 */

package api

import (
	"archive/tar"
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"strings"

	log "github.com/sirupsen/logrus"
)

const (
	DOCKERFILE   = "Dockerfile"
	DOCKERIGNORE = ".dockerignore"
)

func (client *DockerClient) BuildImage(path, tag string) error {
	v := url.Values{}
	v.Set("rm", "1")
	if tag != "" {
		v.Set("t", tag)
	}

	uri := fmt.Sprintf("/v%s/build?%s", API_VERSION, v.Encode())

	dockerfile := path

	fi, err := os.Lstat(dockerfile)
	if err != nil {
		return err
	}

	fm := fi.Mode()
	if fm.IsDir() {
		dockerfile = filepath.Join(dockerfile, DOCKERFILE)
		if _, err := os.Stat(dockerfile); os.IsNotExist(err) {
			return fmt.Errorf("No Dockerfile found in %s", path)
		}
	}

	var (
		root     = filepath.Dir(dockerfile)
		filename = filepath.Base(dockerfile)
	)

	ignore, err := ioutil.ReadFile(filepath.Join(root, DOCKERIGNORE))
	if err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("Error reading .dockerignore: %s", err)
	}

	var excludes []string
	for _, pattern := range strings.Split(string(ignore), "\n") {
		pattern = strings.TrimSpace(pattern)
		if pattern == "" {
			continue
		}
		pattern = filepath.Clean(pattern)
		excludes = append(excludes, pattern)
	}

	fmt.Fprintf(client.out, "Sending build context to Docker daemon\n")
	if log.GetLevel() < log.InfoLevel {
		fmt.Fprintf(client.out, "---> ")
	}

	pipeReader, pipeWriter := io.Pipe()

	go func() {
		var (
			files int64 = 0
			total int64 = 0
		)

		bufWriter := bufio.NewWriterSize(pipeWriter, 32*1024)
		tarWriter := tar.NewWriter(bufWriter)
		tmpWriter := bufio.NewWriterSize(nil, 32*1024)
		defer tmpWriter.Reset(nil)

		seen := make(map[string]bool)

		filepath.Walk(filepath.Join(root, "."), func(filePath string, f os.FileInfo, err error) error {
			if err != nil {
				log.Debugf("Can't stat file %s, error: %s", filePath, err)
				return nil
			}

			relFilePath, err := filepath.Rel(root, filePath)
			if err != nil || (relFilePath == "." && f.IsDir()) {
				return nil
			}

			skip := false

			switch relFilePath {
			default:
				skip, err = func() (bool, error) { // Excluding
					for _, exclude := range excludes {
						matched, err := filepath.Match(exclude, relFilePath)
						if err != nil {
							log.Errorf("Error matching: %s, pattern: %s", relFilePath, exclude)
							return false, err
						}
						if matched {
							if filepath.Clean(relFilePath) == "." {
								log.Errorf("Can't exclude whole path, excluding pattern: %s", exclude)
								continue
							}
							return true, nil
						}
					}
					return false, nil
				}()
				if err != nil {
					log.Debugf("Error matching: %s, %s", relFilePath, err)
					return err
				}
			case DOCKERFILE:
				if filename != DOCKERFILE {
					skip = true
				}
			case DOCKERIGNORE:
			case filename:
			}

			if skip {
				log.WithField("", " Skipped").Debugf("---> %s", relFilePath)
				if f.IsDir() {
					return filepath.SkipDir
				}
				return nil
			}

			if seen[relFilePath] {
				return nil
			}
			seen[relFilePath] = true

			var size int64

			if err := func() error { // Adding a file to tar
				fi, err := os.Lstat(filePath)
				if err != nil {
					log.Errorf("Can't get file info: %s, error: %s", filePath, err)
					return err
				}

				size = fi.Size()

				link := ""
				if (fi.Mode() & os.ModeSymlink) != 0 {
					if link, err = os.Readlink(filePath); err != nil {
						log.Errorf("Can't read link to tar: %s, error: %s", filePath, err)
						return err
					}
				}

				hdr, err := tar.FileInfoHeader(fi, link)
				if err != nil {
					log.Errorf("Can't get file info header to tar: %s, error: %s", filePath, err)
					return err
				}

				name := relFilePath
				if fi.IsDir() && !strings.HasSuffix(name, "/") {
					name = name + "/"
				}
				hdr.Name = name

				if name == filename {
					hdr.Name = DOCKERFILE
				}

				if err := tarWriter.WriteHeader(hdr); err != nil {
					log.Errorf("Can't write tar header, error: %s", err)
					return err
				}

				if hdr.Typeflag == tar.TypeReg {
					file, err := os.Open(filePath)
					if err != nil {
						log.Errorf("Can't open file: %s, error: %s", filePath, err)
						return err
					}

					tmpWriter.Reset(tarWriter)
					defer tmpWriter.Reset(nil)
					_, err = io.Copy(tmpWriter, file)
					file.Close()
					if err != nil {
						log.Errorf("Can't write file to tar: %s, error: %s", filePath, err)
						return err
					}
					err = tmpWriter.Flush()
					if err != nil {
						log.Errorf("Can't flush file to tar, error: %s", err)
						return err
					}
				}

				return nil
			}(); err != nil {
				log.Debugf("Can't add file %s to tar, error: %s", filePath, err)
			}

			files++
			total += size

			if log.GetLevel() < log.InfoLevel {
				fmt.Fprintf(client.out, ".")
			}
			log.WithFields(log.Fields{
				"": fmt.Sprintf(" %7.2f KB", float64(size)/1000),
			}).Infof("---> %s", relFilePath)

			return nil
		})

		if err := tarWriter.Close(); err != nil {
			log.Debugf("Can't close tar writer: %s", err)
		}

		bufWriter.Flush()
		if err := pipeWriter.Close(); err != nil {
			log.Debugf("Can't close pipe writer: %s", err)
		}

		if log.GetLevel() < log.InfoLevel {
			fmt.Fprintf(client.out, "\n")
		}
		fmt.Fprintf(client.out, "---> Sent %d file(s), %.2f KB\n", files, float64(total)/1000)
	}()

	headers := map[string]string{}
	headers["Content-type"] = "application/tar"

	return client.doStreamRequest("POST", uri, pipeReader, headers)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strconv"
	"time"
)

var (
	curmonth, curyear int
	daystr            = [...]string{
		"", "01", "02", "03", "04", "05", "06", "07",
		"08", "09", "10", "11", "12", "13", "14",
		"15", "16", "17", "18", "19", "20", "21",
		"22", "23", "24", "25", "26", "27", "28",
		"29", "30", "31",
	}
	month = [...]int{0,
		31, 29, 31, 30,
		31, 30, 31, 31,
		30, 31, 30, 31,
	}
	monthstr = [...]string{"",
		"January", "February", "March", "April",
		"May", "June", "July", "August", "September",
		"October", "November", "December",
	}
	today = time.Now()
)

const calander = "\n" +
	"  ]====================================[   \n" +
	"  |  Su | Mo | Tu | We | Th | Fr | Sa  |   \n" +
	"  ]====================================[   \n" +
	"   | %S | %S | %S | %S | %S | %S | %S |    \n" +
	"   | %S | %S | %S | %S | %S | %S | %S |    \n" +
	"   | %S | %S | %S | %S | %S | %S | %S |    \n" +
	"   | %S | %S | %S | %S | %S | %S | %S |    \n" +
	"   | %S | %S | %S | %S | %S | %S | %S |    \n" +
	"   | %S | %S | %S | %S | %S | %S | %S |    \n" +
	"  ]====================================[   \n" +
	" <                 %S                   >  \n" +
	"  ]====================================[   \n"

func main() {
	procFlags()
	var p []byte
	cal(curmonth, curyear, &p, 24)
	fmt.Println(p)
}

//	return day of the week
//	of jan 1 of given year
func jan1(yr int) int {

	//	normal gregorian calendar
	//	one extra day per four years

	d := 4 + yr + (yr+3)/4

	// 	julian calendar
	// 	regular gregorian
	// 	less three days per 400

	if yr > 1800 {
		d -= (yr - 1701) / 100
		d += (yr - 1601) / 400
	}

	// 	great calendar changeover instant

	if yr > 1752 {
		d += 3
	}

	return d % 7
}

func cal(m, y int, p *[]byte, w int) {
	d := jan1(y)

	switch jan1((y+1)+7-d) % 7 {

	//non-leap year
	case 1:
		month[2] = 28
		break
	//1752
	default:
		month[9] = 19
		break
	//leap year
	case 2:
	}

	for i := 1; i < m; i++ {
		d += month[i]
	}
	d %= 7
	*p = make([]byte, 128)
	s := (*p)[3*d:]

	for i := 1; i <= month[m]; i++ {

		if i == 3 && month[m] == 19 {
			i += 11
			month[m] += 11
		}
		if i > 9 {
			s[0] = byte(i/10 + 48)
		}
		s = s[1:]
		s[0] = byte(i%10 + 48)
		s = s[2:]
		d++
		if d == 7 {
			d = 0
			s = (*p)[w:]
			p = &s
		}
	}

}

func fatal(err error) {
	fmt.Fprintf(os.Stderr, "%s:\t%s\n", "awk", err.Error())
	os.Exit(2)
}

func procFlags() {
	args := os.Args[1:]
	for _, a := range args {
		if curmonth == 0 &&
			(a[0] >= 'A' && a[0] <= 'z' ||
				(len(a) <= 2 && a[0] >= '0' && a[0] <= '9')) {

			if a[0] < 'a' {
				a = string(a[0]+'a'-'A') + a[1:]
			}
			if len(a) > 1 && a[0] < 'A' {
				switch a[1] {
				case 0:
				case 1:
				case 2:
				default:
					goto YEAR
				}
			}
			switch a {
			case "jan", "january", "1":
				curmonth = 1
			case "feb", "february", "2":
				curmonth = 2
			case "mar", "march", "3":
				curmonth = 3
			case "apr", "april", "4":
				curmonth = 4
			case "may", "5":
				curmonth = 5
			case "jun", "june", "6":
				curmonth = 6
			case "jul", "july", "7":
				curmonth = 7
			case "aug", "august", "8":
				curmonth = 8
			case "sep", "september", "9":
				curmonth = 9
			case "oct", "october", "10":
				curmonth = 10
			case "nov", "november", "11":
				curmonth = 11
			case "dec", "december", "12":
				curmonth = 12
			default:
				fatal(fmt.Errorf("Invalid month argument value: %s", a))
			}
			continue
		}
	YEAR:
		if len(a) <= 4 && a[0] >= '1' && a[0] <= '9' {

			for _, s := range a {
				if s < '0' || s > '9' {
					fatal(fmt.Errorf("Invalid year argument value: %s", a))
				}
			}

			var err error
			curyear, err = strconv.Atoi(a)
			if err != nil {
				fatal(err)
			}
		} else {

			fatal(fmt.Errorf("Invalid argument value: %s", a))
		}

	}
}
<commit_msg>fixed algorithm<commit_after>package main

import (
	"fmt"
	"os"
	"strconv"
	"time"
)

var (
	daystr = [...]string{
		"", "01", "02", "03", "04", "05", "06", "07",
		"08", "09", "10", "11", "12", "13", "14",
		"15", "16", "17", "18", "19", "20", "21",
		"22", "23", "24", "25", "26", "27", "28",
		"29", "30", "31",
	}
	monthstr = [...]string{"",
		"January", "February", "March", "April",
		"May", "June", "July", "August", "September",
		"October", "November", "December",
	}
	daysInMonth = [...]int{0,
		31, 29, 31, 30,
		31, 30, 31, 31,
		30, 31, 30, 31,
	}

	today = time.Now()
)

type year int

const calTemplate = "\n" +
	"  ]====================================[   \n" +
	"  |  Su | Mo | Tu | We | Th | Fr | Sa  |   \n" +
	"  ]====================================[   \n" +
	"   | %v | %v | %v | %v | %v | %v | %v |    \n" +
	"   | %v | %v | %v | %v | %v | %v | %v |    \n" +
	"   | %v | %v | %v | %v | %v | %v | %v |    \n" +
	"   | %v | %v | %v | %v | %v | %v | %v |    \n" +
	"   | %v | %v | %v | %v | %v | %v | %v |    \n" +
	"   | %v | %v | %v | %v | %v | %v | %v |    \n" +
	"  ]====================================[   \n" +
	" <                 %v                   >  \n" +
	"  ]====================================[   \n"

func main() {
	cal(procFlags())

}

func cal(m int, y year) {
	y.numDaysPerMonth()
	fmt.Println(daysInMonth)
	var (
		b          []byte
		d          int
		dayOfFirst = y.jan1()
	)
	for _, v := range daysInMonth {
		d += v
	}

	b = make([]byte, d+dayOfFirst)

	var offsetDays []byte

	switch dayOfFirst {
	case 0:
		offsetDays = []byte{0}
	case 1:
		offsetDays = []byte{31}
	case 2:
		offsetDays = []byte{30, 31}
	case 3:
		offsetDays = []byte{29, 30, 31}
	case 4:
		offsetDays = []byte{28, 29, 30, 31}
	case 5:
		offsetDays = []byte{27, 28, 29, 30, 31}
	case 6:
		offsetDays = []byte{26, 27, 28, 29, 30, 31}
	}
	copy(b, offsetDays)

	var offs = len(offsetDays) - 1
	for mo, ndays := range daysInMonth {
		var _ = mo
		for i := 0; i < ndays; i++ {
			b[offs] = byte(i + 1)
			offs++
		}
	}
	/*
		fmt.Printf(calTemplate,
			daystr[b[offset+0]], daystr[b[offset+1]], daystr[b[offset+2]], daystr[b[offset+3]], daystr[b[offset+4]], daystr[b[offset+5]], daystr[b[offset+6]],
			daystr[b[offset+7]], daystr[b[offset+8]], daystr[b[offset+9]], daystr[b[offset+10]], daystr[b[offset+11]], daystr[b[offset+12]], daystr[b[offset+13]],
			daystr[b[offset+14]], daystr[b[offset+15]], daystr[b[offset+16]], daystr[b[offset+17]], daystr[b[offset+18]], daystr[b[offset+19]], daystr[b[offset+20]],
			daystr[b[offset+21]], daystr[b[offset+22]], daystr[b[offset+23]], daystr[b[offset+24]], daystr[b[offset+25]], daystr[b[offset+26]], daystr[b[offset+27]],
			daystr[b[offset+28]], daystr[b[offset+29]], daystr[b[offset+30]], daystr[b[offset+31]], daystr[b[offset+32]], daystr[b[offset+33]], daystr[b[offset+34]],
			daystr[b[offset+35]], daystr[b[offset+36]], daystr[b[offset+37]], daystr[b[offset+38]], daystr[b[offset+39]], daystr[b[offset+40]], daystr[b[offset+41]],
			daystr[b[offset+42]], daystr[b[offset+43]], daystr[b[offset+44]], daystr[b[offset+45]], daystr[b[offset+46]], daystr[b[offset+47]], daystr[b[offset+48]],
			monthstr[mo])
	}*/

}

func fatal(err error) {
	fmt.Fprintf(os.Stderr, "%s:\t%s\n", "awk", err.Error())
	os.Exit(2)
}

func procFlags() (m int, y year) {
	args := os.Args[1:]
	y = year(today.Year())
	for _, a := range args {
		if m == 0 &&
			(a[0] >= 'A' && a[0] <= 'z' ||
				(len(a) <= 2 && a[0] >= '0' && a[0] <= '9')) {

			if a[0] < 'a' {
				a = string(a[0]+'a'-'A') + a[1:]
			}
			if len(a) > 1 && a[0] < 'A' {
				switch a[1] {
				case 0:
				case 1:
				case 2:
				default:
					goto YEAR
				}
			}
			switch a {
			case "jan", "january", "1":
				m = 1
			case "feb", "february", "2":
				m = 2
			case "mar", "march", "3":
				m = 3
			case "apr", "april", "4":
				m = 4
			case "may", "5":
				m = 5
			case "jun", "june", "6":
				m = 6
			case "jul", "july", "7":
				m = 7
			case "aug", "august", "8":
				m = 8
			case "sep", "september", "9":
				m = 9
			case "oct", "october", "10":
				m = 10
			case "nov", "november", "11":
				m = 11
			case "dec", "december", "12":
				m = 12
			default:
				fatal(fmt.Errorf("Invalid month argument value: %s", a))
			}
			continue
		}
	YEAR:
		if len(a) <= 4 && a[0] >= '1' && a[0] <= '9' {

			for _, s := range a {
				if s < '0' || s > '9' {
					fatal(fmt.Errorf("Invalid year argument value: %s", a))
				}
			}
			if tmp, err := strconv.Atoi(a); err != nil {
				fatal(err)
			} else {
				y = year(tmp)
			}

		} else {

			fatal(fmt.Errorf("Invalid argument value: %s", a))
		}
	}
	return
}

func (yr year) numDaysPerMonth() {
	d := yr.jan1()
	switch (year(int(yr)+1).jan1() + (7 - d)) % 7 {
	case 2:
	case 1:
		daysInMonth[2] = 28
	//1752
	default:
		daysInMonth[9] = 19
	}
}

//	return day of the week
//	of jan 1 of given year
func (yr year) jan1() int {

	//	normal gregorian calendar
	//	one extra day per four years
	y := int(yr)
	d := 4 + y + (y+3)/4

	// 	julian calendar
	// 	regular gregorian
	// 	less three days per 400

	if y > 1800 {
		d -= (y - 1701) / 100
		d += (y - 1601) / 400
	}

	// 	great calendar changeover instant

	if y > 1752 {
		d += 3
	}

	return d % 7
}
<|endoftext|>


<commit_before>package gitosis

import (
	ini "github.com/kless/goconfig/config"
	"io/ioutil"
	. "launchpad.net/gocheck"
	"os"
	"path"
	"syscall"
)

func (s *S) TestAddKeyAddsAKeyFileToTheKeydirDirectoryAndTheMemberToTheGroup(c *C) {
	err := AddGroup("pato-fu")
	c.Assert(err, IsNil)
	err = AddKey("pato-fu", "tolices", "my-key")
	c.Assert(err, IsNil)
	p, err := getKeydirPath()
	c.Assert(err, IsNil)
	filePath := path.Join(p, "tolices_key1.pub")
	file, err := os.Open(filePath)
	c.Assert(err, IsNil)
	defer file.Close()
	content, err := ioutil.ReadAll(file)
	c.Assert(err, IsNil)
	c.Assert(string(content), Equals, "my-key")
	conf, err := ini.ReadDefault(path.Join(s.gitosisRepo, "gitosis.conf"))
	c.Assert(err, IsNil)
	members, err := conf.String("group pato-fu", "members")
	c.Assert(err, IsNil)
	c.Assert(members, Equals, "tolices_key1")
}

func (s *S) TestAddKeyUseKey2IfThereIsAlreadyAKeyForTheMember(c *C) {
	err := AddGroup("pato-fu")
	c.Assert(err, IsNil)
	p, err := getKeydirPath()
	c.Assert(err, IsNil)
	key1Path := path.Join(p, "gol-de-quem_key1.pub")
	f, err := os.OpenFile(key1Path, syscall.O_CREAT, 0644)
	c.Assert(err, IsNil)
	f.Close()
	err = AddKey("pato-fu", "gol-de-quem", "my-key")
	c.Assert(err, IsNil)
	file, err := os.Open(path.Join(p, "gol-de-quem_key2.pub"))
	c.Assert(err, IsNil)
	defer file.Close()
	content, err := ioutil.ReadAll(file)
	c.Assert(err, IsNil)
	c.Assert(string(content), Equals, "my-key")
}

func (s *S) TestAddKeyReturnsErrorIfTheGroupDoesNotExist(c *C) {
	err := AddKey("pato-fu", "sertoes", "my-key")
	c.Assert(err, NotNil)
	c.Assert(err, ErrorMatches, "^Group not found$")
}

func (s *S) TestAddKeyDoesNotReturnErrorIfTheDirectoryExists(c *C) {
	err := AddGroup("pato-fu")
	c.Assert(err, IsNil)
	p, err := getKeydirPath()
	c.Assert(err, IsNil)
	os.MkdirAll(p, 0755)
	err = AddKey("pato-fu", "vida-imbecil", "my-key")
	c.Assert(err, IsNil)
}

func (s *S) TestAddKeyShouldRemoveTheKeyFileIfItFailsToAddTheMemberToGitosisFile(c *C) {
	err := AddGroup("pain-of-salvation")
	c.Assert(err, IsNil)
	err = addMember("pain-of-salvation", "used_key1")
	c.Assert(err, IsNil)
	err = AddKey("pain-of-salvation", "used", "my-key")
	c.Assert(err, NotNil)
	c.Assert(err, ErrorMatches, "^Failed to add member to the group, the key file was not saved$")
	p, err := getKeydirPath()
	c.Assert(err, IsNil)
	filepath := path.Join(p, "used_key1.pub")
	f, err := os.Open(filepath)
	if f != nil {
		defer f.Close()
	}
	c.Assert(err, NotNil)
	c.Assert(os.IsNotExist(err), Equals, true)
}
<commit_msg>gitosis: test for AddKey commit<commit_after>package gitosis

import (
	ini "github.com/kless/goconfig/config"
	"io/ioutil"
	. "launchpad.net/gocheck"
	"os"
	"os/exec"
	"path"
	"syscall"
)

func (s *S) TestAddKeyAddsAKeyFileToTheKeydirDirectoryAndTheMemberToTheGroup(c *C) {
	err := AddGroup("pato-fu")
	c.Assert(err, IsNil)
	err = AddKey("pato-fu", "tolices", "my-key")
	c.Assert(err, IsNil)
	p, err := getKeydirPath()
	c.Assert(err, IsNil)
	filePath := path.Join(p, "tolices_key1.pub")
	file, err := os.Open(filePath)
	c.Assert(err, IsNil)
	defer file.Close()
	content, err := ioutil.ReadAll(file)
	c.Assert(err, IsNil)
	c.Assert(string(content), Equals, "my-key")
	conf, err := ini.ReadDefault(path.Join(s.gitosisRepo, "gitosis.conf"))
	c.Assert(err, IsNil)
	members, err := conf.String("group pato-fu", "members")
	c.Assert(err, IsNil)
	c.Assert(members, Equals, "tolices_key1")
}

func (s *S) TestAddKeyUseKey2IfThereIsAlreadyAKeyForTheMember(c *C) {
	err := AddGroup("pato-fu")
	c.Assert(err, IsNil)
	p, err := getKeydirPath()
	c.Assert(err, IsNil)
	key1Path := path.Join(p, "gol-de-quem_key1.pub")
	f, err := os.OpenFile(key1Path, syscall.O_CREAT, 0644)
	c.Assert(err, IsNil)
	f.Close()
	err = AddKey("pato-fu", "gol-de-quem", "my-key")
	c.Assert(err, IsNil)
	file, err := os.Open(path.Join(p, "gol-de-quem_key2.pub"))
	c.Assert(err, IsNil)
	defer file.Close()
	content, err := ioutil.ReadAll(file)
	c.Assert(err, IsNil)
	c.Assert(string(content), Equals, "my-key")
}

func (s *S) TestAddKeyReturnsErrorIfTheGroupDoesNotExist(c *C) {
	err := AddKey("pato-fu", "sertoes", "my-key")
	c.Assert(err, NotNil)
	c.Assert(err, ErrorMatches, "^Group not found$")
}

func (s *S) TestAddKeyDoesNotReturnErrorIfTheDirectoryExists(c *C) {
	err := AddGroup("pato-fu")
	c.Assert(err, IsNil)
	p, err := getKeydirPath()
	c.Assert(err, IsNil)
	os.MkdirAll(p, 0755)
	err = AddKey("pato-fu", "vida-imbecil", "my-key")
	c.Assert(err, IsNil)
}

func (s *S) TestAddKeyShouldRemoveTheKeyFileIfItFailsToAddTheMemberToGitosisFile(c *C) {
	err := AddGroup("pain-of-salvation")
	c.Assert(err, IsNil)
	err = addMember("pain-of-salvation", "used_key1")
	c.Assert(err, IsNil)
	err = AddKey("pain-of-salvation", "used", "my-key")
	c.Assert(err, NotNil)
	c.Assert(err, ErrorMatches, "^Failed to add member to the group, the key file was not saved$")
	p, err := getKeydirPath()
	c.Assert(err, IsNil)
	filepath := path.Join(p, "used_key1.pub")
	f, err := os.Open(filepath)
	if f != nil {
		defer f.Close()
	}
	c.Assert(err, NotNil)
	c.Assert(os.IsNotExist(err), Equals, true)
}

func (s *S) TestAddKeyShouldCommit(c *C) {
	err := AddGroup("pain-of-salvation")
	c.Assert(err, IsNil)
	err = AddKey("pain-of-salvation", "diffidentia", "my-key")
	c.Assert(err, IsNil)
	pwd, err := os.Getwd()
	c.Assert(err, IsNil)
	os.Chdir(s.gitosisBare)
	bareOutput, err := exec.Command("git", "log", "-1", "--pretty=format:%s").CombinedOutput()
	c.Assert(err, IsNil)
	os.Chdir(pwd)
	commitMsg := "Adding member diffidentia_key1 to group pain-of-salvation"
	c.Assert(string(bareOutput), Equals, commitMsg)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"compress/gzip"
	"crypto/md5"
	"encoding/base64"
	"fmt"
	"io"
	"math/big"
	"mime"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"code.google.com/p/go.net/html"

	"log"

	"github.com/crowdmob/goamz/s3"
	"github.com/wsxiaoys/terminal/color"
)

const (
	SCRIPT = iota
	STYLE
)

const UPLOAD_WORKERS = 20

func hashFile(path string) []byte {
	hash := md5.New()
	io.WriteString(hash, path)
	io.WriteString(hash, "\n")

	// TODO: Encode type?

	ref := must(os.Open(path)).(*os.File)
	defer ref.Close()

	must(io.Copy(hash, ref))

	return hash.Sum(nil)
}

func hashBytes(data []byte) []byte {
	hash := md5.New()
	must(io.Copy(hash, bytes.NewReader(data)))
	return hash.Sum(nil)
}

func hashFiles(files []string) string {
	hash := new(big.Int)
	for _, file := range files {
		val := new(big.Int)
		val.SetBytes(hashFile(file))

		hash = hash.Xor(hash, val)
	}

	return fmt.Sprintf("%x", hash)
}

func getRef() string {
	gitPath := mustString(exec.LookPath("git"))

	cmd := exec.Command(gitPath, "rev-parse", "--verify", "HEAD")

	out := bytes.Buffer{}
	cmd.Stdout = &out
	panicIf(cmd.Run())

	return string(out.Bytes())
}

func guessContentType(file string) string {
	return mime.TypeByExtension(filepath.Ext(file))
}

func uploadFile(bucket *s3.Bucket, reader io.Reader, dest string, includeHash bool, caching int) string {
	buffer := bytes.NewBuffer([]byte{})
	writer := gzip.NewWriter(buffer)
	must(io.Copy(writer, reader))
	writer.Close()

	data := buffer.Bytes()

	hash := hashBytes(data)
	hashPrefix := fmt.Sprintf("%x", hash)[:12]
	s3Opts := s3.Options{
		ContentMD5:      base64.StdEncoding.EncodeToString(hash),
		ContentEncoding: "gzip",
		CacheControl:    fmt.Sprintf("public, max-age=%d", caching),
	}

	if includeHash {
		dest = filepath.Join(hashPrefix, dest)
	}

	log.Printf("Uploading to %s in %s (%s) [%d]\n", dest, bucket.Name, hashPrefix, caching)
	err := bucket.PutReader(dest, buffer, int64(len(data)), guessContentType(dest), s3.PublicRead, s3Opts)
	panicIf(err)

	return dest
}

type FileRef struct {
	LocalPath    string
	RemotePath   string
	UploadedPath string
}

type FileInst struct {
	File     *FileRef
	InstPath string
}

func writeFiles(options Options, includeHash bool, files chan *FileRef) {
	bucket := s3Session.Bucket(options.Bucket)

	for file := range files {
		handle := must(os.Open(file.LocalPath)).(*os.File)
		defer handle.Close()

		var ttl int
		ttl = FOREVER
		if !includeHash {
			ttl = LIMITED
		}

		(*file).UploadedPath = uploadFile(bucket, handle, file.RemotePath, includeHash, ttl)
	}
}

func deployFiles(options Options, includeHash bool, files []*FileRef) {
	ch := make(chan *FileRef)

	wg := new(sync.WaitGroup)
	for i := 0; i < UPLOAD_WORKERS; i++ {
		wg.Add(1)
		go func() {
			writeFiles(options, includeHash, ch)
			wg.Done()
		}()
	}

	for _, file := range files {
		if !includeHash && strings.HasSuffix(file.RemotePath, ".html") {
			panic(fmt.Sprintf("Cowardly refusing to deploy an html file (%s) without versioning.", file.RemotePath))
		}

		ch <- file
	}

	close(ch)

	wg.Wait()
}

func addFiles(form uint8, parent *html.Node, files []string) {
	for _, file := range files {
		node := html.Node{
			Type: html.ElementNode,
		}
		switch form {
		case SCRIPT:
			node.Data = "script"
			node.Attr = []html.Attribute{
				html.Attribute{
					Key: "src",
					Val: file,
				},
			}

		case STYLE:
			node.Data = "link"
			node.Attr = []html.Attribute{
				html.Attribute{
					Key: "rel",
					Val: "stylesheet",
				},
				html.Attribute{
					Key: "href",
					Val: file,
				},
			}
		default:
			panic("Type not understood")
		}

		parent.AppendChild(&node)
	}
}

func isLocal(href string) bool {
	parsed := must(url.Parse(href)).(*url.URL)
	return parsed.Host == ""
}

func formatHref(path string) string {
	if !strings.HasPrefix(path, "/") {
		path = "/" + path
	}
	return path
}

func renderHTML(options Options, file HTMLFile) string {
	handle := must(os.Open(file.File.LocalPath)).(*os.File)
	defer handle.Close()

	doc := must(html.Parse(handle)).(*html.Node)

	var f func(*html.Node)
	f = func(n *html.Node) {
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			f(c)
		}

		if n.Type == html.ElementNode {
			switch n.Data {
			case "script":
				for i, a := range n.Attr {
					if a.Key == "src" {
						for _, dep := range file.Deps {
							if dep.InstPath == a.Val {
								n.Attr[i].Val = formatHref(dep.File.UploadedPath)
								break
							}
						}
					}
				}
			case "link":
				stylesheet := false
				for _, a := range n.Attr {
					if a.Key == "rel" {
						stylesheet = a.Val == "stylesheet"
						break
					}
				}
				if !stylesheet {
					return
				}

				for i, a := range n.Attr {
					if a.Key == "href" {
						for _, dep := range file.Deps {
							if dep.InstPath == a.Val {
								n.Attr[i].Val = formatHref(dep.File.UploadedPath)
								break
							}
						}
					}
				}
			}
		}
	}
	f(doc)

	buf := bytes.NewBuffer([]byte{})
	panicIf(html.Render(buf, doc))

	return buf.String()
}

func parseHTML(options Options, path string) (files []string, base string) {
	files = make([]string, 0)

	handle := must(os.Open(path)).(*os.File)
	defer handle.Close()

	doc := must(html.Parse(handle)).(*html.Node)

	var f func(*html.Node)
	f = func(n *html.Node) {
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			f(c)
		}

		if n.Type == html.ElementNode {
			switch n.Data {
			case "base":
				for _, a := range n.Attr {
					if a.Key == "href" {
						base = a.Val
					}
				}
			case "script":
				for _, a := range n.Attr {
					if a.Key == "src" {
						if isLocal(a.Val) {
							files = append(files, a.Val)
						}
					}
				}
			case "link":
				local := false
				stylesheet := false
				href := ""
				for _, a := range n.Attr {
					switch a.Key {
					case "href":
						local = isLocal(a.Val)
						href = a.Val
					case "rel":
						stylesheet = a.Val == "stylesheet"
					}
				}
				if local && stylesheet {
					files = append(files, href)
				}
			}
		}
	}
	f(doc)

	return
}

func deployHTML(options Options, id string, file HTMLFile) {
	data := renderHTML(options, file)

	internalPath, err := filepath.Rel(options.Root, file.File.LocalPath)
	if err != nil {
		panic(err)
	}

	permPath := filepath.Join(options.Dest, id, internalPath)
	curPath := filepath.Join(options.Dest, internalPath)

	bucket := s3Session.Bucket(options.Bucket)
	uploadFile(bucket, strings.NewReader(data), permPath, false, FOREVER)

	log.Println("Copying", permPath, "to", curPath)
	copyFile(bucket, permPath, curPath, "text/html", LIMITED)
}

func expandFiles(root string, glob string) []string {
	out := make([]string, 0)
	cases := strings.Split(glob, ",")

	for _, pattern := range cases {
		list := must(filepath.Glob(filepath.Join(root, pattern))).([]string)

		for _, file := range list {
			info := must(os.Stat(file)).(os.FileInfo)

			if info.IsDir() {
				filepath.Walk(file, func(path string, info os.FileInfo, err error) error {
					panicIf(err)

					if !info.IsDir() {
						out = append(out, path)
					}

					return nil
				})
			} else {
				out = append(out, file)
			}
		}
	}
	return out
}

func listFiles(options Options) []*FileRef {
	filePaths := expandFiles(options.Root, options.Files)

	files := make([]*FileRef, len(filePaths))
	for i, path := range filePaths {
		remotePath := filepath.Join(options.Dest, mustString(filepath.Rel(options.Root, path)))

		files[i] = &FileRef{
			LocalPath:  path,
			RemotePath: remotePath,
		}
	}

	return files
}

func ignoreFiles(full []*FileRef, rem []*FileRef) []*FileRef {
	out := make([]*FileRef, 0, len(full))

	for _, file := range full {
		ignore := false
		path := filepath.Clean(file.LocalPath)

		for _, remFile := range rem {
			if filepath.Clean(remFile.LocalPath) == path {
				ignore = true
				break
			}
		}

		if !ignore {
			out = append(out, file)
		}
	}

	return out
}

func extractFileList(options Options, pattern string) (files []string) {
	files = make([]string, 0)

	parts := strings.Split(pattern, ",")

	for _, part := range parts {
		matches, err := filepath.Glob(filepath.Join(options.Root, part))
		if err != nil {
			panic(err)
		}
		if matches == nil {
			panic(fmt.Sprintf("Pattern %s did not match any files", part))
		}

		files = append(files, matches...)
	}

	return files
}

func filesWithExtension(files []*FileRef, ext string) (outFiles []*FileRef) {
	outFiles = make([]*FileRef, 0)
	for _, file := range files {
		if filepath.Ext(file.LocalPath) == ext {
			outFiles = append(outFiles, file)
		}
	}

	return
}

type HTMLFile struct {
	File FileRef
	Deps []FileInst
	Base string
}

func (f HTMLFile) GetLocalPath() string {
	return f.File.LocalPath
}

func Deploy(options Options) {
	if s3Session == nil {
		s3Session = openS3(options.AWSKey, options.AWSSecret)
	}

	files := listFiles(options)

	htmlFileRefs := filesWithExtension(files, ".html")

	inclFiles := make(map[string]*FileRef)
	htmlFiles := make([]HTMLFile, len(htmlFileRefs))
	for i, file := range htmlFileRefs {
		dir := filepath.Dir(file.LocalPath)

		rel, err := filepath.Rel(options.Root, dir)
		if err != nil {
			panic(err)
		}

		paths, base := parseHTML(options, file.LocalPath)

		if strings.HasPrefix(strings.ToLower(base), "http") || strings.HasPrefix(base, "//") {
			panic("Absolute base tags are not supported")
		}

		htmlFiles[i] = HTMLFile{
			File: *file,
			Deps: make([]FileInst, len(paths)),
			Base: base,
		}

		for j, path := range paths {
			local := filepath.Join(options.Root, rel, base, path)
			remote := filepath.Join(options.Dest, rel, base, path)

			ref, ok := inclFiles[local]
			if !ok {
				ref = &FileRef{
					LocalPath:  local,
					RemotePath: remote,

					// Filled in after the deploy:
					UploadedPath: "",
				}

				inclFiles[local] = ref
			}

			use := FileInst{
				File:     ref,
				InstPath: path,
			}

			htmlFiles[i].Deps[j] = use
		}
	}

	inclFileList := make([]*FileRef, len(inclFiles))
	i := 0
	for _, ref := range inclFiles {
		inclFileList[i] = ref
		i++
	}

	hashPaths := make([]string, 0)
	for _, item := range inclFileList {
		hashPaths = append(hashPaths, item.LocalPath)
	}
	for _, item := range htmlFiles {
		hashPaths = append(hashPaths, item.File.LocalPath)
	}

	hash := hashFiles(hashPaths)
	id := hash[:12]

	deployFiles(options, true, inclFileList)
	deployFiles(options, false, ignoreFiles(files, htmlFileRefs))

	// Ensure that the new files exist in s3
	// Time based on "Eventual Consistency: How soon is eventual?"
	time.Sleep(1500 * time.Millisecond)

	wg := sync.WaitGroup{}
	for _, file := range htmlFiles {
		wg.Add(1)

		go func(file HTMLFile) {
			defer wg.Done()
			deployHTML(options, id, file)
		}(file)
	}

	wg.Wait()

	color.Printf(`
+------------------------------------+
|         @{g}Deploy Successful!@{|}         |
|                                    |
|       Deploy ID: @{?}%s@{|}      |
+------------------------------------+
`, id)

}

func deployCmd() {
	options, _ := parseOptions()
	loadConfigFile(&options)

	if options.Bucket == "" {
		panic("You must specify a bucket")
	}

	if options.AWSKey == "" || options.AWSSecret == "" {
		panic("You must specify your AWS credentials")
	}

	Deploy(options)
}
<commit_msg>Improve error message<commit_after>package main

import (
	"bytes"
	"compress/gzip"
	"crypto/md5"
	"encoding/base64"
	"fmt"
	"io"
	"math/big"
	"mime"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"code.google.com/p/go.net/html"

	"log"

	"github.com/crowdmob/goamz/s3"
	"github.com/wsxiaoys/terminal/color"
)

const (
	SCRIPT = iota
	STYLE
)

const UPLOAD_WORKERS = 20

func hashFile(path string) []byte {
	hash := md5.New()
	io.WriteString(hash, path)
	io.WriteString(hash, "\n")

	// TODO: Encode type?

	ref := must(os.Open(path)).(*os.File)
	defer ref.Close()

	must(io.Copy(hash, ref))

	return hash.Sum(nil)
}

func hashBytes(data []byte) []byte {
	hash := md5.New()
	must(io.Copy(hash, bytes.NewReader(data)))
	return hash.Sum(nil)
}

func hashFiles(files []string) string {
	hash := new(big.Int)
	for _, file := range files {
		val := new(big.Int)
		val.SetBytes(hashFile(file))

		hash = hash.Xor(hash, val)
	}

	return fmt.Sprintf("%x", hash)
}

func getRef() string {
	gitPath := mustString(exec.LookPath("git"))

	cmd := exec.Command(gitPath, "rev-parse", "--verify", "HEAD")

	out := bytes.Buffer{}
	cmd.Stdout = &out
	panicIf(cmd.Run())

	return string(out.Bytes())
}

func guessContentType(file string) string {
	return mime.TypeByExtension(filepath.Ext(file))
}

func uploadFile(bucket *s3.Bucket, reader io.Reader, dest string, includeHash bool, caching int) string {
	buffer := bytes.NewBuffer([]byte{})
	writer := gzip.NewWriter(buffer)
	must(io.Copy(writer, reader))
	writer.Close()

	data := buffer.Bytes()

	hash := hashBytes(data)
	hashPrefix := fmt.Sprintf("%x", hash)[:12]
	s3Opts := s3.Options{
		ContentMD5:      base64.StdEncoding.EncodeToString(hash),
		ContentEncoding: "gzip",
		CacheControl:    fmt.Sprintf("public, max-age=%d", caching),
	}

	if includeHash {
		dest = filepath.Join(hashPrefix, dest)
	}

	log.Printf("Uploading to %s in %s (%s) [%d]\n", dest, bucket.Name, hashPrefix, caching)
	err := bucket.PutReader(dest, buffer, int64(len(data)), guessContentType(dest), s3.PublicRead, s3Opts)
	panicIf(err)

	return dest
}

type FileRef struct {
	LocalPath    string
	RemotePath   string
	UploadedPath string
}

type FileInst struct {
	File     *FileRef
	InstPath string
}

func writeFiles(options Options, includeHash bool, files chan *FileRef) {
	bucket := s3Session.Bucket(options.Bucket)

	for file := range files {
		handle := must(os.Open(file.LocalPath)).(*os.File)
		defer handle.Close()

		var ttl int
		ttl = FOREVER
		if !includeHash {
			ttl = LIMITED
		}

		(*file).UploadedPath = uploadFile(bucket, handle, file.RemotePath, includeHash, ttl)
	}
}

func deployFiles(options Options, includeHash bool, files []*FileRef) {
	ch := make(chan *FileRef)

	wg := new(sync.WaitGroup)
	for i := 0; i < UPLOAD_WORKERS; i++ {
		wg.Add(1)
		go func() {
			writeFiles(options, includeHash, ch)
			wg.Done()
		}()
	}

	for _, file := range files {
		if !includeHash && strings.HasSuffix(file.RemotePath, ".html") {
			panic(fmt.Sprintf("Cowardly refusing to deploy an html file (%s) without versioning.", file.RemotePath))
		}

		ch <- file
	}

	close(ch)

	wg.Wait()
}

func addFiles(form uint8, parent *html.Node, files []string) {
	for _, file := range files {
		node := html.Node{
			Type: html.ElementNode,
		}
		switch form {
		case SCRIPT:
			node.Data = "script"
			node.Attr = []html.Attribute{
				html.Attribute{
					Key: "src",
					Val: file,
				},
			}

		case STYLE:
			node.Data = "link"
			node.Attr = []html.Attribute{
				html.Attribute{
					Key: "rel",
					Val: "stylesheet",
				},
				html.Attribute{
					Key: "href",
					Val: file,
				},
			}
		default:
			panic("Type not understood")
		}

		parent.AppendChild(&node)
	}
}

func isLocal(href string) bool {
	parsed := must(url.Parse(href)).(*url.URL)
	return parsed.Host == ""
}

func formatHref(path string) string {
	if !strings.HasPrefix(path, "/") {
		path = "/" + path
	}
	return path
}

func renderHTML(options Options, file HTMLFile) string {
	handle := must(os.Open(file.File.LocalPath)).(*os.File)
	defer handle.Close()

	doc := must(html.Parse(handle)).(*html.Node)

	var f func(*html.Node)
	f = func(n *html.Node) {
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			f(c)
		}

		if n.Type == html.ElementNode {
			switch n.Data {
			case "script":
				for i, a := range n.Attr {
					if a.Key == "src" {
						for _, dep := range file.Deps {
							if dep.InstPath == a.Val {
								n.Attr[i].Val = formatHref(dep.File.UploadedPath)
								break
							}
						}
					}
				}
			case "link":
				stylesheet := false
				for _, a := range n.Attr {
					if a.Key == "rel" {
						stylesheet = a.Val == "stylesheet"
						break
					}
				}
				if !stylesheet {
					return
				}

				for i, a := range n.Attr {
					if a.Key == "href" {
						for _, dep := range file.Deps {
							if dep.InstPath == a.Val {
								n.Attr[i].Val = formatHref(dep.File.UploadedPath)
								break
							}
						}
					}
				}
			}
		}
	}
	f(doc)

	buf := bytes.NewBuffer([]byte{})
	panicIf(html.Render(buf, doc))

	return buf.String()
}

func parseHTML(options Options, path string) (files []string, base string) {
	files = make([]string, 0)

	handle := must(os.Open(path)).(*os.File)
	defer handle.Close()

	doc := must(html.Parse(handle)).(*html.Node)

	var f func(*html.Node)
	f = func(n *html.Node) {
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			f(c)
		}

		if n.Type == html.ElementNode {
			switch n.Data {
			case "base":
				for _, a := range n.Attr {
					if a.Key == "href" {
						base = a.Val
					}
				}
			case "script":
				for _, a := range n.Attr {
					if a.Key == "src" {
						if isLocal(a.Val) {
							files = append(files, a.Val)
						}
					}
				}
			case "link":
				local := false
				stylesheet := false
				href := ""
				for _, a := range n.Attr {
					switch a.Key {
					case "href":
						local = isLocal(a.Val)
						href = a.Val
					case "rel":
						stylesheet = a.Val == "stylesheet"
					}
				}
				if local && stylesheet {
					files = append(files, href)
				}
			}
		}
	}
	f(doc)

	return
}

func deployHTML(options Options, id string, file HTMLFile) {
	data := renderHTML(options, file)

	internalPath, err := filepath.Rel(options.Root, file.File.LocalPath)
	if err != nil {
		panic(err)
	}

	permPath := filepath.Join(options.Dest, id, internalPath)
	curPath := filepath.Join(options.Dest, internalPath)

	bucket := s3Session.Bucket(options.Bucket)
	uploadFile(bucket, strings.NewReader(data), permPath, false, FOREVER)

	log.Println("Copying", permPath, "to", curPath)
	copyFile(bucket, permPath, curPath, "text/html", LIMITED)
}

func expandFiles(root string, glob string) []string {
	out := make([]string, 0)
	cases := strings.Split(glob, ",")

	for _, pattern := range cases {
		list := must(filepath.Glob(filepath.Join(root, pattern))).([]string)

		for _, file := range list {
			info := must(os.Stat(file)).(os.FileInfo)

			if info.IsDir() {
				filepath.Walk(file, func(path string, info os.FileInfo, err error) error {
					panicIf(err)

					if !info.IsDir() {
						out = append(out, path)
					}

					return nil
				})
			} else {
				out = append(out, file)
			}
		}
	}
	return out
}

func listFiles(options Options) []*FileRef {
	filePaths := expandFiles(options.Root, options.Files)

	files := make([]*FileRef, len(filePaths))
	for i, path := range filePaths {
		remotePath := filepath.Join(options.Dest, mustString(filepath.Rel(options.Root, path)))

		files[i] = &FileRef{
			LocalPath:  path,
			RemotePath: remotePath,
		}
	}

	return files
}

func ignoreFiles(full []*FileRef, rem []*FileRef) []*FileRef {
	out := make([]*FileRef, 0, len(full))

	for _, file := range full {
		ignore := false
		path := filepath.Clean(file.LocalPath)

		for _, remFile := range rem {
			if filepath.Clean(remFile.LocalPath) == path {
				ignore = true
				break
			}
		}

		if !ignore {
			out = append(out, file)
		}
	}

	return out
}

func extractFileList(options Options, pattern string) (files []string) {
	files = make([]string, 0)

	parts := strings.Split(pattern, ",")

	for _, part := range parts {
		matches, err := filepath.Glob(filepath.Join(options.Root, part))
		if err != nil {
			panic(err)
		}
		if matches == nil {
			panic(fmt.Sprintf("Pattern %s did not match any files", part))
		}

		files = append(files, matches...)
	}

	return files
}

func filesWithExtension(files []*FileRef, ext string) (outFiles []*FileRef) {
	outFiles = make([]*FileRef, 0)
	for _, file := range files {
		if filepath.Ext(file.LocalPath) == ext {
			outFiles = append(outFiles, file)
		}
	}

	return
}

type HTMLFile struct {
	File FileRef
	Deps []FileInst
	Base string
}

func (f HTMLFile) GetLocalPath() string {
	return f.File.LocalPath
}

func Deploy(options Options) {
	if s3Session == nil {
		s3Session = openS3(options.AWSKey, options.AWSSecret)
	}

	files := listFiles(options)

	htmlFileRefs := filesWithExtension(files, ".html")

	if len(htmlFileRefs) == 0 {
		panic("Error: No HTML files found")
	}

	inclFiles := make(map[string]*FileRef)
	htmlFiles := make([]HTMLFile, len(htmlFileRefs))
	for i, file := range htmlFileRefs {
		dir := filepath.Dir(file.LocalPath)

		rel, err := filepath.Rel(options.Root, dir)
		if err != nil {
			panic(err)
		}

		paths, base := parseHTML(options, file.LocalPath)

		if strings.HasPrefix(strings.ToLower(base), "http") || strings.HasPrefix(base, "//") {
			panic("Absolute base tags are not supported")
		}

		htmlFiles[i] = HTMLFile{
			File: *file,
			Deps: make([]FileInst, len(paths)),
			Base: base,
		}

		for j, path := range paths {
			local := filepath.Join(options.Root, rel, base, path)
			remote := filepath.Join(options.Dest, rel, base, path)

			ref, ok := inclFiles[local]
			if !ok {
				ref = &FileRef{
					LocalPath:  local,
					RemotePath: remote,

					// Filled in after the deploy:
					UploadedPath: "",
				}

				inclFiles[local] = ref
			}

			use := FileInst{
				File:     ref,
				InstPath: path,
			}

			htmlFiles[i].Deps[j] = use
		}
	}

	inclFileList := make([]*FileRef, len(inclFiles))
	i := 0
	for _, ref := range inclFiles {
		inclFileList[i] = ref
		i++
	}

	hashPaths := make([]string, 0)
	for _, item := range inclFileList {
		hashPaths = append(hashPaths, item.LocalPath)
	}
	for _, item := range htmlFiles {
		hashPaths = append(hashPaths, item.File.LocalPath)
	}

	hash := hashFiles(hashPaths)
	id := hash[:12]

	deployFiles(options, true, inclFileList)
	deployFiles(options, false, ignoreFiles(files, htmlFileRefs))

	// Ensure that the new files exist in s3
	// Time based on "Eventual Consistency: How soon is eventual?"
	time.Sleep(1500 * time.Millisecond)

	wg := sync.WaitGroup{}
	for _, file := range htmlFiles {
		wg.Add(1)

		go func(file HTMLFile) {
			defer wg.Done()
			deployHTML(options, id, file)
		}(file)
	}

	wg.Wait()

	color.Printf(`
+------------------------------------+
|         @{g}Deploy Successful!@{|}         |
|                                    |
|       Deploy ID: @{?}%s@{|}      |
+------------------------------------+
`, id)

}

func deployCmd() {
	options, _ := parseOptions()
	loadConfigFile(&options)

	if options.Bucket == "" {
		panic("You must specify a bucket")
	}

	if options.AWSKey == "" || options.AWSSecret == "" {
		panic("You must specify your AWS credentials")
	}

	Deploy(options)
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package scheduling

import (
	"fmt"
	"strconv"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	compute "google.golang.org/api/compute/v1"
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/uuid"
	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
	"k8s.io/kubernetes/test/e2e/framework"
)

var _ = SIGDescribe("Multi-AZ Cluster Volumes", func() {
	f := framework.NewDefaultFramework("multi-az")
	var zoneCount int
	var err error
	image := framework.ServeHostnameImage
	BeforeEach(func() {
		framework.SkipUnlessProviderIs("gce", "gke")
		if zoneCount <= 0 {
			zoneCount, err = getZoneCount(f.ClientSet)
			Expect(err).NotTo(HaveOccurred())
		}
		By(fmt.Sprintf("Checking for multi-zone cluster.  Zone count = %d", zoneCount))
		msg := fmt.Sprintf("Zone count is %d, only run for multi-zone clusters, skipping test", zoneCount)
		framework.SkipUnlessAtLeast(zoneCount, 2, msg)
		// TODO: SkipUnlessDefaultScheduler() // Non-default schedulers might not spread
	})
	It("should schedule pods in the same zones as statically provisioned PVs", func() {
		PodsUseStaticPVsOrFail(f, (2*zoneCount)+1, image)
	})

	It("should only be allowed to provision PDs in zones where nodes exist", func() {
		OnlyAllowNodeZones(f, zoneCount, image)
	})
})

// OnlyAllowNodeZones tests that GetAllCurrentZones returns only zones with Nodes
func OnlyAllowNodeZones(f *framework.Framework, zoneCount int, image string) {
	gceCloud, err := framework.GetGCECloud()
	Expect(err).NotTo(HaveOccurred())

	// Get all the zones that the nodes are in
	expectedZones, err := gceCloud.GetAllZonesFromCloudProvider()
	Expect(err).NotTo(HaveOccurred())
	framework.Logf("Expected zones: %v\n", expectedZones)

	// Get all the zones in this current region
	region := gceCloud.Region()
	allZonesInRegion, err := gceCloud.ListZonesInRegion(region)
	Expect(err).NotTo(HaveOccurred())

	var extraZone string
	for _, zone := range allZonesInRegion {
		if !expectedZones.Has(zone.Name) {
			extraZone = zone.Name
			break
		}
	}
	Expect(extraZone).NotTo(Equal(""), fmt.Sprintf("No extra zones available in region %s", region))

	By(fmt.Sprintf("starting a compute instance in unused zone: %v\n", extraZone))
	project := framework.TestContext.CloudConfig.ProjectID
	zone := extraZone
	myuuid := string(uuid.NewUUID())
	name := "compute-" + myuuid
	imageURL := "https://www.googleapis.com/compute/v1/projects/debian-cloud/global/images/debian-7-wheezy-v20140606"

	rb := &compute.Instance{
		MachineType: "zones/" + zone + "/machineTypes/f1-micro",
		Disks: []*compute.AttachedDisk{
			{
				AutoDelete: true,
				Boot:       true,
				Type:       "PERSISTENT",
				InitializeParams: &compute.AttachedDiskInitializeParams{
					DiskName:    "my-root-pd-" + myuuid,
					SourceImage: imageURL,
				},
			},
		},
		NetworkInterfaces: []*compute.NetworkInterface{
			{
				AccessConfigs: []*compute.AccessConfig{
					{
						Type: "ONE_TO_ONE_NAT",
						Name: "External NAT",
					},
				},
				Network: "/global/networks/default",
			},
		},
		Name: name,
	}

	err = gceCloud.InsertInstance(project, zone, rb)
	Expect(err).NotTo(HaveOccurred())

	defer func() {
		// Teardown of the compute instance
		framework.Logf("Deleting compute resource: %v", name)
		err := gceCloud.DeleteInstance(project, zone, name)
		Expect(err).NotTo(HaveOccurred())
	}()

	By("Creating zoneCount+1 PVCs and making sure PDs are only provisioned in zones with nodes")
	// Create some (zoneCount+1) PVCs with names of form "pvc-x" where x is 1...zoneCount+1
	// This will exploit ChooseZoneForVolume in pkg/volume/util.go to provision them in all the zones it "sees"
	var pvcList []*v1.PersistentVolumeClaim
	c := f.ClientSet
	ns := f.Namespace.Name

	for index := 1; index <= zoneCount+1; index++ {
		pvc := newNamedDefaultClaim(ns, index)
		pvc, err = framework.CreatePVC(c, ns, pvc)
		Expect(err).NotTo(HaveOccurred())
		pvcList = append(pvcList, pvc)

		// Defer the cleanup
		defer func() {
			framework.Logf("deleting claim %q/%q", pvc.Namespace, pvc.Name)
			err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Delete(pvc.Name, nil)
			if err != nil {
				framework.Failf("Error deleting claim %q. Error: %v", pvc.Name, err)
			}
		}()
	}

	// Wait for all claims bound
	for _, claim := range pvcList {
		err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, c, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout)
		Expect(err).NotTo(HaveOccurred())
	}

	pvZones := sets.NewString()
	By("Checking that PDs have been provisioned in only the expected zones")
	for _, claim := range pvcList {
		// Get a new copy of the claim to have all fields populated
		claim, err = c.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{})
		Expect(err).NotTo(HaveOccurred())

		// Get the related PV
		pv, err := c.CoreV1().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{})
		Expect(err).NotTo(HaveOccurred())

		pvZone, ok := pv.ObjectMeta.Labels[kubeletapis.LabelZoneFailureDomain]
		Expect(ok).To(BeTrue(), "PV has no LabelZone to be found")
		pvZones.Insert(pvZone)
	}
	Expect(pvZones.Equal(expectedZones)).To(BeTrue(), fmt.Sprintf("PDs provisioned in unwanted zones. We want zones: %v, got: %v", expectedZones, pvZones))
}

type staticPVTestConfig struct {
	pvSource *v1.PersistentVolumeSource
	pv       *v1.PersistentVolume
	pvc      *v1.PersistentVolumeClaim
	pod      *v1.Pod
}

// Check that the pods using statically created PVs get scheduled to the same zone that the PV is in.
func PodsUseStaticPVsOrFail(f *framework.Framework, podCount int, image string) {
	var err error
	c := f.ClientSet
	ns := f.Namespace.Name

	zones, err := getZoneNames(c)
	Expect(err).NotTo(HaveOccurred())

	By("Creating static PVs across zones")
	configs := make([]*staticPVTestConfig, podCount)
	for i := range configs {
		configs[i] = &staticPVTestConfig{}
	}

	defer func() {
		By("Cleaning up pods and PVs")
		for _, config := range configs {
			framework.DeletePodOrFail(c, ns, config.pod.Name)
		}
		for _, config := range configs {
			framework.WaitForPodNoLongerRunningInNamespace(c, config.pod.Name, ns)
			framework.PVPVCCleanup(c, ns, config.pv, config.pvc)
			err = framework.DeletePVSource(config.pvSource)
			Expect(err).NotTo(HaveOccurred())
		}
	}()

	for i, config := range configs {
		zone := zones[i%len(zones)]
		config.pvSource, err = framework.CreatePVSource(zone)
		Expect(err).NotTo(HaveOccurred())

		pvConfig := framework.PersistentVolumeConfig{
			NamePrefix: "multizone-pv",
			PVSource:   *config.pvSource,
			Prebind:    nil,
		}
		className := ""
		pvcConfig := framework.PersistentVolumeClaimConfig{StorageClassName: &className}

		config.pv, config.pvc, err = framework.CreatePVPVC(c, pvConfig, pvcConfig, ns, true)
		Expect(err).NotTo(HaveOccurred())
	}

	By("Waiting for all PVCs to be bound")
	for _, config := range configs {
		framework.WaitOnPVandPVC(c, ns, config.pv, config.pvc)
	}

	By("Creating pods for each static PV")
	for _, config := range configs {
		podConfig := framework.MakePod(ns, nil, []*v1.PersistentVolumeClaim{config.pvc}, false, "")
		config.pod, err = c.CoreV1().Pods(ns).Create(podConfig)
		Expect(err).NotTo(HaveOccurred())
	}

	By("Waiting for all pods to be running")
	for _, config := range configs {
		err = framework.WaitForPodRunningInNamespace(c, config.pod)
		Expect(err).NotTo(HaveOccurred())
	}
}

func newNamedDefaultClaim(ns string, index int) *v1.PersistentVolumeClaim {
	claim := v1.PersistentVolumeClaim{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "pvc-" + strconv.Itoa(index),
			Namespace: ns,
		},
		Spec: v1.PersistentVolumeClaimSpec{
			AccessModes: []v1.PersistentVolumeAccessMode{
				v1.ReadWriteOnce,
			},
			Resources: v1.ResourceRequirements{
				Requests: v1.ResourceList{
					v1.ResourceName(v1.ResourceStorage): resource.MustParse("1Gi"),
				},
			},
		},
	}

	return &claim
}
<commit_msg>Tag multi-az cluster volume e2e test with sig-storage<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package scheduling

import (
	"fmt"
	"strconv"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	compute "google.golang.org/api/compute/v1"
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/uuid"
	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
	"k8s.io/kubernetes/test/e2e/framework"
)

var _ = SIGDescribe("Multi-AZ Cluster Volumes [sig-storage]", func() {
	f := framework.NewDefaultFramework("multi-az")
	var zoneCount int
	var err error
	image := framework.ServeHostnameImage
	BeforeEach(func() {
		framework.SkipUnlessProviderIs("gce", "gke")
		if zoneCount <= 0 {
			zoneCount, err = getZoneCount(f.ClientSet)
			Expect(err).NotTo(HaveOccurred())
		}
		By(fmt.Sprintf("Checking for multi-zone cluster.  Zone count = %d", zoneCount))
		msg := fmt.Sprintf("Zone count is %d, only run for multi-zone clusters, skipping test", zoneCount)
		framework.SkipUnlessAtLeast(zoneCount, 2, msg)
		// TODO: SkipUnlessDefaultScheduler() // Non-default schedulers might not spread
	})
	It("should schedule pods in the same zones as statically provisioned PVs", func() {
		PodsUseStaticPVsOrFail(f, (2*zoneCount)+1, image)
	})

	It("should only be allowed to provision PDs in zones where nodes exist", func() {
		OnlyAllowNodeZones(f, zoneCount, image)
	})
})

// OnlyAllowNodeZones tests that GetAllCurrentZones returns only zones with Nodes
func OnlyAllowNodeZones(f *framework.Framework, zoneCount int, image string) {
	gceCloud, err := framework.GetGCECloud()
	Expect(err).NotTo(HaveOccurred())

	// Get all the zones that the nodes are in
	expectedZones, err := gceCloud.GetAllZonesFromCloudProvider()
	Expect(err).NotTo(HaveOccurred())
	framework.Logf("Expected zones: %v\n", expectedZones)

	// Get all the zones in this current region
	region := gceCloud.Region()
	allZonesInRegion, err := gceCloud.ListZonesInRegion(region)
	Expect(err).NotTo(HaveOccurred())

	var extraZone string
	for _, zone := range allZonesInRegion {
		if !expectedZones.Has(zone.Name) {
			extraZone = zone.Name
			break
		}
	}
	Expect(extraZone).NotTo(Equal(""), fmt.Sprintf("No extra zones available in region %s", region))

	By(fmt.Sprintf("starting a compute instance in unused zone: %v\n", extraZone))
	project := framework.TestContext.CloudConfig.ProjectID
	zone := extraZone
	myuuid := string(uuid.NewUUID())
	name := "compute-" + myuuid
	imageURL := "https://www.googleapis.com/compute/v1/projects/debian-cloud/global/images/debian-7-wheezy-v20140606"

	rb := &compute.Instance{
		MachineType: "zones/" + zone + "/machineTypes/f1-micro",
		Disks: []*compute.AttachedDisk{
			{
				AutoDelete: true,
				Boot:       true,
				Type:       "PERSISTENT",
				InitializeParams: &compute.AttachedDiskInitializeParams{
					DiskName:    "my-root-pd-" + myuuid,
					SourceImage: imageURL,
				},
			},
		},
		NetworkInterfaces: []*compute.NetworkInterface{
			{
				AccessConfigs: []*compute.AccessConfig{
					{
						Type: "ONE_TO_ONE_NAT",
						Name: "External NAT",
					},
				},
				Network: "/global/networks/default",
			},
		},
		Name: name,
	}

	err = gceCloud.InsertInstance(project, zone, rb)
	Expect(err).NotTo(HaveOccurred())

	defer func() {
		// Teardown of the compute instance
		framework.Logf("Deleting compute resource: %v", name)
		err := gceCloud.DeleteInstance(project, zone, name)
		Expect(err).NotTo(HaveOccurred())
	}()

	By("Creating zoneCount+1 PVCs and making sure PDs are only provisioned in zones with nodes")
	// Create some (zoneCount+1) PVCs with names of form "pvc-x" where x is 1...zoneCount+1
	// This will exploit ChooseZoneForVolume in pkg/volume/util.go to provision them in all the zones it "sees"
	var pvcList []*v1.PersistentVolumeClaim
	c := f.ClientSet
	ns := f.Namespace.Name

	for index := 1; index <= zoneCount+1; index++ {
		pvc := newNamedDefaultClaim(ns, index)
		pvc, err = framework.CreatePVC(c, ns, pvc)
		Expect(err).NotTo(HaveOccurred())
		pvcList = append(pvcList, pvc)

		// Defer the cleanup
		defer func() {
			framework.Logf("deleting claim %q/%q", pvc.Namespace, pvc.Name)
			err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Delete(pvc.Name, nil)
			if err != nil {
				framework.Failf("Error deleting claim %q. Error: %v", pvc.Name, err)
			}
		}()
	}

	// Wait for all claims bound
	for _, claim := range pvcList {
		err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, c, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout)
		Expect(err).NotTo(HaveOccurred())
	}

	pvZones := sets.NewString()
	By("Checking that PDs have been provisioned in only the expected zones")
	for _, claim := range pvcList {
		// Get a new copy of the claim to have all fields populated
		claim, err = c.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{})
		Expect(err).NotTo(HaveOccurred())

		// Get the related PV
		pv, err := c.CoreV1().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{})
		Expect(err).NotTo(HaveOccurred())

		pvZone, ok := pv.ObjectMeta.Labels[kubeletapis.LabelZoneFailureDomain]
		Expect(ok).To(BeTrue(), "PV has no LabelZone to be found")
		pvZones.Insert(pvZone)
	}
	Expect(pvZones.Equal(expectedZones)).To(BeTrue(), fmt.Sprintf("PDs provisioned in unwanted zones. We want zones: %v, got: %v", expectedZones, pvZones))
}

type staticPVTestConfig struct {
	pvSource *v1.PersistentVolumeSource
	pv       *v1.PersistentVolume
	pvc      *v1.PersistentVolumeClaim
	pod      *v1.Pod
}

// Check that the pods using statically created PVs get scheduled to the same zone that the PV is in.
func PodsUseStaticPVsOrFail(f *framework.Framework, podCount int, image string) {
	var err error
	c := f.ClientSet
	ns := f.Namespace.Name

	zones, err := getZoneNames(c)
	Expect(err).NotTo(HaveOccurred())

	By("Creating static PVs across zones")
	configs := make([]*staticPVTestConfig, podCount)
	for i := range configs {
		configs[i] = &staticPVTestConfig{}
	}

	defer func() {
		By("Cleaning up pods and PVs")
		for _, config := range configs {
			framework.DeletePodOrFail(c, ns, config.pod.Name)
		}
		for _, config := range configs {
			framework.WaitForPodNoLongerRunningInNamespace(c, config.pod.Name, ns)
			framework.PVPVCCleanup(c, ns, config.pv, config.pvc)
			err = framework.DeletePVSource(config.pvSource)
			Expect(err).NotTo(HaveOccurred())
		}
	}()

	for i, config := range configs {
		zone := zones[i%len(zones)]
		config.pvSource, err = framework.CreatePVSource(zone)
		Expect(err).NotTo(HaveOccurred())

		pvConfig := framework.PersistentVolumeConfig{
			NamePrefix: "multizone-pv",
			PVSource:   *config.pvSource,
			Prebind:    nil,
		}
		className := ""
		pvcConfig := framework.PersistentVolumeClaimConfig{StorageClassName: &className}

		config.pv, config.pvc, err = framework.CreatePVPVC(c, pvConfig, pvcConfig, ns, true)
		Expect(err).NotTo(HaveOccurred())
	}

	By("Waiting for all PVCs to be bound")
	for _, config := range configs {
		framework.WaitOnPVandPVC(c, ns, config.pv, config.pvc)
	}

	By("Creating pods for each static PV")
	for _, config := range configs {
		podConfig := framework.MakePod(ns, nil, []*v1.PersistentVolumeClaim{config.pvc}, false, "")
		config.pod, err = c.CoreV1().Pods(ns).Create(podConfig)
		Expect(err).NotTo(HaveOccurred())
	}

	By("Waiting for all pods to be running")
	for _, config := range configs {
		err = framework.WaitForPodRunningInNamespace(c, config.pod)
		Expect(err).NotTo(HaveOccurred())
	}
}

func newNamedDefaultClaim(ns string, index int) *v1.PersistentVolumeClaim {
	claim := v1.PersistentVolumeClaim{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "pvc-" + strconv.Itoa(index),
			Namespace: ns,
		},
		Spec: v1.PersistentVolumeClaimSpec{
			AccessModes: []v1.PersistentVolumeAccessMode{
				v1.ReadWriteOnce,
			},
			Resources: v1.ResourceRequirements{
				Requests: v1.ResourceList{
					v1.ResourceName(v1.ResourceStorage): resource.MustParse("1Gi"),
				},
			},
		},
	}

	return &claim
}
<|endoftext|>


<commit_before>package broadcast

import (
	"sync"
	"time"
)

const (
	broadcastMainChanBufferSize = 64
	broadcastChanBufferSize     = 32
	broadcastSendTimeout        = time.Millisecond * 100
)

type BroadcastMessage string

type GroupBroadcast struct {
	chStop chan struct{}
	chMain chan BroadcastMessage
	chs    []chan BroadcastMessage
	chsMux *sync.RWMutex

	flagStarted bool
}

func NewGroupBroadcast() *GroupBroadcast {
	return &GroupBroadcast{
		chStop: make(chan struct{}),
		chMain: make(chan BroadcastMessage, broadcastMainChanBufferSize),
		chs:    make([]chan BroadcastMessage, 0),
		chsMux: &sync.RWMutex{},
	}
}

func (gb *GroupBroadcast) BroadcastMessage(message BroadcastMessage) {
	select {
	case gb.chMain <- message:
	case <-gb.chStop:
	}
}

func (gb *GroupBroadcast) Start(stop <-chan struct{}) {
	if gb.flagStarted {
		return
	}
	gb.flagStarted = true

	go func() {
		select {
		case <-stop:
		}
		gb.stop()
	}()

	go func() {
		for {
			select {
			case message, ok := <-gb.chMain:
				if !ok {
					return
				}
				gb.broadcast(message)
			case <-gb.chStop:
				return
			}
		}
	}()
}

func (gb *GroupBroadcast) broadcast(message BroadcastMessage) {
	gb.chsMux.RLock()
	defer gb.chsMux.RUnlock()

	for _, ch := range gb.chs {
		select {
		case ch <- message:
		case <-gb.chStop:
		}
	}
}

func (gb *GroupBroadcast) createChan() chan BroadcastMessage {
	ch := make(chan BroadcastMessage, broadcastChanBufferSize)

	gb.chsMux.Lock()
	gb.chs = append(gb.chs, ch)
	gb.chsMux.Unlock()

	return ch
}

func (gb *GroupBroadcast) deleteChan(ch chan BroadcastMessage) {
	gb.chsMux.Lock()
	for i := range gb.chs {
		if gb.chs[i] == ch {
			gb.chs = append(gb.chs[:i], gb.chs[i+1:]...)
			close(ch)
			break
		}
	}
	gb.chsMux.Unlock()
}

func (gb *GroupBroadcast) ListenMessages(stop <-chan struct{}, buffer uint) <-chan BroadcastMessage {
	ch := gb.createChan()
	chOut := make(chan BroadcastMessage, buffer)

	go func() {
		defer close(chOut)
		defer gb.deleteChan(ch)

		for {
			select {
			case <-stop:
				return
			case <-gb.chStop:
				return
			case message, ok := <-ch:
				if !ok {
					return
				}
				gb.send(chOut, message, stop, broadcastSendTimeout)
			}
		}
	}()

	return chOut
}

func (gb *GroupBroadcast) send(ch chan BroadcastMessage, message BroadcastMessage, stop <-chan struct{}, timeout time.Duration) {
	var timer = time.NewTimer(timeout)
	defer timer.Stop()
	if cap(ch) == 0 {
		select {
		case ch <- message:
		case <-gb.chStop:
		case <-stop:
		case <-timer.C:
		}
	} else {
		for {
			select {
			case ch <- message:
				return
			case <-gb.chStop:
				return
			case <-stop:
				return
			case <-timer.C:
				return
			default:
				if len(ch) == cap(ch) {
					<-ch
				}
			}
		}
	}
}
func (gb *GroupBroadcast) stop() {
	close(gb.chStop)
	close(gb.chMain)

	gb.chsMux.Lock()
	defer gb.chsMux.Unlock()

	for _, ch := range gb.chs {
		close(ch)
	}

	gb.chs = gb.chs[:0]
}
<commit_msg>Create ticker in send in GroupBroadcast<commit_after>package broadcast

import (
	"sync"
	"time"
)

const (
	broadcastMainChanBufferSize = 64
	broadcastChanBufferSize     = 32
	broadcastSendTimeout        = time.Millisecond * 100
)

type BroadcastMessage string

type GroupBroadcast struct {
	chStop chan struct{}
	chMain chan BroadcastMessage
	chs    []chan BroadcastMessage
	chsMux *sync.RWMutex

	flagStarted bool
}

func NewGroupBroadcast() *GroupBroadcast {
	return &GroupBroadcast{
		chStop: make(chan struct{}),
		chMain: make(chan BroadcastMessage, broadcastMainChanBufferSize),
		chs:    make([]chan BroadcastMessage, 0),
		chsMux: &sync.RWMutex{},
	}
}

func (gb *GroupBroadcast) BroadcastMessage(message BroadcastMessage) {
	select {
	case gb.chMain <- message:
	case <-gb.chStop:
	}
}

func (gb *GroupBroadcast) Start(stop <-chan struct{}) {
	if gb.flagStarted {
		return
	}
	gb.flagStarted = true

	go func() {
		select {
		case <-stop:
		}
		gb.stop()
	}()

	go func() {
		for {
			select {
			case message, ok := <-gb.chMain:
				if !ok {
					return
				}
				gb.broadcast(message)
			case <-gb.chStop:
				return
			}
		}
	}()
}

func (gb *GroupBroadcast) broadcast(message BroadcastMessage) {
	gb.chsMux.RLock()
	defer gb.chsMux.RUnlock()

	for _, ch := range gb.chs {
		select {
		case ch <- message:
		case <-gb.chStop:
		}
	}
}

func (gb *GroupBroadcast) createChan() chan BroadcastMessage {
	ch := make(chan BroadcastMessage, broadcastChanBufferSize)

	gb.chsMux.Lock()
	gb.chs = append(gb.chs, ch)
	gb.chsMux.Unlock()

	return ch
}

func (gb *GroupBroadcast) deleteChan(ch chan BroadcastMessage) {
	gb.chsMux.Lock()
	for i := range gb.chs {
		if gb.chs[i] == ch {
			gb.chs = append(gb.chs[:i], gb.chs[i+1:]...)
			close(ch)
			break
		}
	}
	gb.chsMux.Unlock()
}

func (gb *GroupBroadcast) ListenMessages(stop <-chan struct{}, buffer uint) <-chan BroadcastMessage {
	ch := gb.createChan()
	chOut := make(chan BroadcastMessage, buffer)

	go func() {
		defer close(chOut)
		defer gb.deleteChan(ch)

		for {
			select {
			case <-stop:
				return
			case <-gb.chStop:
				return
			case message, ok := <-ch:
				if !ok {
					return
				}
				gb.send(chOut, message, stop, broadcastSendTimeout)
			}
		}
	}()

	return chOut
}

func (gb *GroupBroadcast) send(ch chan BroadcastMessage, message BroadcastMessage, stop <-chan struct{}, timeout time.Duration) {
	const tickSize = 5

	var timer = time.NewTimer(timeout)
	defer timer.Stop()

	var ticker = time.NewTicker(timeout / tickSize)
	defer ticker.Stop()

	if cap(ch) == 0 {
		select {
		case ch <- message:
		case <-gb.chStop:
		case <-stop:
		case <-timer.C:
		}
	} else {
		for {
			select {
			case ch <- message:
				return
			case <-gb.chStop:
				return
			case <-stop:
				return
			case <-timer.C:
				return
			case <-ticker.C:
				if len(ch) == cap(ch) {
					<-ch
				}
			}
		}
	}
}
func (gb *GroupBroadcast) stop() {
	close(gb.chStop)
	close(gb.chMain)

	gb.chsMux.Lock()
	defer gb.chsMux.Unlock()

	for _, ch := range gb.chs {
		close(ch)
	}

	gb.chs = gb.chs[:0]
}
<|endoftext|>


<commit_before>package database

import (
	"database/sql"
	"log"
	"net/url"
	"strings"

	"github.com/TF2Stadium/Helen/models"
	_ "github.com/lib/pq"
)

var (
	db *sql.DB
)

func Connect(dburl, database, username, password string) {
	DBUrl := url.URL{
		Scheme:   "postgres",
		Host:     dburl,
		Path:     database,
		RawQuery: "sslmode=disable",
	}

	log.Printf("Connecting to DB on %s", DBUrl.String())

	DBUrl.User = url.UserPassword(username, password)
	var err error

	db, err = sql.Open("postgres", DBUrl.String())
	if err != nil {
		log.Fatal(err)
	}
}

func IsAllowed(userid uint32, lobbyid uint, channelname string) (bool, string) {
	var lobbyType, slot int
	db.QueryRow("SELECT type FROM lobbies WHERE id = $1", lobbyid).Scan(&lobbyType)
	err := db.QueryRow("SELECT slot FROM lobby_slots WHERE player_id = $1 AND lobby_id = $2", userid, lobbyid).Scan(&slot)
	if err == sql.ErrNoRows {
		return false, "You're not in this lobby"
	} else if err != nil {
		log.Println(err)
		return false, "Internal fumble error"
	}

	if channelname[0] == 'L' { // channel name is "Lobby..."
		return true, ""
	}

	//channel name is either "RED" or "BLU"
	team, _, err := models.LobbyGetSlotInfoString(models.LobbyType(lobbyType), slot)
	if err != nil {
		log.Println(err)
	}

	if team != strings.ToLower(channelname) {
		return false, "You're in team " + strings.ToUpper(team) + ", not " + channelname
	}

	return true, ""
}

func IsLobbyClosed(lobbyid uint) bool {
	var state int
	db.QueryRow("SELECT state FROM lobbies where id = $1", lobbyid).Scan(&state)
	return state != 5
}

func GetSteamID(userid uint32) string {
	var steamid string
	db.QueryRow("SELECT steam_id FROM players WHERE id = $1", userid).Scan(&steamid)
	return steamid
}
<commit_msg>Remove IsLobbyClosed<commit_after>package database

import (
	"database/sql"
	"log"
	"net/url"
	"strings"

	"github.com/TF2Stadium/Helen/models"
	_ "github.com/lib/pq"
)

var (
	db *sql.DB
)

func Connect(dburl, database, username, password string) {
	DBUrl := url.URL{
		Scheme:   "postgres",
		Host:     dburl,
		Path:     database,
		RawQuery: "sslmode=disable",
	}

	log.Printf("Connecting to DB on %s", DBUrl.String())

	DBUrl.User = url.UserPassword(username, password)
	var err error

	db, err = sql.Open("postgres", DBUrl.String())
	if err != nil {
		log.Fatal(err)
	}
}

func IsAllowed(userid uint32, lobbyid uint, channelname string) (bool, string) {
	var lobbyType, slot int
	db.QueryRow("SELECT type FROM lobbies WHERE id = $1", lobbyid).Scan(&lobbyType)
	err := db.QueryRow("SELECT slot FROM lobby_slots WHERE player_id = $1 AND lobby_id = $2", userid, lobbyid).Scan(&slot)
	if err == sql.ErrNoRows {
		return false, "You're not in this lobby"
	} else if err != nil {
		log.Println(err)
		return false, "Internal fumble error"
	}

	if channelname[0] == 'L' { // channel name is "Lobby..."
		return true, ""
	}

	//channel name is either "RED" or "BLU"
	team, _, err := models.LobbyGetSlotInfoString(models.LobbyType(lobbyType), slot)
	if err != nil {
		log.Println(err)
	}

	if team != strings.ToLower(channelname) {
		return false, "You're in team " + strings.ToUpper(team) + ", not " + channelname
	}

	return true, ""
}

func GetSteamID(userid uint32) string {
	var steamid string
	db.QueryRow("SELECT steam_id FROM players WHERE id = $1", userid).Scan(&steamid)
	return steamid
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package azure

import (
	"sync/atomic"
	"testing"
	"time"
)

func TestCacheReturnsSameObject(t *testing.T) {
	type cacheTestingStruct struct{}
	c := newTimedcache(1 * time.Minute)
	o1 := cacheTestingStruct{}
	get1, _ := c.GetOrCreate("b1", func() interface{} {
		return o1
	})
	o2 := cacheTestingStruct{}
	get2, _ := c.GetOrCreate("b1", func() interface{} {
		return o2
	})
	if get1 != get2 {
		t.Error("Get not equal")
	}
}

func TestCacheCallsCreateFuncOnce(t *testing.T) {
	var callsCount uint32
	f1 := func() interface{} {
		atomic.AddUint32(&callsCount, 1)
		return 1
	}
	c := newTimedcache(500 * time.Millisecond)
	for index := 0; index < 20; index++ {
		_, _ = c.GetOrCreate("b1", f1)
	}

	if callsCount != 1 {
		t.Error("Count not match")
	}
	time.Sleep(500 * time.Millisecond)
	c.GetOrCreate("b1", f1)
	if callsCount != 2 {
		t.Error("Count not match")
	}
}

func TestCacheExpires(t *testing.T) {
	f1 := func() interface{} {
		return 1
	}
	c := newTimedcache(500 * time.Millisecond)
	get1, _ := c.GetOrCreate("b1", f1)
	if get1 != 1 {
		t.Error("Value not equal")
	}
	time.Sleep(500 * time.Millisecond)
	get1, _ = c.GetOrCreate("b1", nil)
	if get1 != nil {
		t.Error("value not expired")
	}
}

func TestCacheDelete(t *testing.T) {
	f1 := func() interface{} {
		return 1
	}
	c := newTimedcache(500 * time.Millisecond)
	get1, _ := c.GetOrCreate("b1", f1)
	if get1 != 1 {
		t.Error("Value not equal")
	}
	get1, _ = c.GetOrCreate("b1", nil)
	if get1 != 1 {
		t.Error("Value not equal")
	}
	c.Delete("b1")
	get1, _ = c.GetOrCreate("b1", nil)
	if get1 != nil {
		t.Error("value not deleted")
	}
}
<commit_msg>New unit tests for timedCache<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package azure

import (
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

var (
	fakeCacheTTL = 2 * time.Second
)

type fakeDataObj struct{}

type fakeDataSource struct {
	data map[string]*fakeDataObj
	lock sync.Mutex
}

func (fake *fakeDataSource) get(key string) (interface{}, error) {
	fake.lock.Lock()
	defer fake.lock.Unlock()

	if v, ok := fake.data[key]; ok {
		return v, nil
	}

	return nil, nil
}

func (fake *fakeDataSource) set(data map[string]*fakeDataObj) {
	fake.lock.Lock()
	defer fake.lock.Unlock()

	fake.data = data
}

func newFakeCache(t *testing.T) (*fakeDataSource, *timedCache) {
	dataSource := &fakeDataSource{
		data: make(map[string]*fakeDataObj),
	}
	getter := dataSource.get
	cache, err := newTimedcache(fakeCacheTTL, getter)
	assert.NoError(t, err)
	return dataSource, cache
}

func TestCacheGet(t *testing.T) {
	val := &fakeDataObj{}
	cases := []struct {
		name     string
		data     map[string]*fakeDataObj
		key      string
		expected interface{}
	}{
		{
			name:     "cache should return nil for empty data source",
			key:      "key1",
			expected: nil,
		},
		{
			name:     "cache should return nil for non exist key",
			data:     map[string]*fakeDataObj{"key2": val},
			key:      "key1",
			expected: nil,
		},
		{
			name:     "cache should return data for existing key",
			data:     map[string]*fakeDataObj{"key1": val},
			key:      "key1",
			expected: val,
		},
	}

	for _, c := range cases {
		dataSource, cache := newFakeCache(t)
		dataSource.set(c.data)
		val, err := cache.Get(c.key)
		assert.NoError(t, err, c.name)
		assert.Equal(t, c.expected, val, c.name)
	}
}

func TestCacheDelete(t *testing.T) {
	key := "key1"
	val := &fakeDataObj{}
	data := map[string]*fakeDataObj{
		key: val,
	}
	dataSource, cache := newFakeCache(t)
	dataSource.set(data)

	v, err := cache.Get(key)
	assert.NoError(t, err)
	assert.Equal(t, val, v, "cache should get correct data")

	dataSource.set(nil)
	cache.Delete(key)
	v, err = cache.Get(key)
	assert.NoError(t, err)
	assert.Equal(t, nil, v, "cache should get nil after data is removed")
}

func TestCacheExpired(t *testing.T) {
	key := "key1"
	val := &fakeDataObj{}
	data := map[string]*fakeDataObj{
		key: val,
	}
	dataSource, cache := newFakeCache(t)
	dataSource.set(data)

	v, err := cache.Get(key)
	assert.NoError(t, err)
	assert.Equal(t, val, v, "cache should get correct data")

	time.Sleep(fakeCacheTTL)
	v, err = cache.Get(key)
	assert.NoError(t, err)
	assert.Equal(t, val, v, "cache should get correct data even after expired")
}
<|endoftext|>