<commit_before>package docker

import (
	"context"
	"fmt"
	"log"
	"net/http"
	"regexp"
	"strings"

	"github.com/ViBiOh/dashboard/auth"
	"github.com/ViBiOh/dashboard/jsonHttp"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/filters"
	"github.com/docker/docker/api/types/network"
	"gopkg.in/yaml.v2"
)

const minMemory = 16777216
const maxMemory = 805306368
const defaultTag = `:latest`
const deploySuffix = `_deploy`
const networkMode = `traefik`
const linkSeparator = `:`

var imageTag = regexp.MustCompile(`^\S*?:\S+$`)

type dockerComposeService struct {
	Image       string
	Command     []string
	Environment map[string]string
	Labels      map[string]string
	Links       []string
	Ports       []string
	ReadOnly    bool  `yaml:"read_only"`
	CPUShares   int64 `yaml:"cpu_shares"`
	MemoryLimit int64 `yaml:"mem_limit"`
}

type dockerCompose struct {
	Version  string
	Services map[string]dockerComposeService
}

type deployedService struct {
	ID   string
	Name string
}

func getConfig(service *dockerComposeService, user *auth.User, appName string) *container.Config {
	environments := make([]string, len(service.Environment))
	for key, value := range service.Environment {
		environments = append(environments, key+`=`+value)
	}

	if service.Labels == nil {
		service.Labels = make(map[string]string)
	}

	service.Labels[ownerLabel] = user.Username
	service.Labels[appLabel] = appName

	config := container.Config{
		Image:  service.Image,
		Labels: service.Labels,
		Env:    environments,
	}

	if len(service.Command) != 0 {
		config.Cmd = service.Command
	}

	return &config
}

func getHostConfig(service *dockerComposeService) *container.HostConfig {
	hostConfig := container.HostConfig{
		LogConfig: container.LogConfig{Type: `json-file`, Config: map[string]string{
			`max-size`: `50m`,
		}},
		NetworkMode:   networkMode,
		RestartPolicy: container.RestartPolicy{Name: `on-failure`, MaximumRetryCount: 5},
		Resources: container.Resources{
			CPUShares: 128,
			Memory:    minMemory,
		},
		SecurityOpt: []string{`no-new-privileges`},
	}

	if service.ReadOnly {
		hostConfig.ReadonlyRootfs = service.ReadOnly
	}

	if service.CPUShares != 0 {
		hostConfig.Resources.CPUShares = service.CPUShares
	}

	if service.MemoryLimit != 0 {
		if service.MemoryLimit <= maxMemory {
			hostConfig.Resources.Memory = service.MemoryLimit
		} else {
			hostConfig.Resources.Memory = maxMemory
		}
	}

	return &hostConfig
}

func getNetworkConfig(service *dockerComposeService, deployedServices map[string]deployedService) *network.NetworkingConfig {
	traefikConfig := network.EndpointSettings{}

	for _, link := range service.Links {
		linkParts := strings.Split(link, linkSeparator)

		target := linkParts[0]
		if linkedService, ok := (deployedServices)[target]; ok {
			target = getFinalName(linkedService.Name)
		}

		alias := linkParts[0]
		if len(linkParts) > 1 {
			alias = linkParts[1]
		}

		traefikConfig.Links = append(traefikConfig.Links, target+linkSeparator+alias)
	}

	return &network.NetworkingConfig{
		EndpointsConfig: map[string]*network.EndpointSettings{
			networkMode: &traefikConfig,
		},
	}
}

func pullImage(image string, user *auth.User) error {
	if !imageTag.MatchString(image) {
		image = image + defaultTag
	}

	log.Printf(`[%s] Starting pull of image %s`, user.Username, image)
	pull, err := docker.ImagePull(context.Background(), image, types.ImagePullOptions{})
	if err != nil {
		return fmt.Errorf(`[%s] Error while pulling image: %v`, user.Username, err)
	}

	readBody(pull)
	log.Printf(`[%s] Ending pull of image %s`, user.Username, image)
	return nil
}

func cleanContainers(containers []types.Container, user *auth.User) {
	for _, container := range containers {
		log.Printf(`[%s] Stopping containers %s`, user.Username, strings.Join(container.Names, `, `))
		stopContainer(container.ID)
	}

	for _, container := range containers {
		log.Printf(`[%s] Deleting containers %s`, user.Username, strings.Join(container.Names, `, `))
		rmContainer(container.ID)
	}
}

func renameDeployedContainers(containers map[string]deployedService, user *auth.User) error {
	for service, container := range containers {
		if err := docker.ContainerRename(context.Background(), container.ID, getFinalName(container.Name)); err != nil {
			return fmt.Errorf(`[%s] Error while renaming container %s: %v`, user.Username, service, err)
		}
	}

	return nil
}

func getServiceFullName(app string, service string) string {
	return app + `_` + service + deploySuffix
}

func getFinalName(serviceFullName string) string {
	return strings.TrimSuffix(serviceFullName, deploySuffix)
}

func deleteServices(appName []byte, services map[string]deployedService, user *auth.User) {
	log.Printf(`[%s] Deleting services for %s`, user.Username, appName)
	for service, container := range services {
		if err := stopContainer(container.ID); err != nil {
			log.Printf(`[%s] Error while stopping service %s for %s: %v`, user.Username, service, appName, err)
		}

		if err := rmContainer(container.ID); err != nil {
			log.Printf(`[%s] Error while deleting service %s for %s: %v`, user.Username, service, appName, err)
		}
	}
}

func startServices(appName []byte, services map[string]deployedService, user *auth.User) error {
	log.Printf(`[%s] Starting services for %s`, user.Username, appName)
	for service, container := range services {
		if err := startContainer(container.ID); err != nil {
			return fmt.Errorf(`[%s] Error while starting service %s for %s: %v`, user.Username, service, appName, err)
		}
	}

	return nil
}

func inspectServices(services map[string]deployedService, user *auth.User) []*types.ContainerJSON {
	containers := make([]*types.ContainerJSON, 0, len(services))

	for service, container := range services {
		infos, err := inspectContainer(container.ID)
		if err != nil {
			log.Printf(`[%s] Error while inspecting container %s: %v`, user.Username, service, err)
		}

		containers = append(containers, &infos)
	}

	return containers
}

func areContainersHealthy(ctx context.Context, user *auth.User, appName []byte, containers []*types.ContainerJSON) bool {
	containersIdsWithHealthcheck := make([]*string, 0, len(containers))
	for _, container := range containers {
		if container.Config.Healthcheck != nil && len(container.Config.Healthcheck.Test) != 0 {
			containersIdsWithHealthcheck = append(containersIdsWithHealthcheck, &container.ID)
		}
	}

	if len(containersIdsWithHealthcheck) == 0 {
		return true
	}

	filtersArgs := filters.NewArgs()
	if err := healthyStatusFilters(user, &filtersArgs, containersIdsWithHealthcheck); err != nil {
		log.Printf(`[%s] Error while defining healthy filters: %v`, user.Username, err)
		return true
	}

	timeoutCtx, cancel := context.WithTimeout(ctx, DeployTimeout)
	defer cancel()

	messages, errors := docker.Events(timeoutCtx, types.EventsOptions{Filters: filtersArgs})
	healthyContainers := make(map[string]bool, len(containersIdsWithHealthcheck))

	for {
		select {
		case <-ctx.Done():
			return false
		case message := <-messages:
			healthyContainers[message.ID] = true
			log.Printf(`[%s] Container %s for %s is healthy`, user.Username, appName, message.From)

			if len(healthyContainers) == len(containersIdsWithHealthcheck) {
				return true
			}
		case err := <-errors:
			log.Printf(`[%s] Error while reading healthy events: %v`, user.Username, err)
			return false
		}
	}
}

func finishDeploy(ctx context.Context, cancel context.CancelFunc, user *auth.User, appName []byte, services map[string]deployedService, oldContainers []types.Container) {
	defer cancel()
	defer func() {
		backgroundMutex.Lock()
		defer backgroundMutex.Unlock()

		backgroundTasks[string(appName)] = false
	}()

	log.Printf(`[%s] Waiting for %s to start...`, user.Username, appName)

	if areContainersHealthy(ctx, user, appName, inspectServices(services, user)) {
		log.Printf(`[%s] Health check succeeded for %s`, user.Username, appName)
		cleanContainers(oldContainers, user)

		if err := renameDeployedContainers(services, user); err != nil {
			log.Print(err)
		}
		log.Printf(`[%s] Succeeded to deploy %s`, user.Username, appName)
	} else {
		log.Printf(`[%s] Health check failed for %s`, user.Username, appName)
		deleteServices(appName, services, user)
		log.Printf(`[%s] Failed to deploy %s`, user.Username, appName)
	}
}

func createContainer(user *auth.User, appName []byte, serviceName string, services map[string]deployedService, service *dockerComposeService) (*deployedService, error) {
	if err := pullImage(service.Image, user); err != nil {
		return nil, err
	}

	serviceFullName := getServiceFullName(string(appName), serviceName)
	log.Printf(`[%s] Creating service %s for %s`, user.Username, serviceName, appName)

	createdContainer, err := docker.ContainerCreate(context.Background(), getConfig(service, user, string(appName)), getHostConfig(service), getNetworkConfig(service, services), serviceFullName)
	if err != nil {
		err = fmt.Errorf(`[%s] Error while creating service %s for %s: %v`, user.Username, serviceName, appName, err)
		return nil, err
	}

	return &deployedService{ID: createdContainer.ID, Name: serviceFullName}, nil
}

func composeFailed(w http.ResponseWriter, user *auth.User, appName []byte, err error) {
	errorHandler(w, err)
	if err != nil {
		log.Printf(`[%s] Failed to deploy %s: %v`, user.Username, appName, err)
	} else {
		log.Printf(`[%s] Failed to deploy %s`, user.Username, appName)
	}
}

func composeHandler(w http.ResponseWriter, user *auth.User, appName []byte, composeFile []byte) {
	if len(appName) == 0 || len(composeFile) == 0 {
		badRequest(w, fmt.Errorf(`[%s] An application name and a compose file are required`, user.Username))
		return
	}

	compose := dockerCompose{}
	if err := yaml.Unmarshal(composeFile, &compose); err != nil {
		errorHandler(w, fmt.Errorf(`[%s] Error while unmarshalling compose file: %v`, user.Username, err))
		return
	}

	appNameStr := string(appName)
	backgroundMutex.Lock()

	if value, ok := backgroundTasks[appNameStr]; ok && value {
		backgroundMutex.Unlock()
		composeFailed(w, user, appName, fmt.Errorf(`Application already in deployment`))
		return
	}

	backgroundTasks[appNameStr] = true
	backgroundMutex.Unlock()

	log.Printf(`[%s] Deploying %s`, user.Username, appName)

	oldContainers, err := listContainers(user, &appNameStr)
	if err != nil {
		composeFailed(w, user, appName, err)
		return
	}

	if len(oldContainers) > 0 && oldContainers[0].Labels[ownerLabel] != user.Username {
		composeFailed(w, user, appName, fmt.Errorf(`Application not owned`))
		forbidden(w)
	}

	newServices := make(map[string]deployedService)
	for serviceName, service := range compose.Services {
		if deployedService, err := createContainer(user, appName, serviceName, newServices, &service); err != nil {
			break
		} else {
			newServices[serviceName] = *deployedService
		}
	}

	ctx, cancel := context.WithCancel(context.Background())
	go finishDeploy(ctx, cancel, user, appName, newServices, oldContainers)

	if err == nil {
		err = startServices(appName, newServices, user)
	}

	if err != nil {
		cancel()
		composeFailed(w, user, appName, err)
	} else {
		jsonHttp.ResponseJSON(w, results{newServices})
	}
}
<commit_msg>Update compose.go<commit_after>package docker

import (
	"context"
	"fmt"
	"log"
	"net/http"
	"regexp"
	"strings"

	"github.com/ViBiOh/dashboard/auth"
	"github.com/ViBiOh/dashboard/jsonHttp"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/filters"
	"github.com/docker/docker/api/types/network"
	"gopkg.in/yaml.v2"
)

const minMemory = 16777216
const maxMemory = 805306368
const defaultTag = `:latest`
const deploySuffix = `_deploy`
const networkMode = `traefik`
const linkSeparator = `:`

var imageTag = regexp.MustCompile(`^\S*?:\S+$`)

type dockerComposeService struct {
	Image       string
	Command     []string
	Environment map[string]string
	Labels      map[string]string
	Links       []string
	Ports       []string
	ReadOnly    bool  `yaml:"read_only"`
	CPUShares   int64 `yaml:"cpu_shares"`
	MemoryLimit int64 `yaml:"mem_limit"`
}

type dockerCompose struct {
	Version  string
	Services map[string]dockerComposeService
}

type deployedService struct {
	ID   string
	Name string
}

func getConfig(service *dockerComposeService, user *auth.User, appName string) *container.Config {
	environments := make([]string, len(service.Environment))
	for key, value := range service.Environment {
		environments = append(environments, key+`=`+value)
	}

	if service.Labels == nil {
		service.Labels = make(map[string]string)
	}

	service.Labels[ownerLabel] = user.Username
	service.Labels[appLabel] = appName

	config := container.Config{
		Image:  service.Image,
		Labels: service.Labels,
		Env:    environments,
	}

	if len(service.Command) != 0 {
		config.Cmd = service.Command
	}

	return &config
}

func getHostConfig(service *dockerComposeService) *container.HostConfig {
	hostConfig := container.HostConfig{
		LogConfig: container.LogConfig{Type: `json-file`, Config: map[string]string{
			`max-size`: `50m`,
		}},
		NetworkMode:   networkMode,
		RestartPolicy: container.RestartPolicy{Name: `on-failure`, MaximumRetryCount: 5},
		Resources: container.Resources{
			CPUShares: 128,
			Memory:    minMemory,
		},
		SecurityOpt: []string{`no-new-privileges`},
	}

	if service.ReadOnly {
		hostConfig.ReadonlyRootfs = service.ReadOnly
	}

	if service.CPUShares != 0 {
		hostConfig.Resources.CPUShares = service.CPUShares
	}

	if service.MemoryLimit != 0 {
		if service.MemoryLimit <= maxMemory {
			hostConfig.Resources.Memory = service.MemoryLimit
		} else {
			hostConfig.Resources.Memory = maxMemory
		}
	}

	return &hostConfig
}

func getNetworkConfig(service *dockerComposeService, deployedServices map[string]deployedService) *network.NetworkingConfig {
	traefikConfig := network.EndpointSettings{}

	for _, link := range service.Links {
		linkParts := strings.Split(link, linkSeparator)

		target := linkParts[0]
		if linkedService, ok := (deployedServices)[target]; ok {
			target = getFinalName(linkedService.Name)
		}

		alias := linkParts[0]
		if len(linkParts) > 1 {
			alias = linkParts[1]
		}

		traefikConfig.Links = append(traefikConfig.Links, target+linkSeparator+alias)
	}

	return &network.NetworkingConfig{
		EndpointsConfig: map[string]*network.EndpointSettings{
			networkMode: &traefikConfig,
		},
	}
}

func pullImage(image string, user *auth.User) error {
	if !imageTag.MatchString(image) {
		image = image + defaultTag
	}

	log.Printf(`[%s] Starting pull of image %s`, user.Username, image)
	pull, err := docker.ImagePull(context.Background(), image, types.ImagePullOptions{})
	if err != nil {
		return fmt.Errorf(`[%s] Error while pulling image: %v`, user.Username, err)
	}

	readBody(pull)
	log.Printf(`[%s] Ending pull of image %s`, user.Username, image)
	return nil
}

func cleanContainers(containers []types.Container, user *auth.User) {
	for _, container := range containers {
		log.Printf(`[%s] Stopping containers %s`, user.Username, strings.Join(container.Names, `, `))
		stopContainer(container.ID)
	}

	for _, container := range containers {
		log.Printf(`[%s] Deleting containers %s`, user.Username, strings.Join(container.Names, `, `))
		rmContainer(container.ID)
	}
}

func renameDeployedContainers(containers map[string]deployedService, user *auth.User) error {
	for service, container := range containers {
		if err := docker.ContainerRename(context.Background(), container.ID, getFinalName(container.Name)); err != nil {
			return fmt.Errorf(`[%s] Error while renaming container %s: %v`, user.Username, service, err)
		}
	}

	return nil
}

func getServiceFullName(app string, service string) string {
	return app + `_` + service + deploySuffix
}

func getFinalName(serviceFullName string) string {
	return strings.TrimSuffix(serviceFullName, deploySuffix)
}

func deleteServices(appName []byte, services map[string]deployedService, user *auth.User) {
	log.Printf(`[%s] Deleting services for %s`, user.Username, appName)
	for service, container := range services {
		if infos, err := inspectContainer(container.ID); err != nil {
			log.Printf(`[%s] Error while inspecting service %s for %s: %v`, user.Username, service, appName, err)
		} else if infos.State.Health != nil {
			log.Printf(`[%s] Healthcheck output for %s: %v`, user.Username, service, infos.State.Health.Log)
		}

		if err := stopContainer(container.ID); err != nil {
			log.Printf(`[%s] Error while stopping service %s for %s: %v`, user.Username, service, appName, err)
		}

		if err := rmContainer(container.ID); err != nil {
			log.Printf(`[%s] Error while deleting service %s for %s: %v`, user.Username, service, appName, err)
		}
	}
}

func startServices(appName []byte, services map[string]deployedService, user *auth.User) error {
	log.Printf(`[%s] Starting services for %s`, user.Username, appName)
	for service, container := range services {
		if err := startContainer(container.ID); err != nil {
			return fmt.Errorf(`[%s] Error while starting service %s for %s: %v`, user.Username, service, appName, err)
		}
	}

	return nil
}

func inspectServices(services map[string]deployedService, user *auth.User) []*types.ContainerJSON {
	containers := make([]*types.ContainerJSON, 0, len(services))

	for service, container := range services {
		infos, err := inspectContainer(container.ID)
		if err != nil {
			log.Printf(`[%s] Error while inspecting container %s: %v`, user.Username, service, err)
		}

		containers = append(containers, &infos)
	}

	return containers
}

func areContainersHealthy(ctx context.Context, user *auth.User, appName []byte, containers []*types.ContainerJSON) bool {
	containersIdsWithHealthcheck := make([]*string, 0, len(containers))
	for _, container := range containers {
		if container.Config.Healthcheck != nil && len(container.Config.Healthcheck.Test) != 0 {
			containersIdsWithHealthcheck = append(containersIdsWithHealthcheck, &container.ID)
		}
	}

	if len(containersIdsWithHealthcheck) == 0 {
		return true
	}

	filtersArgs := filters.NewArgs()
	if err := healthyStatusFilters(user, &filtersArgs, containersIdsWithHealthcheck); err != nil {
		log.Printf(`[%s] Error while defining healthy filters: %v`, user.Username, err)
		return true
	}

	timeoutCtx, cancel := context.WithTimeout(ctx, DeployTimeout)
	defer cancel()

	messages, errors := docker.Events(timeoutCtx, types.EventsOptions{Filters: filtersArgs})
	healthyContainers := make(map[string]bool, len(containersIdsWithHealthcheck))

	for {
		select {
		case <-ctx.Done():
			return false
		case message := <-messages:
			healthyContainers[message.ID] = true
			log.Printf(`[%s] Container %s for %s is healthy`, user.Username, appName, message.From)

			if len(healthyContainers) == len(containersIdsWithHealthcheck) {
				return true
			}
		case err := <-errors:
			log.Printf(`[%s] Error while reading healthy events: %v`, user.Username, err)
			return false
		}
	}
}

func finishDeploy(ctx context.Context, cancel context.CancelFunc, user *auth.User, appName []byte, services map[string]deployedService, oldContainers []types.Container) {
	defer cancel()
	defer func() {
		backgroundMutex.Lock()
		defer backgroundMutex.Unlock()

		backgroundTasks[string(appName)] = false
	}()

	log.Printf(`[%s] Waiting for %s to start...`, user.Username, appName)

	if areContainersHealthy(ctx, user, appName, inspectServices(services, user)) {
		log.Printf(`[%s] Health check succeeded for %s`, user.Username, appName)
		cleanContainers(oldContainers, user)

		if err := renameDeployedContainers(services, user); err != nil {
			log.Print(err)
		}
		log.Printf(`[%s] Succeeded to deploy %s`, user.Username, appName)
	} else {
		log.Printf(`[%s] Health check failed for %s`, user.Username, appName)
		deleteServices(appName, services, user)
		log.Printf(`[%s] Failed to deploy %s`, user.Username, appName)
	}
}

func createContainer(user *auth.User, appName []byte, serviceName string, services map[string]deployedService, service *dockerComposeService) (*deployedService, error) {
	if err := pullImage(service.Image, user); err != nil {
		return nil, err
	}

	serviceFullName := getServiceFullName(string(appName), serviceName)
	log.Printf(`[%s] Creating service %s for %s`, user.Username, serviceName, appName)

	createdContainer, err := docker.ContainerCreate(context.Background(), getConfig(service, user, string(appName)), getHostConfig(service), getNetworkConfig(service, services), serviceFullName)
	if err != nil {
		err = fmt.Errorf(`[%s] Error while creating service %s for %s: %v`, user.Username, serviceName, appName, err)
		return nil, err
	}

	return &deployedService{ID: createdContainer.ID, Name: serviceFullName}, nil
}

func composeFailed(w http.ResponseWriter, user *auth.User, appName []byte, err error) {
	errorHandler(w, err)
	if err != nil {
		log.Printf(`[%s] Failed to deploy %s: %v`, user.Username, appName, err)
	} else {
		log.Printf(`[%s] Failed to deploy %s`, user.Username, appName)
	}
}

func composeHandler(w http.ResponseWriter, user *auth.User, appName []byte, composeFile []byte) {
	if len(appName) == 0 || len(composeFile) == 0 {
		badRequest(w, fmt.Errorf(`[%s] An application name and a compose file are required`, user.Username))
		return
	}

	compose := dockerCompose{}
	if err := yaml.Unmarshal(composeFile, &compose); err != nil {
		errorHandler(w, fmt.Errorf(`[%s] Error while unmarshalling compose file: %v`, user.Username, err))
		return
	}

	appNameStr := string(appName)
	backgroundMutex.Lock()

	if value, ok := backgroundTasks[appNameStr]; ok && value {
		backgroundMutex.Unlock()
		composeFailed(w, user, appName, fmt.Errorf(`Application already in deployment`))
		return
	}

	backgroundTasks[appNameStr] = true
	backgroundMutex.Unlock()

	log.Printf(`[%s] Deploying %s`, user.Username, appName)

	oldContainers, err := listContainers(user, &appNameStr)
	if err != nil {
		composeFailed(w, user, appName, err)
		return
	}

	if len(oldContainers) > 0 && oldContainers[0].Labels[ownerLabel] != user.Username {
		composeFailed(w, user, appName, fmt.Errorf(`Application not owned`))
		forbidden(w)
	}

	newServices := make(map[string]deployedService)
	for serviceName, service := range compose.Services {
		if deployedService, err := createContainer(user, appName, serviceName, newServices, &service); err != nil {
			break
		} else {
			newServices[serviceName] = *deployedService
		}
	}

	ctx, cancel := context.WithCancel(context.Background())
	go finishDeploy(ctx, cancel, user, appName, newServices, oldContainers)

	if err == nil {
		err = startServices(appName, newServices, user)
	}

	if err != nil {
		cancel()
		composeFailed(w, user, appName, err)
	} else {
		jsonHttp.ResponseJSON(w, results{newServices})
	}
}
<|endoftext|>


<commit_before>package docker

import (
	"context"
	"fmt"
	"github.com/ViBiOh/dashboard/auth"
	"github.com/ViBiOh/dashboard/jsonHttp"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/filters"
	"github.com/docker/docker/api/types/network"
	"gopkg.in/yaml.v2"
	"log"
	"net/http"
	"regexp"
	"strings"
	"time"
)

const minMemory = 16777216
const maxMemory = 805306368
const defaultTag = `:latest`
const deploySuffix = `_deploy`
const networkMode = `traefik`
const linkSeparator = `:`
const healthcheckTimeout = 5 * time.Minute

var imageTag = regexp.MustCompile(`^\S*?:\S+$`)

type dockerComposeService struct {
	Image       string
	Command     []string
	Environment map[string]string
	Labels      map[string]string
	Links       []string
	Ports       []string
	ReadOnly    bool  `yaml:"read_only"`
	CPUShares   int64 `yaml:"cpu_shares"`
	MemoryLimit int64 `yaml:"mem_limit"`
}

type dockerCompose struct {
	Version  string
	Services map[string]dockerComposeService
}

type deployedService struct {
	ID   string
	Name string
}

func getConfig(service *dockerComposeService, user *auth.User, appName string) *container.Config {
	environments := make([]string, len(service.Environment))
	for key, value := range service.Environment {
		environments = append(environments, key+`=`+value)
	}

	if service.Labels == nil {
		service.Labels = make(map[string]string)
	}

	service.Labels[ownerLabel] = user.Username
	service.Labels[appLabel] = appName

	config := container.Config{
		Image:  service.Image,
		Labels: service.Labels,
		Env:    environments,
	}

	if len(service.Command) != 0 {
		config.Cmd = service.Command
	}

	return &config
}

func getHostConfig(service *dockerComposeService) *container.HostConfig {
	hostConfig := container.HostConfig{
		LogConfig: container.LogConfig{Type: `json-file`, Config: map[string]string{
			`max-size`: `50m`,
		}},
		NetworkMode:   networkMode,
		RestartPolicy: container.RestartPolicy{Name: `on-failure`, MaximumRetryCount: 5},
		Resources: container.Resources{
			CPUShares: 128,
			Memory:    minMemory,
		},
		SecurityOpt: []string{`no-new-privileges`},
	}

	if service.ReadOnly {
		hostConfig.ReadonlyRootfs = service.ReadOnly
	}

	if service.CPUShares != 0 {
		hostConfig.Resources.CPUShares = service.CPUShares
	}

	if service.MemoryLimit != 0 {
		if service.MemoryLimit <= maxMemory {
			hostConfig.Resources.Memory = service.MemoryLimit
		} else {
			hostConfig.Resources.Memory = maxMemory
		}
	}

	return &hostConfig
}

func getNetworkConfig(service *dockerComposeService, deployedServices *map[string]deployedService) *network.NetworkingConfig {
	traefikConfig := network.EndpointSettings{}

	for _, link := range service.Links {
		linkParts := strings.Split(link, linkSeparator)

		target := linkParts[0]
		if linkedService, ok := (*deployedServices)[target]; ok {
			target = getFinalName(linkedService.Name)
		}

		alias := linkParts[0]
		if len(linkParts) > 1 {
			alias = linkParts[1]
		}

		traefikConfig.Links = append(traefikConfig.Links, target+linkSeparator+alias)
	}

	return &network.NetworkingConfig{
		EndpointsConfig: map[string]*network.EndpointSettings{
			networkMode: &traefikConfig,
		},
	}
}

func pullImage(image string, user *auth.User) error {
	if !imageTag.MatchString(image) {
		image = image + defaultTag
	}

	log.Printf(`[%s] Starting pull of image %s`, user.Username, image)
	pull, err := docker.ImagePull(context.Background(), image, types.ImagePullOptions{})
	if err != nil {
		return fmt.Errorf(`[%s] Error while pulling image: %v`, user.Username, err)
	}

	readBody(pull)
	log.Printf(`[%s] Ending pull of image %s`, user.Username, image)
	return nil
}

func cleanContainers(containers *[]types.Container, user *auth.User) {
	for _, container := range *containers {
		log.Printf(`[%s] Stopping containers %s`, user.Username, strings.Join(container.Names, `, `))
		stopContainer(container.ID)
		log.Printf(`[%s] Deleting containers %s`, user.Username, strings.Join(container.Names, `, `))
		rmContainer(container.ID)
	}
}

func renameDeployedContainers(containers *map[string]deployedService, user *auth.User) error {
	for service, container := range *containers {
		if err := docker.ContainerRename(context.Background(), container.ID, getFinalName(container.Name)); err != nil {
			return fmt.Errorf(`[%s] Error while renaming container %s: %v`, user.Username, service, err)
		}
	}

	return nil
}

func getServiceFullName(app string, service string) string {
	return app + `_` + service + deploySuffix
}

func getFinalName(serviceFullName string) string {
	return strings.TrimSuffix(serviceFullName, deploySuffix)
}

func deleteServices(appName []byte, services map[string]deployedService, user *auth.User) {
	log.Printf(`[%s] Deleting services for %s`, user.Username, appName)
	for service, container := range services {
		if err := rmContainer(container.ID); err != nil {
			log.Printf(`[%s] Error while deleting service %s for %s: %v`, user.Username, service, appName, err)
		}
	}
}

func startServices(appName []byte, services map[string]deployedService, user *auth.User) error {
	log.Printf(`[%s] Starting services for %s`, user.Username, appName)
	for service, container := range services {
		if err := startContainer(container.ID); err != nil {
			return fmt.Errorf(`[%s] Error while starting service %s for %s: %v`, user.Username, service, appName, err)
		}
	}

	return nil
}

func inspectServices(services map[string]deployedService, user *auth.User) []*types.ContainerJSON {
	containers := make([]*types.ContainerJSON, 0, len(services))

	for service, container := range services {
		infos, err := inspectContainer(container.ID)
		if err != nil {
			log.Printf(`[%s] Error while inspecting container %s: %v`, user.Username, service, err)
		}

		containers = append(containers, &infos)
	}

	return containers
}

func healthyListener(ctx context.Context, user *auth.User, containers []*types.ContainerJSON) bool {
	containersIdsWithHealthcheck := make([]*string, 0, len(containers))
	for _, container := range containers {
		if container.Config.Healthcheck != nil && len(container.Config.Healthcheck.Test) != 0 {
			containersIdsWithHealthcheck = append(containersIdsWithHealthcheck, &container.ID)
		}
	}

	if len(containersIdsWithHealthcheck) == 0 {
		return true
	}

	filtersArgs := filters.NewArgs()
	if err := healthyStatusFilters(user, &filtersArgs, containersIdsWithHealthcheck); err != nil {
		log.Printf(`[%s] Error while defining healthy filters: %v`, user.Username, err)
		return true
	}

	timeoutCtx, cancel := context.WithTimeout(ctx, healthcheckTimeout)
	defer cancel()

	messages, err := docker.Events(timeoutCtx, types.EventsOptions{Filters: filtersArgs})
	if err != nil {
		log.Printf(`[%s] Error while reading healthy events: %v`, user.Username, err)
		return false
	}

	healthyContainers := make(map[string]bool, len(containersIdsWithHealthcheck))
	for {
		select {
		case <-ctx.Done():
			return false
		case message := <-messages:
			healthyContainers[message.ID] = true

			if len(healthyContainers) == len(containersIdsWithHealthcheck) {
				return true
			}
		}
	}
}

func createAppHandler(w http.ResponseWriter, user *auth.User, appName []byte, composeFile []byte) {
	if len(appName) == 0 || len(composeFile) == 0 {
		badRequest(w, fmt.Errorf(`[%s] An application name and a compose file are required`, user.Username))
		return
	}

	compose := dockerCompose{}
	if err := yaml.Unmarshal(composeFile, &compose); err != nil {
		errorHandler(w, fmt.Errorf(`[%s] Error while unmarshalling compose file: %v`, user.Username, err))
		return
	}

	appNameStr := string(appName)
	log.Printf(`[%s] Deploying %s`, user.Username, appNameStr)

	ownerContainers, err := listContainers(user, &appNameStr)
	if err != nil {
		errorHandler(w, err)
		return
	}

	if len(ownerContainers) > 0 && ownerContainers[0].Labels[ownerLabel] != user.Username {
		forbidden(w)
	}

	deployedServices := make(map[string]deployedService)

	for serviceName, service := range compose.Services {
		if err := pullImage(service.Image, user); err != nil {
			break
		}

		serviceFullName := getServiceFullName(appNameStr, serviceName)
		log.Printf(`[%s] Creating service %s for %s`, user.Username, serviceName, appName)

		createdContainer, err := docker.ContainerCreate(context.Background(), getConfig(&service, user, appNameStr), getHostConfig(&service), getNetworkConfig(&service, &deployedServices), serviceFullName)
		if err != nil {
			err = fmt.Errorf(`[%s] Error while creating service %s for %s: %v`, user.Username, serviceName, appName, err)
			break
		}

		deployedServices[serviceName] = deployedService{ID: createdContainer.ID, Name: serviceFullName}
	}

	ctx, cancel := context.WithCancel(context.Background())

	go func() {
		addCounter(1)
		defer addCounter(-1)
		defer cancel()

		log.Printf(`[%s] Waiting for %s to start...`, user.Username, appName)

		if healthyListener(ctx, user, inspectServices(deployedServices, user)) {
			log.Printf(`[%s] Health check succeeded for %s`, user.Username, appName)
			cleanContainers(&ownerContainers, user)

			if err := renameDeployedContainers(&deployedServices, user); err != nil {
				log.Print(err)
			}
		} else {
			log.Printf(`[%s] Health check failed for %s`, user.Username, appName)
			deleteServices(appName, deployedServices, user)
		}
	}()

	if err == nil {
		err = startServices(appName, deployedServices, user)
	}

	if err != nil {
		deleteServices(appName, deployedServices, user)
		errorHandler(w, err)
		cancel()
		return
	}

	jsonHttp.ResponseJSON(w, results{deployedServices})
}
<commit_msg>Refactoring function<commit_after>package docker

import (
	"context"
	"fmt"
	"github.com/ViBiOh/dashboard/auth"
	"github.com/ViBiOh/dashboard/jsonHttp"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/filters"
	"github.com/docker/docker/api/types/network"
	"gopkg.in/yaml.v2"
	"log"
	"net/http"
	"regexp"
	"strings"
	"time"
)

const minMemory = 16777216
const maxMemory = 805306368
const defaultTag = `:latest`
const deploySuffix = `_deploy`
const networkMode = `traefik`
const linkSeparator = `:`
const healthcheckTimeout = 5 * time.Minute

var imageTag = regexp.MustCompile(`^\S*?:\S+$`)

type dockerComposeService struct {
	Image       string
	Command     []string
	Environment map[string]string
	Labels      map[string]string
	Links       []string
	Ports       []string
	ReadOnly    bool  `yaml:"read_only"`
	CPUShares   int64 `yaml:"cpu_shares"`
	MemoryLimit int64 `yaml:"mem_limit"`
}

type dockerCompose struct {
	Version  string
	Services map[string]dockerComposeService
}

type deployedService struct {
	ID   string
	Name string
}

func getConfig(service *dockerComposeService, user *auth.User, appName string) *container.Config {
	environments := make([]string, len(service.Environment))
	for key, value := range service.Environment {
		environments = append(environments, key+`=`+value)
	}

	if service.Labels == nil {
		service.Labels = make(map[string]string)
	}

	service.Labels[ownerLabel] = user.Username
	service.Labels[appLabel] = appName

	config := container.Config{
		Image:  service.Image,
		Labels: service.Labels,
		Env:    environments,
	}

	if len(service.Command) != 0 {
		config.Cmd = service.Command
	}

	return &config
}

func getHostConfig(service *dockerComposeService) *container.HostConfig {
	hostConfig := container.HostConfig{
		LogConfig: container.LogConfig{Type: `json-file`, Config: map[string]string{
			`max-size`: `50m`,
		}},
		NetworkMode:   networkMode,
		RestartPolicy: container.RestartPolicy{Name: `on-failure`, MaximumRetryCount: 5},
		Resources: container.Resources{
			CPUShares: 128,
			Memory:    minMemory,
		},
		SecurityOpt: []string{`no-new-privileges`},
	}

	if service.ReadOnly {
		hostConfig.ReadonlyRootfs = service.ReadOnly
	}

	if service.CPUShares != 0 {
		hostConfig.Resources.CPUShares = service.CPUShares
	}

	if service.MemoryLimit != 0 {
		if service.MemoryLimit <= maxMemory {
			hostConfig.Resources.Memory = service.MemoryLimit
		} else {
			hostConfig.Resources.Memory = maxMemory
		}
	}

	return &hostConfig
}

func getNetworkConfig(service *dockerComposeService, deployedServices *map[string]deployedService) *network.NetworkingConfig {
	traefikConfig := network.EndpointSettings{}

	for _, link := range service.Links {
		linkParts := strings.Split(link, linkSeparator)

		target := linkParts[0]
		if linkedService, ok := (*deployedServices)[target]; ok {
			target = getFinalName(linkedService.Name)
		}

		alias := linkParts[0]
		if len(linkParts) > 1 {
			alias = linkParts[1]
		}

		traefikConfig.Links = append(traefikConfig.Links, target+linkSeparator+alias)
	}

	return &network.NetworkingConfig{
		EndpointsConfig: map[string]*network.EndpointSettings{
			networkMode: &traefikConfig,
		},
	}
}

func pullImage(image string, user *auth.User) error {
	if !imageTag.MatchString(image) {
		image = image + defaultTag
	}

	log.Printf(`[%s] Starting pull of image %s`, user.Username, image)
	pull, err := docker.ImagePull(context.Background(), image, types.ImagePullOptions{})
	if err != nil {
		return fmt.Errorf(`[%s] Error while pulling image: %v`, user.Username, err)
	}

	readBody(pull)
	log.Printf(`[%s] Ending pull of image %s`, user.Username, image)
	return nil
}

func cleanContainers(containers []types.Container, user *auth.User) {
	for _, container := range containers {
		log.Printf(`[%s] Stopping containers %s`, user.Username, strings.Join(container.Names, `, `))
		stopContainer(container.ID)
		log.Printf(`[%s] Deleting containers %s`, user.Username, strings.Join(container.Names, `, `))
		rmContainer(container.ID)
	}
}

func renameDeployedContainers(containers map[string]deployedService, user *auth.User) error {
	for service, container := range containers {
		if err := docker.ContainerRename(context.Background(), container.ID, getFinalName(container.Name)); err != nil {
			return fmt.Errorf(`[%s] Error while renaming container %s: %v`, user.Username, service, err)
		}
	}

	return nil
}

func getServiceFullName(app string, service string) string {
	return app + `_` + service + deploySuffix
}

func getFinalName(serviceFullName string) string {
	return strings.TrimSuffix(serviceFullName, deploySuffix)
}

func deleteServices(appName []byte, services map[string]deployedService, user *auth.User) {
	log.Printf(`[%s] Deleting services for %s`, user.Username, appName)
	for service, container := range services {
		if err := rmContainer(container.ID); err != nil {
			log.Printf(`[%s] Error while deleting service %s for %s: %v`, user.Username, service, appName, err)
		}
	}
}

func startServices(appName []byte, services map[string]deployedService, user *auth.User) error {
	log.Printf(`[%s] Starting services for %s`, user.Username, appName)
	for service, container := range services {
		if err := startContainer(container.ID); err != nil {
			return fmt.Errorf(`[%s] Error while starting service %s for %s: %v`, user.Username, service, appName, err)
		}
	}

	return nil
}

func inspectServices(services map[string]deployedService, user *auth.User) []*types.ContainerJSON {
	containers := make([]*types.ContainerJSON, 0, len(services))

	for service, container := range services {
		infos, err := inspectContainer(container.ID)
		if err != nil {
			log.Printf(`[%s] Error while inspecting container %s: %v`, user.Username, service, err)
		}

		containers = append(containers, &infos)
	}

	return containers
}

func areContainersHealthy(ctx context.Context, user *auth.User, containers []*types.ContainerJSON) bool {
	containersIdsWithHealthcheck := make([]*string, 0, len(containers))
	for _, container := range containers {
		if container.Config.Healthcheck != nil && len(container.Config.Healthcheck.Test) != 0 {
			containersIdsWithHealthcheck = append(containersIdsWithHealthcheck, &container.ID)
		}
	}

	if len(containersIdsWithHealthcheck) == 0 {
		return true
	}

	filtersArgs := filters.NewArgs()
	if err := healthyStatusFilters(user, &filtersArgs, containersIdsWithHealthcheck); err != nil {
		log.Printf(`[%s] Error while defining healthy filters: %v`, user.Username, err)
		return true
	}

	timeoutCtx, cancel := context.WithTimeout(ctx, healthcheckTimeout)
	defer cancel()

	messages, err := docker.Events(timeoutCtx, types.EventsOptions{Filters: filtersArgs})
	if err != nil {
		log.Printf(`[%s] Error while reading healthy events: %v`, user.Username, err)
		return false
	}

	healthyContainers := make(map[string]bool, len(containersIdsWithHealthcheck))
	for {
		select {
		case <-ctx.Done():
			return false
		case message := <-messages:
			healthyContainers[message.ID] = true

			if len(healthyContainers) == len(containersIdsWithHealthcheck) {
				return true
			}
		}
	}
}

func finishDeploy(ctx context.Context, cancel context.CancelFunc, user *auth.User, appName []byte, services map[string]deployedService, oldContainers []types.Container) {
	addCounter(1)
	defer addCounter(-1)
	defer cancel()

	log.Printf(`[%s] Waiting for %s to start...`, user.Username, appName)

	if areContainersHealthy(ctx, user, inspectServices(services, user)) {
		log.Printf(`[%s] Health check succeeded for %s`, user.Username, appName)
		cleanContainers(oldContainers, user)

		if err := renameDeployedContainers(services, user); err != nil {
			log.Print(err)
		}
	} else {
		log.Printf(`[%s] Health check failed for %s`, user.Username, appName)
		deleteServices(appName, services, user)
	}
}

func createAppHandler(w http.ResponseWriter, user *auth.User, appName []byte, composeFile []byte) {
	if len(appName) == 0 || len(composeFile) == 0 {
		badRequest(w, fmt.Errorf(`[%s] An application name and a compose file are required`, user.Username))
		return
	}

	compose := dockerCompose{}
	if err := yaml.Unmarshal(composeFile, &compose); err != nil {
		errorHandler(w, fmt.Errorf(`[%s] Error while unmarshalling compose file: %v`, user.Username, err))
		return
	}

	appNameStr := string(appName)
	log.Printf(`[%s] Deploying %s`, user.Username, appNameStr)

	ownerContainers, err := listContainers(user, &appNameStr)
	if err != nil {
		errorHandler(w, err)
		return
	}

	if len(ownerContainers) > 0 && ownerContainers[0].Labels[ownerLabel] != user.Username {
		forbidden(w)
	}

	deployedServices := make(map[string]deployedService)

	for serviceName, service := range compose.Services {
		if err := pullImage(service.Image, user); err != nil {
			break
		}

		serviceFullName := getServiceFullName(appNameStr, serviceName)
		log.Printf(`[%s] Creating service %s for %s`, user.Username, serviceName, appName)

		createdContainer, err := docker.ContainerCreate(context.Background(), getConfig(&service, user, appNameStr), getHostConfig(&service), getNetworkConfig(&service, &deployedServices), serviceFullName)
		if err != nil {
			err = fmt.Errorf(`[%s] Error while creating service %s for %s: %v`, user.Username, serviceName, appName, err)
			break
		}

		deployedServices[serviceName] = deployedService{ID: createdContainer.ID, Name: serviceFullName}
	}

	ctx, cancel := context.WithCancel(context.Background())
	go finishDeploy(ctx, cancel, user, appName, deployedServices, ownerContainers)

	if err == nil {
		err = startServices(appName, deployedServices, user)
	}

	if err != nil {
		deleteServices(appName, deployedServices, user)
		errorHandler(w, err)
		cancel()
		return
	}

	jsonHttp.ResponseJSON(w, results{deployedServices})
}
<|endoftext|>


<commit_before>package middleware

import (
	"bytes"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"strings"
	"text/template"
	"time"
)

// This file contains the context and functions available for
// use in the templates.

// Context is the context with which Caddy templates are executed.
type Context struct {
	Root http.FileSystem
	Req  *http.Request
	// This is used to access information about the URL.
	URL *url.URL
}

// Include returns the contents of filename relative to the site root
func (c Context) Include(filename string) (string, error) {
	file, err := c.Root.Open(filename)
	if err != nil {
		return "", err
	}
	defer file.Close()

	body, err := ioutil.ReadAll(file)
	if err != nil {
		return "", err
	}

	tpl, err := template.New(filename).Parse(string(body))
	if err != nil {
		return "", err
	}

	var buf bytes.Buffer
	err = tpl.Execute(&buf, c)
	if err != nil {
		return "", err
	}

	return buf.String(), nil
}

// Now returns the current timestamp in the specified format.
func (c Context) Now(format string) string {
	return time.Now().Format(format)
}

// Cookie gets the value of a cookie with name name.
func (c Context) Cookie(name string) string {
	cookies := c.Req.Cookies()
	for _, cookie := range cookies {
		if cookie.Name == name {
			return cookie.Value
		}
	}
	return ""
}

// Header gets the value of a request header with field name.
func (c Context) Header(name string) string {
	return c.Req.Header.Get(name)
}

// IP gets the (remote) IP address of the client making the request.
func (c Context) IP() string {
	ip, _, err := net.SplitHostPort(c.Req.RemoteAddr)
	if err != nil {
		return c.Req.RemoteAddr
	}
	return ip
}

// URI returns the raw, unprocessed request URI (including query
// string and hash) obtained directly from the Request-Line of
// the HTTP request.
func (c Context) URI() string {
	return c.Req.RequestURI
}

// Host returns the hostname portion of the Host header
// from the HTTP request.
func (c Context) Host() (string, error) {
	host, _, err := net.SplitHostPort(c.Req.Host)
	if err != nil {
		return "", err
	}
	return host, nil
}

// Port returns the port portion of the Host header if specified.
func (c Context) Port() (string, error) {
	_, port, err := net.SplitHostPort(c.Req.Host)
	if err != nil {
		return "", err
	}
	return port, nil
}

// Method returns the method (GET, POST, etc.) of the request.
func (c Context) Method() string {
	return c.Req.Method
}

// PathMatches returns true if the path portion of the request
// URL matches pattern.
func (c Context) PathMatches(pattern string) bool {
	return Path(c.Req.URL.Path).Matches(pattern)
}

// Truncate truncates the input string to the given length. If
// input is shorter than length, the entire string is returned.
func (c Context) Truncate(input string, length int) string {
	if len(input) > length {
		return input[:length]
	}
	return input
}

// Replace replaces instances of find in input with replacement.
func (c Context) Replace(input, find, replacement string) string {
	return strings.Replace(input, find, replacement, -1)
}
<commit_msg>New template action NowDate to get the time.Time of Now()<commit_after>package middleware

import (
	"bytes"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"strings"
	"text/template"
	"time"
)

// This file contains the context and functions available for
// use in the templates.

// Context is the context with which Caddy templates are executed.
type Context struct {
	Root http.FileSystem
	Req  *http.Request
	// This is used to access information about the URL.
	URL *url.URL
}

// Include returns the contents of filename relative to the site root
func (c Context) Include(filename string) (string, error) {
	file, err := c.Root.Open(filename)
	if err != nil {
		return "", err
	}
	defer file.Close()

	body, err := ioutil.ReadAll(file)
	if err != nil {
		return "", err
	}

	tpl, err := template.New(filename).Parse(string(body))
	if err != nil {
		return "", err
	}

	var buf bytes.Buffer
	err = tpl.Execute(&buf, c)
	if err != nil {
		return "", err
	}

	return buf.String(), nil
}

// Now returns the current timestamp in the specified format.
func (c Context) Now(format string) string {
	return time.Now().Format(format)
}

// NowDate returns the current date/time that can be used
// in other time functions.
func (c Context) NowDate() time.Time {
	return time.Now()
}

// Cookie gets the value of a cookie with name name.
func (c Context) Cookie(name string) string {
	cookies := c.Req.Cookies()
	for _, cookie := range cookies {
		if cookie.Name == name {
			return cookie.Value
		}
	}
	return ""
}

// Header gets the value of a request header with field name.
func (c Context) Header(name string) string {
	return c.Req.Header.Get(name)
}

// IP gets the (remote) IP address of the client making the request.
func (c Context) IP() string {
	ip, _, err := net.SplitHostPort(c.Req.RemoteAddr)
	if err != nil {
		return c.Req.RemoteAddr
	}
	return ip
}

// URI returns the raw, unprocessed request URI (including query
// string and hash) obtained directly from the Request-Line of
// the HTTP request.
func (c Context) URI() string {
	return c.Req.RequestURI
}

// Host returns the hostname portion of the Host header
// from the HTTP request.
func (c Context) Host() (string, error) {
	host, _, err := net.SplitHostPort(c.Req.Host)
	if err != nil {
		return "", err
	}
	return host, nil
}

// Port returns the port portion of the Host header if specified.
func (c Context) Port() (string, error) {
	_, port, err := net.SplitHostPort(c.Req.Host)
	if err != nil {
		return "", err
	}
	return port, nil
}

// Method returns the method (GET, POST, etc.) of the request.
func (c Context) Method() string {
	return c.Req.Method
}

// PathMatches returns true if the path portion of the request
// URL matches pattern.
func (c Context) PathMatches(pattern string) bool {
	return Path(c.Req.URL.Path).Matches(pattern)
}

// Truncate truncates the input string to the given length. If
// input is shorter than length, the entire string is returned.
func (c Context) Truncate(input string, length int) string {
	if len(input) > length {
		return input[:length]
	}
	return input
}

// Replace replaces instances of find in input with replacement.
func (c Context) Replace(input, find, replacement string) string {
	return strings.Replace(input, find, replacement, -1)
}
<|endoftext|>


<commit_before>package main

import (
	"testing"
)

// TODO
func TestThis(t *testing.T) {
}
<commit_msg>test pangu-axe<commit_after>package main

import (
	"os"
	"testing"
)

func TestText(t *testing.T) {
	os.Args = []string{NAME, "text", "新八的構造成分有95%是眼鏡、3%是水、2%是垃圾"}
	main()
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package gcsfake

import (
	"bytes"
	"crypto/md5"
	"errors"
	"fmt"
	"hash/crc32"
	"io"
	"io/ioutil"
	"net/http"
	"sort"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

var crc32Table = crc32.MakeTable(crc32.Castagnoli)

// Create an in-memory bucket with the given name and empty contents.
func NewFakeBucket(name string) gcs.Bucket {
	b := &bucket{name: name}
	b.mu = syncutil.NewInvariantMutex(func() { b.checkInvariants() })
	return b
}

type fakeObject struct {
	// A storage.Object representing a GCS entry for this object.
	entry storage.Object

	// The contents of the object. These never change.
	contents string
}

// A slice of objects compared by name.
type fakeObjectSlice []fakeObject

func (s fakeObjectSlice) Len() int {
	return len(s)
}

func (s fakeObjectSlice) Less(i, j int) bool {
	return s[i].entry.Name < s[j].entry.Name
}

func (s fakeObjectSlice) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

// Return the smallest i such that s[i].entry.Name >= name, or len(s) if there
// is no such i.
func (s fakeObjectSlice) lowerBound(name string) int {
	pred := func(i int) bool {
		return s[i].entry.Name >= name
	}

	return sort.Search(len(s), pred)
}

// Return the smallest i such that s[i].entry.Name == name, or len(s) if there
// is no such i.
func (s fakeObjectSlice) find(name string) int {
	lb := s.lowerBound(name)
	if lb < len(s) && s[lb].entry.Name == name {
		return lb
	}

	return len(s)
}

// Return the smallest string that is lexicographically larger than prefix and
// does not have prefix as a prefix. For the sole case where this is not
// possible (all strings consisting solely of 0xff bytes, including the empty
// string), return the empty string.
func prefixSuccessor(prefix string) string {
	// Attempt to increment the last byte. If that is a 0xff byte, erase it and
	// recurse. If we hit an empty string, then we know our task is impossible.
	limit := []byte(prefix)
	for len(limit) > 0 {
		b := limit[len(limit)-1]
		if b != 0xff {
			limit[len(limit)-1]++
			break
		}

		limit = limit[:len(limit)-1]
	}

	return string(limit)
}

// Return the smallest i such that prefix < s[i].entry.Name and
// !strings.HasPrefix(s[i].entry.Name, prefix).
func (s fakeObjectSlice) prefixUpperBound(prefix string) int {
	successor := prefixSuccessor(prefix)
	if successor == "" {
		return len(s)
	}

	return s.lowerBound(successor)
}

type bucket struct {
	name string
	mu   syncutil.InvariantMutex

	// The set of extant objects.
	//
	// INVARIANT: Strictly increasing.
	objects fakeObjectSlice // GUARDED_BY(mu)

	// The most recent generation number that was minted. The next object will
	// receive generation prevGeneration + 1.
	//
	// INVARIANT: This is an upper bound for generation numbers in objects.
	prevGeneration int64 // GUARDED_BY(mu)
}

// SHARED_LOCKS_REQUIRED(b.mu)
func (b *bucket) checkInvariants() {
	// Make sure 'objects' is strictly increasing.
	for i := 1; i < len(b.objects); i++ {
		objA := b.objects[i-1]
		objB := b.objects[i]
		if !(objA.entry.Name < objB.entry.Name) {
			panic(
				fmt.Sprintf(
					"Object names are not strictly increasing: %v vs. %v",
					objA.entry.Name,
					objB.entry.Name))
		}
	}

	// Make sure prevGeneration is an upper bound for object generation numbers.
	for _, o := range b.objects {
		if !(o.entry.Generation <= b.prevGeneration) {
			panic(
				fmt.Sprintf(
					"Object generation %v exceeds %v",
					o.entry.Generation,
					b.prevGeneration))
		}
	}
}

func (b *bucket) Name() string {
	return b.name
}

// LOCKS_EXCLUDED(b.mu)
func (b *bucket) ListObjects(
	ctx context.Context,
	query *storage.Query) (listing *storage.Objects, err error) {
	b.mu.RLock()
	defer b.mu.RUnlock()

	// Set up the result object.
	listing = new(storage.Objects)

	// Handle nil queries.
	if query == nil {
		query = &storage.Query{}
	}

	// Handle defaults.
	maxResults := query.MaxResults
	if maxResults == 0 {
		maxResults = 1000
	}

	// Find where in the space of object names to start.
	nameStart := query.Prefix
	if query.Cursor != "" && query.Cursor > nameStart {
		nameStart = query.Cursor
	}

	// Find the range of indexes within the array to scan.
	indexStart := b.objects.lowerBound(nameStart)
	prefixLimit := b.objects.prefixUpperBound(query.Prefix)
	indexLimit := minInt(indexStart+maxResults, prefixLimit)

	// Scan the array.
	var lastResultWasPrefix bool
	for i := indexStart; i < indexLimit; i++ {
		var o fakeObject = b.objects[i]
		name := o.entry.Name

		// Search for a delimiter if necessary.
		if query.Delimiter != "" {
			// Search only in the part after the prefix.
			nameMinusQueryPrefix := name[len(query.Prefix):]

			delimiterIndex := strings.Index(nameMinusQueryPrefix, query.Delimiter)
			if delimiterIndex >= 0 {
				resultPrefixLimit := delimiterIndex

				// Transform to an index within name.
				resultPrefixLimit += len(query.Prefix)

				// Include the delimiter in the result.
				resultPrefixLimit += len(query.Delimiter)

				// Save the result, but only if it's not a duplicate.
				resultPrefix := name[:resultPrefixLimit]
				if len(listing.Prefixes) == 0 ||
					listing.Prefixes[len(listing.Prefixes)-1] != resultPrefix {
					listing.Prefixes = append(listing.Prefixes, resultPrefix)
				}

				lastResultWasPrefix = true
				continue
			}
		}

		lastResultWasPrefix = false

		// Otherwise, return as an object result. Make a copy to avoid handing back
		// internal state.
		var oCopy storage.Object = o.entry
		listing.Results = append(listing.Results, &oCopy)
	}

	// Set up a cursor for where to start the next scan if we didn't exhaust the
	// results.
	if indexLimit < prefixLimit {
		listing.Next = &storage.Query{}
		*listing.Next = *query

		// Ion is if the final object we visited was returned as an element in
		// listing.Prefixes, we want to skip all other objects that would result in
		// the same so we don't return duplicate elements in listing.Prefixes
		// accross requests.
		if lastResultWasPrefix {
			lastResultPrefix := listing.Prefixes[len(listing.Prefixes)-1]
			listing.Next.Cursor = prefixSuccessor(lastResultPrefix)

			// Check an assumption: prefixSuccessor cannot result in the empty string
			// above because object names must be non-empty UTF-8 strings, and there
			// is no valid non-empty UTF-8 string that consists of entirely 0xff
			// bytes.
			if listing.Next.Cursor == "" {
				err = errors.New("Unexpected empty string from prefixSuccessor")
				return
			}
		} else {
			// Otherwise, we'll start scanning at the next object.
			listing.Next.Cursor = b.objects[indexLimit].entry.Name
		}
	}

	return
}

// LOCKS_EXCLUDED(b.mu)
func (b *bucket) NewReader(
	ctx context.Context,
	objectName string) (io.ReadCloser, error) {
	b.mu.RLock()
	defer b.mu.RUnlock()

	index := b.objects.find(objectName)
	if index == len(b.objects) {
		return nil, errors.New("object doesn't exist.")
	}

	return ioutil.NopCloser(strings.NewReader(b.objects[index].contents)), nil
}

func (b *bucket) CreateObject(
	ctx context.Context,
	req *gcs.CreateObjectRequest) (o *storage.Object, err error) {
	// Check that the object name is legal.
	name := req.Attrs.Name
	if len(name) == 0 || len(name) > 1024 {
		return nil, errors.New("Invalid object name: length must be in [1, 1024]")
	}

	if !utf8.ValidString(name) {
		return nil, errors.New("Invalid object name: not valid UTF-8")
	}

	for _, r := range name {
		if r == 0x0a || r == 0x0d {
			return nil, errors.New("Invalid object name: must not contain CR or LF")
		}
	}

	// Snarf the object contents.
	buf := new(bytes.Buffer)
	if _, err = io.Copy(buf, req.Contents); err != nil {
		return
	}

	contents := buf.String()

	// Lock and proceed.
	b.mu.Lock()
	defer b.mu.Unlock()

	obj, err := b.addObjectLocked(req, contents)
	if err != nil {
		return
	}

	o = &obj
	return
}

// LOCKS_EXCLUDED(b.mu)
func (b *bucket) UpdateObject(
	ctx context.Context,
	req *gcs.UpdateObjectRequest) (o *storage.Object, err error) {
	b.mu.Lock()
	defer b.mu.Unlock()

	// Match real GCS in not allowing the removal of ContentType.
	if req.ContentType != nil && *req.ContentType == "" {
		err = errors.New("The ContentType field is required and cannot be removed.")
		return
	}

	// Does the object exist?
	index := b.objects.find(req.Name)
	if index == len(b.objects) {
		err = errors.New("Object not found.")
		return
	}

	var obj *storage.Object = &b.objects[index].entry

	// Update the entry's basic fields according to the request.
	if req.ContentType != nil {
		obj.ContentType = *req.ContentType
	}

	if req.ContentEncoding != nil {
		obj.ContentEncoding = *req.ContentEncoding
	}

	if req.ContentLanguage != nil {
		obj.ContentLanguage = *req.ContentLanguage
	}

	if req.CacheControl != nil {
		obj.CacheControl = *req.CacheControl
	}

	// Update the user metadata if necessary.
	if len(req.Metadata) > 0 {
		if obj.Metadata == nil {
			obj.Metadata = make(map[string]string)
		}

		for k, v := range req.Metadata {
			if v == nil {
				delete(obj.Metadata, k)
				continue
			}

			obj.Metadata[k] = *v
		}
	}

	// Bump up the entry generation number.
	obj.MetaGeneration++

	// Make a copy to avoid handing back internal state.
	var objCopy storage.Object = *obj
	o = &objCopy

	return
}

// LOCKS_EXCLUDED(b.mu)
func (b *bucket) DeleteObject(
	ctx context.Context,
	name string) error {
	b.mu.Lock()
	defer b.mu.Unlock()

	// Do we possess the object with the given name?
	index := b.objects.find(name)
	if index == len(b.objects) {
		return errors.New("Object Not Found.")
	}

	// Remove the object.
	b.objects = append(b.objects[:index], b.objects[index+1:]...)

	return nil
}

// Create an object struct for the given attributes and contents.
//
// EXCLUSIVE_LOCKS_REQUIRED(b.mu)
func (b *bucket) mintObject(
	attrs *storage.ObjectAttrs,
	contents string) (o fakeObject) {
	// Set up basic info.
	b.prevGeneration++
	o.entry = storage.Object{
		Bucket:          b.Name(),
		Name:            attrs.Name,
		ContentType:     attrs.ContentType,
		ContentLanguage: attrs.ContentLanguage,
		CacheControl:    attrs.CacheControl,
		Owner:           "user-fake",
		Size:            int64(len(contents)),
		ContentEncoding: attrs.ContentEncoding,
		CRC32C:          crc32.Checksum([]byte(contents), crc32Table),
		MediaLink:       "http://localhost/download/storage/fake/" + attrs.Name,
		Metadata:        attrs.Metadata,
		Generation:      b.prevGeneration,
		MetaGeneration:  1,
		StorageClass:    "STANDARD",
		Updated:         time.Now(),
	}

	// Fill in the MD5 field.
	md5Array := md5.Sum([]byte(contents))
	o.entry.MD5 = md5Array[:]

	// Set up contents.
	o.contents = contents

	// Match the real GCS client library's behavior of sniffing content types
	// when not explicitly specified.
	if o.entry.ContentType == "" {
		o.entry.ContentType = http.DetectContentType([]byte(contents))
	}

	return
}

// Add a record and return a copy of the minted entry.
//
// EXCLUSIVE_LOCKS_REQUIRED(b.mu)
func (b *bucket) addObjectLocked(
	req *gcs.CreateObjectRequest,
	contents string) (entry storage.Object, err error) {
	// Create an object record from the given attributes.
	var o fakeObject = b.mintObject(&req.Attrs, contents)

	// Replace an entry in or add an entry to our list of objects.
	existingIndex := b.objects.find(req.Attrs.Name)
	if existingIndex < len(b.objects) {
		b.objects[existingIndex] = o
	} else {
		b.objects = append(b.objects, o)
		sort.Sort(b.objects)
	}

	entry = o.entry
	return
}

func minInt(a, b int) int {
	if a < b {
		return a
	}

	return b
}

func maxInt(a, b int) int {
	if a > b {
		return a
	}

	return b
}
<commit_msg>Implemented precondition checking in the fake.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package gcsfake

import (
	"bytes"
	"crypto/md5"
	"errors"
	"fmt"
	"hash/crc32"
	"io"
	"io/ioutil"
	"net/http"
	"sort"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

var crc32Table = crc32.MakeTable(crc32.Castagnoli)

// Create an in-memory bucket with the given name and empty contents.
func NewFakeBucket(name string) gcs.Bucket {
	b := &bucket{name: name}
	b.mu = syncutil.NewInvariantMutex(func() { b.checkInvariants() })
	return b
}

type fakeObject struct {
	// A storage.Object representing a GCS entry for this object.
	entry storage.Object

	// The contents of the object. These never change.
	contents string
}

// A slice of objects compared by name.
type fakeObjectSlice []fakeObject

func (s fakeObjectSlice) Len() int {
	return len(s)
}

func (s fakeObjectSlice) Less(i, j int) bool {
	return s[i].entry.Name < s[j].entry.Name
}

func (s fakeObjectSlice) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

// Return the smallest i such that s[i].entry.Name >= name, or len(s) if there
// is no such i.
func (s fakeObjectSlice) lowerBound(name string) int {
	pred := func(i int) bool {
		return s[i].entry.Name >= name
	}

	return sort.Search(len(s), pred)
}

// Return the smallest i such that s[i].entry.Name == name, or len(s) if there
// is no such i.
func (s fakeObjectSlice) find(name string) int {
	lb := s.lowerBound(name)
	if lb < len(s) && s[lb].entry.Name == name {
		return lb
	}

	return len(s)
}

// Return the smallest string that is lexicographically larger than prefix and
// does not have prefix as a prefix. For the sole case where this is not
// possible (all strings consisting solely of 0xff bytes, including the empty
// string), return the empty string.
func prefixSuccessor(prefix string) string {
	// Attempt to increment the last byte. If that is a 0xff byte, erase it and
	// recurse. If we hit an empty string, then we know our task is impossible.
	limit := []byte(prefix)
	for len(limit) > 0 {
		b := limit[len(limit)-1]
		if b != 0xff {
			limit[len(limit)-1]++
			break
		}

		limit = limit[:len(limit)-1]
	}

	return string(limit)
}

// Return the smallest i such that prefix < s[i].entry.Name and
// !strings.HasPrefix(s[i].entry.Name, prefix).
func (s fakeObjectSlice) prefixUpperBound(prefix string) int {
	successor := prefixSuccessor(prefix)
	if successor == "" {
		return len(s)
	}

	return s.lowerBound(successor)
}

type bucket struct {
	name string
	mu   syncutil.InvariantMutex

	// The set of extant objects.
	//
	// INVARIANT: Strictly increasing.
	objects fakeObjectSlice // GUARDED_BY(mu)

	// The most recent generation number that was minted. The next object will
	// receive generation prevGeneration + 1.
	//
	// INVARIANT: This is an upper bound for generation numbers in objects.
	prevGeneration int64 // GUARDED_BY(mu)
}

// SHARED_LOCKS_REQUIRED(b.mu)
func (b *bucket) checkInvariants() {
	// Make sure 'objects' is strictly increasing.
	for i := 1; i < len(b.objects); i++ {
		objA := b.objects[i-1]
		objB := b.objects[i]
		if !(objA.entry.Name < objB.entry.Name) {
			panic(
				fmt.Sprintf(
					"Object names are not strictly increasing: %v vs. %v",
					objA.entry.Name,
					objB.entry.Name))
		}
	}

	// Make sure prevGeneration is an upper bound for object generation numbers.
	for _, o := range b.objects {
		if !(o.entry.Generation <= b.prevGeneration) {
			panic(
				fmt.Sprintf(
					"Object generation %v exceeds %v",
					o.entry.Generation,
					b.prevGeneration))
		}
	}
}

func (b *bucket) Name() string {
	return b.name
}

// LOCKS_EXCLUDED(b.mu)
func (b *bucket) ListObjects(
	ctx context.Context,
	query *storage.Query) (listing *storage.Objects, err error) {
	b.mu.RLock()
	defer b.mu.RUnlock()

	// Set up the result object.
	listing = new(storage.Objects)

	// Handle nil queries.
	if query == nil {
		query = &storage.Query{}
	}

	// Handle defaults.
	maxResults := query.MaxResults
	if maxResults == 0 {
		maxResults = 1000
	}

	// Find where in the space of object names to start.
	nameStart := query.Prefix
	if query.Cursor != "" && query.Cursor > nameStart {
		nameStart = query.Cursor
	}

	// Find the range of indexes within the array to scan.
	indexStart := b.objects.lowerBound(nameStart)
	prefixLimit := b.objects.prefixUpperBound(query.Prefix)
	indexLimit := minInt(indexStart+maxResults, prefixLimit)

	// Scan the array.
	var lastResultWasPrefix bool
	for i := indexStart; i < indexLimit; i++ {
		var o fakeObject = b.objects[i]
		name := o.entry.Name

		// Search for a delimiter if necessary.
		if query.Delimiter != "" {
			// Search only in the part after the prefix.
			nameMinusQueryPrefix := name[len(query.Prefix):]

			delimiterIndex := strings.Index(nameMinusQueryPrefix, query.Delimiter)
			if delimiterIndex >= 0 {
				resultPrefixLimit := delimiterIndex

				// Transform to an index within name.
				resultPrefixLimit += len(query.Prefix)

				// Include the delimiter in the result.
				resultPrefixLimit += len(query.Delimiter)

				// Save the result, but only if it's not a duplicate.
				resultPrefix := name[:resultPrefixLimit]
				if len(listing.Prefixes) == 0 ||
					listing.Prefixes[len(listing.Prefixes)-1] != resultPrefix {
					listing.Prefixes = append(listing.Prefixes, resultPrefix)
				}

				lastResultWasPrefix = true
				continue
			}
		}

		lastResultWasPrefix = false

		// Otherwise, return as an object result. Make a copy to avoid handing back
		// internal state.
		var oCopy storage.Object = o.entry
		listing.Results = append(listing.Results, &oCopy)
	}

	// Set up a cursor for where to start the next scan if we didn't exhaust the
	// results.
	if indexLimit < prefixLimit {
		listing.Next = &storage.Query{}
		*listing.Next = *query

		// Ion is if the final object we visited was returned as an element in
		// listing.Prefixes, we want to skip all other objects that would result in
		// the same so we don't return duplicate elements in listing.Prefixes
		// accross requests.
		if lastResultWasPrefix {
			lastResultPrefix := listing.Prefixes[len(listing.Prefixes)-1]
			listing.Next.Cursor = prefixSuccessor(lastResultPrefix)

			// Check an assumption: prefixSuccessor cannot result in the empty string
			// above because object names must be non-empty UTF-8 strings, and there
			// is no valid non-empty UTF-8 string that consists of entirely 0xff
			// bytes.
			if listing.Next.Cursor == "" {
				err = errors.New("Unexpected empty string from prefixSuccessor")
				return
			}
		} else {
			// Otherwise, we'll start scanning at the next object.
			listing.Next.Cursor = b.objects[indexLimit].entry.Name
		}
	}

	return
}

// LOCKS_EXCLUDED(b.mu)
func (b *bucket) NewReader(
	ctx context.Context,
	objectName string) (io.ReadCloser, error) {
	b.mu.RLock()
	defer b.mu.RUnlock()

	index := b.objects.find(objectName)
	if index == len(b.objects) {
		return nil, errors.New("object doesn't exist.")
	}

	return ioutil.NopCloser(strings.NewReader(b.objects[index].contents)), nil
}

func (b *bucket) CreateObject(
	ctx context.Context,
	req *gcs.CreateObjectRequest) (o *storage.Object, err error) {
	// Check that the object name is legal.
	name := req.Attrs.Name
	if len(name) == 0 || len(name) > 1024 {
		return nil, errors.New("Invalid object name: length must be in [1, 1024]")
	}

	if !utf8.ValidString(name) {
		return nil, errors.New("Invalid object name: not valid UTF-8")
	}

	for _, r := range name {
		if r == 0x0a || r == 0x0d {
			return nil, errors.New("Invalid object name: must not contain CR or LF")
		}
	}

	// Snarf the object contents.
	buf := new(bytes.Buffer)
	if _, err = io.Copy(buf, req.Contents); err != nil {
		return
	}

	contents := buf.String()

	// Lock and proceed.
	b.mu.Lock()
	defer b.mu.Unlock()

	obj, err := b.addObjectLocked(req, contents)
	if err != nil {
		return
	}

	o = &obj
	return
}

// LOCKS_EXCLUDED(b.mu)
func (b *bucket) UpdateObject(
	ctx context.Context,
	req *gcs.UpdateObjectRequest) (o *storage.Object, err error) {
	b.mu.Lock()
	defer b.mu.Unlock()

	// Match real GCS in not allowing the removal of ContentType.
	if req.ContentType != nil && *req.ContentType == "" {
		err = errors.New("The ContentType field is required and cannot be removed.")
		return
	}

	// Does the object exist?
	index := b.objects.find(req.Name)
	if index == len(b.objects) {
		err = errors.New("Object not found.")
		return
	}

	var obj *storage.Object = &b.objects[index].entry

	// Update the entry's basic fields according to the request.
	if req.ContentType != nil {
		obj.ContentType = *req.ContentType
	}

	if req.ContentEncoding != nil {
		obj.ContentEncoding = *req.ContentEncoding
	}

	if req.ContentLanguage != nil {
		obj.ContentLanguage = *req.ContentLanguage
	}

	if req.CacheControl != nil {
		obj.CacheControl = *req.CacheControl
	}

	// Update the user metadata if necessary.
	if len(req.Metadata) > 0 {
		if obj.Metadata == nil {
			obj.Metadata = make(map[string]string)
		}

		for k, v := range req.Metadata {
			if v == nil {
				delete(obj.Metadata, k)
				continue
			}

			obj.Metadata[k] = *v
		}
	}

	// Bump up the entry generation number.
	obj.MetaGeneration++

	// Make a copy to avoid handing back internal state.
	var objCopy storage.Object = *obj
	o = &objCopy

	return
}

// LOCKS_EXCLUDED(b.mu)
func (b *bucket) DeleteObject(
	ctx context.Context,
	name string) error {
	b.mu.Lock()
	defer b.mu.Unlock()

	// Do we possess the object with the given name?
	index := b.objects.find(name)
	if index == len(b.objects) {
		return errors.New("Object Not Found.")
	}

	// Remove the object.
	b.objects = append(b.objects[:index], b.objects[index+1:]...)

	return nil
}

// Create an object struct for the given attributes and contents.
//
// EXCLUSIVE_LOCKS_REQUIRED(b.mu)
func (b *bucket) mintObject(
	attrs *storage.ObjectAttrs,
	contents string) (o fakeObject) {
	// Set up basic info.
	b.prevGeneration++
	o.entry = storage.Object{
		Bucket:          b.Name(),
		Name:            attrs.Name,
		ContentType:     attrs.ContentType,
		ContentLanguage: attrs.ContentLanguage,
		CacheControl:    attrs.CacheControl,
		Owner:           "user-fake",
		Size:            int64(len(contents)),
		ContentEncoding: attrs.ContentEncoding,
		CRC32C:          crc32.Checksum([]byte(contents), crc32Table),
		MediaLink:       "http://localhost/download/storage/fake/" + attrs.Name,
		Metadata:        attrs.Metadata,
		Generation:      b.prevGeneration,
		MetaGeneration:  1,
		StorageClass:    "STANDARD",
		Updated:         time.Now(),
	}

	// Fill in the MD5 field.
	md5Array := md5.Sum([]byte(contents))
	o.entry.MD5 = md5Array[:]

	// Set up contents.
	o.contents = contents

	// Match the real GCS client library's behavior of sniffing content types
	// when not explicitly specified.
	if o.entry.ContentType == "" {
		o.entry.ContentType = http.DetectContentType([]byte(contents))
	}

	return
}

// Add a record and return a copy of the minted entry.
//
// EXCLUSIVE_LOCKS_REQUIRED(b.mu)
func (b *bucket) addObjectLocked(
	req *gcs.CreateObjectRequest,
	contents string) (entry storage.Object, err error) {
	// Find any existing record for this name.
	existingIndex := b.objects.find(req.Attrs.Name)

	var existingRecord *fakeObject
	if existingIndex < len(b.objects) {
		existingRecord = &b.objects[existingIndex]
	}

	// Check preconditions.
	if req.GenerationPrecondition != nil {
		if *req.GenerationPrecondition == 0 && existingRecord != nil {
			err = errors.New("Precondition failed: object exists.")
			return
		}

		if *req.GenerationPrecondition > 0 {
			if existingRecord == nil {
				err = errors.New("Precondition failed: object doesn't exist.")
				return
			}

			if existingRecord.entry.Generation != *req.GenerationPrecondition {
				err = fmt.Errorf(
					"Precondition failed: object has generation %v",
					existingRecord.entry.Generation)

				return
			}
		}
	}

	// Create an object record from the given attributes.
	var o fakeObject = b.mintObject(&req.Attrs, contents)

	// Replace an entry in or add an entry to our list of objects.
	if existingIndex < len(b.objects) {
		b.objects[existingIndex] = o
	} else {
		b.objects = append(b.objects, o)
		sort.Sort(b.objects)
	}

	entry = o.entry
	return
}

func minInt(a, b int) int {
	if a < b {
		return a
	}

	return b
}

func maxInt(a, b int) int {
	if a > b {
		return a
	}

	return b
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/golang/gddo/log"

	"github.com/spf13/pflag"
	"github.com/spf13/viper"
)

const (
	gaeProjectEnvVar = "GAE_LONG_APP_ID"
)

const (
	// Server Config
	ConfigProject           = "project"
	ConfigTrustProxyHeaders = "trust_proxy_headers"
	ConfigBindAddress       = "http"
	ConfigAssetsDir         = "assets"
	ConfigRobotThreshold    = "robot"

	// Database Config
	ConfigDBServer      = "db-server"
	ConfigDBIdleTimeout = "db-idle-timeout"
	ConfigDBLog         = "db-log"

	// Display Config
	ConfigSidebar        = "sidebar"
	ConfigSourcegraphURL = "sourcegraph_url"
	ConfigDefaultGOOS    = "default_goos"

	// Crawl Config
	ConfigMaxAge          = "max_age"
	ConfigGetTimeout      = "get_timeout"
	ConfigFirstGetTimeout = "first_get_timeout"
	ConfigGithubInterval  = "github_interval"
	ConfigCrawlInterval   = "crawl_interval"
	ConfigDialTimeout     = "dial_timeout"
	ConfigRequestTimeout  = "request_timeout"
	ConfigMemcacheAddr    = "memcache_addr"
)

// Initialize configuration
func init() {
	ctx := context.Background()

	// Automatically detect if we are on App Engine.
	if os.Getenv(gaeProjectEnvVar) != "" {
		viper.Set("on_appengine", true)
	} else {
		viper.Set("on_appengine", false)
	}

	// Setup command line flags
	flags := buildFlags()
	flags.Parse(os.Args)
	if err := viper.BindPFlags(flags); err != nil {
		panic(err)
	}

	// Also fetch from enviorment
	viper.SetEnvPrefix("gddo")
	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
	viper.AutomaticEnv()

	// Automatically get project ID from env on Google App Engine
	viper.BindEnv(ConfigProject, gaeProjectEnvVar)

	// Read from config.
	readViperConfig(ctx)

	log.Info(ctx, "config values loaded", "values", viper.AllSettings())
}

func buildFlags() *pflag.FlagSet {
	flags := pflag.NewFlagSet("default", pflag.ExitOnError)

	flags.StringP("config", "c", "", "path to motd config file")
	flags.String("project", "", "Google Cloud Platform project used for Google services")
	// TODO(stephenmw): flags.Bool("enable-admin-pages", false, "When true, enables /admin pages")
	flags.Float64(ConfigRobotThreshold, 100, "Request counter threshold for robots.")
	flags.String(ConfigAssetsDir, filepath.Join(defaultBase("github.com/golang/gddo/gddo-server"), "assets"), "Base directory for templates and static files.")
	flags.Duration(ConfigGetTimeout, 8*time.Second, "Time to wait for package update from the VCS.")
	flags.Duration(ConfigFirstGetTimeout, 5*time.Second, "Time to wait for first fetch of package from the VCS.")
	flags.Duration(ConfigMaxAge, 24*time.Hour, "Update package documents older than this age.")
	flags.String(ConfigBindAddress, ":8080", "Listen for HTTP connections on this address.")
	flags.Bool(ConfigSidebar, false, "Enable package page sidebar.")
	flags.String(ConfigDefaultGOOS, "", "Default GOOS to use when building package documents.")
	flags.Bool(ConfigTrustProxyHeaders, false, "If enabled, identify the remote address of the request using X-Real-Ip in header.")
	flags.String(ConfigSourcegraphURL, "https://sourcegraph.com", "Link to global uses on Sourcegraph based at this URL (no need for trailing slash).")
	flags.Duration(ConfigGithubInterval, 0, "Github updates crawler sleeps for this duration between fetches. Zero disables the crawler.")
	flags.Duration(ConfigCrawlInterval, 0, "Package updater sleeps for this duration between package updates. Zero disables updates.")
	flags.Duration(ConfigDialTimeout, 5*time.Second, "Timeout for dialing an HTTP connection.")
	flags.Duration(ConfigRequestTimeout, 20*time.Second, "Time out for roundtripping an HTTP request.")
	flags.String(ConfigDBServer, "redis://127.0.0.1:6379", "URI of Redis server.")
	flags.Duration(ConfigDBIdleTimeout, 250*time.Second, "Close Redis connections after remaining idle for this duration.")
	flags.Bool(ConfigDBLog, false, "Log database commands")
	flags.String(ConfigMemcacheAddr, "", "Address in the format host:port gddo uses to point to the memcache backend.")

	return flags
}

// readViperConfig finds and then parses a config file. It will log.Fatal if the
// config file was specified or could not parse. Otherwise it will only warn
// that it failed to load the config.
func readViperConfig(ctx context.Context) {
	viper.AddConfigPath(".")
	viper.AddConfigPath("/etc")
	viper.SetConfigName("gddo")
	if viper.GetString("config") != "" {
		viper.SetConfigFile(viper.GetString("config"))
	}

	if err := viper.ReadInConfig(); err != nil {
		// If a config exists but could not be parsed, we should bail.
		if _, ok := err.(viper.ConfigParseError); ok {
			log.Fatal(ctx, "failed to parse config", "error", err)
		}

		// If the user specified a config file location in flags or env and
		// we failed to load it, we should bail. If not, it is just a warning.
		if viper.GetString("config") != "" {
			log.Fatal(ctx, "failed to load configuration file", "error", err)
		} else {
			log.Warn(ctx, "failed to load configuration file", "error", err)
		}
	} else {
		log.Info(ctx, "loaded configuration file successfully", "path", viper.ConfigFileUsed())
	}
}
<commit_msg>gddo-server: Change the env var name to get GCloud Project ID<commit_after>package main

import (
	"context"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/golang/gddo/log"

	"github.com/spf13/pflag"
	"github.com/spf13/viper"
)

const (
	gaeProjectEnvVar = "GCLOUD_PROJECT"
)

const (
	// Server Config
	ConfigProject           = "project"
	ConfigTrustProxyHeaders = "trust_proxy_headers"
	ConfigBindAddress       = "http"
	ConfigAssetsDir         = "assets"
	ConfigRobotThreshold    = "robot"

	// Database Config
	ConfigDBServer      = "db-server"
	ConfigDBIdleTimeout = "db-idle-timeout"
	ConfigDBLog         = "db-log"

	// Display Config
	ConfigSidebar        = "sidebar"
	ConfigSourcegraphURL = "sourcegraph_url"
	ConfigDefaultGOOS    = "default_goos"

	// Crawl Config
	ConfigMaxAge          = "max_age"
	ConfigGetTimeout      = "get_timeout"
	ConfigFirstGetTimeout = "first_get_timeout"
	ConfigGithubInterval  = "github_interval"
	ConfigCrawlInterval   = "crawl_interval"
	ConfigDialTimeout     = "dial_timeout"
	ConfigRequestTimeout  = "request_timeout"
	ConfigMemcacheAddr    = "memcache_addr"
)

// Initialize configuration
func init() {
	ctx := context.Background()

	// Automatically detect if we are on App Engine.
	if os.Getenv(gaeProjectEnvVar) != "" {
		viper.Set("on_appengine", true)
	} else {
		viper.Set("on_appengine", false)
	}

	// Setup command line flags
	flags := buildFlags()
	flags.Parse(os.Args)
	if err := viper.BindPFlags(flags); err != nil {
		panic(err)
	}

	// Also fetch from enviorment
	viper.SetEnvPrefix("gddo")
	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
	viper.AutomaticEnv()

	// Automatically get project ID from env on Google App Engine
	viper.BindEnv(ConfigProject, gaeProjectEnvVar)

	// Read from config.
	readViperConfig(ctx)

	log.Info(ctx, "config values loaded", "values", viper.AllSettings())
}

func buildFlags() *pflag.FlagSet {
	flags := pflag.NewFlagSet("default", pflag.ExitOnError)

	flags.StringP("config", "c", "", "path to motd config file")
	flags.String("project", "", "Google Cloud Platform project used for Google services")
	// TODO(stephenmw): flags.Bool("enable-admin-pages", false, "When true, enables /admin pages")
	flags.Float64(ConfigRobotThreshold, 100, "Request counter threshold for robots.")
	flags.String(ConfigAssetsDir, filepath.Join(defaultBase("github.com/golang/gddo/gddo-server"), "assets"), "Base directory for templates and static files.")
	flags.Duration(ConfigGetTimeout, 8*time.Second, "Time to wait for package update from the VCS.")
	flags.Duration(ConfigFirstGetTimeout, 5*time.Second, "Time to wait for first fetch of package from the VCS.")
	flags.Duration(ConfigMaxAge, 24*time.Hour, "Update package documents older than this age.")
	flags.String(ConfigBindAddress, ":8080", "Listen for HTTP connections on this address.")
	flags.Bool(ConfigSidebar, false, "Enable package page sidebar.")
	flags.String(ConfigDefaultGOOS, "", "Default GOOS to use when building package documents.")
	flags.Bool(ConfigTrustProxyHeaders, false, "If enabled, identify the remote address of the request using X-Real-Ip in header.")
	flags.String(ConfigSourcegraphURL, "https://sourcegraph.com", "Link to global uses on Sourcegraph based at this URL (no need for trailing slash).")
	flags.Duration(ConfigGithubInterval, 0, "Github updates crawler sleeps for this duration between fetches. Zero disables the crawler.")
	flags.Duration(ConfigCrawlInterval, 0, "Package updater sleeps for this duration between package updates. Zero disables updates.")
	flags.Duration(ConfigDialTimeout, 5*time.Second, "Timeout for dialing an HTTP connection.")
	flags.Duration(ConfigRequestTimeout, 20*time.Second, "Time out for roundtripping an HTTP request.")
	flags.String(ConfigDBServer, "redis://127.0.0.1:6379", "URI of Redis server.")
	flags.Duration(ConfigDBIdleTimeout, 250*time.Second, "Close Redis connections after remaining idle for this duration.")
	flags.Bool(ConfigDBLog, false, "Log database commands")
	flags.String(ConfigMemcacheAddr, "", "Address in the format host:port gddo uses to point to the memcache backend.")

	return flags
}

// readViperConfig finds and then parses a config file. It will log.Fatal if the
// config file was specified or could not parse. Otherwise it will only warn
// that it failed to load the config.
func readViperConfig(ctx context.Context) {
	viper.AddConfigPath(".")
	viper.AddConfigPath("/etc")
	viper.SetConfigName("gddo")
	if viper.GetString("config") != "" {
		viper.SetConfigFile(viper.GetString("config"))
	}

	if err := viper.ReadInConfig(); err != nil {
		// If a config exists but could not be parsed, we should bail.
		if _, ok := err.(viper.ConfigParseError); ok {
			log.Fatal(ctx, "failed to parse config", "error", err)
		}

		// If the user specified a config file location in flags or env and
		// we failed to load it, we should bail. If not, it is just a warning.
		if viper.GetString("config") != "" {
			log.Fatal(ctx, "failed to load configuration file", "error", err)
		} else {
			log.Warn(ctx, "failed to load configuration file", "error", err)
		}
	} else {
		log.Info(ctx, "loaded configuration file successfully", "path", viper.ConfigFileUsed())
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"github.com/01org/ciao/ciao-controller/types"
	"github.com/01org/ciao/payloads"
	"github.com/01org/ciao/ssntp/uuid"
)

func validateVMWorkload(req types.Workload) error {
	// FWType must be either EFI or legacy.
	if req.FWType != string(payloads.EFI) && req.FWType != payloads.Legacy {
		return types.ErrBadRequest
	}

	// Must have storage for VMs
	if len(req.Storage) == 0 {
		return types.ErrBadRequest
	}

	return nil
}

func validateContainerWorkload(req types.Workload) error {
	// we should reject anything with ImageID set, but
	// we'll just ignore it.
	if req.ImageName == "" {
		return types.ErrBadRequest
	}

	return nil
}

func validateWorkloadStorage(req types.Workload) error {
	bootableCount := 0
	for i := range req.Storage {
		// check that a workload type is specified
		if req.Storage[i].SourceType == "" {
			return types.ErrBadRequest
		}

		// you may not request a sized volume unless it's empty.
		if req.Storage[i].Size > 0 && req.Storage[i].SourceType != types.Empty {
			return types.ErrBadRequest
		}

		// you may not request a bootable empty volume.
		if req.Storage[i].Bootable && req.Storage[i].SourceType == types.Empty {
			return types.ErrBadRequest
		}

		if req.Storage[i].ID != "" {
			// validate that the id is at least valid
			// uuid4.
			_, err := uuid.Parse(req.Storage[i].ID)
			if err != nil {
				return types.ErrBadRequest
			}

			// If we have an ID we must have a type to get it from
			if req.Storage[i].SourceType != types.Empty {
				return types.ErrBadRequest
			}
		} else {
			if req.Storage[i].SourceType == types.Empty {
				// you many not use a source ID with empty.
				if req.Storage[i].SourceID != "" {
					return types.ErrBadRequest
				}
			} else {
				// you must specify an ID with volume/image
				if req.Storage[i].SourceID == "" {
					return types.ErrBadRequest
				}
			}
		}

		if req.Storage[i].Bootable {
			bootableCount++
		}
	}

	// must be at least one bootable volume
	if req.VMType == payloads.QEMU && bootableCount == 0 {
		return types.ErrBadRequest
	}

	return nil
}

// this is probably an insufficient amount of checking.
func validateWorkloadRequest(req types.Workload) error {
	// ID must be blank.
	if req.ID != "" {
		return types.ErrBadRequest
	}

	if req.VMType == payloads.QEMU {
		err := validateVMWorkload(req)
		if err != nil {
			return err
		}
	} else {
		err := validateContainerWorkload(req)
		if err != nil {
			return err
		}
	}

	if req.ImageID != "" {
		// validate that the image id is at least valid
		// uuid4.
		_, err := uuid.Parse(req.ImageID)
		if err != nil {
			return types.ErrBadRequest
		}
	}

	if req.Config == "" {
		return types.ErrBadRequest
	}

	if len(req.Storage) > 0 {
		err := validateWorkloadStorage(req)
		if err != nil {
			return err
		}
	}

	return nil
}

func (c *controller) CreateWorkload(req types.Workload) (types.Workload, error) {
	err := validateWorkloadRequest(req)
	if err != nil {
		return req, err
	}

	// create a workload storage resource for this new workload.
	if req.ImageID != "" {
		// validate that the image id is at least valid
		// uuid4.
		_, err = uuid.Parse(req.ImageID)
		if err != nil {
			return req, err
		}

		storage := types.StorageResource{
			Bootable:   true,
			Ephemeral:  true,
			SourceType: types.ImageService,
			SourceID:   req.ImageID,
		}

		req.ImageID = ""
		req.Storage = append(req.Storage, storage)
	}

	req.ID = uuid.Generate().String()

	err = c.ds.AddWorkload(req)
	return req, err
}
<commit_msg>ciao-controller: Remove redundant if checks<commit_after>// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"github.com/01org/ciao/ciao-controller/types"
	"github.com/01org/ciao/payloads"
	"github.com/01org/ciao/ssntp/uuid"
)

func validateVMWorkload(req types.Workload) error {
	// FWType must be either EFI or legacy.
	if req.FWType != string(payloads.EFI) && req.FWType != payloads.Legacy {
		return types.ErrBadRequest
	}

	// Must have storage for VMs
	if len(req.Storage) == 0 {
		return types.ErrBadRequest
	}

	return nil
}

func validateContainerWorkload(req types.Workload) error {
	// we should reject anything with ImageID set, but
	// we'll just ignore it.
	if req.ImageName == "" {
		return types.ErrBadRequest
	}

	return nil
}

func validateWorkloadStorage(req types.Workload) error {
	bootableCount := 0
	for i := range req.Storage {
		// check that a workload type is specified
		if req.Storage[i].SourceType == "" {
			return types.ErrBadRequest
		}

		// you may not request a sized volume unless it's empty.
		if req.Storage[i].Size > 0 && req.Storage[i].SourceType != types.Empty {
			return types.ErrBadRequest
		}

		// you may not request a bootable empty volume.
		if req.Storage[i].Bootable && req.Storage[i].SourceType == types.Empty {
			return types.ErrBadRequest
		}

		if req.Storage[i].ID != "" {
			// validate that the id is at least valid
			// uuid4.
			_, err := uuid.Parse(req.Storage[i].ID)
			if err != nil {
				return types.ErrBadRequest
			}

			// If we have an ID we must have a type to get it from
			if req.Storage[i].SourceType != types.Empty {
				return types.ErrBadRequest
			}
		} else {
			// you may only use no source id with empty type
			if req.Storage[i].SourceType != types.Empty {
				return types.ErrBadRequest
			}
		}

		if req.Storage[i].Bootable {
			bootableCount++
		}
	}

	// must be at least one bootable volume
	if req.VMType == payloads.QEMU && bootableCount == 0 {
		return types.ErrBadRequest
	}

	return nil
}

// this is probably an insufficient amount of checking.
func validateWorkloadRequest(req types.Workload) error {
	// ID must be blank.
	if req.ID != "" {
		return types.ErrBadRequest
	}

	if req.VMType == payloads.QEMU {
		err := validateVMWorkload(req)
		if err != nil {
			return err
		}
	} else {
		err := validateContainerWorkload(req)
		if err != nil {
			return err
		}
	}

	if req.ImageID != "" {
		// validate that the image id is at least valid
		// uuid4.
		_, err := uuid.Parse(req.ImageID)
		if err != nil {
			return types.ErrBadRequest
		}
	}

	if req.Config == "" {
		return types.ErrBadRequest
	}

	if len(req.Storage) > 0 {
		err := validateWorkloadStorage(req)
		if err != nil {
			return err
		}
	}

	return nil
}

func (c *controller) CreateWorkload(req types.Workload) (types.Workload, error) {
	err := validateWorkloadRequest(req)
	if err != nil {
		return req, err
	}

	// create a workload storage resource for this new workload.
	if req.ImageID != "" {
		// validate that the image id is at least valid
		// uuid4.
		_, err = uuid.Parse(req.ImageID)
		if err != nil {
			return req, err
		}

		storage := types.StorageResource{
			Bootable:   true,
			Ephemeral:  true,
			SourceType: types.ImageService,
			SourceID:   req.ImageID,
		}

		req.ImageID = ""
		req.Storage = append(req.Storage, storage)
	}

	req.ID = uuid.Generate().String()

	err = c.ds.AddWorkload(req)
	return req, err
}
<|endoftext|>