<commit_before>/*
Copyright 2016 Skippbox, Ltd All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package compose

import (
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"k8s.io/kubernetes/pkg/api"

	"github.com/Sirupsen/logrus"
	"github.com/docker/libcompose/config"
	"github.com/docker/libcompose/docker"
	"github.com/docker/libcompose/lookup"
	"github.com/docker/libcompose/project"
	"github.com/skippbox/kompose/pkg/kobject"
)

type Compose struct {
}

// load Environment Variable from compose file
func loadEnvVarsfromCompose(e map[string]string) []kobject.EnvVar {
	envs := []kobject.EnvVar{}
	for k, v := range e {
		envs = append(envs, kobject.EnvVar{
			Name:  k,
			Value: v,
		})
	}
	return envs
}

// Load Ports from compose file
func loadPortsFromCompose(composePorts []string) ([]kobject.Ports, string) {
	ports := []kobject.Ports{}
	character := ":"
	for _, port := range composePorts {
		p := api.ProtocolTCP
		if strings.Contains(port, character) {
			hostPort := port[0:strings.Index(port, character)]
			hostPort = strings.TrimSpace(hostPort)
			hostPortInt, err := strconv.Atoi(hostPort)
			if err != nil {
				return nil, "Invalid host port of " + port
			}
			containerPort := port[strings.Index(port, character)+1:]
			containerPort = strings.TrimSpace(containerPort)
			containerPortInt, err := strconv.Atoi(containerPort)
			if err != nil {
				return nil, "Invalid container port of " + port
			}
			ports = append(ports, kobject.Ports{
				HostPort:      int32(hostPortInt),
				ContainerPort: int32(containerPortInt),
				Protocol:      p,
			})
		} else {
			containerPortInt, err := strconv.Atoi(port)
			if err != nil {
				return nil, "Invalid container port of " + port
			}
			ports = append(ports, kobject.Ports{
				ContainerPort: int32(containerPortInt),
				Protocol:      p,
			})
		}

	}
	return ports, ""
}

// load Docker Compose file into KomposeObject
func (c *Compose) LoadFile(file string) kobject.KomposeObject {
	komposeObject := kobject.KomposeObject{
		ServiceConfigs: make(map[string]kobject.ServiceConfig),
	}
	context := &docker.Context{}
	if file == "" {
		file = "docker-compose.yml"
	}
	context.ComposeFiles = []string{file}

	if context.ResourceLookup == nil {
		context.ResourceLookup = &lookup.FileResourceLookup{}
	}

	if context.EnvironmentLookup == nil {
		cwd, err := os.Getwd()
		if err != nil {
			return kobject.KomposeObject{}
		}
		context.EnvironmentLookup = &lookup.ComposableEnvLookup{
			Lookups: []config.EnvironmentLookup{
				&lookup.EnvfileLookup{
					Path: filepath.Join(cwd, ".env"),
				},
				&lookup.OsEnvLookup{},
			},
		}
	}

	// load compose file into composeObject
	composeObject := project.NewProject(&context.Context, nil, nil)
	err := composeObject.Parse()
	if err != nil {
		logrus.Fatalf("Failed to load compose file", err)
	}

	// transform composeObject into komposeObject
	composeServiceNames := composeObject.ServiceConfigs.Keys()

	// volume config and network config are not supported
	if len(composeObject.NetworkConfigs) > 0 {
		logrus.Warningf("Unsupported network configuration of compose v2 - ignoring")
	}
	if len(composeObject.VolumeConfigs) > 0 {
		logrus.Warningf("Unsupported volume configuration of compose v2 - ignoring")
	}

	networksWarningFound := false

	for _, name := range composeServiceNames {
		if composeServiceConfig, ok := composeObject.ServiceConfigs.Get(name); ok {
			//FIXME: networks always contains one default element, even it isn't declared in compose v2.
			if composeServiceConfig.Networks != nil && len(composeServiceConfig.Networks.Networks) > 0 &&
				composeServiceConfig.Networks.Networks[0].Name != "default" &&
				!networksWarningFound {
				logrus.Warningf("Unsupported key networks - ignoring")
				networksWarningFound = true
			}
			kobject.CheckUnsupportedKey(composeServiceConfig)
			serviceConfig := kobject.ServiceConfig{}
			serviceConfig.Image = composeServiceConfig.Image
			serviceConfig.ContainerName = composeServiceConfig.ContainerName

			// load environment variables
			envs := loadEnvVarsfromCompose(composeServiceConfig.Environment.ToMap())
			serviceConfig.Environment = envs

			// load ports
			ports, err := loadPortsFromCompose(composeServiceConfig.Ports)
			if err != "" {
				logrus.Fatalf("Failed to load ports from compose file: " + err)
			}
			serviceConfig.Port = ports

			serviceConfig.WorkingDir = composeServiceConfig.WorkingDir
			serviceConfig.Volumes = composeServiceConfig.Volumes

			// convert compose labels to annotations
			serviceConfig.Annotations = map[string]string(composeServiceConfig.Labels)

			serviceConfig.CPUSet = composeServiceConfig.CPUSet
			serviceConfig.CPUShares = composeServiceConfig.CPUShares
			serviceConfig.CPUQuota = composeServiceConfig.CPUQuota
			serviceConfig.CapAdd = composeServiceConfig.CapAdd
			serviceConfig.CapDrop = composeServiceConfig.CapDrop
			serviceConfig.Expose = composeServiceConfig.Expose
			serviceConfig.Privileged = composeServiceConfig.Privileged
			serviceConfig.Restart = composeServiceConfig.Restart
			serviceConfig.User = composeServiceConfig.User

			komposeObject.ServiceConfigs[name] = serviceConfig
		}
	}

	return komposeObject
}
<commit_msg>some items in libcompose ServiceConfig changed type<commit_after>/*
Copyright 2016 Skippbox, Ltd All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package compose

import (
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"k8s.io/kubernetes/pkg/api"

	"github.com/Sirupsen/logrus"
	"github.com/docker/libcompose/config"
	"github.com/docker/libcompose/docker"
	"github.com/docker/libcompose/lookup"
	"github.com/docker/libcompose/project"
	"github.com/skippbox/kompose/pkg/kobject"
)

type Compose struct {
}

// load Environment Variable from compose file
func loadEnvVarsfromCompose(e map[string]string) []kobject.EnvVar {
	envs := []kobject.EnvVar{}
	for k, v := range e {
		envs = append(envs, kobject.EnvVar{
			Name:  k,
			Value: v,
		})
	}
	return envs
}

// Load Ports from compose file
func loadPortsFromCompose(composePorts []string) ([]kobject.Ports, string) {
	ports := []kobject.Ports{}
	character := ":"
	for _, port := range composePorts {
		p := api.ProtocolTCP
		if strings.Contains(port, character) {
			hostPort := port[0:strings.Index(port, character)]
			hostPort = strings.TrimSpace(hostPort)
			hostPortInt, err := strconv.Atoi(hostPort)
			if err != nil {
				return nil, "Invalid host port of " + port
			}
			containerPort := port[strings.Index(port, character)+1:]
			containerPort = strings.TrimSpace(containerPort)
			containerPortInt, err := strconv.Atoi(containerPort)
			if err != nil {
				return nil, "Invalid container port of " + port
			}
			ports = append(ports, kobject.Ports{
				HostPort:      int32(hostPortInt),
				ContainerPort: int32(containerPortInt),
				Protocol:      p,
			})
		} else {
			containerPortInt, err := strconv.Atoi(port)
			if err != nil {
				return nil, "Invalid container port of " + port
			}
			ports = append(ports, kobject.Ports{
				ContainerPort: int32(containerPortInt),
				Protocol:      p,
			})
		}

	}
	return ports, ""
}

// load Docker Compose file into KomposeObject
func (c *Compose) LoadFile(file string) kobject.KomposeObject {
	komposeObject := kobject.KomposeObject{
		ServiceConfigs: make(map[string]kobject.ServiceConfig),
	}
	context := &docker.Context{}
	if file == "" {
		file = "docker-compose.yml"
	}
	context.ComposeFiles = []string{file}

	if context.ResourceLookup == nil {
		context.ResourceLookup = &lookup.FileResourceLookup{}
	}

	if context.EnvironmentLookup == nil {
		cwd, err := os.Getwd()
		if err != nil {
			return kobject.KomposeObject{}
		}
		context.EnvironmentLookup = &lookup.ComposableEnvLookup{
			Lookups: []config.EnvironmentLookup{
				&lookup.EnvfileLookup{
					Path: filepath.Join(cwd, ".env"),
				},
				&lookup.OsEnvLookup{},
			},
		}
	}

	// load compose file into composeObject
	composeObject := project.NewProject(&context.Context, nil, nil)
	err := composeObject.Parse()
	if err != nil {
		logrus.Fatalf("Failed to load compose file", err)
	}

	// transform composeObject into komposeObject
	composeServiceNames := composeObject.ServiceConfigs.Keys()

	// volume config and network config are not supported
	if len(composeObject.NetworkConfigs) > 0 {
		logrus.Warningf("Unsupported network configuration of compose v2 - ignoring")
	}
	if len(composeObject.VolumeConfigs) > 0 {
		logrus.Warningf("Unsupported volume configuration of compose v2 - ignoring")
	}

	networksWarningFound := false

	for _, name := range composeServiceNames {
		if composeServiceConfig, ok := composeObject.ServiceConfigs.Get(name); ok {
			//FIXME: networks always contains one default element, even it isn't declared in compose v2.
			if composeServiceConfig.Networks != nil && len(composeServiceConfig.Networks.Networks) > 0 &&
				composeServiceConfig.Networks.Networks[0].Name != "default" &&
				!networksWarningFound {
				logrus.Warningf("Unsupported key networks - ignoring")
				networksWarningFound = true
			}
			kobject.CheckUnsupportedKey(composeServiceConfig)
			serviceConfig := kobject.ServiceConfig{}
			serviceConfig.Image = composeServiceConfig.Image
			serviceConfig.ContainerName = composeServiceConfig.ContainerName

			// load environment variables
			envs := loadEnvVarsfromCompose(composeServiceConfig.Environment.ToMap())
			serviceConfig.Environment = envs

			// load ports
			ports, err := loadPortsFromCompose(composeServiceConfig.Ports)
			if err != "" {
				logrus.Fatalf("Failed to load ports from compose file: " + err)
			}
			serviceConfig.Port = ports

			serviceConfig.WorkingDir = composeServiceConfig.WorkingDir

			if composeServiceConfig.Volumes != nil {
				for _, volume := range composeServiceConfig.Volumes.Volumes {
					serviceConfig.Volumes = append(serviceConfig.Volumes, volume.String())
				}
			}

			// convert compose labels to annotations
			serviceConfig.Annotations = map[string]string(composeServiceConfig.Labels)

			serviceConfig.CPUSet = composeServiceConfig.CPUSet
			serviceConfig.CPUShares = int64(composeServiceConfig.CPUShares)
			serviceConfig.CPUQuota = int64(composeServiceConfig.CPUQuota)
			serviceConfig.CapAdd = composeServiceConfig.CapAdd
			serviceConfig.CapDrop = composeServiceConfig.CapDrop
			serviceConfig.Expose = composeServiceConfig.Expose
			serviceConfig.Privileged = composeServiceConfig.Privileged
			serviceConfig.Restart = composeServiceConfig.Restart
			serviceConfig.User = composeServiceConfig.User

			komposeObject.ServiceConfigs[name] = serviceConfig
		}
	}

	return komposeObject
}
<|endoftext|>


<commit_before>package manifest

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"
)

func TestGetManifestWithoutInterval(t *testing.T) {
	handler := func(w http.ResponseWriter, r *http.Request) {
		data := `{
			"targets": [
				{
					"url": "http://www.canary.io",
					"name": "canary"
				}
			]
		}`

		fmt.Fprintf(w, data)
	}

	ts := httptest.NewServer(http.HandlerFunc(handler))
	defer ts.Close()

	m, err := GetManifest(ts.URL, 42)
	if err != nil {
		t.Fatal(err)
	}

	if len(m.Targets) != 1 {
		t.Fatalf("%d targets found, but expected 1", len(m.Targets))
	}

	target := m.Targets[0]
	if target.Name != "canary" {
		t.Fatalf("expected name to be equal to 'canary', go %s", target.URL)
	}

	if target.URL != "http://www.canary.io" {
		t.Fatalf("expected URL to be equal to 'http://www.canary.io', got %s", target.URL)
	}

	if target.Interval != 42 {
		t.Fatal("expected Interval to be equal to 42 when undefined in the manifest json, got %d", target.Interval)
	}
}

func TestGetManifestWithInterval(t *testing.T) {
	handler := func(w http.ResponseWriter, r *http.Request) {
		data := `{
			"targets": [
				{
					"url": "http://www.canary.io",
					"name": "canary",
					"interval": 2
				},
				{
					"url": "http://www.github.com",
					"name": "github",
					"interval": 4
				}
			]
		}`

		fmt.Fprintf(w, data)
	}

	ts := httptest.NewServer(http.HandlerFunc(handler))
	defer ts.Close()

	m, err := GetManifest(ts.URL, 42)
	if err != nil {
		t.Fatal(err)
	}

	first_target := m.Targets[0]

	if first_target.Interval != 2 {
		t.Fatalf("expected Interval on the first target to be equal to the manifest json definition of 2, got %d", first_target.Interval)
	}

	second_target := m.Targets[1]

	if second_target.Interval != 4 {
		t.Fatalf("expected Interval on the second target to be equal to the manifest json definition of 4, got %d", second_target.Interval)
	}
}

func TestGetManifestWithTags(t *testing.T) {
	handler := func(w http.ResponseWriter, r *http.Request) {
		data := `{
			"targets": [
				{
					"url": "http://www.canary.io",
					"name": "canary"
				},
				{
					"url": "http://www.github.com",
					"name": "github",
					"tags": [ "tag1", "tag2" ]
				}
			]
		}`

		fmt.Fprintf(w, data)
	}

	ts := httptest.NewServer(http.HandlerFunc(handler))
	defer ts.Close()

	m, err := GetManifest(ts.URL, 42)
	if err != nil {
		t.Fatal(err)
	}

	first_target := m.Targets[0]

	if len(first_target.Tags) != 0 {
		t.Fatalf("expected Tags on the first target to be empty, got %v", first_target.Tags)
	}

	second_target := m.Targets[1]

	if len(second_target.Tags) != 2 {
		t.Fatalf("expected number of Tags on the second target to be equal to the manifest json definition of 2, got %d", len(second_target.Tags))
	} else {
		if second_target.Tags[0] != "tag1" {
			t.Fatalf("expected first element of Tags on the second target to be equal to the manifest json definition of 'tag1', got %s", second_target.Tags[0])
		}

		if second_target.Tags[1] != "tag2" {
			t.Fatalf("expected first element of Tags on the second target to be equal to the manifest json definition of 'tag2', got %s", second_target.Tags[1])
		}
	}
}

func TestGetManifestWithAttributes(t *testing.T) {
	handler := func(w http.ResponseWriter, r *http.Request) {
		data := `{
			"targets": [
				{
					"url": "http://www.canary.io",
					"name": "canary"
				},
				{
					"url": "http://www.github.com",
					"name": "github",
					"attributes": {
						"foo": "bar",
						"baz": "bap"
					}
				}
			]
		}`

		fmt.Fprintf(w, data)
	}

	ts := httptest.NewServer(http.HandlerFunc(handler))
	defer ts.Close()

	m, err := GetManifest(ts.URL, 42)
	if err != nil {
		t.Fatal(err)
	}

	first_target := m.Targets[0]

	if first_target.Attributes != nil {
		t.Fatalf("expected Attributes on the first target to be empty, got %v", first_target.Attributes)
	}

	second_target := m.Targets[1]

	if second_target.Attributes == nil {
		t.Fatalf("expected Attributes on the second target to be equal to the manifest json definition, got %v", second_target.Attributes)
	} else {
		foo := second_target.Attributes["foo"]

		if foo != "bar" {
			t.Fatalf("expected 'foo' element of Attributes on the second target to be equal to the manifest json definition of 'bar', got %s", foo)
		}

		baz := second_target.Attributes["baz"]
		if baz != "bap" {
			t.Fatalf("expected 'baz' element of Attributes on the second target to be equal to the manifest json definition of 'bap', got %s", baz)
		}
	}
}

func TestGetManifestRampup(t *testing.T) {
	handler := func(w http.ResponseWriter, r *http.Request) {
		data := `{
			"targets": [
				{
					"url": "http://www.canary.io",
					"name": "canary"
				},
				{
					"url": "http://www.github.com",
					"name": "github"
				},
				{
					"url": "http://www.google.com",
					"name": "google"
				},
				{
					"url": "http://www.youtube.com",
					"name": "youtube"
				}
			]
		}`

		fmt.Fprintf(w, data)
	}

	ts := httptest.NewServer(http.HandlerFunc(handler))
	defer ts.Close()

	m, err := GetManifest(ts.URL, 42)
	if err != nil {
		t.Fatal(err)
	}

	// m.StartDelays will the same length as m.Targets
	if len(m.Targets) != len(m.StartDelays) {
		t.Fatalf("expected length of m.StartDelays (%d) to match length of m.Targets (%d)", len(m.Targets), len(m.StartDelays))
	}

	// without calling GenerateRampupDelays, StartDelays should all be zero.
	for index, value := range m.StartDelays {
		if value != 0.0 {
			t.Fatalf("Expected initial start delay to be 0.0, got %d for index %d", value, index)
		}
	}

	// Calling GenerateRampupDelays will update m.StartDelays to an even distribution of
	// millisecond delays for the passed Sampling interval (in seconds).
	m.GenerateRampupDelays(10)

	if m.StartDelays[0] != 0.0 {
		t.Fatalf("The first start delay should be 0.0 even after generation, got %d", m.StartDelays[0])
	}

	if m.StartDelays[1] != 2500.0 {
		t.Fatalf("The second start delay should be 2500.0 ms after generation, got %d", m.StartDelays[1])
	}

	if m.StartDelays[2] != 5000.0 {
		t.Fatalf("The second start delay should be 5000.0 ms after generation, got %d", m.StartDelays[2])
	}

	if m.StartDelays[3] != 7500.0 {
		t.Fatalf("The second start delay should be 7500.0 ms after generation, got %d", m.StartDelays[3])
	}
}
<commit_msg>Add test for json deserialization of CaptureHeader<commit_after>package manifest

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"
)

func TestGetManifestWithoutInterval(t *testing.T) {
	handler := func(w http.ResponseWriter, r *http.Request) {
		data := `{
			"targets": [
				{
					"url": "http://www.canary.io",
					"name": "canary"
				}
			]
		}`

		fmt.Fprintf(w, data)
	}

	ts := httptest.NewServer(http.HandlerFunc(handler))
	defer ts.Close()

	m, err := GetManifest(ts.URL, 42)
	if err != nil {
		t.Fatal(err)
	}

	if len(m.Targets) != 1 {
		t.Fatalf("%d targets found, but expected 1", len(m.Targets))
	}

	target := m.Targets[0]
	if target.Name != "canary" {
		t.Fatalf("expected name to be equal to 'canary', go %s", target.URL)
	}

	if target.URL != "http://www.canary.io" {
		t.Fatalf("expected URL to be equal to 'http://www.canary.io', got %s", target.URL)
	}

	if target.Interval != 42 {
		t.Fatal("expected Interval to be equal to 42 when undefined in the manifest json, got %d", target.Interval)
	}
}

func TestGetManifestWithInterval(t *testing.T) {
	handler := func(w http.ResponseWriter, r *http.Request) {
		data := `{
			"targets": [
				{
					"url": "http://www.canary.io",
					"name": "canary",
					"interval": 2
				},
				{
					"url": "http://www.github.com",
					"name": "github",
					"interval": 4
				}
			]
		}`

		fmt.Fprintf(w, data)
	}

	ts := httptest.NewServer(http.HandlerFunc(handler))
	defer ts.Close()

	m, err := GetManifest(ts.URL, 42)
	if err != nil {
		t.Fatal(err)
	}

	first_target := m.Targets[0]

	if first_target.Interval != 2 {
		t.Fatalf("expected Interval on the first target to be equal to the manifest json definition of 2, got %d", first_target.Interval)
	}

	second_target := m.Targets[1]

	if second_target.Interval != 4 {
		t.Fatalf("expected Interval on the second target to be equal to the manifest json definition of 4, got %d", second_target.Interval)
	}
}

func TestGetManifestWithTags(t *testing.T) {
	handler := func(w http.ResponseWriter, r *http.Request) {
		data := `{
			"targets": [
				{
					"url": "http://www.canary.io",
					"name": "canary"
				},
				{
					"url": "http://www.github.com",
					"name": "github",
					"tags": [ "tag1", "tag2" ]
				}
			]
		}`

		fmt.Fprintf(w, data)
	}

	ts := httptest.NewServer(http.HandlerFunc(handler))
	defer ts.Close()

	m, err := GetManifest(ts.URL, 42)
	if err != nil {
		t.Fatal(err)
	}

	first_target := m.Targets[0]

	if len(first_target.Tags) != 0 {
		t.Fatalf("expected Tags on the first target to be empty, got %v", first_target.Tags)
	}

	second_target := m.Targets[1]

	if len(second_target.Tags) != 2 {
		t.Fatalf("expected number of Tags on the second target to be equal to the manifest json definition of 2, got %d", len(second_target.Tags))
	} else {
		if second_target.Tags[0] != "tag1" {
			t.Fatalf("expected first element of Tags on the second target to be equal to the manifest json definition of 'tag1', got %s", second_target.Tags[0])
		}

		if second_target.Tags[1] != "tag2" {
			t.Fatalf("expected first element of Tags on the second target to be equal to the manifest json definition of 'tag2', got %s", second_target.Tags[1])
		}
	}
}

func TestGetManifestWithAttributes(t *testing.T) {
	handler := func(w http.ResponseWriter, r *http.Request) {
		data := `{
			"targets": [
				{
					"url": "http://www.canary.io",
					"name": "canary"
				},
				{
					"url": "http://www.github.com",
					"name": "github",
					"attributes": {
						"foo": "bar",
						"baz": "bap"
					}
				}
			]
		}`

		fmt.Fprintf(w, data)
	}

	ts := httptest.NewServer(http.HandlerFunc(handler))
	defer ts.Close()

	m, err := GetManifest(ts.URL, 42)
	if err != nil {
		t.Fatal(err)
	}

	first_target := m.Targets[0]

	if first_target.Attributes != nil {
		t.Fatalf("expected Attributes on the first target to be empty, got %v", first_target.Attributes)
	}

	second_target := m.Targets[1]

	if second_target.Attributes == nil {
		t.Fatalf("expected Attributes on the second target to be equal to the manifest json definition, got %v", second_target.Attributes)
	} else {
		foo := second_target.Attributes["foo"]

		if foo != "bar" {
			t.Fatalf("expected 'foo' element of Attributes on the second target to be equal to the manifest json definition of 'bar', got %s", foo)
		}

		baz := second_target.Attributes["baz"]
		if baz != "bap" {
			t.Fatalf("expected 'baz' element of Attributes on the second target to be equal to the manifest json definition of 'bap', got %s", baz)
		}
	}
}

func TestGetManifestRampup(t *testing.T) {
	handler := func(w http.ResponseWriter, r *http.Request) {
		data := `{
			"targets": [
				{
					"url": "http://www.canary.io",
					"name": "canary"
				},
				{
					"url": "http://www.github.com",
					"name": "github"
				},
				{
					"url": "http://www.google.com",
					"name": "google"
				},
				{
					"url": "http://www.youtube.com",
					"name": "youtube"
				}
			]
		}`

		fmt.Fprintf(w, data)
	}

	ts := httptest.NewServer(http.HandlerFunc(handler))
	defer ts.Close()

	m, err := GetManifest(ts.URL, 42)
	if err != nil {
		t.Fatal(err)
	}

	// m.StartDelays will the same length as m.Targets
	if len(m.Targets) != len(m.StartDelays) {
		t.Fatalf("expected length of m.StartDelays (%d) to match length of m.Targets (%d)", len(m.Targets), len(m.StartDelays))
	}

	// without calling GenerateRampupDelays, StartDelays should all be zero.
	for index, value := range m.StartDelays {
		if value != 0.0 {
			t.Fatalf("Expected initial start delay to be 0.0, got %d for index %d", value, index)
		}
	}

	// Calling GenerateRampupDelays will update m.StartDelays to an even distribution of
	// millisecond delays for the passed Sampling interval (in seconds).
	m.GenerateRampupDelays(10)

	if m.StartDelays[0] != 0.0 {
		t.Fatalf("The first start delay should be 0.0 even after generation, got %d", m.StartDelays[0])
	}

	if m.StartDelays[1] != 2500.0 {
		t.Fatalf("The second start delay should be 2500.0 ms after generation, got %d", m.StartDelays[1])
	}

	if m.StartDelays[2] != 5000.0 {
		t.Fatalf("The second start delay should be 5000.0 ms after generation, got %d", m.StartDelays[2])
	}

	if m.StartDelays[3] != 7500.0 {
		t.Fatalf("The second start delay should be 7500.0 ms after generation, got %d", m.StartDelays[3])
	}
}

func TestGetManifestWithCapturedHeaders(t *testing.T) {
	handler := func(w http.ResponseWriter, r *http.Request) {
		data := `{
			"targets": [
				{
					"url": "http://www.canary.io",
					"name": "canary"
				},
				{
					"url": "http://www.github.com",
					"name": "github",
					"captureHeaders": [ "Server" ]
				}
			]
		}`

		fmt.Fprintf(w, data)
	}

	ts := httptest.NewServer(http.HandlerFunc(handler))
	defer ts.Close()

	m, err := GetManifest(ts.URL, 42)
	if err != nil {
		t.Fatal(err)
	}

	first_target := m.Targets[0]

	if len(first_target.CaptureHeaders) != 0 {
		t.Fatalf("expected CaptureHeaders on the first target to be empty, got %v", first_target.CaptureHeaders)
	}

	second_target := m.Targets[1]

	if len(second_target.CaptureHeaders) != 1 {
		t.Fatalf("expected Attributes on the second target to be equal to the manifest json definition, got %v", second_target.CaptureHeaders)
	} else {
		h := second_target.CaptureHeaders[0]
		if h != "Server" {
			t.Fatalf("expected first element of CaptureHeaders on the second target to be equal to the manifest json definition of 'Server', got %s", h)
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1alpha1

import (
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/intstr"
)

type NodeClassResourceType string

const (
	NodeClassResourceFile      NodeClassResourceType = "File"
	NodeClassResourceReference NodeClassResourceType = "Reference"
)

type NodeClassResource struct {
	// Type is the type of the resource
	Type NodeClassResourceType `json:"type,omitempty"`

	// Name is the name of the resource
	// It can be used as a key for overriding
	Name string `json:"name,omitempty"`

	// For File type
	// Path is the full path of the file including filename
	// +optional
	Path string `json:"path,omitempty"`
	// Owner is the owner of the file
	// +optional
	Owner string `json:"owner,omitempty"`
	// Permission is the permission of the file
	// +optional
	Permission string `json:"permission,omitempty"`
	// Template is the template or content of the file
	// +optional
	Template string `json:"template,omitempty"`

	// For Reference type
	// Reference points to an Object in registry that the controller will use when
	// provisioning the node
	// Ex. a secret ref to be used as ssh credentials.
	// +optional
	Reference *corev1.ObjectReference `json:"reference,omitempty"`
}

// +genclient
// +genclient:nonNamespaced
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// NodeClass describes the parameters for a class of nodes that can be
// provisioned by the specific controller
type NodeClass struct {
	metav1.TypeMeta `json:",inline"`

	// Standard object's metadata.
	// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata
	// +optional
	metav1.ObjectMeta `json:"metadata,omitempty"`

	// NodeController specifes which node controller should provision and
	// control this class of nodes
	NodeController string `json:"nodeController,omitempty"`

	// Config is free form data containing overridable parameters that the controller
	// will use
	// +optional
	Config runtime.RawExtension `json:"config,omitempty"`

	// Resources is a list of file-like resources that the controller can use
	// +optional
	Resources []NodeClassResource `json:"resources,omitempty"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// NodeClassList is a collection of node classes.
type NodeClassList struct {
	metav1.TypeMeta `json:",inline"`
	// Standard list metadata
	// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata
	// +optional
	metav1.ListMeta `json:"metadata,omitempty"`

	// Items is the list of NodeClasses
	Items []NodeClass `json:"items"`
}

// NodeSetSpec is the spec of a node set.
type NodeSetSpec struct {
	// NodeSelector is a selector of node labels which a node must have to be in the NodeSet.
	// More info: http://kubernetes.io/docs/user-guide/node-selection/README
	// +optional
	NodeSelector map[string]string `json:"nodeSelector,omitempty"`

	// NodeClass specifies which NodeClass the nodes created by this NodeSet should
	// belong to
	NodeClass string `json:"nodeClass,omitempty"`

	// NodeSetController specifies a NodeSet controller to be in charge of this NodeSet
	// If not specified, the default one that comes with the NodeController in NodeClass
	// will be used.
	// +optional
	NodeSetController string `json:"nodeSetController,omitempty"`

	// Replicas is the number of desired replicas.
	Replicas int32 `json:"replicas,omitempty"`

	// The maximum number of Nodes that can be unavailable during update
	// This can be an absolute number (ex. 3) or a percentage of total replicas
	// The absolute number will be calculated from percentage by rounding down.
	// This can't be 0 if MaxSurge is 0.
	// By default, a value of 0 is used
	// +optional
	MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"`

	// The maximum number of Nodes that can be created above the desired replicas.
	// This can be an absolute number (ex. 3) or a percentage of total replicas
	// The absolute number will be calculated from percentage by rounding up.
	// This can't be 0 if MaxUnavailble is 0.
	// A default of 1 is used if not set.
	// +optional
	MaxSurge *intstr.IntOrString `json:"maxSurge,omitempty"`

	// Config contains free form structural data that overrides those in NodeClass.
	// +optional
	Config runtime.RawExtension `json:"config,omitempty"`
}

type NodeSetConditionType string

const (
	// This is added when the NodeSet controllers failed to create or delete a node during
	// reconciliation.
	NodeSetReplicaFailure NodeSetConditionType = "ReplicaFailure"
)

// NodeSetCondition describes the state of a NodeSet at a certain point.
type NodeSetCondition struct {
	// Type of NodeSet condition.
	Type NodeSetConditionType `json:"type"`

	// Status of the condition, one of True, False, Unknown.
	Status corev1.ConditionStatus `json:"status"`

	// The last time the condition transitioned from one status to another.
	// +optional
	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"`

	// The reason for the condition's last transition.
	// +optional
	Reason string `json:"reason,omitempty"`

	// A human readable message indicating details about the transition.
	// +optional
	Message string `json:"message,omitempty"`
}

// NodeSetStatus is the status of a nodeset
type NodeSetStatus struct {
	// Replicas is the number of actual replicas.
	Replicas int32 `json:"replicas"`

	// The number of replicas with running status for this NodeSet.
	// +optional
	RunningReplicas int32 `json:"runningReplicas,omitempty"`

	// ObservedGeneration is the most recent generation observed by the controller.
	// +optional
	ObservedGeneration int64 `json:"observedGeneration,omitempty"`

	// Represents the latest available observations of the NodeSet's current state.
	// +optional
	Conditions []NodeSetCondition `json:"conditions,omitempty"`
}

// +genclient
// +genclient:nonNamespaced
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// NodeSet is a set of nodes of the same class.
type NodeSet struct {
	metav1.TypeMeta `json:",inline"`

	// Standard object's metadata.
	// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata
	// +optional
	metav1.ObjectMeta `json:"metadata,omitempty"`

	// Specification of desired behaviour of the NodeSet.
	// +optional
	Spec NodeSetSpec `json:"spec,omitempty"`

	// Most recently observed status of the NodeSet.
	// This data may not be up to date.
	// Populated by the system.
	// Read-only.
	// +optional
	Status NodeSetStatus `json:"status,omitempty"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// NodeSetList is a collection of nodesets.
type NodeSetList struct {
	metav1.TypeMeta `json:",inline"`
	// Standard list metadata
	// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata
	// +optional
	metav1.ListMeta `json:"metadata,omitempty"`

	// Items is the list of NodeSets
	Items []NodeSet `json:"items"`
}
<commit_msg>Add plurals to types<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1alpha1

import (
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/intstr"
)

type NodeClassResourceType string

const (
	NodeClassResourceFile      NodeClassResourceType = "File"
	NodeClassResourceReference NodeClassResourceType = "Reference"

	NodeClassResourcePlural = "nodeclasses"
	NodeSetResourcePlural   = "nodesets"
)

type NodeClassResource struct {
	// Type is the type of the resource
	Type NodeClassResourceType `json:"type,omitempty"`

	// Name is the name of the resource
	// It can be used as a key for overriding
	Name string `json:"name,omitempty"`

	// For File type
	// Path is the full path of the file including filename
	// +optional
	Path string `json:"path,omitempty"`
	// Owner is the owner of the file
	// +optional
	Owner string `json:"owner,omitempty"`
	// Permission is the permission of the file
	// +optional
	Permission string `json:"permission,omitempty"`
	// Template is the template or content of the file
	// +optional
	Template string `json:"template,omitempty"`

	// For Reference type
	// Reference points to an Object in registry that the controller will use when
	// provisioning the node
	// Ex. a secret ref to be used as ssh credentials.
	// +optional
	Reference *corev1.ObjectReference `json:"reference,omitempty"`
}

// +genclient
// +genclient:nonNamespaced
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// NodeClass describes the parameters for a class of nodes that can be
// provisioned by the specific controller
type NodeClass struct {
	metav1.TypeMeta `json:",inline"`

	// Standard object's metadata.
	// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata
	// +optional
	metav1.ObjectMeta `json:"metadata,omitempty"`

	// NodeController specifes which node controller should provision and
	// control this class of nodes
	NodeController string `json:"nodeController,omitempty"`

	// Config is free form data containing overridable parameters that the controller
	// will use
	// +optional
	Config runtime.RawExtension `json:"config,omitempty"`

	// Resources is a list of file-like resources that the controller can use
	// +optional
	Resources []NodeClassResource `json:"resources,omitempty"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// NodeClassList is a collection of node classes.
type NodeClassList struct {
	metav1.TypeMeta `json:",inline"`
	// Standard list metadata
	// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata
	// +optional
	metav1.ListMeta `json:"metadata,omitempty"`

	// Items is the list of NodeClasses
	Items []NodeClass `json:"items"`
}

// NodeSetSpec is the spec of a node set.
type NodeSetSpec struct {
	// NodeSelector is a selector of node labels which a node must have to be in the NodeSet.
	// More info: http://kubernetes.io/docs/user-guide/node-selection/README
	// +optional
	NodeSelector map[string]string `json:"nodeSelector,omitempty"`

	// NodeClass specifies which NodeClass the nodes created by this NodeSet should
	// belong to
	NodeClass string `json:"nodeClass,omitempty"`

	// NodeSetController specifies a NodeSet controller to be in charge of this NodeSet
	// If not specified, the default one that comes with the NodeController in NodeClass
	// will be used.
	// +optional
	NodeSetController string `json:"nodeSetController,omitempty"`

	// Replicas is the number of desired replicas.
	Replicas int32 `json:"replicas,omitempty"`

	// The maximum number of Nodes that can be unavailable during update
	// This can be an absolute number (ex. 3) or a percentage of total replicas
	// The absolute number will be calculated from percentage by rounding down.
	// This can't be 0 if MaxSurge is 0.
	// By default, a value of 0 is used
	// +optional
	MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"`

	// The maximum number of Nodes that can be created above the desired replicas.
	// This can be an absolute number (ex. 3) or a percentage of total replicas
	// The absolute number will be calculated from percentage by rounding up.
	// This can't be 0 if MaxUnavailble is 0.
	// A default of 1 is used if not set.
	// +optional
	MaxSurge *intstr.IntOrString `json:"maxSurge,omitempty"`

	// Config contains free form structural data that overrides those in NodeClass.
	// +optional
	Config runtime.RawExtension `json:"config,omitempty"`
}

type NodeSetConditionType string

const (
	// This is added when the NodeSet controllers failed to create or delete a node during
	// reconciliation.
	NodeSetReplicaFailure NodeSetConditionType = "ReplicaFailure"
)

// NodeSetCondition describes the state of a NodeSet at a certain point.
type NodeSetCondition struct {
	// Type of NodeSet condition.
	Type NodeSetConditionType `json:"type"`

	// Status of the condition, one of True, False, Unknown.
	Status corev1.ConditionStatus `json:"status"`

	// The last time the condition transitioned from one status to another.
	// +optional
	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"`

	// The reason for the condition's last transition.
	// +optional
	Reason string `json:"reason,omitempty"`

	// A human readable message indicating details about the transition.
	// +optional
	Message string `json:"message,omitempty"`
}

// NodeSetStatus is the status of a nodeset
type NodeSetStatus struct {
	// Replicas is the number of actual replicas.
	Replicas int32 `json:"replicas"`

	// The number of replicas with running status for this NodeSet.
	// +optional
	RunningReplicas int32 `json:"runningReplicas,omitempty"`

	// ObservedGeneration is the most recent generation observed by the controller.
	// +optional
	ObservedGeneration int64 `json:"observedGeneration,omitempty"`

	// Represents the latest available observations of the NodeSet's current state.
	// +optional
	Conditions []NodeSetCondition `json:"conditions,omitempty"`
}

// +genclient
// +genclient:nonNamespaced
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// NodeSet is a set of nodes of the same class.
type NodeSet struct {
	metav1.TypeMeta `json:",inline"`

	// Standard object's metadata.
	// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata
	// +optional
	metav1.ObjectMeta `json:"metadata,omitempty"`

	// Specification of desired behaviour of the NodeSet.
	// +optional
	Spec NodeSetSpec `json:"spec,omitempty"`

	// Most recently observed status of the NodeSet.
	// This data may not be up to date.
	// Populated by the system.
	// Read-only.
	// +optional
	Status NodeSetStatus `json:"status,omitempty"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// NodeSetList is a collection of nodesets.
type NodeSetList struct {
	metav1.TypeMeta `json:",inline"`
	// Standard list metadata
	// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata
	// +optional
	metav1.ListMeta `json:"metadata,omitempty"`

	// Items is the list of NodeSets
	Items []NodeSet `json:"items"`
}
<|endoftext|>


<commit_before>// Copyright 2020 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fuse

import (
	"syscall"

	"gvisor.dev/gvisor/pkg/abi/linux"
	"gvisor.dev/gvisor/pkg/context"
	"gvisor.dev/gvisor/pkg/log"
	"gvisor.dev/gvisor/pkg/sentry/kernel"
	"gvisor.dev/gvisor/pkg/sentry/kernel/auth"
	"gvisor.dev/gvisor/pkg/sentry/vfs"
	"gvisor.dev/gvisor/pkg/sync"
	"gvisor.dev/gvisor/pkg/syserror"
	"gvisor.dev/gvisor/pkg/usermem"
	"gvisor.dev/gvisor/pkg/waiter"
)

const fuseDevMinor = 229

// fuseDevice implements vfs.Device for /dev/fuse.
type fuseDevice struct{}

// Open implements vfs.Device.Open.
func (fuseDevice) Open(ctx context.Context, mnt *vfs.Mount, vfsd *vfs.Dentry, opts vfs.OpenOptions) (*vfs.FileDescription, error) {
	if !kernel.FUSEEnabled {
		return nil, syserror.ENOENT
	}

	var fd DeviceFD
	if err := fd.vfsfd.Init(&fd, opts.Flags, mnt, vfsd, &vfs.FileDescriptionOptions{
		UseDentryMetadata: true,
	}); err != nil {
		return nil, err
	}
	return &fd.vfsfd, nil
}

// DeviceFD implements vfs.FileDescriptionImpl for /dev/fuse.
type DeviceFD struct {
	vfsfd vfs.FileDescription
	vfs.FileDescriptionDefaultImpl
	vfs.DentryMetadataFileDescriptionImpl
	vfs.NoLockFD

	// mounted specifies whether a FUSE filesystem was mounted using the DeviceFD.
	mounted bool

	// nextOpID is used to create new requests.
	nextOpID linux.FUSEOpID

	// queue is the list of requests that need to be processed by the FUSE server.
	queue requestList

	// numActiveRequests is the number of requests made by the Sentry that has
	// yet to be responded to.
	numActiveRequests uint64

	// completions is used to map a request to its response. A Writer will use this
	// to notify the caller of a completed response.
	completions map[linux.FUSEOpID]*futureResponse

	writeCursor uint32

	// writeBuf is the memory buffer used to copy in the FUSE out header from
	// userspace.
	writeBuf []byte

	// writeCursorFR current FR being copied from server.
	writeCursorFR *futureResponse

	// mu protects all the queues, maps, buffers and cursors and nextOpID.
	mu sync.Mutex

	// waitQueue is used to notify interested parties when the device becomes
	// readable or writable.
	waitQueue waiter.Queue

	// fullQueueCh is a channel used to synchronize the readers with the writers.
	// Writers (inbound requests to the filesystem) block if there are too many
	// unprocessed in-flight requests.
	fullQueueCh chan struct{}

	// fs is the FUSE filesystem that this FD is being used for.
	fs *filesystem
}

// Release implements vfs.FileDescriptionImpl.Release.
func (fd *DeviceFD) Release(context.Context) {
	fd.fs.conn.connected = false
}

// PRead implements vfs.FileDescriptionImpl.PRead.
func (fd *DeviceFD) PRead(ctx context.Context, dst usermem.IOSequence, offset int64, opts vfs.ReadOptions) (int64, error) {
	// Operations on /dev/fuse don't make sense until a FUSE filesystem is mounted.
	if !fd.mounted {
		return 0, syserror.EPERM
	}

	return 0, syserror.ENOSYS
}

// Read implements vfs.FileDescriptionImpl.Read.
func (fd *DeviceFD) Read(ctx context.Context, dst usermem.IOSequence, opts vfs.ReadOptions) (int64, error) {
	// Operations on /dev/fuse don't make sense until a FUSE filesystem is mounted.
	if !fd.mounted {
		return 0, syserror.EPERM
	}

	// We require that any Read done on this filesystem have a sane minimum
	// read buffer. It must have the capacity for the fixed parts of any request
	// header (Linux uses the request header and the FUSEWriteIn header for this
	// calculation) + the negotiated MaxWrite room for the data.
	minBuffSize := linux.FUSE_MIN_READ_BUFFER
	inHdrLen := uint32((*linux.FUSEHeaderIn)(nil).SizeBytes())
	writeHdrLen := uint32((*linux.FUSEWriteIn)(nil).SizeBytes())
	negotiatedMinBuffSize := inHdrLen + writeHdrLen + fd.fs.conn.maxWrite
	if minBuffSize < negotiatedMinBuffSize {
		minBuffSize = negotiatedMinBuffSize
	}

	// If the read buffer is too small, error out.
	if dst.NumBytes() < int64(minBuffSize) {
		return 0, syserror.EINVAL
	}

	fd.mu.Lock()
	defer fd.mu.Unlock()
	return fd.readLocked(ctx, dst, opts)
}

// readLocked implements the reading of the fuse device while locked with DeviceFD.mu.
func (fd *DeviceFD) readLocked(ctx context.Context, dst usermem.IOSequence, opts vfs.ReadOptions) (int64, error) {
	if fd.queue.Empty() {
		return 0, syserror.ErrWouldBlock
	}

	var readCursor uint32
	var bytesRead int64
	for {
		req := fd.queue.Front()
		if dst.NumBytes() < int64(req.hdr.Len) {
			// The request is too large. Cannot process it. All requests must be smaller than the
			// negotiated size as specified by Connection.MaxWrite set as part of the FUSE_INIT
			// handshake.
			errno := -int32(syscall.EIO)
			if req.hdr.Opcode == linux.FUSE_SETXATTR {
				errno = -int32(syscall.E2BIG)
			}

			// Return the error to the calling task.
			if err := fd.sendError(ctx, errno, req); err != nil {
				return 0, err
			}

			// We're done with this request.
			fd.queue.Remove(req)
			if req.hdr.Opcode == linux.FUSE_RELEASE {
				fd.numActiveRequests -= 1
			}

			// Restart the read as this request was invalid.
			log.Warningf("fuse.DeviceFD.Read: request found was too large. Restarting read.")
			return fd.readLocked(ctx, dst, opts)
		}

		n, err := dst.CopyOut(ctx, req.data[readCursor:])
		if err != nil {
			return 0, err
		}
		readCursor += uint32(n)
		bytesRead += int64(n)

		if readCursor >= req.hdr.Len {
			// Fully done with this req, remove it from the queue.
			fd.queue.Remove(req)
			if req.hdr.Opcode == linux.FUSE_RELEASE {
				fd.numActiveRequests -= 1
			}
			break
		}
	}

	return bytesRead, nil
}

// PWrite implements vfs.FileDescriptionImpl.PWrite.
func (fd *DeviceFD) PWrite(ctx context.Context, src usermem.IOSequence, offset int64, opts vfs.WriteOptions) (int64, error) {
	// Operations on /dev/fuse don't make sense until a FUSE filesystem is mounted.
	if !fd.mounted {
		return 0, syserror.EPERM
	}

	return 0, syserror.ENOSYS
}

// Write implements vfs.FileDescriptionImpl.Write.
func (fd *DeviceFD) Write(ctx context.Context, src usermem.IOSequence, opts vfs.WriteOptions) (int64, error) {
	fd.mu.Lock()
	defer fd.mu.Unlock()
	return fd.writeLocked(ctx, src, opts)
}

// writeLocked implements writing to the fuse device while locked with DeviceFD.mu.
func (fd *DeviceFD) writeLocked(ctx context.Context, src usermem.IOSequence, opts vfs.WriteOptions) (int64, error) {
	// Operations on /dev/fuse don't make sense until a FUSE filesystem is mounted.
	if !fd.mounted {
		return 0, syserror.EPERM
	}

	var cn, n int64
	hdrLen := uint32((*linux.FUSEHeaderOut)(nil).SizeBytes())

	for src.NumBytes() > 0 {
		if fd.writeCursorFR != nil {
			// Already have common header, and we're now copying the payload.
			wantBytes := fd.writeCursorFR.hdr.Len

			// Note that the FR data doesn't have the header. Copy it over if its necessary.
			if fd.writeCursorFR.data == nil {
				fd.writeCursorFR.data = make([]byte, wantBytes)
			}

			bytesCopied, err := src.CopyIn(ctx, fd.writeCursorFR.data[fd.writeCursor:wantBytes])
			if err != nil {
				return 0, err
			}
			src = src.DropFirst(bytesCopied)

			cn = int64(bytesCopied)
			n += cn
			fd.writeCursor += uint32(cn)
			if fd.writeCursor == wantBytes {
				// Done reading this full response. Clean up and unblock the
				// initiator.
				break
			}

			// Check if we have more data in src.
			continue
		}

		// Assert that the header isn't read into the writeBuf yet.
		if fd.writeCursor >= hdrLen {
			return 0, syserror.EINVAL
		}

		// We don't have the full common response header yet.
		wantBytes := hdrLen - fd.writeCursor
		bytesCopied, err := src.CopyIn(ctx, fd.writeBuf[fd.writeCursor:wantBytes])
		if err != nil {
			return 0, err
		}
		src = src.DropFirst(bytesCopied)

		cn = int64(bytesCopied)
		n += cn
		fd.writeCursor += uint32(cn)
		if fd.writeCursor == hdrLen {
			// Have full header in the writeBuf. Use it to fetch the actual futureResponse
			// from the device's completions map.
			var hdr linux.FUSEHeaderOut
			hdr.UnmarshalBytes(fd.writeBuf)

			// We have the header now and so the writeBuf has served its purpose.
			// We could reset it manually here but instead of doing that, at the
			// end of the write, the writeCursor will be set to 0 thereby allowing
			// the next request to overwrite whats in the buffer,

			fut, ok := fd.completions[hdr.Unique]
			if !ok {
				// Server sent us a response for a request we never sent?
				return 0, syserror.EINVAL
			}

			delete(fd.completions, hdr.Unique)

			// Copy over the header into the future response. The rest of the payload
			// will be copied over to the FR's data in the next iteration.
			fut.hdr = &hdr
			fd.writeCursorFR = fut

			// Next iteration will now try read the complete request, if src has
			// any data remaining. Otherwise we're done.
		}
	}

	if fd.writeCursorFR != nil {
		if err := fd.sendResponse(ctx, fd.writeCursorFR); err != nil {
			return 0, err
		}

		// Ready the device for the next request.
		fd.writeCursorFR = nil
		fd.writeCursor = 0
	}

	return n, nil
}

// Readiness implements vfs.FileDescriptionImpl.Readiness.
func (fd *DeviceFD) Readiness(mask waiter.EventMask) waiter.EventMask {
	fd.mu.Lock()
	defer fd.mu.Unlock()
	return fd.readinessLocked(mask)
}

// readinessLocked implements checking the readiness of the fuse device while
// locked with DeviceFD.mu.
func (fd *DeviceFD) readinessLocked(mask waiter.EventMask) waiter.EventMask {
	var ready waiter.EventMask
	ready |= waiter.EventOut // FD is always writable
	if !fd.queue.Empty() {
		// Have reqs available, FD is readable.
		ready |= waiter.EventIn
	}

	return ready & mask
}

// EventRegister implements waiter.Waitable.EventRegister.
func (fd *DeviceFD) EventRegister(e *waiter.Entry, mask waiter.EventMask) {
	fd.waitQueue.EventRegister(e, mask)
}

// EventUnregister implements waiter.Waitable.EventUnregister.
func (fd *DeviceFD) EventUnregister(e *waiter.Entry) {
	fd.waitQueue.EventUnregister(e)
}

// Seek implements vfs.FileDescriptionImpl.Seek.
func (fd *DeviceFD) Seek(ctx context.Context, offset int64, whence int32) (int64, error) {
	// Operations on /dev/fuse don't make sense until a FUSE filesystem is mounted.
	if !fd.mounted {
		return 0, syserror.EPERM
	}

	return 0, syserror.ENOSYS
}

// sendResponse sends a response to the waiting task (if any).
func (fd *DeviceFD) sendResponse(ctx context.Context, fut *futureResponse) error {
	// See if the running task need to perform some action before returning.
	// Since we just finished writing the future, we can be sure that
	// getResponse generates a populated response.
	if err := fd.noReceiverAction(ctx, fut.getResponse()); err != nil {
		return err
	}

	// Signal that the queue is no longer full.
	select {
	case fd.fullQueueCh <- struct{}{}:
	default:
	}
	fd.numActiveRequests -= 1

	// Signal the task waiting on a response.
	close(fut.ch)
	return nil
}

// sendError sends an error response to the waiting task (if any).
func (fd *DeviceFD) sendError(ctx context.Context, errno int32, req *Request) error {
	// Return the error to the calling task.
	outHdrLen := uint32((*linux.FUSEHeaderOut)(nil).SizeBytes())
	respHdr := linux.FUSEHeaderOut{
		Len:    outHdrLen,
		Error:  errno,
		Unique: req.hdr.Unique,
	}

	fut, ok := fd.completions[respHdr.Unique]
	if !ok {
		// Server sent us a response for a request we never sent?
		return syserror.EINVAL
	}
	delete(fd.completions, respHdr.Unique)

	fut.hdr = &respHdr
	if err := fd.sendResponse(ctx, fut); err != nil {
		return err
	}

	return nil
}

// noReceiverAction has the calling kernel.Task do some action if its known that no
// receiver is going to be waiting on the future channel. This is to be used by:
// FUSE_INIT.
func (fd *DeviceFD) noReceiverAction(ctx context.Context, r *Response) error {
	switch r.opcode {
	case linux.FUSE_INIT:
		creds := auth.CredentialsFromContext(ctx)
		rootUserNs := kernel.KernelFromContext(ctx).RootUserNamespace()
		return fd.fs.conn.InitRecv(r, creds.HasCapabilityIn(linux.CAP_SYS_ADMIN, rootUserNs))
		// TODO(gvisor.dev/issue/3247): support async read: correctly process the response using information from r.options.
	}

	return nil
}
<commit_msg>FUSE device: clean up readLocked<commit_after>// Copyright 2020 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fuse

import (
	"syscall"

	"gvisor.dev/gvisor/pkg/abi/linux"
	"gvisor.dev/gvisor/pkg/context"
	"gvisor.dev/gvisor/pkg/sentry/kernel"
	"gvisor.dev/gvisor/pkg/sentry/kernel/auth"
	"gvisor.dev/gvisor/pkg/sentry/vfs"
	"gvisor.dev/gvisor/pkg/sync"
	"gvisor.dev/gvisor/pkg/syserror"
	"gvisor.dev/gvisor/pkg/usermem"
	"gvisor.dev/gvisor/pkg/waiter"
)

const fuseDevMinor = 229

// fuseDevice implements vfs.Device for /dev/fuse.
type fuseDevice struct{}

// Open implements vfs.Device.Open.
func (fuseDevice) Open(ctx context.Context, mnt *vfs.Mount, vfsd *vfs.Dentry, opts vfs.OpenOptions) (*vfs.FileDescription, error) {
	if !kernel.FUSEEnabled {
		return nil, syserror.ENOENT
	}

	var fd DeviceFD
	if err := fd.vfsfd.Init(&fd, opts.Flags, mnt, vfsd, &vfs.FileDescriptionOptions{
		UseDentryMetadata: true,
	}); err != nil {
		return nil, err
	}
	return &fd.vfsfd, nil
}

// DeviceFD implements vfs.FileDescriptionImpl for /dev/fuse.
type DeviceFD struct {
	vfsfd vfs.FileDescription
	vfs.FileDescriptionDefaultImpl
	vfs.DentryMetadataFileDescriptionImpl
	vfs.NoLockFD

	// mounted specifies whether a FUSE filesystem was mounted using the DeviceFD.
	mounted bool

	// nextOpID is used to create new requests.
	nextOpID linux.FUSEOpID

	// queue is the list of requests that need to be processed by the FUSE server.
	queue requestList

	// numActiveRequests is the number of requests made by the Sentry that has
	// yet to be responded to.
	numActiveRequests uint64

	// completions is used to map a request to its response. A Writer will use this
	// to notify the caller of a completed response.
	completions map[linux.FUSEOpID]*futureResponse

	writeCursor uint32

	// writeBuf is the memory buffer used to copy in the FUSE out header from
	// userspace.
	writeBuf []byte

	// writeCursorFR current FR being copied from server.
	writeCursorFR *futureResponse

	// mu protects all the queues, maps, buffers and cursors and nextOpID.
	mu sync.Mutex

	// waitQueue is used to notify interested parties when the device becomes
	// readable or writable.
	waitQueue waiter.Queue

	// fullQueueCh is a channel used to synchronize the readers with the writers.
	// Writers (inbound requests to the filesystem) block if there are too many
	// unprocessed in-flight requests.
	fullQueueCh chan struct{}

	// fs is the FUSE filesystem that this FD is being used for.
	fs *filesystem
}

// Release implements vfs.FileDescriptionImpl.Release.
func (fd *DeviceFD) Release(context.Context) {
	fd.fs.conn.connected = false
}

// PRead implements vfs.FileDescriptionImpl.PRead.
func (fd *DeviceFD) PRead(ctx context.Context, dst usermem.IOSequence, offset int64, opts vfs.ReadOptions) (int64, error) {
	// Operations on /dev/fuse don't make sense until a FUSE filesystem is mounted.
	if !fd.mounted {
		return 0, syserror.EPERM
	}

	return 0, syserror.ENOSYS
}

// Read implements vfs.FileDescriptionImpl.Read.
func (fd *DeviceFD) Read(ctx context.Context, dst usermem.IOSequence, opts vfs.ReadOptions) (int64, error) {
	// Operations on /dev/fuse don't make sense until a FUSE filesystem is mounted.
	if !fd.mounted {
		return 0, syserror.EPERM
	}

	// We require that any Read done on this filesystem have a sane minimum
	// read buffer. It must have the capacity for the fixed parts of any request
	// header (Linux uses the request header and the FUSEWriteIn header for this
	// calculation) + the negotiated MaxWrite room for the data.
	minBuffSize := linux.FUSE_MIN_READ_BUFFER
	inHdrLen := uint32((*linux.FUSEHeaderIn)(nil).SizeBytes())
	writeHdrLen := uint32((*linux.FUSEWriteIn)(nil).SizeBytes())
	negotiatedMinBuffSize := inHdrLen + writeHdrLen + fd.fs.conn.maxWrite
	if minBuffSize < negotiatedMinBuffSize {
		minBuffSize = negotiatedMinBuffSize
	}

	// If the read buffer is too small, error out.
	if dst.NumBytes() < int64(minBuffSize) {
		return 0, syserror.EINVAL
	}

	fd.mu.Lock()
	defer fd.mu.Unlock()
	return fd.readLocked(ctx, dst, opts)
}

// readLocked implements the reading of the fuse device while locked with DeviceFD.mu.
//
// Preconditions: dst is large enough for any reasonable request.
func (fd *DeviceFD) readLocked(ctx context.Context, dst usermem.IOSequence, opts vfs.ReadOptions) (int64, error) {
	var req *Request

	// Find the first valid request.
	// For the normal case this loop only execute once.
	for !fd.queue.Empty() {
		req = fd.queue.Front()

		if int64(req.hdr.Len) <= dst.NumBytes() {
			break
		}

		// The request is too large. Cannot process it. All requests must be smaller than the
		// negotiated size as specified by Connection.MaxWrite set as part of the FUSE_INIT
		// handshake.
		errno := -int32(syscall.EIO)
		if req.hdr.Opcode == linux.FUSE_SETXATTR {
			errno = -int32(syscall.E2BIG)
		}

		// Return the error to the calling task.
		if err := fd.sendError(ctx, errno, req); err != nil {
			return 0, err
		}

		// We're done with this request.
		fd.queue.Remove(req)
		req = nil
	}

	if req == nil {
		return 0, syserror.ErrWouldBlock
	}

	// We already checked the size: dst must be able to fit the whole request.
	// Now we write the marshalled header, the payload,
	// and the potential additional payload
	// to the user memory IOSequence.

	n, err := dst.CopyOut(ctx, req.data)
	if err != nil {
		return 0, err
	}
	if n != len(req.data) {
		return 0, syserror.EIO
	}

	// Fully done with this req, remove it from the queue.
	fd.queue.Remove(req)
	if req.hdr.Opcode == linux.FUSE_RELEASE {
		fd.numActiveRequests -= 1
	}

	return int64(n), nil
}

// PWrite implements vfs.FileDescriptionImpl.PWrite.
func (fd *DeviceFD) PWrite(ctx context.Context, src usermem.IOSequence, offset int64, opts vfs.WriteOptions) (int64, error) {
	// Operations on /dev/fuse don't make sense until a FUSE filesystem is mounted.
	if !fd.mounted {
		return 0, syserror.EPERM
	}

	return 0, syserror.ENOSYS
}

// Write implements vfs.FileDescriptionImpl.Write.
func (fd *DeviceFD) Write(ctx context.Context, src usermem.IOSequence, opts vfs.WriteOptions) (int64, error) {
	fd.mu.Lock()
	defer fd.mu.Unlock()
	return fd.writeLocked(ctx, src, opts)
}

// writeLocked implements writing to the fuse device while locked with DeviceFD.mu.
func (fd *DeviceFD) writeLocked(ctx context.Context, src usermem.IOSequence, opts vfs.WriteOptions) (int64, error) {
	// Operations on /dev/fuse don't make sense until a FUSE filesystem is mounted.
	if !fd.mounted {
		return 0, syserror.EPERM
	}

	var cn, n int64
	hdrLen := uint32((*linux.FUSEHeaderOut)(nil).SizeBytes())

	for src.NumBytes() > 0 {
		if fd.writeCursorFR != nil {
			// Already have common header, and we're now copying the payload.
			wantBytes := fd.writeCursorFR.hdr.Len

			// Note that the FR data doesn't have the header. Copy it over if its necessary.
			if fd.writeCursorFR.data == nil {
				fd.writeCursorFR.data = make([]byte, wantBytes)
			}

			bytesCopied, err := src.CopyIn(ctx, fd.writeCursorFR.data[fd.writeCursor:wantBytes])
			if err != nil {
				return 0, err
			}
			src = src.DropFirst(bytesCopied)

			cn = int64(bytesCopied)
			n += cn
			fd.writeCursor += uint32(cn)
			if fd.writeCursor == wantBytes {
				// Done reading this full response. Clean up and unblock the
				// initiator.
				break
			}

			// Check if we have more data in src.
			continue
		}

		// Assert that the header isn't read into the writeBuf yet.
		if fd.writeCursor >= hdrLen {
			return 0, syserror.EINVAL
		}

		// We don't have the full common response header yet.
		wantBytes := hdrLen - fd.writeCursor
		bytesCopied, err := src.CopyIn(ctx, fd.writeBuf[fd.writeCursor:wantBytes])
		if err != nil {
			return 0, err
		}
		src = src.DropFirst(bytesCopied)

		cn = int64(bytesCopied)
		n += cn
		fd.writeCursor += uint32(cn)
		if fd.writeCursor == hdrLen {
			// Have full header in the writeBuf. Use it to fetch the actual futureResponse
			// from the device's completions map.
			var hdr linux.FUSEHeaderOut
			hdr.UnmarshalBytes(fd.writeBuf)

			// We have the header now and so the writeBuf has served its purpose.
			// We could reset it manually here but instead of doing that, at the
			// end of the write, the writeCursor will be set to 0 thereby allowing
			// the next request to overwrite whats in the buffer,

			fut, ok := fd.completions[hdr.Unique]
			if !ok {
				// Server sent us a response for a request we never sent?
				return 0, syserror.EINVAL
			}

			delete(fd.completions, hdr.Unique)

			// Copy over the header into the future response. The rest of the payload
			// will be copied over to the FR's data in the next iteration.
			fut.hdr = &hdr
			fd.writeCursorFR = fut

			// Next iteration will now try read the complete request, if src has
			// any data remaining. Otherwise we're done.
		}
	}

	if fd.writeCursorFR != nil {
		if err := fd.sendResponse(ctx, fd.writeCursorFR); err != nil {
			return 0, err
		}

		// Ready the device for the next request.
		fd.writeCursorFR = nil
		fd.writeCursor = 0
	}

	return n, nil
}

// Readiness implements vfs.FileDescriptionImpl.Readiness.
func (fd *DeviceFD) Readiness(mask waiter.EventMask) waiter.EventMask {
	fd.mu.Lock()
	defer fd.mu.Unlock()
	return fd.readinessLocked(mask)
}

// readinessLocked implements checking the readiness of the fuse device while
// locked with DeviceFD.mu.
func (fd *DeviceFD) readinessLocked(mask waiter.EventMask) waiter.EventMask {
	var ready waiter.EventMask
	ready |= waiter.EventOut // FD is always writable
	if !fd.queue.Empty() {
		// Have reqs available, FD is readable.
		ready |= waiter.EventIn
	}

	return ready & mask
}

// EventRegister implements waiter.Waitable.EventRegister.
func (fd *DeviceFD) EventRegister(e *waiter.Entry, mask waiter.EventMask) {
	fd.waitQueue.EventRegister(e, mask)
}

// EventUnregister implements waiter.Waitable.EventUnregister.
func (fd *DeviceFD) EventUnregister(e *waiter.Entry) {
	fd.waitQueue.EventUnregister(e)
}

// Seek implements vfs.FileDescriptionImpl.Seek.
func (fd *DeviceFD) Seek(ctx context.Context, offset int64, whence int32) (int64, error) {
	// Operations on /dev/fuse don't make sense until a FUSE filesystem is mounted.
	if !fd.mounted {
		return 0, syserror.EPERM
	}

	return 0, syserror.ENOSYS
}

// sendResponse sends a response to the waiting task (if any).
func (fd *DeviceFD) sendResponse(ctx context.Context, fut *futureResponse) error {
	// See if the running task need to perform some action before returning.
	// Since we just finished writing the future, we can be sure that
	// getResponse generates a populated response.
	if err := fd.noReceiverAction(ctx, fut.getResponse()); err != nil {
		return err
	}

	// Signal that the queue is no longer full.
	select {
	case fd.fullQueueCh <- struct{}{}:
	default:
	}
	fd.numActiveRequests -= 1

	// Signal the task waiting on a response.
	close(fut.ch)
	return nil
}

// sendError sends an error response to the waiting task (if any).
func (fd *DeviceFD) sendError(ctx context.Context, errno int32, req *Request) error {
	// Return the error to the calling task.
	outHdrLen := uint32((*linux.FUSEHeaderOut)(nil).SizeBytes())
	respHdr := linux.FUSEHeaderOut{
		Len:    outHdrLen,
		Error:  errno,
		Unique: req.hdr.Unique,
	}

	fut, ok := fd.completions[respHdr.Unique]
	if !ok {
		// Server sent us a response for a request we never sent?
		return syserror.EINVAL
	}
	delete(fd.completions, respHdr.Unique)

	fut.hdr = &respHdr
	if err := fd.sendResponse(ctx, fut); err != nil {
		return err
	}

	return nil
}

// noReceiverAction has the calling kernel.Task do some action if its known that no
// receiver is going to be waiting on the future channel. This is to be used by:
// FUSE_INIT.
func (fd *DeviceFD) noReceiverAction(ctx context.Context, r *Response) error {
	switch r.opcode {
	case linux.FUSE_INIT:
		creds := auth.CredentialsFromContext(ctx)
		rootUserNs := kernel.KernelFromContext(ctx).RootUserNamespace()
		return fd.fs.conn.InitRecv(r, creds.HasCapabilityIn(linux.CAP_SYS_ADMIN, rootUserNs))
		// TODO(gvisor.dev/issue/3247): support async read: correctly process the response using information from r.options.
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2018 The OpenPitrix Authors. All rights reserved.
// Use of this source code is governed by a Apache license
// that can be found in the LICENSE file.

package app

import (
	"context"
	"fmt"
	"sort"
	"time"

	"openpitrix.io/openpitrix/pkg/constants"
	"openpitrix.io/openpitrix/pkg/db"
	"openpitrix.io/openpitrix/pkg/logger"
	"openpitrix.io/openpitrix/pkg/models"
	"openpitrix.io/openpitrix/pkg/pb"
	"openpitrix.io/openpitrix/pkg/pi"
	"openpitrix.io/openpitrix/pkg/repoiface"
	"openpitrix.io/openpitrix/pkg/repoiface/wrapper"
	"openpitrix.io/openpitrix/pkg/sender"
	"openpitrix.io/openpitrix/pkg/service/category/categoryutil"
	"openpitrix.io/openpitrix/pkg/util/stringutil"
)

type repoProxy struct {
	repo *pb.Repo
}

func newRepoProxy(repo *pb.Repo) *repoProxy {
	vp := &repoProxy{}
	vp.repo = repo
	return vp
}

func (rp *repoProxy) deleteAppVersions(ctx context.Context) error {
	repoId := rp.repo.RepoId.GetValue()
	err := clearApps(ctx, repoId, []string{})
	if err != nil {
		return err
	}
	return clearRepoAppVersions(ctx, repoId, []string{})
}

func (rp *repoProxy) SyncRepo(ctx context.Context) error {
	repo := rp.repo
	if repo.Status.GetValue() == constants.StatusDeleted {
		return rp.deleteAppVersions(ctx)
	}
	reader, err := repoiface.NewReader(ctx, repo)
	if err != nil {
		return err
	}
	indexFile, err := reader.GetIndex(ctx)
	if err != nil {
		return err
	}
	var appIds []string
	for appName, appVersions := range indexFile.GetEntries() {
		var appId string
		logger.Debug(ctx, "Start index app [%s]", appName)
		logger.Debug(ctx, "App [%s] has [%d] versions", appName, appVersions.Len())
		if len(appVersions) == 0 {
			return fmt.Errorf("failed to sync app [%s], no versions", appName)
		}
		sort.Sort(appVersions)
		appId, err := rp.syncAppInfo(ctx, appVersions[0])
		if err != nil {
			logger.Error(ctx, "Failed to sync app [%s] to app info", appName)
			return err
		}
		logger.Info(ctx, "Sync [%s] to app [%s] success", appName, appId)
		var versionIds []string
		for _, appVersion := range appVersions {
			var versionId string
			versionId, err = rp.syncAppVersionInfo(ctx, appId, appVersion)
			if err != nil {
				logger.Error(ctx, "Failed to sync app version [%s] to app version", appVersion.GetAppVersion())
				return err
			}
			logger.Debug(ctx, "App version [%s] sync to app version [%s]", appVersion.GetVersion(), versionId)
			versionIds = append(versionIds, versionId)
		}
		err = clearAppVersions(ctx, appId, versionIds)
		if err != nil {
			return err
		}
		err = resortAppVersions(ctx, appId)
		if err != nil {
			return err
		}
		err = syncAppStatus(ctx, appId)
		if err != nil {
			return err
		}
		appIds = append(appIds, appId)
	}
	err = clearApps(ctx, rp.repo.RepoId.GetValue(), appIds)
	if err != nil {
		return err
	}
	return nil
}

func (rp *repoProxy) syncAppInfo(ctx context.Context, appIface wrapper.VersionInterface) (string, error) {
	chartName := appIface.GetName()
	repoId := rp.repo.GetRepoId().GetValue()

	var enabledCategoryIds []string
	var disabledCategoryIds []string

	for _, c := range rp.repo.GetCategorySet() {
		switch c.Status.GetValue() {
		case constants.StatusEnabled:
			enabledCategoryIds = append(enabledCategoryIds, c.CategoryId.GetValue())
		case constants.StatusDisabled:
			disabledCategoryIds = append(disabledCategoryIds, c.CategoryId.GetValue())
		}
	}
	if len(enabledCategoryIds) == 0 {
		enabledCategoryIds = append(enabledCategoryIds, models.UncategorizedId)
	}

	var appId string
	var app = &models.App{}
	err := pi.Global().DB(ctx).
		Select(models.AppColumns...).
		From(constants.TableApp).
		Where(db.Eq(constants.ColumnActive, false)).
		Where(db.Eq(constants.ColumnRepoId, repoId)).
		Where(db.Eq(constants.ColumnChartName, chartName)).
		LoadOne(&app)
	if err != nil {
		// insert new
		if err != db.ErrNotFound {
			return appId, err
		}
		app = models.NewApp(
			chartName,
			sender.OwnerPath(rp.repo.GetOwnerPath().GetValue()),
			rp.repo.GetOwner().GetValue(),
		)
		app.RepoId = repoId
		app.ChartName = chartName
		app.Description = appIface.GetDescription()
		app.Home = appIface.GetHome()
		app.Icon = appIface.GetIcon()
		app.Screenshots = appIface.GetScreenshots()
		app.Maintainers = appIface.GetMaintainers()
		app.Keywords = appIface.GetKeywords()
		app.Sources = appIface.GetSources()
		app.CreateTime = appIface.GetCreateTime()

		_, err = pi.Global().DB(ctx).
			InsertInto(constants.TableApp).
			Record(app).
			Exec()
		if err != nil {
			return appId, err
		}

		err = categoryutil.SyncResourceCategories(
			ctx,
			pi.Global().DB(ctx),
			app.AppId,
			enabledCategoryIds,
		)
		if err != nil {
			return appId, err
		}

		return app.AppId, err
	}

	appId = app.AppId
	var updateAttr = make(map[string]interface{})
	if app.Description != appIface.GetDescription() {
		updateAttr[constants.ColumnDescription] = appIface.GetDescription()
	}
	if app.Home != appIface.GetHome() {
		updateAttr[constants.ColumnHome] = appIface.GetHome()
	}
	if app.Icon != appIface.GetIcon() {
		updateAttr[constants.ColumnIcon] = appIface.GetIcon()
	}
	if app.Screenshots != appIface.GetScreenshots() {
		updateAttr[constants.ColumnScreenshots] = appIface.GetScreenshots()
	}
	if app.Maintainers != appIface.GetMaintainers() {
		updateAttr[constants.ColumnMaintainers] = appIface.GetMaintainers()
	}
	if app.Keywords != appIface.GetKeywords() {
		updateAttr[constants.ColumnKeywords] = appIface.GetKeywords()
	}
	if app.Sources != appIface.GetSources() {
		updateAttr[constants.ColumnSources] = appIface.GetSources()
	}
	if app.CreateTime != appIface.GetCreateTime() {
		updateAttr[constants.ColumnCreateTime] = appIface.GetCreateTime()
	}
	if len(updateAttr) > 0 {
		_, err = pi.Global().DB(ctx).
			Update(constants.TableApp).
			SetMap(updateAttr).
			Set(constants.ColumnUpdateTime, time.Now()).
			Where(db.Eq(constants.ColumnAppId, appId)).
			Where(db.Eq(constants.ColumnActive, false)).
			Exec()
		if err != nil {
			return appId, err
		}
	}

	// update exists, only need sync categories
	appCategories, err := getAppCategories(ctx, appId)
	if err != nil {
		return appId, err
	}
	var categoryMap = make(map[string]bool)
	for _, c := range appCategories {
		categoryId := c.GetCategoryId().GetValue()
		// app follow repo's categories:
		// if repo *disable* some categories, app MUST *disable* it
		// if repo *enable*  some categories, app MUST *enable*  it
		if c.GetStatus().GetValue() == constants.StatusEnabled {
			if !stringutil.StringIn(categoryId, disabledCategoryIds) {
				categoryMap[categoryId] = true
			}
		}
	}
	for _, c := range enabledCategoryIds {
		categoryMap[c] = true
	}
	var categoryIds []string
	for c := range categoryMap {
		if c == models.UncategorizedId && len(categoryMap) > 1 {
			continue
		}
		categoryIds = append(categoryIds, c)
	}
	err = categoryutil.SyncResourceCategories(
		ctx,
		pi.Global().DB(ctx),
		appId,
		categoryIds,
	)
	return app.AppId, err
}

func (rp *repoProxy) syncAppVersionInfo(ctx context.Context, appId string, versionInterface wrapper.VersionInterface) (string, error) {
	versionName := versionInterface.GetVersionName()
	var appVersion = &models.AppVersion{}
	var versionId = ""
	var status = getAppDefaultStatus(rp.repo)
	if status == constants.StatusActive {
		defer func() {
			if versionId != "" {
				err := syncAppVersion(ctx, versionId)
				if err != nil {
					logger.Error(ctx, "Active app version [%s] failed: %+v", versionId, err)
				}
			}
		}()
	}
	err := pi.Global().DB(ctx).
		Select(models.AppVersionColumns...).
		From(constants.TableAppVersion).
		Where(db.Eq(constants.ColumnAppId, appId)).
		Where(db.Eq(constants.ColumnName, versionName)).
		Where(db.Eq(constants.ColumnActive, false)).
		LoadOne(&appVersion)
	if err != nil {
		if err != db.ErrNotFound {
			return versionId, err
		}
		// not found version, create new
		appVersion = models.NewAppVersion(
			appId,
			versionName,
			versionInterface.GetDescription(),
			sender.OwnerPath(rp.repo.GetOwnerPath().GetValue()),
		)

		appVersion.PackageName = versionInterface.GetPackageName()
		appVersion.Status = status
		appVersion.Type = rp.repo.Type.GetValue()

		_, err = pi.Global().DB(ctx).
			InsertInto(constants.TableAppVersion).
			Record(appVersion).
			Exec()
		if err != nil {
			return versionId, err
		}
	}
	// update exists
	versionId = appVersion.VersionId
	var updateAttr = make(map[string]interface{})

	if appVersion.Status != rp.repo.GetAppDefaultStatus().GetValue() {
		updateAttr[constants.ColumnStatus] = getAppVersionStatus(
			status,
			appVersion.Status,
		)
	}

	if appVersion.PackageName != versionInterface.GetPackageName() {
		updateAttr[constants.ColumnPackageName] = versionInterface.GetPackageName()
	}
	if len(updateAttr) == 0 {
		return versionId, nil
	}
	_, err = pi.Global().DB(ctx).
		Update(constants.TableAppVersion).
		SetMap(updateAttr).
		Set(constants.ColumnUpdateTime, time.Now()).
		Where(db.Eq(constants.ColumnVersionId, versionId)).
		Where(db.Eq(constants.ColumnActive, false)).
		Exec()
	return versionId, err
}
<commit_msg>fix: add createTime of version<commit_after>// Copyright 2018 The OpenPitrix Authors. All rights reserved.
// Use of this source code is governed by a Apache license
// that can be found in the LICENSE file.

package app

import (
	"context"
	"fmt"
	"sort"
	"time"

	"openpitrix.io/openpitrix/pkg/constants"
	"openpitrix.io/openpitrix/pkg/db"
	"openpitrix.io/openpitrix/pkg/logger"
	"openpitrix.io/openpitrix/pkg/models"
	"openpitrix.io/openpitrix/pkg/pb"
	"openpitrix.io/openpitrix/pkg/pi"
	"openpitrix.io/openpitrix/pkg/repoiface"
	"openpitrix.io/openpitrix/pkg/repoiface/wrapper"
	"openpitrix.io/openpitrix/pkg/sender"
	"openpitrix.io/openpitrix/pkg/service/category/categoryutil"
	"openpitrix.io/openpitrix/pkg/util/stringutil"
)

type repoProxy struct {
	repo *pb.Repo
}

func newRepoProxy(repo *pb.Repo) *repoProxy {
	vp := &repoProxy{}
	vp.repo = repo
	return vp
}

func (rp *repoProxy) deleteAppVersions(ctx context.Context) error {
	repoId := rp.repo.RepoId.GetValue()
	err := clearApps(ctx, repoId, []string{})
	if err != nil {
		return err
	}
	return clearRepoAppVersions(ctx, repoId, []string{})
}

func (rp *repoProxy) SyncRepo(ctx context.Context) error {
	repo := rp.repo
	if repo.Status.GetValue() == constants.StatusDeleted {
		return rp.deleteAppVersions(ctx)
	}
	reader, err := repoiface.NewReader(ctx, repo)
	if err != nil {
		return err
	}
	indexFile, err := reader.GetIndex(ctx)
	if err != nil {
		return err
	}
	var appIds []string
	for appName, appVersions := range indexFile.GetEntries() {
		var appId string
		logger.Debug(ctx, "Start index app [%s]", appName)
		logger.Debug(ctx, "App [%s] has [%d] versions", appName, appVersions.Len())
		if len(appVersions) == 0 {
			return fmt.Errorf("failed to sync app [%s], no versions", appName)
		}
		sort.Sort(appVersions)
		appId, err := rp.syncAppInfo(ctx, appVersions[0])
		if err != nil {
			logger.Error(ctx, "Failed to sync app [%s] to app info", appName)
			return err
		}
		logger.Info(ctx, "Sync [%s] to app [%s] success", appName, appId)
		var versionIds []string
		for _, appVersion := range appVersions {
			var versionId string
			versionId, err = rp.syncAppVersionInfo(ctx, appId, appVersion)
			if err != nil {
				logger.Error(ctx, "Failed to sync app version [%s] to app version", appVersion.GetAppVersion())
				return err
			}
			logger.Debug(ctx, "App version [%s] sync to app version [%s]", appVersion.GetVersion(), versionId)
			versionIds = append(versionIds, versionId)
		}
		err = clearAppVersions(ctx, appId, versionIds)
		if err != nil {
			return err
		}
		err = resortAppVersions(ctx, appId)
		if err != nil {
			return err
		}
		err = syncAppStatus(ctx, appId)
		if err != nil {
			return err
		}
		appIds = append(appIds, appId)
	}
	err = clearApps(ctx, rp.repo.RepoId.GetValue(), appIds)
	if err != nil {
		return err
	}
	return nil
}

func (rp *repoProxy) syncAppInfo(ctx context.Context, appIface wrapper.VersionInterface) (string, error) {
	chartName := appIface.GetName()
	repoId := rp.repo.GetRepoId().GetValue()

	var enabledCategoryIds []string
	var disabledCategoryIds []string

	for _, c := range rp.repo.GetCategorySet() {
		switch c.Status.GetValue() {
		case constants.StatusEnabled:
			enabledCategoryIds = append(enabledCategoryIds, c.CategoryId.GetValue())
		case constants.StatusDisabled:
			disabledCategoryIds = append(disabledCategoryIds, c.CategoryId.GetValue())
		}
	}
	if len(enabledCategoryIds) == 0 {
		enabledCategoryIds = append(enabledCategoryIds, models.UncategorizedId)
	}

	var appId string
	var app = &models.App{}
	err := pi.Global().DB(ctx).
		Select(models.AppColumns...).
		From(constants.TableApp).
		Where(db.Eq(constants.ColumnActive, false)).
		Where(db.Eq(constants.ColumnRepoId, repoId)).
		Where(db.Eq(constants.ColumnChartName, chartName)).
		LoadOne(&app)
	if err != nil {
		// insert new
		if err != db.ErrNotFound {
			return appId, err
		}
		app = models.NewApp(
			chartName,
			sender.OwnerPath(rp.repo.GetOwnerPath().GetValue()),
			rp.repo.GetOwner().GetValue(),
		)
		app.RepoId = repoId
		app.ChartName = chartName
		app.Description = appIface.GetDescription()
		app.Home = appIface.GetHome()
		app.Icon = appIface.GetIcon()
		app.Screenshots = appIface.GetScreenshots()
		app.Maintainers = appIface.GetMaintainers()
		app.Keywords = appIface.GetKeywords()
		app.Sources = appIface.GetSources()
		app.CreateTime = appIface.GetCreateTime()

		_, err = pi.Global().DB(ctx).
			InsertInto(constants.TableApp).
			Record(app).
			Exec()
		if err != nil {
			return appId, err
		}

		err = categoryutil.SyncResourceCategories(
			ctx,
			pi.Global().DB(ctx),
			app.AppId,
			enabledCategoryIds,
		)
		if err != nil {
			return appId, err
		}

		return app.AppId, err
	}

	appId = app.AppId
	var updateAttr = make(map[string]interface{})
	if app.Description != appIface.GetDescription() {
		updateAttr[constants.ColumnDescription] = appIface.GetDescription()
	}
	if app.Home != appIface.GetHome() {
		updateAttr[constants.ColumnHome] = appIface.GetHome()
	}
	if app.Icon != appIface.GetIcon() {
		updateAttr[constants.ColumnIcon] = appIface.GetIcon()
	}
	if app.Screenshots != appIface.GetScreenshots() {
		updateAttr[constants.ColumnScreenshots] = appIface.GetScreenshots()
	}
	if app.Maintainers != appIface.GetMaintainers() {
		updateAttr[constants.ColumnMaintainers] = appIface.GetMaintainers()
	}
	if app.Keywords != appIface.GetKeywords() {
		updateAttr[constants.ColumnKeywords] = appIface.GetKeywords()
	}
	if app.Sources != appIface.GetSources() {
		updateAttr[constants.ColumnSources] = appIface.GetSources()
	}
	if app.CreateTime != appIface.GetCreateTime() {
		updateAttr[constants.ColumnCreateTime] = appIface.GetCreateTime()
	}
	if len(updateAttr) > 0 {
		_, err = pi.Global().DB(ctx).
			Update(constants.TableApp).
			SetMap(updateAttr).
			Set(constants.ColumnUpdateTime, time.Now()).
			Where(db.Eq(constants.ColumnAppId, appId)).
			Where(db.Eq(constants.ColumnActive, false)).
			Exec()
		if err != nil {
			return appId, err
		}
	}

	// update exists, only need sync categories
	appCategories, err := getAppCategories(ctx, appId)
	if err != nil {
		return appId, err
	}
	var categoryMap = make(map[string]bool)
	for _, c := range appCategories {
		categoryId := c.GetCategoryId().GetValue()
		// app follow repo's categories:
		// if repo *disable* some categories, app MUST *disable* it
		// if repo *enable*  some categories, app MUST *enable*  it
		if c.GetStatus().GetValue() == constants.StatusEnabled {
			if !stringutil.StringIn(categoryId, disabledCategoryIds) {
				categoryMap[categoryId] = true
			}
		}
	}
	for _, c := range enabledCategoryIds {
		categoryMap[c] = true
	}
	var categoryIds []string
	for c := range categoryMap {
		if c == models.UncategorizedId && len(categoryMap) > 1 {
			continue
		}
		categoryIds = append(categoryIds, c)
	}
	err = categoryutil.SyncResourceCategories(
		ctx,
		pi.Global().DB(ctx),
		appId,
		categoryIds,
	)
	return app.AppId, err
}

func (rp *repoProxy) syncAppVersionInfo(ctx context.Context, appId string, versionInterface wrapper.VersionInterface) (string, error) {
	versionName := versionInterface.GetVersionName()
	var appVersion = &models.AppVersion{}
	var versionId = ""
	var status = getAppDefaultStatus(rp.repo)
	if status == constants.StatusActive {
		defer func() {
			if versionId != "" {
				err := syncAppVersion(ctx, versionId)
				if err != nil {
					logger.Error(ctx, "Active app version [%s] failed: %+v", versionId, err)
				}
			}
		}()
	}
	err := pi.Global().DB(ctx).
		Select(models.AppVersionColumns...).
		From(constants.TableAppVersion).
		Where(db.Eq(constants.ColumnAppId, appId)).
		Where(db.Eq(constants.ColumnName, versionName)).
		Where(db.Eq(constants.ColumnActive, false)).
		LoadOne(&appVersion)
	if err != nil {
		if err != db.ErrNotFound {
			return versionId, err
		}
		// not found version, create new
		appVersion = models.NewAppVersion(
			appId,
			versionName,
			versionInterface.GetDescription(),
			sender.OwnerPath(rp.repo.GetOwnerPath().GetValue()),
		)

		appVersion.PackageName = versionInterface.GetPackageName()
		appVersion.CreateTime = versionInterface.GetCreateTime()
		appVersion.Status = status
		appVersion.Type = rp.repo.Type.GetValue()

		_, err = pi.Global().DB(ctx).
			InsertInto(constants.TableAppVersion).
			Record(appVersion).
			Exec()
		if err != nil {
			return versionId, err
		}
	}
	// update exists
	versionId = appVersion.VersionId
	var updateAttr = make(map[string]interface{})

	if appVersion.Status != rp.repo.GetAppDefaultStatus().GetValue() {
		updateAttr[constants.ColumnStatus] = getAppVersionStatus(
			status,
			appVersion.Status,
		)
	}

	if appVersion.PackageName != versionInterface.GetPackageName() {
		updateAttr[constants.ColumnPackageName] = versionInterface.GetPackageName()
	}
	if len(updateAttr) == 0 {
		return versionId, nil
	}
	_, err = pi.Global().DB(ctx).
		Update(constants.TableAppVersion).
		SetMap(updateAttr).
		Set(constants.ColumnUpdateTime, time.Now()).
		Where(db.Eq(constants.ColumnVersionId, versionId)).
		Where(db.Eq(constants.ColumnActive, false)).
		Exec()
	return versionId, err
}
<|endoftext|>