<commit_before>package composebuilder_test

import (
	"io/ioutil"
	"strings"

	"github.com/Originate/exosphere/src/docker/composebuilder"
	"github.com/Originate/exosphere/src/types"
	"github.com/Originate/exosphere/src/types/context"
	"github.com/Originate/exosphere/src/util"
	"github.com/Originate/exosphere/test/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("composebuilder", func() {
	var _ = Describe("GetApplicationDockerCompose", func() {
		It("should return the proper docker configs for development", func() {
			appDir, err := ioutil.TempDir("", "")
			Expect(err).NotTo(HaveOccurred())
			err = helpers.CheckoutApp(appDir, "complex-setup-app")
			Expect(err).NotTo(HaveOccurred())
			internalServices := []string{"html-server", "todo-service", "users-service"}
			externalServices := []string{"external-service"}
			internalDependencies := []string{"exocom"}
			externalDependencies := []string{"mongo"}
			appContext, err := context.GetAppContext(appDir)
			Expect(err).NotTo(HaveOccurred())

			dockerCompose, err := composebuilder.GetApplicationDockerCompose(composebuilder.ApplicationOptions{
				AppContext: appContext,
				BuildMode: types.BuildMode{
					Type:        types.BuildModeTypeLocal,
					Environment: types.BuildModeEnvironmentDevelopment,
				},
			})
			Expect(err).NotTo(HaveOccurred())

			By("generate an image name for each dependency and external service")
			for _, serviceRole := range util.JoinStringSlices(internalDependencies, externalDependencies, externalServices) {
				Expect(len(dockerCompose.Services[serviceRole].Image)).ToNot(Equal(0))
			}

			By("should have the correct build command for each internal service and dependency")
			for _, serviceRole := range internalServices {
				Expect(dockerCompose.Services[serviceRole].Command).To(Equal(`echo "does not run"`))
			}
			Expect(dockerCompose.Services["exocom"].Command).To(Equal(""))

			By("should include 'exocom' in the dependencies of every service")
			for _, serviceRole := range append(internalServices, externalServices...) {
				exists := util.DoesStringArrayContain(dockerCompose.Services[serviceRole].DependsOn, "exocom")
				Expect(exists).To(Equal(true))
			}

			By("should include external dependencies as dependencies")
			exists := util.DoesStringArrayContain(dockerCompose.Services["todo-service"].DependsOn, "mongo")
			Expect(exists).To(Equal(true))

			By("should properly reserve ports for services")
			actualApiPort := dockerCompose.Services["api-service"].Ports
			expectedApiPort := []string{"3000:80"}
			Expect(actualApiPort).To(Equal(expectedApiPort))

			actualExternalServicePort := dockerCompose.Services["external-service"].Ports
			expectedExternalServicePort := []string{"3010:5000"}
			Expect(actualExternalServicePort).To(Equal(expectedExternalServicePort))

			actualHtmlPort := dockerCompose.Services["html-server"].Ports
			expectedHtmlPort := []string{"3020:80"}
			Expect(actualHtmlPort).To(Equal(expectedHtmlPort))

			By("should inject the proper service endpoint environment variables")
			expectedApiEndpointKey := "API_SERVICE_EXTERNAL_ORIGIN"
			expectedApiEndpointValue := "http://localhost:3000"
			expectedHtmlEndpointKey := "HTML_SERVER_EXTERNAL_ORIGIN"
			expectedHtmlEndpointValue := "http://localhost:3020"

			skipServices := []string{"api-service", "exocom", "mongo"}
			for serviceRole, dockerConfig := range dockerCompose.Services {
				if util.DoesStringArrayContain(skipServices, serviceRole) {
					continue
				}
				Expect(dockerConfig.Environment[expectedApiEndpointKey]).To(Equal(expectedApiEndpointValue))
			}
			skipServices = []string{"html-server", "exocom", "mongo"}
			for serviceRole, dockerConfig := range dockerCompose.Services {
				if util.DoesStringArrayContain(skipServices, serviceRole) {
					continue
				}
				Expect(dockerConfig.Environment[expectedHtmlEndpointKey]).To(Equal(expectedHtmlEndpointValue))
			}
			nonPublicServiceKeys := []string{
				"USERS_SERVICE_EXTERNAL_ORIGIN",
				"TODO_SERVICE_EXTERNAL_ORIGIN",
			}
			for _, dockerConfig := range dockerCompose.Services {
				for _, nonPublicKey := range nonPublicServiceKeys {
					Expect(dockerConfig.Environment[nonPublicKey]).To(Equal(""))
				}
			}

			By("should include the correct exocom environment variables")
			environment := dockerCompose.Services["exocom"].Environment
			expectedServiceRoutes := []string{
				`{"receives":["todo.create"],"role":"todo-service","sends":["todo.created"]}`,
				`{"namespace":"mongo","receives":["mongo.list","mongo.create"],"role":"users-service","sends":["mongo.listed","mongo.created"]}`,
				`{"receives":["todo.created"],"role":"html-server","sends":["todo.create"]}`,
				`{"receives":["users.listed","users.created"],"role":"external-service","sends":["users.list","users.create"]}`,
			}
			for _, serviceRoute := range expectedServiceRoutes {
				Expect(strings.Contains(environment["SERVICE_ROUTES"], serviceRoute))
			}

			By("should include exocom environment variables in every services' environment")
			for _, serviceRole := range append(internalServices, externalServices...) {
				environment := dockerCompose.Services[serviceRole].Environment
				Expect(environment["EXOCOM_HOST"]).To(Equal("exocom"))
			}

			By("should generate a volume path for an external dependency that mounts a volume")
			Expect(len(dockerCompose.Services["mongo"].Volumes)).NotTo(Equal(0))

			By("should have the specified image for the external service")
			serviceRole := "external-service"
			imageName := "originate/test-web-server:0.0.1"
			Expect(dockerCompose.Services[serviceRole].Image).To(Equal(imageName))

			By("should have the ports for the external dependency defined in application.yml")
			serviceRole = "mongo"
			ports := []string{"4000:4000"}
			Expect(dockerCompose.Services[serviceRole].Ports).To(Equal(ports))
			Expect(len(dockerCompose.Services[serviceRole].Volumes)).NotTo(Equal(0))
		})
	})

	var _ = Describe("compiles the docker compose project name properly", func() {
		expected := "spacetweet123"

		It("converts all characters to lowercase", func() {
			actual := composebuilder.GetDockerComposeProjectName("SpaceTweet123")
			Expect(actual).To(Equal(expected))
		})

		It("strips non-alphanumeric characters", func() {
			actual := composebuilder.GetDockerComposeProjectName("$Space-Tweet_123")
			Expect(actual).To(Equal(expected))
		})

		It("strips whitespace characters", func() {
			actual := composebuilder.GetDockerComposeProjectName("Space   Tweet  123")
			Expect(actual).To(Equal(expected))
		})
	})
})
<commit_msg>cleanup more service routes (#862)<commit_after>package composebuilder_test

import (
	"encoding/json"
	"io/ioutil"

	"github.com/Originate/exosphere/src/docker/composebuilder"
	"github.com/Originate/exosphere/src/types"
	"github.com/Originate/exosphere/src/types/context"
	"github.com/Originate/exosphere/src/util"
	"github.com/Originate/exosphere/test/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("composebuilder", func() {
	var _ = Describe("GetApplicationDockerCompose", func() {
		It("should return the proper docker configs for development", func() {
			appDir, err := ioutil.TempDir("", "")
			Expect(err).NotTo(HaveOccurred())
			err = helpers.CheckoutApp(appDir, "complex-setup-app")
			Expect(err).NotTo(HaveOccurred())
			internalServices := []string{"html-server", "todo-service", "users-service"}
			externalServices := []string{"external-service"}
			internalDependencies := []string{"exocom"}
			externalDependencies := []string{"mongo"}
			appContext, err := context.GetAppContext(appDir)
			Expect(err).NotTo(HaveOccurred())

			dockerCompose, err := composebuilder.GetApplicationDockerCompose(composebuilder.ApplicationOptions{
				AppContext: appContext,
				BuildMode: types.BuildMode{
					Type:        types.BuildModeTypeLocal,
					Environment: types.BuildModeEnvironmentDevelopment,
				},
			})
			Expect(err).NotTo(HaveOccurred())

			By("generate an image name for each dependency and external service")
			for _, serviceRole := range util.JoinStringSlices(internalDependencies, externalDependencies, externalServices) {
				Expect(len(dockerCompose.Services[serviceRole].Image)).ToNot(Equal(0))
			}

			By("should have the correct build command for each internal service and dependency")
			for _, serviceRole := range internalServices {
				Expect(dockerCompose.Services[serviceRole].Command).To(Equal(`echo "does not run"`))
			}
			Expect(dockerCompose.Services["exocom"].Command).To(Equal(""))

			By("should include 'exocom' in the dependencies of every service")
			for _, serviceRole := range append(internalServices, externalServices...) {
				exists := util.DoesStringArrayContain(dockerCompose.Services[serviceRole].DependsOn, "exocom")
				Expect(exists).To(Equal(true))
			}

			By("should include external dependencies as dependencies")
			exists := util.DoesStringArrayContain(dockerCompose.Services["todo-service"].DependsOn, "mongo")
			Expect(exists).To(Equal(true))

			By("should properly reserve ports for services")
			actualApiPort := dockerCompose.Services["api-service"].Ports
			expectedApiPort := []string{"3000:80"}
			Expect(actualApiPort).To(Equal(expectedApiPort))

			actualExternalServicePort := dockerCompose.Services["external-service"].Ports
			expectedExternalServicePort := []string{"3010:5000"}
			Expect(actualExternalServicePort).To(Equal(expectedExternalServicePort))

			actualHtmlPort := dockerCompose.Services["html-server"].Ports
			expectedHtmlPort := []string{"3020:80"}
			Expect(actualHtmlPort).To(Equal(expectedHtmlPort))

			By("should inject the proper service endpoint environment variables")
			expectedApiEndpointKey := "API_SERVICE_EXTERNAL_ORIGIN"
			expectedApiEndpointValue := "http://localhost:3000"
			expectedHtmlEndpointKey := "HTML_SERVER_EXTERNAL_ORIGIN"
			expectedHtmlEndpointValue := "http://localhost:3020"

			skipServices := []string{"api-service", "exocom", "mongo"}
			for serviceRole, dockerConfig := range dockerCompose.Services {
				if util.DoesStringArrayContain(skipServices, serviceRole) {
					continue
				}
				Expect(dockerConfig.Environment[expectedApiEndpointKey]).To(Equal(expectedApiEndpointValue))
			}
			skipServices = []string{"html-server", "exocom", "mongo"}
			for serviceRole, dockerConfig := range dockerCompose.Services {
				if util.DoesStringArrayContain(skipServices, serviceRole) {
					continue
				}
				Expect(dockerConfig.Environment[expectedHtmlEndpointKey]).To(Equal(expectedHtmlEndpointValue))
			}
			nonPublicServiceKeys := []string{
				"USERS_SERVICE_EXTERNAL_ORIGIN",
				"TODO_SERVICE_EXTERNAL_ORIGIN",
			}
			for _, dockerConfig := range dockerCompose.Services {
				for _, nonPublicKey := range nonPublicServiceKeys {
					Expect(dockerConfig.Environment[nonPublicKey]).To(Equal(""))
				}
			}

			By("should include the correct exocom environment variables")
			environment := dockerCompose.Services["exocom"].Environment
			serviceData, err := json.Marshal(map[string]map[string]interface{}{
				"api-service":      {},
				"external-service": {},
				"html-server": {
					"receives": []interface{}{"todo.created"},
					"sends":    []interface{}{"todo.create"},
				},
				"todo-service": {
					"receives": []interface{}{"todo.create"},
					"sends":    []interface{}{"todo.created"},
				},
				"users-service": {
					"receives": []interface{}{"mongo.list", "mongo.create"},
					"sends":    []interface{}{"mongo.listed", "mongo.created"},
					"translations": []interface{}{
						map[string]interface{}{
							"internal": "mongo create",
							"public":   "users create",
						},
					},
				},
			})
			Expect(err).NotTo(HaveOccurred())
			Expect(environment).To(Equal(map[string]string{
				"SERVICE_DATA": string(serviceData),
			}))

			By("should include exocom environment variables in every services' environment")
			for _, serviceRole := range append(internalServices, externalServices...) {
				environment := dockerCompose.Services[serviceRole].Environment
				Expect(environment["EXOCOM_HOST"]).To(Equal("exocom"))
			}

			By("should generate a volume path for an external dependency that mounts a volume")
			Expect(len(dockerCompose.Services["mongo"].Volumes)).NotTo(Equal(0))

			By("should have the specified image for the external service")
			serviceRole := "external-service"
			imageName := "originate/test-web-server:0.0.1"
			Expect(dockerCompose.Services[serviceRole].Image).To(Equal(imageName))

			By("should have the ports for the external dependency defined in application.yml")
			serviceRole = "mongo"
			ports := []string{"4000:4000"}
			Expect(dockerCompose.Services[serviceRole].Ports).To(Equal(ports))
			Expect(len(dockerCompose.Services[serviceRole].Volumes)).NotTo(Equal(0))
		})
	})

	var _ = Describe("compiles the docker compose project name properly", func() {
		expected := "spacetweet123"

		It("converts all characters to lowercase", func() {
			actual := composebuilder.GetDockerComposeProjectName("SpaceTweet123")
			Expect(actual).To(Equal(expected))
		})

		It("strips non-alphanumeric characters", func() {
			actual := composebuilder.GetDockerComposeProjectName("$Space-Tweet_123")
			Expect(actual).To(Equal(expected))
		})

		It("strips whitespace characters", func() {
			actual := composebuilder.GetDockerComposeProjectName("Space   Tweet  123")
			Expect(actual).To(Equal(expected))
		})
	})
})
<|endoftext|>


<commit_before>//
// Copyright 2021, Kordian Bruck
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package gitlab

import (
	"fmt"
	"time"
)

// PackagesService handles communication with the packages related methods
// of the GitLab API.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/packages.html
type PackagesService struct {
	client *Client
}

// Package represents a GitLab single package
//
// GitLab API docs: https://docs.gitlab.com/ee/api/packages.html
type Package struct {
	ID        int        `json:"id"`
	Name      string     `json:"name"`
	Version   string     `json:"version"`
	Type      string     `json:"package_type"`
	CreatedAt *time.Time `json:"created_at"`
}

func (s Package) String() string {
	return Stringify(s)
}

// PackageFile represents one file contained within a package
//
// GitLab API docs: https://docs.gitlab.com/ee/api/packages.html
type PackageFile struct {
	ID        int        `json:"id"`
	PackageId int        `json:"package_id"`
	CreatedAt *time.Time `json:"created_at"`
	FileName  string     `json:"file_name"`
	Size      int        `json:"size"`
	MD5       string     `json:"file_md5"`
	SHA1      string     `json:"file_sha1"`
}

func (s PackageFile) String() string {
	return Stringify(s)
}

// ListProjectPackagesOptions are the parameters available in a ListProjectPackages() Operation
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/packages.html#within-a-project
type ListProjectPackagesOptions struct {
	ListOptions
	OrderBy            string `json:"order_by,omitempty"`
	Sort               string `json:"sort,omitempty"`
	Type               string `json:"package_type,omitempty"`
	Name               string `json:"package_name,omitempty"`
	IncludeVersionless bool   `json:"include_versionless,omitempty"`
}

// ListProjectPackages gets a list of packages in a project.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/packages.html#within-a-project
func (s *PackagesService) ListProjectPackages(pid interface{}, opt *ListProjectPackagesOptions, options ...RequestOptionFunc) ([]*Package, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/packages", pathEscape(project))

	req, err := s.client.NewRequest("GET", u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	var packages []*Package
	resp, err := s.client.Do(req, &packages)
	if err != nil {
		return nil, resp, err
	}

	return packages, resp, err
}

// DeleteRegistryRepository deletes a repository in a registry.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/packages.html#delete-a-project-package
func (s *PackagesService) DeleteProjectPackage(pid interface{}, packageId int, options ...RequestOptionFunc) (*Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, err
	}
	u := fmt.Sprintf("projects/%s/packages/%d", pathEscape(project), packageId)

	req, err := s.client.NewRequest("DELETE", u, nil, options)
	if err != nil {
		return nil, err
	}

	return s.client.Do(req, nil)
}

// ListPackageFilesOptions represents the available
// ListPackageFiles() options.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/packages.html#list-package-files
type ListPackageFilesOptions ListOptions

// ListPackageFiles gets a list of files that are within a package
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/packages.html#list-package-files
func (s *PackagesService) ListPackageFiles(pid interface{}, packageId int, opt *ListPackageFilesOptions, options ...RequestOptionFunc) ([]*PackageFile, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/packages/%d/package_files",
		pathEscape(project),
		packageId,
	)

	req, err := s.client.NewRequest("GET", u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	var tags []*PackageFile
	resp, err := s.client.Do(req, &tags)
	if err != nil {
		return nil, resp, err
	}

	return tags, resp, err
}
<commit_msg>Options for list are passed by query, because GET. So add correct naming for those in the struct.<commit_after>//
// Copyright 2021, Kordian Bruck
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package gitlab

import (
	"fmt"
	"time"
)

// PackagesService handles communication with the packages related methods
// of the GitLab API.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/packages.html
type PackagesService struct {
	client *Client
}

// Package represents a GitLab single package
//
// GitLab API docs: https://docs.gitlab.com/ee/api/packages.html
type Package struct {
	ID        int        `json:"id"`
	Name      string     `json:"name"`
	Version   string     `json:"version"`
	Type      string     `json:"package_type"`
	CreatedAt *time.Time `json:"created_at"`
}

func (s Package) String() string {
	return Stringify(s)
}

// PackageFile represents one file contained within a package
//
// GitLab API docs: https://docs.gitlab.com/ee/api/packages.html
type PackageFile struct {
	ID        int        `json:"id"`
	PackageId int        `json:"package_id"`
	CreatedAt *time.Time `json:"created_at"`
	FileName  string     `json:"file_name"`
	Size      int        `json:"size"`
	MD5       string     `json:"file_md5"`
	SHA1      string     `json:"file_sha1"`
}

func (s PackageFile) String() string {
	return Stringify(s)
}

// ListProjectPackagesOptions are the parameters available in a ListProjectPackages() Operation
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/packages.html#within-a-project
type ListProjectPackagesOptions struct {
	ListOptions
	OrderBy            string `url:"order_by,omitempty" json:"order_by,omitempty"`
	Sort               string `url:"sort,omitempty" json:"sort,omitempty"`
	Type               string `url:"package_type,omitempty" json:"package_type,omitempty"`
	Name               string `url:"package_name,omitempty" json:"package_name,omitempty"`
	IncludeVersionless bool   `url:"include_versionless,omitempty" json:"include_versionless,omitempty"`
}

// ListProjectPackages gets a list of packages in a project.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/packages.html#within-a-project
func (s *PackagesService) ListProjectPackages(pid interface{}, opt *ListProjectPackagesOptions, options ...RequestOptionFunc) ([]*Package, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/packages", pathEscape(project))

	req, err := s.client.NewRequest("GET", u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	var packages []*Package
	resp, err := s.client.Do(req, &packages)
	if err != nil {
		return nil, resp, err
	}

	return packages, resp, err
}

// DeleteRegistryRepository deletes a repository in a registry.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/packages.html#delete-a-project-package
func (s *PackagesService) DeleteProjectPackage(pid interface{}, packageId int, options ...RequestOptionFunc) (*Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, err
	}
	u := fmt.Sprintf("projects/%s/packages/%d", pathEscape(project), packageId)

	req, err := s.client.NewRequest("DELETE", u, nil, options)
	if err != nil {
		return nil, err
	}

	return s.client.Do(req, nil)
}

// ListPackageFilesOptions represents the available
// ListPackageFiles() options.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/packages.html#list-package-files
type ListPackageFilesOptions ListOptions

// ListPackageFiles gets a list of files that are within a package
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/packages.html#list-package-files
func (s *PackagesService) ListPackageFiles(pid interface{}, packageId int, opt *ListPackageFilesOptions, options ...RequestOptionFunc) ([]*PackageFile, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/packages/%d/package_files",
		pathEscape(project),
		packageId,
	)

	req, err := s.client.NewRequest("GET", u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	var tags []*PackageFile
	resp, err := s.client.Do(req, &tags)
	if err != nil {
		return nil, resp, err
	}

	return tags, resp, err
}
<|endoftext|>


<commit_before>package paginate

import (
	"fmt"
	"net/url"
	"strconv"
)

const (
	ASC  = 1
	DESC = -1
)

type Item interface {
	OrderValue(order string) string
}

type Interface interface {
	Equal(order string, i, j int) bool
	Value(order string, i int) string
	Len() int
}

type Page struct {
	Items []Item
}

func (p *Page) Equal(order string, i, j int) bool {
	return p.Value(order, i) == p.Value(order, j)
}

func (p *Page) Value(order string, i int) string {
	return p.Items[i].OrderValue(order)
}

func (p *Page) Len() int {
	return len(p.Items)
}

type Cursor struct {
	Value     string
	Offset    int
	Count     int
	Order     string
	Direction int
}

type Config Cursor

func (c Cursor) DirectionString() string {
	if c.Direction == ASC {
		return "asc"
	} else {
		return "desc"
	}
}

func NewCursorFromQuery(query string) (Cursor, error) {
	c := Cursor{}
	m, err := url.ParseQuery(query)
	if err != nil {
		return c, err
	}

	if v, ok := m["value"]; ok {
		c.Value = v[0]
	}
	if v, ok := m["offset"]; ok {
		offset, err := strconv.Atoi(v[0])
		if err != nil {
			return c, err
		}
		c.Offset = offset
	}
	if v, ok := m["count"]; ok {
		count, err := strconv.Atoi(v[0])
		if err != nil {
			return c, err
		}
		c.Count = count
	}
	if v, ok := m["order"]; ok {
		c.Order = v[0]
	}
	if v, ok := m["direction"]; ok {
		switch {
		case v[0] == "desc":
			c.Direction = DESC
		case v[0] == "asc":
			c.Direction = ASC
		default:
			return c, fmt.Errorf("'%s' in not a supported direction, use asc or desc", v)
		}
	}
	return c, nil
}

type Pagination struct {
	Cursor
	config Config
}

func (p *Pagination) max(items Interface) int {
	if items.Len() <= p.Count {
		return items.Len() - 1
	} else {
		return p.Count
	}
}

func (p *Pagination) equalCount(items Interface, order string, max int) int {
	c := 0
	for i := 0; i < p.Count; i++ {
		if items.Equal(order, i, max) {
			c += 1
		}
	}
	return c
}

func NewPagination(cursor Cursor, config Config) *Pagination {
	if cursor.Value == "" {
		cursor.Value = config.Value
	}
	if cursor.Offset == 0 {
		cursor.Offset = config.Offset
	}
	if cursor.Count == 0 {
		cursor.Count = config.Count
	}
	if cursor.Order == "" {
		cursor.Order = config.Order
	}
	if cursor.Direction == 0 {
		cursor.Direction = config.Direction
	}
	return &Pagination{cursor, config}
}

func FromUrl(rawurl *url.URL, config Config) (*Pagination, error) {
	cursor, err := NewCursorFromQuery(rawurl.RawQuery)
	if err != nil {
		return nil, err
	}
	return NewPagination(cursor, config), nil
}

func (p *Pagination) after(items Interface, last, direction int) *Pagination {
	if items.Len() == 0 {
		return nil
	}
	value := items.Value(p.Order, last)
	offset := p.equalCount(items, p.Order, last)
	if offset == p.Count && value == p.Value {
		offset += p.Offset
	}
	cursor := Cursor{value, offset, p.Count, p.Order, direction}
	return NewPagination(cursor, p.config)
}

func (p *Pagination) Prev(items Interface) *Pagination {
	min := 0
	return p.after(items, min, p.Direction*-1)
}

func (p *Pagination) Next(items Interface, next_page_prefetched bool) *Pagination {
	if next_page_prefetched && items.Len() <= p.Count {
		return nil
	}
	max := p.max(items)
	return p.after(items, max, p.Direction)
}

func (p *Pagination) ToUrl(baseurl *url.URL) (*url.URL, error) {
	query, err := url.ParseQuery(baseurl.RawQuery)
	if err != nil {
		return nil, err
	}
	query.Set("value", p.Value)
	query.Set("offset", strconv.Itoa(p.Offset))
	query.Set("count", strconv.Itoa(p.Count))
	query.Set("order", p.Order)
	query.Set("direction", p.DirectionString())
	newurl, err := url.Parse(baseurl.String())
	if err != nil {
		return nil, err
	}
	newurl.RawQuery = query.Encode()
	return newurl, nil
}
<commit_msg>Simplify everything<commit_after>package paginate

import (
	"fmt"
	"net/url"
	"strconv"
)

const (
	ASC  = 1
	DESC = -1
)

type Pager interface {
	PaginationValue(p *Pagination) string
}

type Cursor struct {
	Value     string
	Offset    int
	Count     int
	Order     string
	Direction int
}

type Pagination struct {
	Cursor
	defaults Cursor
}

func NewCursorFromQuery(query string) (Cursor, error) {
	c := Cursor{}
	m, err := url.ParseQuery(query)
	if err != nil {
		return c, err
	}

	if v, ok := m["value"]; ok {
		c.Value = v[0]
	}
	if v, ok := m["offset"]; ok {
		offset, err := strconv.Atoi(v[0])
		if err != nil {
			return c, err
		}
		c.Offset = offset
	}
	if v, ok := m["count"]; ok {
		count, err := strconv.Atoi(v[0])
		if err != nil {
			return c, err
		}
		c.Count = count
	}
	if v, ok := m["order"]; ok {
		c.Order = v[0]
	}
	if v, ok := m["direction"]; ok {
		direction, err := strconv.Atoi(v[0])
		if err != nil {
			return c, err
		}
		if direction == ASC || direction == DESC {
			c.Direction = direction
		} else {
			return c, fmt.Errorf("'%s' in not a supported direction, use -1 (ASC) or 1 (DESC)", direction)
		}
	}
	return c, nil
}

func (p *Pagination) max(items []Pager) int {
	if len(items) <= p.Count {
		return len(items) - 1
	} else {
		return p.Count
	}
}

func (p *Pagination) equalCount(items []Pager, max int) int {
	c := 0
	for i := 0; i < p.Count; i++ {
		if items[i].PaginationValue(p) == items[max].PaginationValue(p) {
			c += 1
		}
	}
	return c
}

func NewPagination(cursor, defaults Cursor) *Pagination {
	if cursor.Value == "" {
		cursor.Value = defaults.Value
	}
	if cursor.Offset == 0 {
		cursor.Offset = defaults.Offset
	}
	if cursor.Count == 0 {
		cursor.Count = defaults.Count
	}
	if cursor.Order == "" {
		cursor.Order = defaults.Order
	}
	if cursor.Direction == 0 {
		cursor.Direction = defaults.Direction
	}
	return &Pagination{cursor, defaults}
}

func FromUrl(rawurl *url.URL, defaults Cursor) (*Pagination, error) {
	cursor, err := NewCursorFromQuery(rawurl.RawQuery)
	if err != nil {
		return nil, err
	}
	return NewPagination(cursor, defaults), nil
}

func (p *Pagination) after(items []Pager, last, direction int) *Pagination {
	if len(items) == 0 {
		return nil
	}
	value := items[last].PaginationValue(p)
	offset := p.equalCount(items, last)
	if offset == p.Count && value == p.Value {
		offset += p.Offset
	}
	cursor := Cursor{value, offset, p.Count, p.Order, direction}
	return NewPagination(cursor, p.defaults)
}

func (p *Pagination) Prev(items []Pager) *Pagination {
	min := 0
	return p.after(items, min, p.Direction*-1)
}

func (p *Pagination) Next(items []Pager, next_page_prefetched bool) *Pagination {
	if next_page_prefetched && len(items) <= p.Count {
		return nil
	}
	max := p.max(items)
	return p.after(items, max, p.Direction)
}

func (p *Pagination) ToUrl(baseurl *url.URL) (*url.URL, error) {
	query, err := url.ParseQuery(baseurl.RawQuery)
	if err != nil {
		return nil, err
	}
	query.Set("value", p.Value)
	query.Set("offset", strconv.Itoa(p.Offset))
	query.Set("count", strconv.Itoa(p.Count))
	query.Set("order", p.Order)
	query.Set("direction", strconv.Itoa(p.Direction))
	newurl, err := url.Parse(baseurl.String())
	if err != nil {
		return nil, err
	}
	newurl.RawQuery = query.Encode()
	return newurl, nil
}
<|endoftext|>


<commit_before>package bild

import (
	"runtime"
	"sync"
)

// ParallelizationEnabled sets if package should use goroutines when possible
var ParallelizationEnabled = true

func parallelize(size int, fn func(start, end int)) {
	if !ParallelizationEnabled {
		fn(0, size)
	} else {
		var wg sync.WaitGroup
		procs := runtime.GOMAXPROCS(0)
		partSize := size / procs
		wg.Add(procs)

		for i := 0; i < procs; i++ {
			i := i // needed for the closure to work
			go func() {
				defer wg.Done()
				fn(partSize*i, partSize*(i+1))
			}()
		}

		wg.Wait()
	}
}
<commit_msg>fixed left < partSize case<commit_after>package bild

import (
	"runtime"
	"sync"
)

// ParallelizationEnabled sets if package should use goroutines when possible
var ParallelizationEnabled = true

func parallelize(size int, fn func(start, end int)) {
	if !ParallelizationEnabled {
		fn(0, size)
	} else {
		var wg sync.WaitGroup
		procs := runtime.GOMAXPROCS(0)

		counter := size
		partSize := size / procs
		for counter > 0 {
			start := counter - partSize
			end := counter
			if start < 0 {
				start = 0
			}
			counter -= partSize
			wg.Add(1)
			go func() {
				defer wg.Done()
				fn(start, end)
			}()
		}

		wg.Wait()
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package fs/testing provides fake implementations of the fs package.
//
// These implementations can be used to mock out the file system in tests.
package testing

import (
	"fmt"
	"github.com/globocom/tsuru/fs"
	"os"
	"strings"
	"syscall"
)

// FakeFile representss a fake instance of the File interface.
//
// Methods from FakeFile act like methods in os.File, but instead of working in
// a real file, them work in an internal string.
//
// An instance of FakeFile is returned by RecordingFs.Open method.
type FakeFile struct {
	content string
	current int64
	r       *strings.Reader
}

func (f *FakeFile) reader() *strings.Reader {
	if f.r == nil {
		f.r = strings.NewReader(f.content)
	}
	return f.r
}

func (f *FakeFile) Close() error {
	f.current = 0
	return nil
}

func (f *FakeFile) Read(p []byte) (n int, err error) {
	n, err = f.reader().Read(p)
	f.current += int64(n)
	return
}

func (f *FakeFile) ReadAt(p []byte, off int64) (n int, err error) {
	n, err = f.reader().ReadAt(p, off)
	f.current += off + int64(n)
	return
}

func (f *FakeFile) Seek(offset int64, whence int) (int64, error) {
	var err error
	f.current, err = f.reader().Seek(offset, whence)
	return f.current, err
}

func (f *FakeFile) Stat() (fi os.FileInfo, err error) {
	return
}

func (f *FakeFile) Write(p []byte) (n int, err error) {
	n = len(p)
	f.content = f.content[:f.current] + string(p)
	return
}

func (f *FakeFile) WriteString(s string) (ret int, err error) {
	ret = len(s)
	f.content = s
	return
}

func (f *FakeFile) Truncate(size int64) error {
	f.content = f.content[:size]
	return nil
}

// RecordingFs implements the Fs interface providing a "recording" file system.
//
// A recording file system is a file system that does not execute any action,
// just record them.
//
// All methods from RecordingFs never return errors.
type RecordingFs struct {
	actions []string
	files   map[string]*FakeFile

	// FileContent is used to provide content for files opened using
	// RecordingFs.
	FileContent string
}

// HasAction checks if a given action was executed in the filesystem.
//
// For example, when you call the Open method with the "/tmp/file.txt"
// argument, RecordingFs will store locally the action "open /tmp/file.txt" and
// you can check it calling HasAction:
//
//     rfs.Open("/tmp/file.txt")
//     rfs.HasAction("open /tmp/file.txt") // true
func (r *RecordingFs) HasAction(action string) bool {
	for _, a := range r.actions {
		if action == a {
			return true
		}
	}
	return false
}

func (r *RecordingFs) open(name string, read bool) (fs.File, error) {
	if r.files == nil {
		r.files = make(map[string]*FakeFile)
		if r.FileContent == "" && read {
			return nil, syscall.ENOENT
		}
	} else if f, ok := r.files[name]; ok {
		f.r = nil
		return f, nil
	} else if r.FileContent == "" && read {
		return nil, syscall.ENOENT
	}
	fil := &FakeFile{content: r.FileContent}
	r.files[name] = fil
	return fil, nil
}

// Create records the action "create <name>" and returns an instance of
// FakeFile and nil error.
func (r *RecordingFs) Create(name string) (fs.File, error) {
	r.actions = append(r.actions, "create "+name)
	return r.open(name, false)
}

// Mkdir records the action "mkdir <name> with mode <perm>" and returns nil.
func (r *RecordingFs) Mkdir(name string, perm os.FileMode) error {
	r.actions = append(r.actions, fmt.Sprintf("mkdir %s with mode %#o", name, perm))
	return nil
}

// MkdirAll records the action "mkdirall <path> with mode <perm>" and returns
// nil.
func (r *RecordingFs) MkdirAll(path string, perm os.FileMode) error {
	r.actions = append(r.actions, fmt.Sprintf("mkdirall %s with mode %#o", path, perm))
	return nil
}

// Open records the action "open <name>" and returns an instance of FakeFile
// and nil error.
func (r *RecordingFs) Open(name string) (fs.File, error) {
	r.actions = append(r.actions, "open "+name)
	return r.open(name, true)
}

// OpenFile records the action "openfile <name> with mode <perm>" and returns
// an instance of FakeFile and nil error.
func (r *RecordingFs) OpenFile(name string, flag int, perm os.FileMode) (fs.File, error) {
	r.actions = append(r.actions, fmt.Sprintf("openfile %s with mode %#o", name, perm))
	read := flag&syscall.O_CREAT != syscall.O_CREAT &&
		flag&syscall.O_APPEND != syscall.O_APPEND &&
		flag&syscall.O_RDWR != syscall.O_RDWR &&
		flag&syscall.O_TRUNC != syscall.O_TRUNC &&
		flag&syscall.O_WRONLY != syscall.O_WRONLY
	f, err := r.open(name, read)
	if flag&syscall.O_TRUNC == syscall.O_TRUNC {
		f.Truncate(0)
	}
	return f, err
}

func (r *RecordingFs) deleteFile(name string) {
	if r.files != nil {
		delete(r.files, name)
	}
}

// Remove records the action "remove <name>" and returns nil.
func (r *RecordingFs) Remove(name string) error {
	r.actions = append(r.actions, "remove "+name)
	r.deleteFile(name)
	return nil
}

// RemoveAll records the action "removeall <path>" and returns nil.
func (r *RecordingFs) RemoveAll(path string) error {
	r.actions = append(r.actions, "removeall "+path)
	r.deleteFile(path)
	return nil
}

// Stat records the action "stat <name>" and returns nil, nil.
func (r *RecordingFs) Stat(name string) (os.FileInfo, error) {
	r.actions = append(r.actions, "stat "+name)
	return nil, nil
}

// FailureFs is like RecordingFs, except that it returns ENOENT on Open,
// OpenFile and Remove.
type FailureFs struct {
	RecordingFs
}

// Open is used to simulate ENOENT.
func (r *FailureFs) Open(name string) (fs.File, error) {
	r.RecordingFs.Open(name)
	err := os.PathError{Err: syscall.ENOENT, Path: name}
	return nil, &err
}

func (r *FailureFs) Remove(name string) error {
	r.RecordingFs.Remove(name)
	return &os.PathError{Err: syscall.ENOENT, Path: name}
}

func (r *FailureFs) RemoveAll(path string) error {
	r.RecordingFs.RemoveAll(path)
	return &os.PathError{Err: syscall.ENOENT, Path: path}
}

func (r *FailureFs) OpenFile(name string, flag int, perm os.FileMode) (fs.File, error) {
	r.RecordingFs.OpenFile(name, flag, perm)
	return r.Open(name)
}
<commit_msg>fs/testing: update license headers<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package fs/testing provides fake implementations of the fs package.
//
// These implementations can be used to mock out the file system in tests.
package testing

import (
	"fmt"
	"github.com/globocom/tsuru/fs"
	"os"
	"strings"
	"syscall"
)

// FakeFile representss a fake instance of the File interface.
//
// Methods from FakeFile act like methods in os.File, but instead of working in
// a real file, them work in an internal string.
//
// An instance of FakeFile is returned by RecordingFs.Open method.
type FakeFile struct {
	content string
	current int64
	r       *strings.Reader
}

func (f *FakeFile) reader() *strings.Reader {
	if f.r == nil {
		f.r = strings.NewReader(f.content)
	}
	return f.r
}

func (f *FakeFile) Close() error {
	f.current = 0
	return nil
}

func (f *FakeFile) Read(p []byte) (n int, err error) {
	n, err = f.reader().Read(p)
	f.current += int64(n)
	return
}

func (f *FakeFile) ReadAt(p []byte, off int64) (n int, err error) {
	n, err = f.reader().ReadAt(p, off)
	f.current += off + int64(n)
	return
}

func (f *FakeFile) Seek(offset int64, whence int) (int64, error) {
	var err error
	f.current, err = f.reader().Seek(offset, whence)
	return f.current, err
}

func (f *FakeFile) Stat() (fi os.FileInfo, err error) {
	return
}

func (f *FakeFile) Write(p []byte) (n int, err error) {
	n = len(p)
	f.content = f.content[:f.current] + string(p)
	return
}

func (f *FakeFile) WriteString(s string) (ret int, err error) {
	ret = len(s)
	f.content = s
	return
}

func (f *FakeFile) Truncate(size int64) error {
	f.content = f.content[:size]
	return nil
}

// RecordingFs implements the Fs interface providing a "recording" file system.
//
// A recording file system is a file system that does not execute any action,
// just record them.
//
// All methods from RecordingFs never return errors.
type RecordingFs struct {
	actions []string
	files   map[string]*FakeFile

	// FileContent is used to provide content for files opened using
	// RecordingFs.
	FileContent string
}

// HasAction checks if a given action was executed in the filesystem.
//
// For example, when you call the Open method with the "/tmp/file.txt"
// argument, RecordingFs will store locally the action "open /tmp/file.txt" and
// you can check it calling HasAction:
//
//     rfs.Open("/tmp/file.txt")
//     rfs.HasAction("open /tmp/file.txt") // true
func (r *RecordingFs) HasAction(action string) bool {
	for _, a := range r.actions {
		if action == a {
			return true
		}
	}
	return false
}

func (r *RecordingFs) open(name string, read bool) (fs.File, error) {
	if r.files == nil {
		r.files = make(map[string]*FakeFile)
		if r.FileContent == "" && read {
			return nil, syscall.ENOENT
		}
	} else if f, ok := r.files[name]; ok {
		f.r = nil
		return f, nil
	} else if r.FileContent == "" && read {
		return nil, syscall.ENOENT
	}
	fil := &FakeFile{content: r.FileContent}
	r.files[name] = fil
	return fil, nil
}

// Create records the action "create <name>" and returns an instance of
// FakeFile and nil error.
func (r *RecordingFs) Create(name string) (fs.File, error) {
	r.actions = append(r.actions, "create "+name)
	return r.open(name, false)
}

// Mkdir records the action "mkdir <name> with mode <perm>" and returns nil.
func (r *RecordingFs) Mkdir(name string, perm os.FileMode) error {
	r.actions = append(r.actions, fmt.Sprintf("mkdir %s with mode %#o", name, perm))
	return nil
}

// MkdirAll records the action "mkdirall <path> with mode <perm>" and returns
// nil.
func (r *RecordingFs) MkdirAll(path string, perm os.FileMode) error {
	r.actions = append(r.actions, fmt.Sprintf("mkdirall %s with mode %#o", path, perm))
	return nil
}

// Open records the action "open <name>" and returns an instance of FakeFile
// and nil error.
func (r *RecordingFs) Open(name string) (fs.File, error) {
	r.actions = append(r.actions, "open "+name)
	return r.open(name, true)
}

// OpenFile records the action "openfile <name> with mode <perm>" and returns
// an instance of FakeFile and nil error.
func (r *RecordingFs) OpenFile(name string, flag int, perm os.FileMode) (fs.File, error) {
	r.actions = append(r.actions, fmt.Sprintf("openfile %s with mode %#o", name, perm))
	read := flag&syscall.O_CREAT != syscall.O_CREAT &&
		flag&syscall.O_APPEND != syscall.O_APPEND &&
		flag&syscall.O_RDWR != syscall.O_RDWR &&
		flag&syscall.O_TRUNC != syscall.O_TRUNC &&
		flag&syscall.O_WRONLY != syscall.O_WRONLY
	f, err := r.open(name, read)
	if flag&syscall.O_TRUNC == syscall.O_TRUNC {
		f.Truncate(0)
	}
	return f, err
}

func (r *RecordingFs) deleteFile(name string) {
	if r.files != nil {
		delete(r.files, name)
	}
}

// Remove records the action "remove <name>" and returns nil.
func (r *RecordingFs) Remove(name string) error {
	r.actions = append(r.actions, "remove "+name)
	r.deleteFile(name)
	return nil
}

// RemoveAll records the action "removeall <path>" and returns nil.
func (r *RecordingFs) RemoveAll(path string) error {
	r.actions = append(r.actions, "removeall "+path)
	r.deleteFile(path)
	return nil
}

// Stat records the action "stat <name>" and returns nil, nil.
func (r *RecordingFs) Stat(name string) (os.FileInfo, error) {
	r.actions = append(r.actions, "stat "+name)
	return nil, nil
}

// FailureFs is like RecordingFs, except that it returns ENOENT on Open,
// OpenFile and Remove.
type FailureFs struct {
	RecordingFs
}

// Open is used to simulate ENOENT.
func (r *FailureFs) Open(name string) (fs.File, error) {
	r.RecordingFs.Open(name)
	err := os.PathError{Err: syscall.ENOENT, Path: name}
	return nil, &err
}

func (r *FailureFs) Remove(name string) error {
	r.RecordingFs.Remove(name)
	return &os.PathError{Err: syscall.ENOENT, Path: name}
}

func (r *FailureFs) RemoveAll(path string) error {
	r.RecordingFs.RemoveAll(path)
	return &os.PathError{Err: syscall.ENOENT, Path: path}
}

func (r *FailureFs) OpenFile(name string, flag int, perm os.FileMode) (fs.File, error) {
	r.RecordingFs.OpenFile(name, flag, perm)
	return r.Open(name)
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build freebsd openbsd netbsd darwin linux

package fsnotify

import (
	"os"
	"testing"
	"time"
)

func TestFsnotifyFakeSymlink(t *testing.T) {
	// Create an fsnotify watcher instance and initialize it
	watcher, err := NewWatcher()
	if err != nil {
		t.Fatalf("NewWatcher() failed: %s", err)
	}

	const testDir string = "_test"

	// Create directory to watch
	if os.Mkdir(testDir, 0777) != nil {
		t.Fatalf("Failed to create test directory: %s", err)
	}
	defer os.RemoveAll(testDir)

	var errorsReceived = 0
	// Receive errors on the error channel on a separate goroutine
	go func() {
		for errors := range watcher.Error {
			t.Logf("Received error: %s", errors)
			errorsReceived++
		}
	}()

	// Count the CREATE events received
	var createEventsReceived = 0
	var otherEventsReceived = 0
	go func() {
		for ev := range watcher.Event {
			t.Logf("event received: %s", ev)
			if ev.IsCreate() {
				createEventsReceived++
			} else {
				otherEventsReceived++
			}
		}
	}()

	// Add a watch for testDir
	err = watcher.Watch(testDir)
	if err != nil {
		t.Fatalf("Watcher.Watch() failed: %s", err)
	}

	if os.Symlink("_test/zzz", "_test/zzznew") != nil {
		t.Fatalf("Failed to create bogus symlink: %s", err)
	}
	t.Logf("Created bogus symlink")

	// We expect this event to be received almost immediately, but let's wait 500 ms to be sure
	time.Sleep(500 * time.Millisecond)

	// Should not be error, just no events for broken links (watching nothing)
	if errorsReceived > 0 {
		t.Fatal("fsnotify errors have been received.")
	}
	if otherEventsReceived > 0 {
		t.Fatal("fsnotify other events received on the broken link")
	}

	// Except for 1 create event (for the link itself)
	if createEventsReceived == 0 {
		t.Fatal("fsnotify create events were not received after 500 ms")
	}
	if createEventsReceived > 1 {
		t.Fatal("fsnotify more create events received than expected")
	}

	// Try closing the fsnotify instance
	t.Log("calling Close()")
	watcher.Close()
}
<commit_msg>Small error reporting fix<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build freebsd openbsd netbsd darwin linux

package fsnotify

import (
	"os"
	"testing"
	"time"
)

func TestFsnotifyFakeSymlink(t *testing.T) {
	// Create an fsnotify watcher instance and initialize it
	watcher, err := NewWatcher()
	if err != nil {
		t.Fatalf("NewWatcher() failed: %s", err)
	}

	const testDir string = "_test"

	// Create directory to watch
	if err := os.Mkdir(testDir, 0777); err != nil {
		t.Fatalf("Failed to create test directory: %s", err)
	}
	defer os.RemoveAll(testDir)

	var errorsReceived = 0
	// Receive errors on the error channel on a separate goroutine
	go func() {
		for errors := range watcher.Error {
			t.Logf("Received error: %s", errors)
			errorsReceived++
		}
	}()

	// Count the CREATE events received
	var createEventsReceived = 0
	var otherEventsReceived = 0
	go func() {
		for ev := range watcher.Event {
			t.Logf("event received: %s", ev)
			if ev.IsCreate() {
				createEventsReceived++
			} else {
				otherEventsReceived++
			}
		}
	}()

	// Add a watch for testDir
	err = watcher.Watch(testDir)
	if err != nil {
		t.Fatalf("Watcher.Watch() failed: %s", err)
	}

	if os.Symlink("_test/zzz", "_test/zzznew") != nil {
		t.Fatalf("Failed to create bogus symlink: %s", err)
	}
	t.Logf("Created bogus symlink")

	// We expect this event to be received almost immediately, but let's wait 500 ms to be sure
	time.Sleep(500 * time.Millisecond)

	// Should not be error, just no events for broken links (watching nothing)
	if errorsReceived > 0 {
		t.Fatal("fsnotify errors have been received.")
	}
	if otherEventsReceived > 0 {
		t.Fatal("fsnotify other events received on the broken link")
	}

	// Except for 1 create event (for the link itself)
	if createEventsReceived == 0 {
		t.Fatal("fsnotify create events were not received after 500 ms")
	}
	if createEventsReceived > 1 {
		t.Fatal("fsnotify more create events received than expected")
	}

	// Try closing the fsnotify instance
	t.Log("calling Close()")
	watcher.Close()
}
<|endoftext|>


<commit_before>package client

import (
	"github.com/Symantec/Dominator/lib/hash"
	proto "github.com/Symantec/Dominator/proto/filegenerator"
	"reflect"
)

func buildMachine(machine Machine) *machineType {
	computedFiles := make(map[string]string, len(machine.ComputedFiles))
	sourceToPaths := make(map[string][]string)
	for _, computedFile := range machine.ComputedFiles {
		computedFiles[computedFile.Pathname] = computedFile.Source
		pathnames := sourceToPaths[computedFile.Source]
		pathnames = append(pathnames, computedFile.Pathname)
		sourceToPaths[computedFile.Source] = pathnames
	}
	return &machineType{
		machine:       machine.Machine,
		computedFiles: computedFiles,
		sourceToPaths: sourceToPaths}
}

func (m *Manager) addMachine(machine *machineType) {
	hostname := machine.machine.Hostname
	_, ok := m.machineMap[hostname]
	if ok {
		panic(hostname + ": already added")
	}
	m.machineMap[hostname] = machine
	m.sendYieldRequests(machine)
}

func (m *Manager) removeMachine(hostname string) {
	if machine, ok := m.machineMap[hostname]; !ok {
		panic(hostname + ": not present")
	} else {
		delete(m.machineMap, hostname)
		close(machine.updateChannel)
	}
}

func (m *Manager) updateMachine(machine *machineType) {
	hostname := machine.machine.Hostname
	if mapMachine, ok := m.machineMap[hostname]; !ok {
		panic(hostname + ": not present")
	} else {
		sendRequests := false
		if machine.machine != mapMachine.machine {
			mapMachine.machine = machine.machine
			sendRequests = true
		}
		if !reflect.DeepEqual(machine.computedFiles, mapMachine.computedFiles) {
			sendRequests = true
			mapMachine.computedFiles = machine.computedFiles
		}
		if !reflect.DeepEqual(machine.sourceToPaths, mapMachine.sourceToPaths) {
			sendRequests = true
			mapMachine.sourceToPaths = machine.sourceToPaths
		}
		if sendRequests {
			m.sendYieldRequests(mapMachine)
		}
	}
}

func (m *Manager) sendYieldRequests(machine *machineType) {
	for sourceName, pathnames := range machine.sourceToPaths {
		request := &proto.ClientRequest{
			YieldRequest: &proto.YieldRequest{machine.machine, pathnames}}
		m.getSource(sourceName).sendChannel <- request
	}
}

func (m *Manager) handleYieldResponse(machine *machineType,
	files []proto.FileInfo) {
	objectsToWaitFor := make(map[hash.Hash]struct{})
	waiterChannel := make(chan hash.Hash)
	for _, file := range files {
		sourceName, ok := machine.computedFiles[file.Pathname]
		if !ok {
			m.logger.Printf("no source name for: %s on: %s\n",
				file.Pathname, machine.machine.Hostname)
			continue
		}
		source, ok := m.sourceMap[sourceName]
		if !ok {
			panic("no source for: " + sourceName)
		}
		hashes := make([]hash.Hash, 1)
		hashes[0] = file.Hash
		if lengths, err := m.objectServer.CheckObjects(hashes); err != nil {
			panic(err)
		} else if lengths[0] < 1 {
			request := &proto.ClientRequest{
				GetObjectRequest: &proto.GetObjectRequest{file.Hash}}
			source.sendChannel <- request
			objectsToWaitFor[file.Hash] = struct{}{}
			if _, ok := m.objectWaiters[file.Hash]; !ok {
				m.objectWaiters[file.Hash] = make([]chan<- hash.Hash, 0, 1)
			}
			m.objectWaiters[file.Hash] = append(m.objectWaiters[file.Hash],
				waiterChannel)
		}
	}
	if len(objectsToWaitFor) > 0 {
		go waitForObjectsAndSendUpdate(waiterChannel, objectsToWaitFor,
			machine.updateChannel, files)
	} else {
		machine.updateChannel <- files
	}
}

func waitForObjectsAndSendUpdate(objectChannel <-chan hash.Hash,
	objectsToWaitFor map[hash.Hash]struct{},
	updateChannel chan<- []proto.FileInfo, files []proto.FileInfo) {
	defer func() {
		recover() // If updateChannel is closed, it means the machine went away.
	}()
	for hashVal := range objectChannel {
		delete(objectsToWaitFor, hashVal)
		if len(objectsToWaitFor) < 1 {
			updateChannel <- files // This will panic if the machine went away.
		}
	}
}
<commit_msg>Fix hang in lib/filegen/client if yield request with zero hash is received.<commit_after>package client

import (
	"github.com/Symantec/Dominator/lib/hash"
	proto "github.com/Symantec/Dominator/proto/filegenerator"
	"reflect"
)

var zeroHash hash.Hash

func buildMachine(machine Machine) *machineType {
	computedFiles := make(map[string]string, len(machine.ComputedFiles))
	sourceToPaths := make(map[string][]string)
	for _, computedFile := range machine.ComputedFiles {
		computedFiles[computedFile.Pathname] = computedFile.Source
		pathnames := sourceToPaths[computedFile.Source]
		pathnames = append(pathnames, computedFile.Pathname)
		sourceToPaths[computedFile.Source] = pathnames
	}
	return &machineType{
		machine:       machine.Machine,
		computedFiles: computedFiles,
		sourceToPaths: sourceToPaths}
}

func (m *Manager) addMachine(machine *machineType) {
	hostname := machine.machine.Hostname
	_, ok := m.machineMap[hostname]
	if ok {
		panic(hostname + ": already added")
	}
	m.machineMap[hostname] = machine
	m.sendYieldRequests(machine)
}

func (m *Manager) removeMachine(hostname string) {
	if machine, ok := m.machineMap[hostname]; !ok {
		panic(hostname + ": not present")
	} else {
		delete(m.machineMap, hostname)
		close(machine.updateChannel)
	}
}

func (m *Manager) updateMachine(machine *machineType) {
	hostname := machine.machine.Hostname
	if mapMachine, ok := m.machineMap[hostname]; !ok {
		panic(hostname + ": not present")
	} else {
		sendRequests := false
		if machine.machine != mapMachine.machine {
			mapMachine.machine = machine.machine
			sendRequests = true
		}
		if !reflect.DeepEqual(machine.computedFiles, mapMachine.computedFiles) {
			sendRequests = true
			mapMachine.computedFiles = machine.computedFiles
		}
		if !reflect.DeepEqual(machine.sourceToPaths, mapMachine.sourceToPaths) {
			sendRequests = true
			mapMachine.sourceToPaths = machine.sourceToPaths
		}
		if sendRequests {
			m.sendYieldRequests(mapMachine)
		}
	}
}

func (m *Manager) sendYieldRequests(machine *machineType) {
	for sourceName, pathnames := range machine.sourceToPaths {
		request := &proto.ClientRequest{
			YieldRequest: &proto.YieldRequest{machine.machine, pathnames}}
		m.getSource(sourceName).sendChannel <- request
	}
}

func (m *Manager) handleYieldResponse(machine *machineType,
	files []proto.FileInfo) {
	objectsToWaitFor := make(map[hash.Hash]struct{})
	waiterChannel := make(chan hash.Hash)
	// Get list of objects to wait for.
	for _, file := range files {
		sourceName, ok := machine.computedFiles[file.Pathname]
		if !ok {
			m.logger.Printf("no source name for: %s on: %s\n",
				file.Pathname, machine.machine.Hostname)
			continue
		}
		source, ok := m.sourceMap[sourceName]
		if !ok {
			panic("no source for: " + sourceName)
		}
		if file.Hash == zeroHash {
			m.logger.Printf("Received zero hash for machine: %s file: %s\n",
				machine.machine.Hostname, file.Pathname)
			continue // No object.
		}
		hashes := []hash.Hash{file.Hash}
		if lengths, err := m.objectServer.CheckObjects(hashes); err != nil {
			panic(err)
		} else if lengths[0] < 1 {
			// Not already in objectserver, so add to the list to wait for.
			request := &proto.ClientRequest{
				GetObjectRequest: &proto.GetObjectRequest{file.Hash}}
			source.sendChannel <- request
			objectsToWaitFor[file.Hash] = struct{}{}
			if _, ok := m.objectWaiters[file.Hash]; !ok {
				m.objectWaiters[file.Hash] = make([]chan<- hash.Hash, 0, 1)
			}
			m.objectWaiters[file.Hash] = append(m.objectWaiters[file.Hash],
				waiterChannel)
		}
	}
	if len(objectsToWaitFor) > 0 {
		go waitForObjectsAndSendUpdate(waiterChannel, objectsToWaitFor,
			machine.updateChannel, files)
	} else {
		machine.updateChannel <- files
	}
}

func waitForObjectsAndSendUpdate(objectChannel <-chan hash.Hash,
	objectsToWaitFor map[hash.Hash]struct{},
	updateChannel chan<- []proto.FileInfo, files []proto.FileInfo) {
	defer func() {
		recover() // If updateChannel is closed, it means the machine went away.
	}()
	for hashVal := range objectChannel {
		delete(objectsToWaitFor, hashVal)
		if len(objectsToWaitFor) < 1 {
			updateChannel <- files // This will panic if the machine went away.
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
)

// A sentinel error returned by ObjectProxy.Sync.
var ErrNotCurrent error = errors.New("Source generation not current.")

// A view on a particular generation of an object in GCS that allows random
// access reads and writes.
//
// Reads may involve reading from a local cache. Writes are buffered locally
// until the Sync method is called, at which time a new generation of the
// object is created.
//
// This type is not safe for concurrent access. The user must provide external
// synchronization.
type ObjectProxy struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	// The name of the GCS object for which we are a proxy. Might not currently
	// exist in the bucket.
	name string

	/////////////////////////
	// Mutable state
	/////////////////////////

	// The specific generation of the object from which our local state is
	// branched. If we have no local state, the contents of this object are
	// exactly our contents. May be zero if our source is a "doesn't exist"
	// generation.
	srcGeneration uint64

	// A local temporary file containing our current contents. When non-nil, this
	// is the authority on our contents. When nil, our contents are defined by
	// the generation identified by srcGeneration.
	localFile *os.File

	// false if localFile is present but its contents may be different from the
	// contents of our source generation. Sync needs to do work iff this is true.
	//
	// INVARIANT: If srcGeneration == 0, then dirty
	// INVARIANT: If dirty, then localFile != nil
	dirty bool
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Create a view on the given GCS object generation, or zero if branching from
// a non-existent object (in which case the initial contents are empty).
func NewObjectProxy(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	srcGeneration uint64) (op *ObjectProxy, err error) {
	// Set up the basic struct.
	op = &ObjectProxy{
		bucket:        bucket,
		name:          name,
		srcGeneration: srcGeneration,
	}

	// For "doesn't exist" source generations, we must establish an empty local
	// file and mark the proxy dirty.
	if srcGeneration == 0 {
		if err = op.ensureLocalFile(ctx); err != nil {
			return
		}

		op.dirty = true
	}

	return
}

// Return the name of the proxied object. This may or may not be an object that
// currently exists in the bucket.
func (op *ObjectProxy) Name() string {
	return op.name
}

// Panic if any internal invariants are violated. Careful users can call this
// at appropriate times to help debug weirdness. Consider using
// syncutil.InvariantMutex to automate the process.
func (op *ObjectProxy) CheckInvariants() {
	// INVARIANT: If srcGeneration == 0, then dirty
	if op.srcGeneration == 0 && !op.dirty {
		panic("Expected dirty.")
	}

	// INVARIANT: If dirty, then localFile != nil
	if op.dirty && op.localFile == nil {
		panic("Expected non-nil localFile.")
	}
}

// Destroy any local file caches, putting the proxy into an indeterminate
// state. Should be used before dropping the final reference to the proxy.
func (op *ObjectProxy) Destroy() {
	panic("TODO")
}

// Return the current size in bytes of the content and an indication of whether
// the proxied object has changed out from under us (in which case Sync will
// fail).
func (op *ObjectProxy) Stat(
	ctx context.Context) (size uint64, clobbered bool, err error) {
	panic("TODO")
}

// Make a random access read into our view of the content. May block for
// network access.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) ReadAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// Serve the read from the file.
	n, err = op.localFile.ReadAt(buf, offset)

	return
}

// Make a random access write into our view of the content. May block for
// network access. Not guaranteed to be reflected remotely until after Sync is
// called successfully.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) WriteAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	panic("TODO")
}

// Truncate our view of the content to the given number of bytes, extending if
// n is greater than the current size. May block for network access. Not
// guaranteed to be reflected remotely until after Sync is called successfully.
func (op *ObjectProxy) Truncate(ctx context.Context, n uint64) (err error) {
	panic("TODO")
}

// If the proxy is dirty due to having been written to or due to having a nil
// source, save its current contents to GCS and return a generation number for
// a generation with exactly those contents. Do so with a precondition such
// that the creation will fail if the source generation is not current. In that
// case, return ErrNotCurrent.
func (op *ObjectProxy) Sync(ctx context.Context) (gen uint64, err error) {
	panic("TODO")
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Set up a local temporary file for the given generation of the given object.
// Special case: generation == 0 means an empty file.
func makeLocalFile(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	generation uint64) (f *os.File, err error) {
	// Create the file.
	f, err = ioutil.TempFile("", "object_proxy")
	if err != nil {
		err = fmt.Errorf("TempFile: %v", err)
		return
	}

	// Fetch its contents if necessary.
	if generation != 0 {
		panic("TODO")
	}

	return
}

// Ensure that op.localFile is non-nil with an authoritative view of op's
// contents.
func (op *ObjectProxy) ensureLocalFile(ctx context.Context) (err error) {
	// Is there anything to do?
	if op.localFile != nil {
		return
	}

	// Set up the file.
	f, err := makeLocalFile(ctx, op.bucket, op.name, op.srcGeneration)
	if err != nil {
		err = fmt.Errorf("makeLocalFile: %v", err)
		return
	}

	op.localFile = f
	return
}
<commit_msg>Implemented ObjectProxy.Destroy.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
)

// A sentinel error returned by ObjectProxy.Sync.
var ErrNotCurrent error = errors.New("Source generation not current.")

// A view on a particular generation of an object in GCS that allows random
// access reads and writes.
//
// Reads may involve reading from a local cache. Writes are buffered locally
// until the Sync method is called, at which time a new generation of the
// object is created.
//
// This type is not safe for concurrent access. The user must provide external
// synchronization.
type ObjectProxy struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	// The name of the GCS object for which we are a proxy. Might not currently
	// exist in the bucket.
	name string

	/////////////////////////
	// Mutable state
	/////////////////////////

	// The specific generation of the object from which our local state is
	// branched. If we have no local state, the contents of this object are
	// exactly our contents. May be zero if our source is a "doesn't exist"
	// generation.
	srcGeneration uint64

	// A local temporary file containing our current contents. When non-nil, this
	// is the authority on our contents. When nil, our contents are defined by
	// the generation identified by srcGeneration.
	localFile *os.File

	// false if localFile is present but its contents may be different from the
	// contents of our source generation. Sync needs to do work iff this is true.
	//
	// INVARIANT: If srcGeneration == 0, then dirty
	// INVARIANT: If dirty, then localFile != nil
	dirty bool
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Create a view on the given GCS object generation, or zero if branching from
// a non-existent object (in which case the initial contents are empty).
func NewObjectProxy(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	srcGeneration uint64) (op *ObjectProxy, err error) {
	// Set up the basic struct.
	op = &ObjectProxy{
		bucket:        bucket,
		name:          name,
		srcGeneration: srcGeneration,
	}

	// For "doesn't exist" source generations, we must establish an empty local
	// file and mark the proxy dirty.
	if srcGeneration == 0 {
		if err = op.ensureLocalFile(ctx); err != nil {
			return
		}

		op.dirty = true
	}

	return
}

// Return the name of the proxied object. This may or may not be an object that
// currently exists in the bucket.
func (op *ObjectProxy) Name() string {
	return op.name
}

// Panic if any internal invariants are violated. Careful users can call this
// at appropriate times to help debug weirdness. Consider using
// syncutil.InvariantMutex to automate the process.
func (op *ObjectProxy) CheckInvariants() {
	// INVARIANT: If srcGeneration == 0, then dirty
	if op.srcGeneration == 0 && !op.dirty {
		panic("Expected dirty.")
	}

	// INVARIANT: If dirty, then localFile != nil
	if op.dirty && op.localFile == nil {
		panic("Expected non-nil localFile.")
	}
}

// Destroy any local file caches, putting the proxy into an indeterminate
// state. Should be used before dropping the final reference to the proxy.
func (op *ObjectProxy) Destroy() (err error) {
	// Make sure that when we exit no invariants are violated.
	defer func() {
		op.srcGeneration = 1
		op.localFile = nil
		op.dirty = false
	}()

	// If we have no local file, there's nothing to do.
	if op.localFile == nil {
		return
	}

	// Close the local file.
	if err = op.localFile.Close(); err != nil {
		err = fmt.Errorf("Close: %v", err)
		return
	}

	return
}

// Return the current size in bytes of the content and an indication of whether
// the proxied object has changed out from under us (in which case Sync will
// fail).
func (op *ObjectProxy) Stat(
	ctx context.Context) (size uint64, clobbered bool, err error) {
	panic("TODO")
}

// Make a random access read into our view of the content. May block for
// network access.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) ReadAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// Serve the read from the file.
	n, err = op.localFile.ReadAt(buf, offset)

	return
}

// Make a random access write into our view of the content. May block for
// network access. Not guaranteed to be reflected remotely until after Sync is
// called successfully.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) WriteAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	panic("TODO")
}

// Truncate our view of the content to the given number of bytes, extending if
// n is greater than the current size. May block for network access. Not
// guaranteed to be reflected remotely until after Sync is called successfully.
func (op *ObjectProxy) Truncate(ctx context.Context, n uint64) (err error) {
	panic("TODO")
}

// If the proxy is dirty due to having been written to or due to having a nil
// source, save its current contents to GCS and return a generation number for
// a generation with exactly those contents. Do so with a precondition such
// that the creation will fail if the source generation is not current. In that
// case, return ErrNotCurrent.
func (op *ObjectProxy) Sync(ctx context.Context) (gen uint64, err error) {
	panic("TODO")
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Set up an unlinked local temporary file for the given generation of the
// given object. Special case: generation == 0 means an empty file.
func makeLocalFile(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	generation uint64) (f *os.File, err error) {
	// Create the file.
	f, err = ioutil.TempFile("", "object_proxy")
	if err != nil {
		err = fmt.Errorf("TempFile: %v", err)
		return
	}

	// Unlink the file so that its inode will be garbage collected when the file
	// is closed.
	if err = os.Remove(f.Name()); err != nil {
		f.Close()
		err = fmt.Errorf("Remove: %v", err)
		return
	}

	// Fetch the object's contents if necessary.
	if generation != 0 {
		panic("TODO")
	}

	return
}

// Ensure that op.localFile is non-nil with an authoritative view of op's
// contents.
func (op *ObjectProxy) ensureLocalFile(ctx context.Context) (err error) {
	// Is there anything to do?
	if op.localFile != nil {
		return
	}

	// Set up the file.
	f, err := makeLocalFile(ctx, op.bucket, op.name, op.srcGeneration)
	if err != nil {
		err = fmt.Errorf("makeLocalFile: %v", err)
		return
	}

	op.localFile = f
	return
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package json

import (
	"bytes"
	"math"
	"rand"
	"testing"
)

// Tests of simple examples.

type example struct {
	compact string
	indent  string
}

var examples = []example{
	{`1`, `1`},
	{`{}`, `{}`},
	{`[]`, `[]`},
	{`{"":2}`, "{\n\t\"\": 2\n}"},
	{`[3]`, "[\n\t3\n]"},
	{`[1,2,3]`, "[\n\t1,\n\t2,\n\t3\n]"},
	{`{"x":1}`, "{\n\t\"x\": 1\n}"},
	{ex1, ex1i},
}

var ex1 = `[true,false,null,"x",1,1.5,0,-5e+2]`

var ex1i = `[
	true,
	false,
	null,
	"x",
	1,
	1.5,
	0,
	-5e+2
]`

func TestCompact(t *testing.T) {
	var buf bytes.Buffer
	for _, tt := range examples {
		buf.Reset()
		if err := Compact(&buf, []byte(tt.compact)); err != nil {
			t.Errorf("Compact(%#q): %v", tt.compact, err)
		} else if s := buf.String(); s != tt.compact {
			t.Errorf("Compact(%#q) = %#q, want original", tt.compact, s)
		}

		buf.Reset()
		if err := Compact(&buf, []byte(tt.indent)); err != nil {
			t.Errorf("Compact(%#q): %v", tt.indent, err)
			continue
		} else if s := buf.String(); s != tt.compact {
			t.Errorf("Compact(%#q) = %#q, want %#q", tt.indent, s, tt.compact)
		}
	}
}

func TestIndent(t *testing.T) {
	var buf bytes.Buffer
	for _, tt := range examples {
		buf.Reset()
		if err := Indent(&buf, []byte(tt.indent), "", "\t"); err != nil {
			t.Errorf("Indent(%#q): %v", tt.indent, err)
		} else if s := buf.String(); s != tt.indent {
			t.Errorf("Indent(%#q) = %#q, want original", tt.indent, s)
		}

		buf.Reset()
		if err := Indent(&buf, []byte(tt.compact), "", "\t"); err != nil {
			t.Errorf("Indent(%#q): %v", tt.compact, err)
			continue
		} else if s := buf.String(); s != tt.indent {
			t.Errorf("Indent(%#q) = %#q, want %#q", tt.compact, s, tt.indent)
		}
	}
}

// Tests of a large random structure.

func TestCompactBig(t *testing.T) {
	initBig()
	var buf bytes.Buffer
	if err := Compact(&buf, jsonBig); err != nil {
		t.Fatalf("Compact: %v", err)
	}
	b := buf.Bytes()
	if bytes.Compare(b, jsonBig) != 0 {
		t.Error("Compact(jsonBig) != jsonBig")
		diff(t, b, jsonBig)
		return
	}
}

func TestIndentBig(t *testing.T) {
	initBig()
	var buf bytes.Buffer
	if err := Indent(&buf, jsonBig, "", "\t"); err != nil {
		t.Fatalf("Indent1: %v", err)
	}
	b := buf.Bytes()
	if len(b) == len(jsonBig) {
		// jsonBig is compact (no unnecessary spaces);
		// indenting should make it bigger
		t.Fatalf("Indent(jsonBig) did not get bigger")
	}

	// should be idempotent
	var buf1 bytes.Buffer
	if err := Indent(&buf1, b, "", "\t"); err != nil {
		t.Fatalf("Indent2: %v", err)
	}
	b1 := buf1.Bytes()
	if bytes.Compare(b1, b) != 0 {
		t.Error("Indent(Indent(jsonBig)) != Indent(jsonBig)")
		diff(t, b1, b)
		return
	}

	// should get back to original
	buf1.Reset()
	if err := Compact(&buf1, b); err != nil {
		t.Fatalf("Compact: %v", err)
	}
	b1 = buf1.Bytes()
	if bytes.Compare(b1, jsonBig) != 0 {
		t.Error("Compact(Indent(jsonBig)) != jsonBig")
		diff(t, b1, jsonBig)
		return
	}
}

func TestNextValueBig(t *testing.T) {
	initBig()
	var scan scanner
	item, rest, err := nextValue(jsonBig, &scan)
	if err != nil {
		t.Fatalf("nextValue: %s", err)
	}
	if len(item) != len(jsonBig) || &item[0] != &jsonBig[0] {
		t.Errorf("invalid item: %d %d", len(item), len(jsonBig))
	}
	if len(rest) != 0 {
		t.Errorf("invalid rest: %d", len(rest))
	}

	item, rest, err = nextValue(append(jsonBig, []byte("HELLO WORLD")...), &scan)
	if err != nil {
		t.Fatalf("nextValue extra: %s", err)
	}
	if len(item) != len(jsonBig) {
		t.Errorf("invalid item: %d %d", len(item), len(jsonBig))
	}
	if string(rest) != "HELLO WORLD" {
		t.Errorf("invalid rest: %d", len(rest))
	}
}

func BenchmarkSkipValue(b *testing.B) {
	initBig()
	var scan scanner
	for i := 0; i < b.N; i++ {
		nextValue(jsonBig, &scan)
	}
	b.SetBytes(int64(len(jsonBig)))
}

func diff(t *testing.T, a, b []byte) {
	for i := 0; ; i++ {
		if i >= len(a) || i >= len(b) || a[i] != b[i] {
			j := i - 10
			if j < 0 {
				j = 0
			}
			t.Errorf("diverge at %d: «%s» vs «%s»", i, trim(a[j:]), trim(b[j:]))
			return
		}
	}
}

func trim(b []byte) []byte {
	if len(b) > 20 {
		return b[0:20]
	}
	return b
}

// Generate a random JSON object.

var jsonBig []byte

const (
	big   = 10000
	small = 100
)

func initBig() {
	n := big
	if testing.Short() {
		n = small
	}
	if len(jsonBig) != n {
		b, err := Marshal(genValue(n))
		if err != nil {
			panic(err)
		}
		jsonBig = b
	}
}

func genValue(n int) interface{} {
	if n > 1 {
		switch rand.Intn(2) {
		case 0:
			return genArray(n)
		case 1:
			return genMap(n)
		}
	}
	switch rand.Intn(3) {
	case 0:
		return rand.Intn(2) == 0
	case 1:
		return rand.NormFloat64()
	case 2:
		return genString(30)
	}
	panic("unreachable")
}

func genString(stddev float64) string {
	n := int(math.Fabs(rand.NormFloat64()*stddev + stddev/2))
	c := make([]int, n)
	for i := range c {
		f := math.Fabs(rand.NormFloat64()*64 + 32)
		if f > 0x10ffff {
			f = 0x10ffff
		}
		c[i] = int(f)
	}
	return string(c)
}

func genArray(n int) []interface{} {
	f := int(math.Fabs(rand.NormFloat64()) * math.Fmin(10, float64(n/2)))
	if f > n {
		f = n
	}
	x := make([]interface{}, int(f))
	for i := range x {
		x[i] = genValue(((i+1)*n)/f - (i*n)/f)
	}
	return x
}

func genMap(n int) map[string]interface{} {
	f := int(math.Fabs(rand.NormFloat64()) * math.Fmin(10, float64(n/2)))
	if f > n {
		f = n
	}
	if n > 0 && f == 0 {
		f = 1
	}
	x := make(map[string]interface{})
	for i := 0; i < f; i++ {
		x[genString(10)] = genValue(((i+1)*n)/f - (i*n)/f)
	}
	return x
}
<commit_msg>json: fix test when rand returns 0.<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package json

import (
	"bytes"
	"math"
	"rand"
	"testing"
)

// Tests of simple examples.

type example struct {
	compact string
	indent  string
}

var examples = []example{
	{`1`, `1`},
	{`{}`, `{}`},
	{`[]`, `[]`},
	{`{"":2}`, "{\n\t\"\": 2\n}"},
	{`[3]`, "[\n\t3\n]"},
	{`[1,2,3]`, "[\n\t1,\n\t2,\n\t3\n]"},
	{`{"x":1}`, "{\n\t\"x\": 1\n}"},
	{ex1, ex1i},
}

var ex1 = `[true,false,null,"x",1,1.5,0,-5e+2]`

var ex1i = `[
	true,
	false,
	null,
	"x",
	1,
	1.5,
	0,
	-5e+2
]`

func TestCompact(t *testing.T) {
	var buf bytes.Buffer
	for _, tt := range examples {
		buf.Reset()
		if err := Compact(&buf, []byte(tt.compact)); err != nil {
			t.Errorf("Compact(%#q): %v", tt.compact, err)
		} else if s := buf.String(); s != tt.compact {
			t.Errorf("Compact(%#q) = %#q, want original", tt.compact, s)
		}

		buf.Reset()
		if err := Compact(&buf, []byte(tt.indent)); err != nil {
			t.Errorf("Compact(%#q): %v", tt.indent, err)
			continue
		} else if s := buf.String(); s != tt.compact {
			t.Errorf("Compact(%#q) = %#q, want %#q", tt.indent, s, tt.compact)
		}
	}
}

func TestIndent(t *testing.T) {
	var buf bytes.Buffer
	for _, tt := range examples {
		buf.Reset()
		if err := Indent(&buf, []byte(tt.indent), "", "\t"); err != nil {
			t.Errorf("Indent(%#q): %v", tt.indent, err)
		} else if s := buf.String(); s != tt.indent {
			t.Errorf("Indent(%#q) = %#q, want original", tt.indent, s)
		}

		buf.Reset()
		if err := Indent(&buf, []byte(tt.compact), "", "\t"); err != nil {
			t.Errorf("Indent(%#q): %v", tt.compact, err)
			continue
		} else if s := buf.String(); s != tt.indent {
			t.Errorf("Indent(%#q) = %#q, want %#q", tt.compact, s, tt.indent)
		}
	}
}

// Tests of a large random structure.

func TestCompactBig(t *testing.T) {
	initBig()
	var buf bytes.Buffer
	if err := Compact(&buf, jsonBig); err != nil {
		t.Fatalf("Compact: %v", err)
	}
	b := buf.Bytes()
	if bytes.Compare(b, jsonBig) != 0 {
		t.Error("Compact(jsonBig) != jsonBig")
		diff(t, b, jsonBig)
		return
	}
}

func TestIndentBig(t *testing.T) {
	initBig()
	var buf bytes.Buffer
	if err := Indent(&buf, jsonBig, "", "\t"); err != nil {
		t.Fatalf("Indent1: %v", err)
	}
	b := buf.Bytes()
	if len(b) == len(jsonBig) {
		// jsonBig is compact (no unnecessary spaces);
		// indenting should make it bigger
		t.Fatalf("Indent(jsonBig) did not get bigger")
	}

	// should be idempotent
	var buf1 bytes.Buffer
	if err := Indent(&buf1, b, "", "\t"); err != nil {
		t.Fatalf("Indent2: %v", err)
	}
	b1 := buf1.Bytes()
	if bytes.Compare(b1, b) != 0 {
		t.Error("Indent(Indent(jsonBig)) != Indent(jsonBig)")
		diff(t, b1, b)
		return
	}

	// should get back to original
	buf1.Reset()
	if err := Compact(&buf1, b); err != nil {
		t.Fatalf("Compact: %v", err)
	}
	b1 = buf1.Bytes()
	if bytes.Compare(b1, jsonBig) != 0 {
		t.Error("Compact(Indent(jsonBig)) != jsonBig")
		diff(t, b1, jsonBig)
		return
	}
}

func TestNextValueBig(t *testing.T) {
	initBig()
	var scan scanner
	item, rest, err := nextValue(jsonBig, &scan)
	if err != nil {
		t.Fatalf("nextValue: %s", err)
	}
	if len(item) != len(jsonBig) || &item[0] != &jsonBig[0] {
		t.Errorf("invalid item: %d %d", len(item), len(jsonBig))
	}
	if len(rest) != 0 {
		t.Errorf("invalid rest: %d", len(rest))
	}

	item, rest, err = nextValue(append(jsonBig, []byte("HELLO WORLD")...), &scan)
	if err != nil {
		t.Fatalf("nextValue extra: %s", err)
	}
	if len(item) != len(jsonBig) {
		t.Errorf("invalid item: %d %d", len(item), len(jsonBig))
	}
	if string(rest) != "HELLO WORLD" {
		t.Errorf("invalid rest: %d", len(rest))
	}
}

func BenchmarkSkipValue(b *testing.B) {
	initBig()
	var scan scanner
	for i := 0; i < b.N; i++ {
		nextValue(jsonBig, &scan)
	}
	b.SetBytes(int64(len(jsonBig)))
}

func diff(t *testing.T, a, b []byte) {
	for i := 0; ; i++ {
		if i >= len(a) || i >= len(b) || a[i] != b[i] {
			j := i - 10
			if j < 0 {
				j = 0
			}
			t.Errorf("diverge at %d: «%s» vs «%s»", i, trim(a[j:]), trim(b[j:]))
			return
		}
	}
}

func trim(b []byte) []byte {
	if len(b) > 20 {
		return b[0:20]
	}
	return b
}

// Generate a random JSON object.

var jsonBig []byte

const (
	big   = 10000
	small = 100
)

func initBig() {
	n := big
	if testing.Short() {
		n = small
	}
	if len(jsonBig) != n {
		b, err := Marshal(genValue(n))
		if err != nil {
			panic(err)
		}
		jsonBig = b
	}
}

func genValue(n int) interface{} {
	if n > 1 {
		switch rand.Intn(2) {
		case 0:
			return genArray(n)
		case 1:
			return genMap(n)
		}
	}
	switch rand.Intn(3) {
	case 0:
		return rand.Intn(2) == 0
	case 1:
		return rand.NormFloat64()
	case 2:
		return genString(30)
	}
	panic("unreachable")
}

func genString(stddev float64) string {
	n := int(math.Fabs(rand.NormFloat64()*stddev + stddev/2))
	c := make([]int, n)
	for i := range c {
		f := math.Fabs(rand.NormFloat64()*64 + 32)
		if f > 0x10ffff {
			f = 0x10ffff
		}
		c[i] = int(f)
	}
	return string(c)
}

func genArray(n int) []interface{} {
	f := int(math.Fabs(rand.NormFloat64()) * math.Fmin(10, float64(n/2)))
	if f > n {
		f = n
	}
	if n > 0 && f == 0 {
		f = 1
	}
	x := make([]interface{}, int(f))
	for i := range x {
		x[i] = genValue(((i+1)*n)/f - (i*n)/f)
	}
	return x
}

func genMap(n int) map[string]interface{} {
	f := int(math.Fabs(rand.NormFloat64()) * math.Fmin(10, float64(n/2)))
	if f > n {
		f = n
	}
	if n > 0 && f == 0 {
		f = 1
	}
	x := make(map[string]interface{})
	for i := 0; i < f; i++ {
		x[genString(10)] = genValue(((i+1)*n)/f - (i*n)/f)
	}
	return x
}
<|endoftext|>


<commit_before>package metrics

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"regexp"
	"strconv"
	"strings"

	"github.com/fweikert/continuous-integration/metrics/clients"
	"github.com/fweikert/continuous-integration/metrics/data"
)

type Flakiness struct {
	client    *clients.GcsClient
	columns   []Column
	gcsBucket string
	gcsSuffix string
	pipelines []*data.PipelineID
}

func (f *Flakiness) Name() string {
	return "flakiness"
}

func (f *Flakiness) Columns() []Column {
	return f.columns
}

func (f *Flakiness) Collect() (data.DataSet, error) {
	result := data.CreateDataSet(GetColumnNames(f.columns))
	for _, pipeline := range f.pipelines {
		gcsPath := f.gcsSuffix + pipeline.Slug
		contents, err := f.client.ReadAllFiles(f.gcsBucket, gcsPath)
		if err != nil {
			return nil, err
		}
		if len(contents) == 0 {
			log.Printf("There is no flakiness data for pipeline %s in GCS location %s/%s", pipeline, f.gcsBucket, gcsPath)
			continue
		}

		for fileName, content := range contents {
			build, err := getBuildNumber(fileName)
			if err != nil {
				return nil, fmt.Errorf("Invalid flaky data file %s in %s/%s: %v", fileName, f.gcsBucket, gcsPath, err)
			}

			messages, err := getFlakyMessages(content)
			if err != nil {
				return nil, fmt.Errorf("Failed to parse flaky data file %s in %s/%s: %v", fileName, f.gcsBucket, gcsPath, err)
			}

			for _, msg := range messages {
				err = result.AddRow(pipeline.Org, pipeline.Slug, build, msg.ID.Summary.Label, len(msg.Summary.Passed), len(msg.Summary.Failed))
				if err != nil {
					return nil, fmt.Errorf("Failed to add flakiness data for build #%d of pipeline %s: %v", build, pipeline, err)
				}
			}
		}
	}
	return result, nil
}

func getFlakyMessages(content []byte) ([]message, error) {
	messages := make([]message, 0)
	for _, line := range bytes.Split(content, []byte("\n")) {
		if len(line) == 0 {
			continue
		}

		var msg message
		err := json.Unmarshal([]byte(line), &msg)
		if err != nil {
			return nil, fmt.Errorf("JSON parse error: %v. Line:\n%s", err, line)
		}
		if msg.Summary.Status == "FLAKY" {
			messages = append(messages, msg)
		}
	}
	return messages, nil
}

func getBuildNumber(fileName string) (int, error) {
	re := regexp.MustCompile(`(\d+).json$`)
	matches := re.FindStringSubmatch(fileName)
	if len(matches) == 2 {
		if build, err := strconv.Atoi(matches[1]); err == nil {
			return build, nil
		}
	}
	return 0, fmt.Errorf("Flakiness data file names must be 'some_path/<build_number.json>. Invalid given value: %s", fileName)
}

type message struct {
	ID struct {
		Summary struct {
			Label string `json:"label"`
		} `json:"testSummary"`
	}
	Summary struct {
		Passed []struct{ uri string } `json:"passed"`
		Failed []struct{ uri string } `json:"failed"`
		Status string                 `json:"overallStatus"`
	} `json:"testSummary"`
}

// CREATE TABLE flakiness (org VARCHAR(255), pipeline VARCHAR(255), build INT, target VARCHAR(255), passed_count INT, failed_count INT, PRIMARY KEY(org, pipeline, build, target));
func CreateFlakiness(client *clients.GcsClient, gcsBucket, gcsBasePath string, pipelines ...*data.PipelineID) *Flakiness {
	columns := []Column{Column{"org", true}, Column{"pipeline", true}, Column{"build", true}, Column{"target", true}, Column{"passed_count", false}, Column{"failed_count", false}}
	gcsSuffix := gcsBasePath
	if !strings.HasSuffix(gcsBasePath, "/") {
		gcsSuffix = gcsBasePath + "/"
	}
	return &Flakiness{client: client, columns: columns, gcsBucket: gcsBucket, gcsSuffix: gcsSuffix, pipelines: pipelines}
}
<commit_msg>Fix quotation marks in error message<commit_after>package metrics

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"regexp"
	"strconv"
	"strings"

	"github.com/fweikert/continuous-integration/metrics/clients"
	"github.com/fweikert/continuous-integration/metrics/data"
)

type Flakiness struct {
	client    *clients.GcsClient
	columns   []Column
	gcsBucket string
	gcsSuffix string
	pipelines []*data.PipelineID
}

func (f *Flakiness) Name() string {
	return "flakiness"
}

func (f *Flakiness) Columns() []Column {
	return f.columns
}

func (f *Flakiness) Collect() (data.DataSet, error) {
	result := data.CreateDataSet(GetColumnNames(f.columns))
	for _, pipeline := range f.pipelines {
		gcsPath := f.gcsSuffix + pipeline.Slug
		contents, err := f.client.ReadAllFiles(f.gcsBucket, gcsPath)
		if err != nil {
			return nil, err
		}
		if len(contents) == 0 {
			log.Printf("There is no flakiness data for pipeline %s in GCS location %s/%s", pipeline, f.gcsBucket, gcsPath)
			continue
		}

		for fileName, content := range contents {
			build, err := getBuildNumber(fileName)
			if err != nil {
				return nil, fmt.Errorf("Invalid flaky data file %s in %s/%s: %v", fileName, f.gcsBucket, gcsPath, err)
			}

			messages, err := getFlakyMessages(content)
			if err != nil {
				return nil, fmt.Errorf("Failed to parse flaky data file %s in %s/%s: %v", fileName, f.gcsBucket, gcsPath, err)
			}

			for _, msg := range messages {
				err = result.AddRow(pipeline.Org, pipeline.Slug, build, msg.ID.Summary.Label, len(msg.Summary.Passed), len(msg.Summary.Failed))
				if err != nil {
					return nil, fmt.Errorf("Failed to add flakiness data for build #%d of pipeline %s: %v", build, pipeline, err)
				}
			}
		}
	}
	return result, nil
}

func getFlakyMessages(content []byte) ([]message, error) {
	messages := make([]message, 0)
	for _, line := range bytes.Split(content, []byte("\n")) {
		if len(line) == 0 {
			continue
		}

		var msg message
		err := json.Unmarshal([]byte(line), &msg)
		if err != nil {
			return nil, fmt.Errorf("JSON parse error: %v. Line:\n%s", err, line)
		}
		if msg.Summary.Status == "FLAKY" {
			messages = append(messages, msg)
		}
	}
	return messages, nil
}

func getBuildNumber(fileName string) (int, error) {
	re := regexp.MustCompile(`(\d+).json$`)
	matches := re.FindStringSubmatch(fileName)
	if len(matches) == 2 {
		if build, err := strconv.Atoi(matches[1]); err == nil {
			return build, nil
		}
	}
	return 0, fmt.Errorf("Flakiness data file names must be 'some_path/<build_number.json>'. Invalid given value: '%s'", fileName)
}

type message struct {
	ID struct {
		Summary struct {
			Label string `json:"label"`
		} `json:"testSummary"`
	}
	Summary struct {
		Passed []struct{ uri string } `json:"passed"`
		Failed []struct{ uri string } `json:"failed"`
		Status string                 `json:"overallStatus"`
	} `json:"testSummary"`
}

// CREATE TABLE flakiness (org VARCHAR(255), pipeline VARCHAR(255), build INT, target VARCHAR(255), passed_count INT, failed_count INT, PRIMARY KEY(org, pipeline, build, target));
func CreateFlakiness(client *clients.GcsClient, gcsBucket, gcsBasePath string, pipelines ...*data.PipelineID) *Flakiness {
	columns := []Column{Column{"org", true}, Column{"pipeline", true}, Column{"build", true}, Column{"target", true}, Column{"passed_count", false}, Column{"failed_count", false}}
	gcsSuffix := gcsBasePath
	if !strings.HasSuffix(gcsBasePath, "/") {
		gcsSuffix = gcsBasePath + "/"
	}
	return &Flakiness{client: client, columns: columns, gcsBucket: gcsBucket, gcsSuffix: gcsSuffix, pipelines: pipelines}
}
<|endoftext|>


<commit_before>// Package flow contains data structure for computation.
// Mostly Dataset operations such as Map/Reduce/Join/Sort etc.
package flow

import (
	"reflect"
)

// reference by task runner on remote mode
var Contexts []*FlowContext

type FlowContext struct {
	Id                int
	Steps             []*Step
	Datasets          []*Dataset
	ChannelBufferSize int // configurable channel buffer size for reading
}

func New() (fc *FlowContext) {
	fc = &FlowContext{Id: len(Contexts)}
	Contexts = append(Contexts, fc)
	return
}

func (fc *FlowContext) newNextDataset(shardSize int, dType reflect.Type) (ret *Dataset) {
	ret = NewDataset(fc, dType)
	if dType != nil {
		ret.SetupShard(shardSize)
	}
	return
}

func (fc *FlowContext) NewStep() (step *Step) {
	step = &Step{Id: len(fc.Steps)}
	fc.Steps = append(fc.Steps, step)
	return
}

// the tasks should run on the source dataset shard
func (f *FlowContext) AddOneToOneStep(input *Dataset, output *Dataset) (step *Step) {
	step = f.NewStep()
	FromStepToDataset(step, output)
	FromDatasetToStep(input, step)

	// setup the network
	if output != nil && len(output.ExternalInputChans) > 0 {
		task := step.NewTask()
		FromTaskToDatasetShard(task, output.GetShards()[0])
	} else {
		for i, shard := range input.GetShards() {
			task := step.NewTask()
			if output != nil && output.Shards != nil {
				FromTaskToDatasetShard(task, output.GetShards()[i])
			}
			FromDatasetShardToTask(shard, task)
		}
	}
	return
}

// the task should run on the destination dataset shard
func (f *FlowContext) AddAllToOneStep(input *Dataset, output *Dataset) (step *Step) {
	step = f.NewStep()
	FromStepToDataset(step, output)
	FromDatasetToStep(input, step)

	// setup the network
	task := step.NewTask()
	if output != nil {
		FromTaskToDatasetShard(task, output.GetShards()[0])
	}
	for _, shard := range input.GetShards() {
		FromDatasetShardToTask(shard, task)
	}
	return
}

// the task should run on the source dataset shard
// input is nil for initial source dataset
func (f *FlowContext) AddOneToAllStep(input *Dataset, output *Dataset) (step *Step) {
	step = f.NewStep()
	FromStepToDataset(step, output)
	FromDatasetToStep(input, step)

	// setup the network
	task := step.NewTask()
	if input != nil {
		FromDatasetShardToTask(input.GetShards()[0], task)
	}
	for _, shard := range output.GetShards() {
		FromTaskToDatasetShard(task, shard)
	}
	return
}

func (f *FlowContext) AddOneToEveryNStep(input *Dataset, n int, output *Dataset) (step *Step) {
	step = f.NewStep()
	FromStepToDataset(step, output)
	FromDatasetToStep(input, step)

	// setup the network
	m := len(input.GetShards())
	for i, inShard := range input.GetShards() {
		task := step.NewTask()
		for k := 0; k < n; k++ {
			FromTaskToDatasetShard(task, output.GetShards()[k*m+i])
		}
		FromDatasetShardToTask(inShard, task)
	}
	return
}

func (f *FlowContext) AddLinkedNToOneStep(input *Dataset, m int, output *Dataset) (step *Step) {
	step = f.NewStep()
	FromStepToDataset(step, output)
	FromDatasetToStep(input, step)

	// setup the network
	for i, outShard := range output.GetShards() {
		task := step.NewTask()
		FromTaskToDatasetShard(task, outShard)
		for k := 0; k < m; k++ {
			FromDatasetShardToTask(input.GetShards()[i*m+k], task)
		}
	}
	return
}

// All dataset should have the same number of shards.
func (f *FlowContext) MergeDatasets1ShardTo1Step(inputs []*Dataset, output *Dataset) (step *Step) {
	step = f.NewStep()
	FromStepToDataset(step, output)
	for _, input := range inputs {
		FromDatasetToStep(input, step)
	}

	// setup the network
	if output != nil {
		for shardId, outShard := range output.Shards {
			task := step.NewTask()
			for _, input := range inputs {
				FromDatasetShardToTask(input.GetShards()[shardId], task)
			}
			FromTaskToDatasetShard(task, outShard)
		}
	}
	return
}

func FromStepToDataset(step *Step, output *Dataset) {
	if output == nil {
		return
	}
	output.Step = step
	step.Output = output
}

func FromDatasetToStep(input *Dataset, step *Step) {
	if input == nil {
		return
	}
	step.Inputs = append(step.Inputs, input)
	input.ReadingSteps = append(input.ReadingSteps, step)
}

func FromDatasetShardToTask(shard *DatasetShard, task *Task) {
	shard.ReadingTasks = append(shard.ReadingTasks, task)
	task.Inputs = append(task.Inputs, shard)
	task.InputChans = append(task.InputChans, make(chan reflect.Value))
}

func FromTaskToDatasetShard(task *Task, shard *DatasetShard) {
	if shard != nil {
		task.Outputs = append(task.Outputs, shard)
	}
}
<commit_msg>just a textual change to trigger travis build<commit_after>// Package flow contains data structure for computation.
// Mostly Dataset operations such as Map/Reduce/Join/Sort etc.
package flow

import (
	"reflect"
)

// reference by task runner on exectutors
var Contexts []*FlowContext

type FlowContext struct {
	Id                int
	Steps             []*Step
	Datasets          []*Dataset
	ChannelBufferSize int // configurable channel buffer size for reading
}

func New() (fc *FlowContext) {
	fc = &FlowContext{Id: len(Contexts)}
	Contexts = append(Contexts, fc)
	return
}

func (fc *FlowContext) newNextDataset(shardSize int, dType reflect.Type) (ret *Dataset) {
	ret = NewDataset(fc, dType)
	if dType != nil {
		ret.SetupShard(shardSize)
	}
	return
}

func (fc *FlowContext) NewStep() (step *Step) {
	step = &Step{Id: len(fc.Steps)}
	fc.Steps = append(fc.Steps, step)
	return
}

// the tasks should run on the source dataset shard
func (f *FlowContext) AddOneToOneStep(input *Dataset, output *Dataset) (step *Step) {
	step = f.NewStep()
	FromStepToDataset(step, output)
	FromDatasetToStep(input, step)

	// setup the network
	if output != nil && len(output.ExternalInputChans) > 0 {
		task := step.NewTask()
		FromTaskToDatasetShard(task, output.GetShards()[0])
	} else {
		for i, shard := range input.GetShards() {
			task := step.NewTask()
			if output != nil && output.Shards != nil {
				FromTaskToDatasetShard(task, output.GetShards()[i])
			}
			FromDatasetShardToTask(shard, task)
		}
	}
	return
}

// the task should run on the destination dataset shard
func (f *FlowContext) AddAllToOneStep(input *Dataset, output *Dataset) (step *Step) {
	step = f.NewStep()
	FromStepToDataset(step, output)
	FromDatasetToStep(input, step)

	// setup the network
	task := step.NewTask()
	if output != nil {
		FromTaskToDatasetShard(task, output.GetShards()[0])
	}
	for _, shard := range input.GetShards() {
		FromDatasetShardToTask(shard, task)
	}
	return
}

// the task should run on the source dataset shard
// input is nil for initial source dataset
func (f *FlowContext) AddOneToAllStep(input *Dataset, output *Dataset) (step *Step) {
	step = f.NewStep()
	FromStepToDataset(step, output)
	FromDatasetToStep(input, step)

	// setup the network
	task := step.NewTask()
	if input != nil {
		FromDatasetShardToTask(input.GetShards()[0], task)
	}
	for _, shard := range output.GetShards() {
		FromTaskToDatasetShard(task, shard)
	}
	return
}

func (f *FlowContext) AddOneToEveryNStep(input *Dataset, n int, output *Dataset) (step *Step) {
	step = f.NewStep()
	FromStepToDataset(step, output)
	FromDatasetToStep(input, step)

	// setup the network
	m := len(input.GetShards())
	for i, inShard := range input.GetShards() {
		task := step.NewTask()
		for k := 0; k < n; k++ {
			FromTaskToDatasetShard(task, output.GetShards()[k*m+i])
		}
		FromDatasetShardToTask(inShard, task)
	}
	return
}

func (f *FlowContext) AddLinkedNToOneStep(input *Dataset, m int, output *Dataset) (step *Step) {
	step = f.NewStep()
	FromStepToDataset(step, output)
	FromDatasetToStep(input, step)

	// setup the network
	for i, outShard := range output.GetShards() {
		task := step.NewTask()
		FromTaskToDatasetShard(task, outShard)
		for k := 0; k < m; k++ {
			FromDatasetShardToTask(input.GetShards()[i*m+k], task)
		}
	}
	return
}

// All dataset should have the same number of shards.
func (f *FlowContext) MergeDatasets1ShardTo1Step(inputs []*Dataset, output *Dataset) (step *Step) {
	step = f.NewStep()
	FromStepToDataset(step, output)
	for _, input := range inputs {
		FromDatasetToStep(input, step)
	}

	// setup the network
	if output != nil {
		for shardId, outShard := range output.Shards {
			task := step.NewTask()
			for _, input := range inputs {
				FromDatasetShardToTask(input.GetShards()[shardId], task)
			}
			FromTaskToDatasetShard(task, outShard)
		}
	}
	return
}

func FromStepToDataset(step *Step, output *Dataset) {
	if output == nil {
		return
	}
	output.Step = step
	step.Output = output
}

func FromDatasetToStep(input *Dataset, step *Step) {
	if input == nil {
		return
	}
	step.Inputs = append(step.Inputs, input)
	input.ReadingSteps = append(input.ReadingSteps, step)
}

func FromDatasetShardToTask(shard *DatasetShard, task *Task) {
	shard.ReadingTasks = append(shard.ReadingTasks, task)
	task.Inputs = append(task.Inputs, shard)
	task.InputChans = append(task.InputChans, make(chan reflect.Value))
}

func FromTaskToDatasetShard(task *Task, shard *DatasetShard) {
	if shard != nil {
		task.Outputs = append(task.Outputs, shard)
	}
}
<|endoftext|>