<commit_before>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package api

import (
	"encoding/hex"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strconv"

	"github.com/juju/errors"
	charmresource "gopkg.in/juju/charm.v6-unstable/resource"
)

const (
	// HTTPEndpointPattern is the URL path pattern registered with
	// the API server. This includes wildcards (starting with ":") that
	// are converted into URL query values by the pattern mux. Also see
	// apiserver/apiserver.go.
	HTTPEndpointPattern = "/services/:service/resources/:resource"

	// HTTPEndpointPath is the URL path, with substitutions, for
	// a resource request.
	HTTPEndpointPath = "/environment/%s/services/%s/resources/%s"
)

// NewEndpointPath returns the API URL path for the identified resource.
func NewEndpointPath(envUUID, service, name string) string {
	return fmt.Sprintf(HTTPEndpointPath, envUUID, service, name)
}

// ExtractEndpointDetails pulls the endpoint wildcard values from
// the provided URL.
func ExtractEndpointDetails(url *url.URL) (service, name string) {
	service = url.Query().Get(":service")
	name = url.Query().Get(":resource")
	return service, name
}

// NewHTTPUploadRequest generates a new HTTP request for the given resource.
func NewHTTPUploadRequest(envUUID, service, name string, r io.ReadSeeker) (*http.Request, error) {
	// TODO(ericsnow) Use the newer GenerateFingerprint()...
	data, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, errors.Trace(err)
	}
	if _, err := r.Seek(0, os.SEEK_SET); err != nil {
		return nil, errors.Trace(err)
	}

	fp, err := charmresource.GenerateFingerprint(data)
	if err != nil {
		return nil, errors.Trace(err)
	}

	method := "PUT"
	// TODO(ericsnow) What about the rest of the URL?
	urlStr := NewEndpointPath(envUUID, service, name)
	req, err := http.NewRequest(method, urlStr, r)
	if err != nil {
		return nil, errors.Trace(err)
	}

	req.Header.Set("Content-Type", "application/octet-stream")
	req.Header.Set("Content-Length", fmt.Sprint(len(data)))
	req.Header.Set("Content-SHA384", fp.String())

	return req, nil
}

// ExtractUploadRequest pulls the required info from the HTTP request.
func ExtractUploadRequest(req *http.Request) (service, name string, size int64, _ charmresource.Fingerprint, _ error) {
	var fp charmresource.Fingerprint

	ctype := req.Header.Get("Content-Type")
	if ctype != "application/octet-stream" {
		return "", "", 0, fp, errors.Errorf("unsupported content type %q", ctype)
	}

	service, name = ExtractEndpointDetails(req.URL)

	fingerprint := req.Header.Get("Content-SHA384") // This parallels "Content-MD5".
	sizeRaw := req.Header.Get("Content-Length")

	// TODO(ericsnow) Use the newer ParseFingerprint().
	fpData, err := hex.DecodeString(fingerprint)
	if err != nil {
		return "", "", 0, fp, errors.Annotate(err, "invalid fingerprint")
	}
	fp, err = charmresource.NewFingerprint(fpData)
	if err != nil {
		return "", "", 0, fp, errors.Annotate(err, "invalid fingerprint")
	}

	size, err = strconv.ParseInt(sizeRaw, 10, 64)
	if err != nil {
		return "", "", 0, fp, errors.Annotate(err, "invalid size")
	}

	return service, name, size, fp, nil
}
<commit_msg>Do not set the request body.<commit_after>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package api

import (
	"encoding/hex"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strconv"

	"github.com/juju/errors"
	charmresource "gopkg.in/juju/charm.v6-unstable/resource"
)

const (
	// HTTPEndpointPattern is the URL path pattern registered with
	// the API server. This includes wildcards (starting with ":") that
	// are converted into URL query values by the pattern mux. Also see
	// apiserver/apiserver.go.
	HTTPEndpointPattern = "/services/:service/resources/:resource"

	// HTTPEndpointPath is the URL path, with substitutions, for
	// a resource request.
	HTTPEndpointPath = "/environment/%s/services/%s/resources/%s"
)

// NewEndpointPath returns the API URL path for the identified resource.
func NewEndpointPath(envUUID, service, name string) string {
	return fmt.Sprintf(HTTPEndpointPath, envUUID, service, name)
}

// ExtractEndpointDetails pulls the endpoint wildcard values from
// the provided URL.
func ExtractEndpointDetails(url *url.URL) (service, name string) {
	service = url.Query().Get(":service")
	name = url.Query().Get(":resource")
	return service, name
}

// NewHTTPUploadRequest generates a new HTTP request for the given resource.
func NewHTTPUploadRequest(envUUID, service, name string, r io.ReadSeeker) (*http.Request, error) {
	// TODO(ericsnow) Use the newer GenerateFingerprint()...
	data, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, errors.Trace(err)
	}
	if _, err := r.Seek(0, os.SEEK_SET); err != nil {
		return nil, errors.Trace(err)
	}

	fp, err := charmresource.GenerateFingerprint(data)
	if err != nil {
		return nil, errors.Trace(err)
	}

	method := "PUT"
	// TODO(ericsnow) What about the rest of the URL?
	urlStr := NewEndpointPath(envUUID, service, name)
	req, err := http.NewRequest(method, urlStr, nil)
	if err != nil {
		return nil, errors.Trace(err)
	}

	req.Header.Set("Content-Type", "application/octet-stream")
	req.Header.Set("Content-Length", fmt.Sprint(len(data)))
	req.Header.Set("Content-SHA384", fp.String())

	return req, nil
}

// ExtractUploadRequest pulls the required info from the HTTP request.
func ExtractUploadRequest(req *http.Request) (service, name string, size int64, _ charmresource.Fingerprint, _ error) {
	var fp charmresource.Fingerprint

	ctype := req.Header.Get("Content-Type")
	if ctype != "application/octet-stream" {
		return "", "", 0, fp, errors.Errorf("unsupported content type %q", ctype)
	}

	service, name = ExtractEndpointDetails(req.URL)

	fingerprint := req.Header.Get("Content-SHA384") // This parallels "Content-MD5".
	sizeRaw := req.Header.Get("Content-Length")

	// TODO(ericsnow) Use the newer ParseFingerprint().
	fpData, err := hex.DecodeString(fingerprint)
	if err != nil {
		return "", "", 0, fp, errors.Annotate(err, "invalid fingerprint")
	}
	fp, err = charmresource.NewFingerprint(fpData)
	if err != nil {
		return "", "", 0, fp, errors.Annotate(err, "invalid fingerprint")
	}

	size, err = strconv.ParseInt(sizeRaw, 10, 64)
	if err != nil {
		return "", "", 0, fp, errors.Annotate(err, "invalid size")
	}

	return service, name, size, fp, nil
}
<|endoftext|>


<commit_before>package import_server

import (
	"crypto/rand"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"path"
	"sort"
	"strings"

	"github.com/google/uuid"

	"github.com/flant/werf/pkg/docker"

	"github.com/flant/logboek"
	"github.com/flant/werf/pkg/config"
	"github.com/flant/werf/pkg/stapel"
)

const rsyncServerPort = "873"

type RsyncServer struct {
	IPAddress              string
	Port                   string
	DockerContainerName    string
	DockerImageName        string
	AuthUser, AuthPassword string
}

func RunRsyncServer(dockerImageName string, tmpDir string) (*RsyncServer, error) {
	logboek.Debug.LogF("RunRsyncServer for docker image %q\n", dockerImageName)

	srv := &RsyncServer{
		Port:                rsyncServerPort,
		DockerContainerName: fmt.Sprintf("import-server-%s", uuid.New().String()),
		AuthUser:            fmt.Sprintf("werf-%s", generateSecureRandomString(4)),
		AuthPassword:        generateSecureRandomString(16),
	}

	stapelContainerName, err := stapel.GetOrCreateContainer()
	if err != nil {
		return nil, err
	}

	secretsFilePath := path.Join(tmpDir, "rsyncd.secrets")
	if err := ioutil.WriteFile(secretsFilePath, []byte(fmt.Sprintf("%s:%s\n", srv.AuthUser, srv.AuthPassword)), 0644); err != nil {
		return nil, fmt.Errorf("unable to write %s: %s", secretsFilePath, err)
	}

	rsyncConfPath := path.Join(tmpDir, "rsyncd.conf")
	if err := ioutil.WriteFile(rsyncConfPath, []byte(fmt.Sprintf(`pid file = /.werf/rsyncd.pid
lock file = /.werf/rsyncd.lock
log file = /.werf/rsyncd.log
uid = root
port = %s

[import]
path = /
comment = Image files to import
read only = true
timeout = 300
auth users = %s
secrets file = /.werf/rsyncd.secrets
strict modes = false
`, rsyncServerPort, srv.AuthUser)), 0644); err != nil {
		return nil, fmt.Errorf("unable to write %s: %s", rsyncConfPath, err)
	}

	runArgs := []string{
		"--detach",
		"--rm",
		"--user=0:0",
		"--workdir=/",
		fmt.Sprintf("--name=%s", srv.DockerContainerName),
		fmt.Sprintf("--volumes-from=%s", stapelContainerName),
		fmt.Sprintf("--volume=%s:/.werf/rsyncd.conf", rsyncConfPath),
		fmt.Sprintf("--volume=%s:/.werf/rsyncd.secrets", secretsFilePath),
		fmt.Sprintf("--expose=%s", rsyncServerPort),
		fmt.Sprintf("--entrypoint=%s", stapel.RsyncBinPath()),
		dockerImageName,
		"--daemon",
		"--no-detach",
		"--config=/.werf/rsyncd.conf",
	}
	logboek.Debug.LogF("Run rsync server command: %q\n", fmt.Sprintf("docker run %s", strings.Join(runArgs, " ")))
	if output, err := docker.CliRun_RecordedOutput(runArgs...); err != nil {
		logboek.LogErrorF("%s", output)
		return nil, err
	}

	logboek.Debug.LogF("Inspect container %s\n", srv.DockerContainerName)

	if inspect, err := docker.ContainerInspect(srv.DockerContainerName); err != nil {
		return nil, fmt.Errorf("unable to inspect import server container %s: %s", srv.DockerContainerName, err)
	} else {
		if inspect.NetworkSettings == nil {
			return nil, fmt.Errorf("unable to get import server container %s ip address: no network settings available in inspect", srv.DockerContainerName)
		}
		srv.IPAddress = inspect.NetworkSettings.IPAddress
	}

	return srv, nil
}

func (srv *RsyncServer) Shutdown() error {
	if output, err := docker.CliRm_RecordedOutput("--force", srv.DockerContainerName); err != nil {
		logboek.LogErrorF("%s", output)
		return fmt.Errorf("unable to remove container %s: %s", srv.DockerContainerName, err)
	}
	return nil
}

func (srv *RsyncServer) GetCopyCommand(importConfig *config.Import) string {
	var args []string

	rsyncImportPathSpec := fmt.Sprintf("rsync://%s@%s:%s/import/%s", srv.AuthUser, srv.IPAddress, srv.Port, importConfig.Add)
	rsyncStatImportPathCommand := fmt.Sprintf("RSYNC_PASSWORD='%s' %s -L %s", srv.AuthPassword, stapel.RsyncBinPath(), rsyncImportPathSpec)

	// save stat output to variable
	args = append(args, fmt.Sprintf("statOutput=$(%s)", rsyncStatImportPathCommand))
	// check command exit code from last subshell
	args = append(args, "[ $? -eq 0 ]")
	// unset old value of IMPORT_PATH_TRAILING_SLASH_OPTIONAL variable from other copy commands
	args = append(args, "unset IMPORT_PATH_TRAILING_SLASH_OPTIONAL")
	// set fileTypeField
	args = append(args, fmt.Sprintf("fileTypeField=$(echo $statOutput | %s -c1)", stapel.HeadBinPath()))
	// check command exit code from last subshell
	args = append(args, "[ $? -eq 0 ]")
	// set optional trailing slash when importing directory so that rsync will automatically
	// merge already existing directory in the target image
	args = append(args, "if [ $fileTypeField = d ] ; then IMPORT_PATH_TRAILING_SLASH_OPTIONAL=/ ; fi")
	// create a parent directory where target file/directory will reside
	args = append(args, fmt.Sprintf("%s -p %s", stapel.MkdirBinPath(), path.Dir(importConfig.To)))

	var rsyncChownOption string
	if importConfig.Owner != "" || importConfig.Group != "" {
		rsyncChownOption = fmt.Sprintf("--chown=%s:%s", importConfig.Owner, importConfig.Group)
	}
	rsyncCommand := fmt.Sprintf("RSYNC_PASSWORD='%s' %s --archive --links --inplace %s", srv.AuthPassword, stapel.RsyncBinPath(), rsyncChownOption)

	if len(importConfig.IncludePaths) != 0 {
		/**
				Если указали include_paths — это означает, что надо копировать
				только указанные пути. Поэтому exclude_paths в приоритете, т.к. в данном режиме
		        exclude_paths может относится только к путям, указанным в include_paths.
		        При этом случай, когда в include_paths указали более специальный путь, чем в exclude_paths,
		        будет обрабатываться в пользу exclude, этот путь не скопируется.
		*/
		for _, p := range importConfig.ExcludePaths {
			rsyncCommand += fmt.Sprintf(" --filter='-/ %s'", path.Join(importConfig.Add, p))
		}

		for _, p := range importConfig.IncludePaths {
			targetPath := path.Join(importConfig.Add, p)

			// Генерируем разрешающее правило для каждого элемента пути
			for _, pathPart := range descentPath(targetPath) {
				rsyncCommand += fmt.Sprintf(" --filter='+/ %s'", pathPart)
			}

			/**
					На данный момент не знаем директорию или файл имел в виду пользователь,
			        поэтому подставляем фильтры для обоих возможных случаев.

					Автоматом подставляем паттерн ** для включения файлов, содержащихся в
			        директории, которую пользователь указал в include_paths.
			*/
			rsyncCommand += fmt.Sprintf(" --filter='+/ %s'", targetPath)
			rsyncCommand += fmt.Sprintf(" --filter='+/ %s'", path.Join(targetPath, "**"))
		}

		// Все что не подошло по include — исключается
		rsyncCommand += fmt.Sprintf(" --filter='-/ %s'", path.Join(importConfig.Add, "**"))
	} else {
		for _, p := range importConfig.ExcludePaths {
			rsyncCommand += fmt.Sprintf(" --filter='-/ %s'", path.Join(importConfig.Add, p))
		}
	}

	rsyncCommand += fmt.Sprintf(" %s$IMPORT_PATH_TRAILING_SLASH_OPTIONAL %s", rsyncImportPathSpec, importConfig.To)
	// run rsync itself
	args = append(args, rsyncCommand)

	command := strings.Join(args, " && ")

	logboek.Debug.LogF("Rsync server copy commands for import: artifact=%q image=%q add=%s to=%s includePaths=%v excludePaths=%v: %q\n", importConfig.ArtifactName, importConfig.ImageName, importConfig.Add, importConfig.To, importConfig.IncludePaths, importConfig.ExcludePaths, command)

	return command
}

func descentPath(filePath string) []string {
	var parts []string

	part := filePath
	for {
		parts = append(parts, part)
		part = path.Dir(part)

		if part == path.Dir(part) {
			break
		}
	}

	sort.Sort(sort.Reverse(sort.StringSlice(parts[:])))

	return parts
}

func generateSecureRandomString(lenght int) string {
	randomBytes := make([]byte, lenght)
	if _, err := rand.Read(randomBytes); err != nil {
		panic(fmt.Sprintf("cannot generate secure random string: %s", err))
	}
	return hex.EncodeToString(randomBytes)
}
<commit_msg>Fix "@ERROR: invalid gid nogroup" on imports stage<commit_after>package import_server

import (
	"crypto/rand"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"path"
	"sort"
	"strings"

	"github.com/google/uuid"

	"github.com/flant/werf/pkg/docker"

	"github.com/flant/logboek"
	"github.com/flant/werf/pkg/config"
	"github.com/flant/werf/pkg/stapel"
)

const rsyncServerPort = "873"

type RsyncServer struct {
	IPAddress              string
	Port                   string
	DockerContainerName    string
	DockerImageName        string
	AuthUser, AuthPassword string
}

func RunRsyncServer(dockerImageName string, tmpDir string) (*RsyncServer, error) {
	logboek.Debug.LogF("RunRsyncServer for docker image %q\n", dockerImageName)

	srv := &RsyncServer{
		Port:                rsyncServerPort,
		DockerContainerName: fmt.Sprintf("import-server-%s", uuid.New().String()),
		AuthUser:            fmt.Sprintf("werf-%s", generateSecureRandomString(4)),
		AuthPassword:        generateSecureRandomString(16),
	}

	stapelContainerName, err := stapel.GetOrCreateContainer()
	if err != nil {
		return nil, err
	}

	secretsFilePath := path.Join(tmpDir, "rsyncd.secrets")
	if err := ioutil.WriteFile(secretsFilePath, []byte(fmt.Sprintf("%s:%s\n", srv.AuthUser, srv.AuthPassword)), 0644); err != nil {
		return nil, fmt.Errorf("unable to write %s: %s", secretsFilePath, err)
	}

	rsyncConfPath := path.Join(tmpDir, "rsyncd.conf")
	if err := ioutil.WriteFile(rsyncConfPath, []byte(fmt.Sprintf(`pid file = /.werf/rsyncd.pid
lock file = /.werf/rsyncd.lock
log file = /.werf/rsyncd.log
uid = root
gid = root
port = %s

[import]
path = /
comment = Image files to import
read only = true
timeout = 300
auth users = %s
secrets file = /.werf/rsyncd.secrets
strict modes = false
`, rsyncServerPort, srv.AuthUser)), 0644); err != nil {
		return nil, fmt.Errorf("unable to write %s: %s", rsyncConfPath, err)
	}

	runArgs := []string{
		"--detach",
		"--rm",
		"--user=0:0",
		"--workdir=/",
		fmt.Sprintf("--name=%s", srv.DockerContainerName),
		fmt.Sprintf("--volumes-from=%s", stapelContainerName),
		fmt.Sprintf("--volume=%s:/.werf/rsyncd.conf", rsyncConfPath),
		fmt.Sprintf("--volume=%s:/.werf/rsyncd.secrets", secretsFilePath),
		fmt.Sprintf("--expose=%s", rsyncServerPort),
		fmt.Sprintf("--entrypoint=%s", stapel.RsyncBinPath()),
		dockerImageName,
		"--daemon",
		"--no-detach",
		"--config=/.werf/rsyncd.conf",
	}
	logboek.Debug.LogF("Run rsync server command: %q\n", fmt.Sprintf("docker run %s", strings.Join(runArgs, " ")))
	if output, err := docker.CliRun_RecordedOutput(runArgs...); err != nil {
		logboek.LogErrorF("%s", output)
		return nil, err
	}

	logboek.Debug.LogF("Inspect container %s\n", srv.DockerContainerName)

	if inspect, err := docker.ContainerInspect(srv.DockerContainerName); err != nil {
		return nil, fmt.Errorf("unable to inspect import server container %s: %s", srv.DockerContainerName, err)
	} else {
		if inspect.NetworkSettings == nil {
			return nil, fmt.Errorf("unable to get import server container %s ip address: no network settings available in inspect", srv.DockerContainerName)
		}
		srv.IPAddress = inspect.NetworkSettings.IPAddress
	}

	return srv, nil
}

func (srv *RsyncServer) Shutdown() error {
	if output, err := docker.CliRm_RecordedOutput("--force", srv.DockerContainerName); err != nil {
		logboek.LogErrorF("%s", output)
		return fmt.Errorf("unable to remove container %s: %s", srv.DockerContainerName, err)
	}
	return nil
}

func (srv *RsyncServer) GetCopyCommand(importConfig *config.Import) string {
	var args []string

	rsyncImportPathSpec := fmt.Sprintf("rsync://%s@%s:%s/import/%s", srv.AuthUser, srv.IPAddress, srv.Port, importConfig.Add)
	rsyncStatImportPathCommand := fmt.Sprintf("RSYNC_PASSWORD='%s' %s -L %s", srv.AuthPassword, stapel.RsyncBinPath(), rsyncImportPathSpec)

	// save stat output to variable
	args = append(args, fmt.Sprintf("statOutput=$(%s)", rsyncStatImportPathCommand))
	// check command exit code from last subshell
	args = append(args, "[ $? -eq 0 ]")
	// unset old value of IMPORT_PATH_TRAILING_SLASH_OPTIONAL variable from other copy commands
	args = append(args, "unset IMPORT_PATH_TRAILING_SLASH_OPTIONAL")
	// set fileTypeField
	args = append(args, fmt.Sprintf("fileTypeField=$(echo $statOutput | %s -c1)", stapel.HeadBinPath()))
	// check command exit code from last subshell
	args = append(args, "[ $? -eq 0 ]")
	// set optional trailing slash when importing directory so that rsync will automatically
	// merge already existing directory in the target image
	args = append(args, "if [ $fileTypeField = d ] ; then IMPORT_PATH_TRAILING_SLASH_OPTIONAL=/ ; fi")
	// create a parent directory where target file/directory will reside
	args = append(args, fmt.Sprintf("%s -p %s", stapel.MkdirBinPath(), path.Dir(importConfig.To)))

	var rsyncChownOption string
	if importConfig.Owner != "" || importConfig.Group != "" {
		rsyncChownOption = fmt.Sprintf("--chown=%s:%s", importConfig.Owner, importConfig.Group)
	}
	rsyncCommand := fmt.Sprintf("RSYNC_PASSWORD='%s' %s --archive --links --inplace %s", srv.AuthPassword, stapel.RsyncBinPath(), rsyncChownOption)

	if len(importConfig.IncludePaths) != 0 {
		/**
				Если указали include_paths — это означает, что надо копировать
				только указанные пути. Поэтому exclude_paths в приоритете, т.к. в данном режиме
		        exclude_paths может относится только к путям, указанным в include_paths.
		        При этом случай, когда в include_paths указали более специальный путь, чем в exclude_paths,
		        будет обрабатываться в пользу exclude, этот путь не скопируется.
		*/
		for _, p := range importConfig.ExcludePaths {
			rsyncCommand += fmt.Sprintf(" --filter='-/ %s'", path.Join(importConfig.Add, p))
		}

		for _, p := range importConfig.IncludePaths {
			targetPath := path.Join(importConfig.Add, p)

			// Генерируем разрешающее правило для каждого элемента пути
			for _, pathPart := range descentPath(targetPath) {
				rsyncCommand += fmt.Sprintf(" --filter='+/ %s'", pathPart)
			}

			/**
					На данный момент не знаем директорию или файл имел в виду пользователь,
			        поэтому подставляем фильтры для обоих возможных случаев.

					Автоматом подставляем паттерн ** для включения файлов, содержащихся в
			        директории, которую пользователь указал в include_paths.
			*/
			rsyncCommand += fmt.Sprintf(" --filter='+/ %s'", targetPath)
			rsyncCommand += fmt.Sprintf(" --filter='+/ %s'", path.Join(targetPath, "**"))
		}

		// Все что не подошло по include — исключается
		rsyncCommand += fmt.Sprintf(" --filter='-/ %s'", path.Join(importConfig.Add, "**"))
	} else {
		for _, p := range importConfig.ExcludePaths {
			rsyncCommand += fmt.Sprintf(" --filter='-/ %s'", path.Join(importConfig.Add, p))
		}
	}

	rsyncCommand += fmt.Sprintf(" %s$IMPORT_PATH_TRAILING_SLASH_OPTIONAL %s", rsyncImportPathSpec, importConfig.To)
	// run rsync itself
	args = append(args, rsyncCommand)

	command := strings.Join(args, " && ")

	logboek.Debug.LogF("Rsync server copy commands for import: artifact=%q image=%q add=%s to=%s includePaths=%v excludePaths=%v: %q\n", importConfig.ArtifactName, importConfig.ImageName, importConfig.Add, importConfig.To, importConfig.IncludePaths, importConfig.ExcludePaths, command)

	return command
}

func descentPath(filePath string) []string {
	var parts []string

	part := filePath
	for {
		parts = append(parts, part)
		part = path.Dir(part)

		if part == path.Dir(part) {
			break
		}
	}

	sort.Sort(sort.Reverse(sort.StringSlice(parts[:])))

	return parts
}

func generateSecureRandomString(lenght int) string {
	randomBytes := make([]byte, lenght)
	if _, err := rand.Read(randomBytes); err != nil {
		panic(fmt.Sprintf("cannot generate secure random string: %s", err))
	}
	return hex.EncodeToString(randomBytes)
}
<|endoftext|>


<commit_before>package ristretto255

import (
	"bytes"
	"encoding/hex"
	"testing"

	"github.com/gtank/ristretto255/internal/edwards25519"
	"github.com/gtank/ristretto255/internal/radix51"
)

func assertFeEqual(value, expect *radix51.FieldElement) {
	if value.Equal(expect) == 1 {
		return
	} else {
		panic("failed equality assertion")
	}
}

type sqrtRatioTest struct {
	u, v     *radix51.FieldElement
	sqrt     *radix51.FieldElement
	choice   int
	negative int
}

// These tests can be found in curve25519-dalek's 'field.rs'
func TestSqrtRatioM1(t *testing.T) {
	var (
		zero, one = radix51.Zero, radix51.One

		// Two is nonsquare in our field, 4 is square
		two  = new(radix51.FieldElement).Add(one, one)
		four = new(radix51.FieldElement).Add(two, two)

		// 2*i
		twoTimesSqrtM1 = new(radix51.FieldElement).Mul(two, sqrtM1)

		sqrt2i = fieldElementFromDecimal(
			"38214883241950591754978413199355411911188925816896391856984770930832735035196")

		invSqrt4 = fieldElementFromDecimal(
			"28948022309329048855892746252171976963317496166410141009864396001978282409974")
	)

	// Check the construction of those magic numbers.
	assertFeEqual(new(radix51.FieldElement).Mul(sqrt2i, sqrt2i), twoTimesSqrtM1)
	assertFeEqual(new(radix51.FieldElement).Mul(new(radix51.FieldElement).Square(invSqrt4), four), one)

	var tests = []sqrtRatioTest{
		{u: zero, v: zero, sqrt: zero, choice: 1, negative: 0},    // 0
		{u: one, v: zero, sqrt: zero, choice: 0, negative: 0},     // 1
		{u: two, v: one, sqrt: sqrt2i, choice: 0, negative: 0},    // 2
		{u: four, v: one, sqrt: two, choice: 1, negative: 0},      // 3
		{u: one, v: four, sqrt: invSqrt4, choice: 1, negative: 0}, // 4
	}

	for idx, tt := range tests {
		sqrt := new(radix51.FieldElement)
		choice := feSqrtRatio(sqrt, tt.u, tt.v)
		if choice != tt.choice || sqrt.Equal(tt.sqrt) != 1 || sqrt.IsNegative() != tt.negative {
			t.Errorf("Failed test %d", idx)
			t.Logf("Got {u: %v, v: %v, sqrt: %v, choice: %d, neg: %d}", tt.u, tt.v, sqrt, choice, sqrt.IsNegative())
		}
	}
}

var (
	// The encoding of Ristretto element that can be represented internally by the Curve25519 base point.
	compressedRistrettoBasepoint, _ = hex.DecodeString("e2f2ae0a6abc4e71a884a961c500515f58e30b6aa582dd8db6a65945e08d2d76")

	// The representative Ristretto basepoint in extended coordinates.
	ristrettoBasepoint = Element{r: edwards25519.ExtendedGroupElement{
		X: radix51.FieldElement([5]uint64{426475514619346, 2063872706840040, 14628272888959, 107677749330612, 288339085807592}),
		Y: radix51.FieldElement([5]uint64{1934594822876571, 2049809580636559, 1991994783322914, 1758681962032007, 380046701118659}),
		Z: radix51.FieldElement([5]uint64{1, 0, 0, 0, 0}),
		T: radix51.FieldElement([5]uint64{410445769351754, 2235400917701188, 1495825632738689, 1351628537510093, 430502003771208}),
	}}
)

func TestRistrettoEncoding(t *testing.T) {
	decodedBasepoint := &Element{}
	err := decodedBasepoint.Decode(compressedRistrettoBasepoint)
	if err != nil {
		t.Fatal(err)
	}

	if decodedBasepoint.Equal(&ristrettoBasepoint) != 1 {
		t.Error("decode succeeded, but got wrong point")
	}

	roundtripBasepoint := decodedBasepoint.Encode(nil)
	if !bytes.Equal(compressedRistrettoBasepoint, roundtripBasepoint) {
		t.Error("decode<>encode roundtrip produced different results")
	}

	encodedBasepoint := ristrettoBasepoint.Encode(nil)
	if !bytes.Equal(compressedRistrettoBasepoint, encodedBasepoint) {
		t.Error("point encode produced different results")
	}
}

func TestRistrettoRoundtrip(t *testing.T) {
	// TODO quickcheck
}
<commit_msg>Add encoding test vectors from the spec.<commit_after>package ristretto255

import (
	"bytes"
	"encoding/hex"
	"testing"

	"github.com/gtank/ristretto255/internal/edwards25519"
	"github.com/gtank/ristretto255/internal/radix51"
)

func assertFeEqual(value, expect *radix51.FieldElement) {
	if value.Equal(expect) == 1 {
		return
	} else {
		panic("failed equality assertion")
	}
}

type sqrtRatioTest struct {
	u, v     *radix51.FieldElement
	sqrt     *radix51.FieldElement
	choice   int
	negative int
}

// These tests can be found in curve25519-dalek's 'field.rs'
func TestSqrtRatioM1(t *testing.T) {
	var (
		zero, one = radix51.Zero, radix51.One

		// Two is nonsquare in our field, 4 is square
		two  = new(radix51.FieldElement).Add(one, one)
		four = new(radix51.FieldElement).Add(two, two)

		// 2*i
		twoTimesSqrtM1 = new(radix51.FieldElement).Mul(two, sqrtM1)

		sqrt2i = fieldElementFromDecimal(
			"38214883241950591754978413199355411911188925816896391856984770930832735035196")

		invSqrt4 = fieldElementFromDecimal(
			"28948022309329048855892746252171976963317496166410141009864396001978282409974")
	)

	// Check the construction of those magic numbers.
	assertFeEqual(new(radix51.FieldElement).Mul(sqrt2i, sqrt2i), twoTimesSqrtM1)
	assertFeEqual(new(radix51.FieldElement).Mul(new(radix51.FieldElement).Square(invSqrt4), four), one)

	var tests = []sqrtRatioTest{
		{u: zero, v: zero, sqrt: zero, choice: 1, negative: 0},    // 0
		{u: one, v: zero, sqrt: zero, choice: 0, negative: 0},     // 1
		{u: two, v: one, sqrt: sqrt2i, choice: 0, negative: 0},    // 2
		{u: four, v: one, sqrt: two, choice: 1, negative: 0},      // 3
		{u: one, v: four, sqrt: invSqrt4, choice: 1, negative: 0}, // 4
	}

	for idx, tt := range tests {
		sqrt := new(radix51.FieldElement)
		choice := feSqrtRatio(sqrt, tt.u, tt.v)
		if choice != tt.choice || sqrt.Equal(tt.sqrt) != 1 || sqrt.IsNegative() != tt.negative {
			t.Errorf("Failed test %d", idx)
			t.Logf("Got {u: %v, v: %v, sqrt: %v, choice: %d, neg: %d}", tt.u, tt.v, sqrt, choice, sqrt.IsNegative())
		}
	}
}

var (
	// The encoding of Ristretto element that can be represented internally by the Curve25519 base point.
	compressedRistrettoBasepoint, _ = hex.DecodeString("e2f2ae0a6abc4e71a884a961c500515f58e30b6aa582dd8db6a65945e08d2d76")

	// The representative Ristretto basepoint in extended coordinates.
	ristrettoBasepoint = Element{r: edwards25519.ExtendedGroupElement{
		X: radix51.FieldElement([5]uint64{426475514619346, 2063872706840040, 14628272888959, 107677749330612, 288339085807592}),
		Y: radix51.FieldElement([5]uint64{1934594822876571, 2049809580636559, 1991994783322914, 1758681962032007, 380046701118659}),
		Z: radix51.FieldElement([5]uint64{1, 0, 0, 0, 0}),
		T: radix51.FieldElement([5]uint64{410445769351754, 2235400917701188, 1495825632738689, 1351628537510093, 430502003771208}),
	}}
)

func TestRistrettoBasepointRoundTrip(t *testing.T) {
	decodedBasepoint := &Element{}
	err := decodedBasepoint.Decode(compressedRistrettoBasepoint)
	if err != nil {
		t.Fatal(err)
	}

	if decodedBasepoint.Equal(&ristrettoBasepoint) != 1 {
		t.Error("decode succeeded, but got wrong point")
	}

	roundtripBasepoint := decodedBasepoint.Encode(nil)
	if !bytes.Equal(compressedRistrettoBasepoint, roundtripBasepoint) {
		t.Error("decode<>encode roundtrip produced different results")
	}

	encodedBasepoint := ristrettoBasepoint.Encode(nil)
	if !bytes.Equal(compressedRistrettoBasepoint, encodedBasepoint) {
		t.Error("point encode produced different results")
	}
}

func TestRistrettoRandomRoundtrip(t *testing.T) {
	// TODO quickcheck
}

func TestRistrettoSmallMultiplesTestVectors(t *testing.T) {
	var testVectors = [16]string{
		// This is the identity point
		"0000000000000000000000000000000000000000000000000000000000000000",
		// This is the basepoint
		"e2f2ae0a6abc4e71a884a961c500515f58e30b6aa582dd8db6a65945e08d2d76",
		// These are small multiples of the basepoint
		"6a493210f7499cd17fecb510ae0cea23a110e8d5b901f8acadd3095c73a3b919",
		"94741f5d5d52755ece4f23f044ee27d5d1ea1e2bd196b462166b16152a9d0259",
		"da80862773358b466ffadfe0b3293ab3d9fd53c5ea6c955358f568322daf6a57",
		"e882b131016b52c1d3337080187cf768423efccbb517bb495ab812c4160ff44e",
		"f64746d3c92b13050ed8d80236a7f0007c3b3f962f5ba793d19a601ebb1df403",
		"44f53520926ec81fbd5a387845beb7df85a96a24ece18738bdcfa6a7822a176d",
		"903293d8f2287ebe10e2374dc1a53e0bc887e592699f02d077d5263cdd55601c",
		"02622ace8f7303a31cafc63f8fc48fdc16e1c8c8d234b2f0d6685282a9076031",
		"20706fd788b2720a1ed2a5dad4952b01f413bcf0e7564de8cdc816689e2db95f",
		"bce83f8ba5dd2fa572864c24ba1810f9522bc6004afe95877ac73241cafdab42",
		"e4549ee16b9aa03099ca208c67adafcafa4c3f3e4e5303de6026e3ca8ff84460",
		"aa52e000df2e16f55fb1032fc33bc42742dad6bd5a8fc0be0167436c5948501f",
		"46376b80f409b29dc2b5f6f0c52591990896e5716f41477cd30085ab7f10301e",
		"e0c418f7c8d9c4cdd7395b93ea124f3ad99021bb681dfc3302a9d99a2e53e64e",
	}

	// TODO need to use Ristretto addition to do this test properly, see
	// https://ristretto.group/test_vectors/ristretto255.html

	for i := 0; i < 16; i++ {
		encoding, err := hex.DecodeString(testVectors[i])
		if err != nil {
			t.Error("Bad hex encoding in test vector")
		}
		// Decode the test vector to a ristretto255 element
		decodedPoint := &Element{}
		err = decodedPoint.Decode(encoding)
		if err != nil {
			t.Error("Could not decode test vector")
		}
		// Re-encode and check round trips
		roundtripPoint := decodedPoint.Encode()
		if !bytes.Equal(encoding, roundtripPoint) {
			t.Errorf("decode<>encode roundtrip failed on test vector %d", i)
		}
		// Check that the test vector encodes i * B
		// TODO add this (need addition, see above)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1beta17

import (
	next "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/util"
	pkgutil "github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
)

// Upgrade upgrades a configuration to the next version.
// Config changes from v1beta17 to v1.0
// 1. Additions:
//    Add EphemeralStorage and ResourceStorage to Kaniko Pod Resource Requirement.
// 2. Removals:
// 3. No updates
func (c *SkaffoldConfig) Upgrade() (util.VersionedConfig, error) {
	var newConfig next.SkaffoldConfig

	pkgutil.CloneThroughJSON(c, &newConfig)
	if err := util.UpgradePipelines(c, &newConfig, upgradeOnePipeline); err != nil {
		return nil, err
	}
	newConfig.APIVersion = next.Version

	return &newConfig, nil
}

// Placeholder for upgrade logic
func upgradeOnePipeline(_, _ interface{}) error {
	return nil
}
<commit_msg>Update pkg/skaffold/schema/v1beta17/upgrade.go<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1beta17

import (
	next "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/util"
	pkgutil "github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
)

// Upgrade upgrades a configuration to the next version.
// Config changes from v1beta17 to v1
// 1. Additions:
//    Add EphemeralStorage and ResourceStorage to Kaniko Pod Resource Requirement.
// 2. Removals:
// 3. No updates
func (c *SkaffoldConfig) Upgrade() (util.VersionedConfig, error) {
	var newConfig next.SkaffoldConfig

	pkgutil.CloneThroughJSON(c, &newConfig)
	if err := util.UpgradePipelines(c, &newConfig, upgradeOnePipeline); err != nil {
		return nil, err
	}
	newConfig.APIVersion = next.Version

	return &newConfig, nil
}

// Placeholder for upgrade logic
func upgradeOnePipeline(_, _ interface{}) error {
	return nil
}
<|endoftext|>


<commit_before>package middleware

import (
	"encoding/json"
	"net/http"

	"context"

	"github.com/remind101/pkg/httpx"
	"github.com/remind101/pkg/reporter"
)

type ErrorHandlerFunc func(context.Context, error, http.ResponseWriter, *http.Request)

type temporaryError interface {
	Temporary() bool // Is the error temporary?
}

type timeoutError interface {
	Timeout() bool // Is the error a timeout?
}

type statusCoder interface {
	StatusCode() int
}

// DefaultErrorHandler is an error handler that will respond with the error
// message and a 500 status.
var DefaultErrorHandler = func(ctx context.Context, err error, w http.ResponseWriter, r *http.Request) {
	writeError(w, err)
}

// ReportingErrorHandler is an error handler that will report the error and respond
// with the error message and a 500 status.
var ReportingErrorHandler = func(ctx context.Context, err error, w http.ResponseWriter, r *http.Request) {
	reporter.Report(ctx, err)
	writeError(w, err)
}

var JSONReportingErrorHandler = func(ctx context.Context, err error, w http.ResponseWriter, r *http.Request) {
	reporter.Report(ctx, err)
	status := statusCodeForError(err)
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(status)

	errorResp := map[string]string{
		"error": err.Error(),
	}

	json.NewEncoder(w).Encode(errorResp)
}

func writeError(w http.ResponseWriter, err error) {
	http.Error(w, err.Error(), statusCodeForError(err))
}

func statusCodeForError(err error) int {
	if e, ok := err.(statusCoder); ok {
		return e.StatusCode()
	}
	if e, ok := err.(temporaryError); ok && e.Temporary() {
		return http.StatusServiceUnavailable
	}

	if e, ok := err.(timeoutError); ok && e.Timeout() {
		return http.StatusServiceUnavailable
	}

	return http.StatusInternalServerError
}

// Error is an httpx.Handler that will handle errors with an ErrorHandler.
type Error struct {
	// ErrorHandler is a function that will be called when a handler returns
	// an error.
	ErrorHandler ErrorHandlerFunc

	// Handler is the wrapped httpx.Handler that will be called.
	handler httpx.Handler
}

func NewError(h httpx.Handler) *Error {
	return &Error{
		handler: h,
	}
}

// HandleError returns a new Error middleware that uses f as the ErrorHandler.
func HandleError(h httpx.Handler, f ErrorHandlerFunc) *Error {
	e := NewError(h)
	e.ErrorHandler = f
	return e
}

// ServeHTTPContext implements the httpx.Handler interface.
func (h *Error) ServeHTTPContext(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
	err := h.handler.ServeHTTPContext(ctx, w, r)

	if err != nil {
		f := h.ErrorHandler
		if f == nil {
			f = DefaultErrorHandler
		}

		f(ctx, err, w, r)
	}

	return nil
}
<commit_msg>Check against root error<commit_after>package middleware

import (
	"encoding/json"
	"net/http"

	"context"

	"github.com/pkg/errors"
	"github.com/remind101/pkg/httpx"
	"github.com/remind101/pkg/reporter"
)

type ErrorHandlerFunc func(context.Context, error, http.ResponseWriter, *http.Request)

type temporaryError interface {
	Temporary() bool // Is the error temporary?
}

type timeoutError interface {
	Timeout() bool // Is the error a timeout?
}

type statusCoder interface {
	StatusCode() int
}

// DefaultErrorHandler is an error handler that will respond with the error
// message and a 500 status.
var DefaultErrorHandler = func(ctx context.Context, err error, w http.ResponseWriter, r *http.Request) {
	writeError(w, err)
}

// ReportingErrorHandler is an error handler that will report the error and respond
// with the error message and a 500 status.
var ReportingErrorHandler = func(ctx context.Context, err error, w http.ResponseWriter, r *http.Request) {
	reporter.Report(ctx, err)
	writeError(w, err)
}

var JSONReportingErrorHandler = func(ctx context.Context, err error, w http.ResponseWriter, r *http.Request) {
	reporter.Report(ctx, err)
	status := statusCodeForError(err)
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(status)

	errorResp := map[string]string{
		"error": err.Error(),
	}

	json.NewEncoder(w).Encode(errorResp)
}

func writeError(w http.ResponseWriter, err error) {
	http.Error(w, err.Error(), statusCodeForError(err))
}

func statusCodeForError(err error) int {
	rootErr := errors.Cause(err)
	if e, ok := rootErr.(statusCoder); ok {
		return e.StatusCode()
	}
	if e, ok := rootErr.(temporaryError); ok && e.Temporary() {
		return http.StatusServiceUnavailable
	}

	if e, ok := rootErr.(timeoutError); ok && e.Timeout() {
		return http.StatusServiceUnavailable
	}

	return http.StatusInternalServerError
}

// Error is an httpx.Handler that will handle errors with an ErrorHandler.
type Error struct {
	// ErrorHandler is a function that will be called when a handler returns
	// an error.
	ErrorHandler ErrorHandlerFunc

	// Handler is the wrapped httpx.Handler that will be called.
	handler httpx.Handler
}

func NewError(h httpx.Handler) *Error {
	return &Error{
		handler: h,
	}
}

// HandleError returns a new Error middleware that uses f as the ErrorHandler.
func HandleError(h httpx.Handler, f ErrorHandlerFunc) *Error {
	e := NewError(h)
	e.ErrorHandler = f
	return e
}

// ServeHTTPContext implements the httpx.Handler interface.
func (h *Error) ServeHTTPContext(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
	err := h.handler.ServeHTTPContext(ctx, w, r)

	if err != nil {
		f := h.ErrorHandler
		if f == nil {
			f = DefaultErrorHandler
		}

		f(ctx, err, w, r)
	}

	return nil
}
<|endoftext|>


<commit_before>// Package azurestoragecache provides an implementation of httpcache.Cache that
// stores and retrieves data using Azure Storage.
package azurestoragecache // import "github.com/PaulARoy/azurestoragecache"

import (
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"bytes"

	vendorstorage "github.com/Azure/azure-sdk-for-go/storage"
)

// Cache objects store and retrieve data using Azure Storage
type Cache struct {
	// Our configuration for Azure Storage
	Config Config

	// The Azure Blob Storage Client
	Client vendorstorage.BlobStorageClient
}

type Config struct {
	// Account configuration for Azure Storage
	AccountName string
	AccountKey string

	// Container name to use to store blob
	ContainerName string
}

var noLogErrors, _ = strconv.ParseBool(os.Getenv("NO_LOG_AZUREBSCACHE_ERRORS"))

func (c *Cache) Get(key string) (resp []byte, ok bool) {
	rdr, err := c.Client.GetBlob(c.Config.ContainerName, key)
	if err != nil {
		return []byte{}, false
	}
	rdr.Close()

	resp, err = ioutil.ReadAll(rdr)
	if err != nil {
		if !noLogErrors {
			log.Printf("azurestoragecache.Get failed: %s", err)
		}
	}
	return resp, err == nil
}

func (c *Cache) Set(key string, block []byte) {
	err := c.Client.CreateBlockBlobFromReader(c.Config.ContainerName,
												key,
												uint64(len(block)),
												bytes.NewReader(block),
												nil)
	if err != nil {
		if !noLogErrors {
			log.Printf("azurestoragecache.Set failed: %s", err)
		}
		return
	}
}

func (c *Cache) Delete(key string) bool {
	res, err := c.Client.DeleteBlobIfExists(c.Config.ContainerName, key, nil)
	if err != nil {
		if !noLogErrors {
			log.Printf("azurestoragecache.Delete failed: %s", err)
		}
		return false
	}
	return res
}

// New returns a new Cache with underlying client for Azure Storage
//
// containerName is the container name for azure blob service
//
// The environment variables AZURESTORAGE_ACCOUNT_NAME and AZURESTORAGE_ACCESS_KEY
// are used as credentials. To use different credentials, construct a Cache object
// manually.
func New(accountName string, accountKey string, containerName string) (*Cache, bool, error) {
	cache := Cache{
		Config: Config{
			AccountName: accountName, // || os.Getenv("AZURESTORAGE_ACCOUNT_NAME"),
			AccountKey: accountKey, // || os.Getenv("AZURESTORAGE_ACCESS_KEY"),
			ContainerName: containerName,
		},
	}

	api, err := vendorstorage.NewBasicClient(cache.Config.AccountName, cache.Config.AccountKey)
	if err != nil {
		return nil, false, err
	}

	cache.Client = api.GetBlobService()

	res, err := cache.Client.CreateContainerIfNotExists(cache.Config.ContainerName,
											vendorstorage.ContainerAccessTypeBlob)
	if err != nil {
		return nil, false, err
	}

	return &cache, res, nil
}<commit_msg>Use blob service each time<commit_after>// Package azurestoragecache provides an implementation of httpcache.Cache that
// stores and retrieves data using Azure Storage.
package azurestoragecache // import "github.com/PaulARoy/azurestoragecache"

import (
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"bytes"

	vendorstorage "github.com/Azure/azure-sdk-for-go/storage"
)

// Cache objects store and retrieve data using Azure Storage
type Cache struct {
	// Our configuration for Azure Storage
	Config Config

	// The Azure Blob Storage Client
	Client vendorstorage.Client
}

type Config struct {
	// Account configuration for Azure Storage
	AccountName string
	AccountKey string

	// Container name to use to store blob
	ContainerName string
}

var noLogErrors, _ = strconv.ParseBool(os.Getenv("NO_LOG_AZUREBSCACHE_ERRORS"))

func (c *Cache) Get(key string) (resp []byte, ok bool) {
	rdr, err := c.Client.GetBlobService().GetBlob(c.Config.ContainerName, key)
	if err != nil {
		return []byte{}, false
	}
	rdr.Close()

	resp, err = ioutil.ReadAll(rdr)
	if err != nil {
		if !noLogErrors {
			log.Printf("azurestoragecache.Get failed: %s", err)
		}
	}
	return resp, err == nil
}

func (c *Cache) Set(key string, block []byte) {
	err := c.Client.GetBlobService().CreateBlockBlobFromReader(c.Config.ContainerName,
												key,
												uint64(len(block)),
												bytes.NewReader(block),
												nil)
	if err != nil {
		if !noLogErrors {
			log.Printf("azurestoragecache.Set failed: %s", err)
		}
		return
	}
}

func (c *Cache) Delete(key string) bool {
	res, err := c.Client.GetBlobService().DeleteBlobIfExists(c.Config.ContainerName, key, nil)
	if err != nil {
		if !noLogErrors {
			log.Printf("azurestoragecache.Delete failed: %s", err)
		}
		return false
	}
	return res
}

// New returns a new Cache with underlying client for Azure Storage
//
// containerName is the container name for azure blob service
//
// The environment variables AZURESTORAGE_ACCOUNT_NAME and AZURESTORAGE_ACCESS_KEY
// are used as credentials. To use different credentials, construct a Cache object
// manually.
func New(accountName string, accountKey string, containerName string) (*Cache, bool, error) {
	cache := Cache{
		Config: Config{
			AccountName: accountName, // || os.Getenv("AZURESTORAGE_ACCOUNT_NAME"),
			AccountKey: accountKey, // || os.Getenv("AZURESTORAGE_ACCESS_KEY"),
			ContainerName: containerName,
		},
	}

	api, err := vendorstorage.NewBasicClient(cache.Config.AccountName, cache.Config.AccountKey)
	if err != nil {
		return nil, false, err
	}

	cache.Client = api

	res, err := cache.Client.GetBlobService().CreateContainerIfNotExists(cache.Config.ContainerName,
											vendorstorage.ContainerAccessTypeBlob)
	if err != nil {
		return nil, false, err
	}

	return &cache, res, nil
}<|endoftext|>


<commit_before>package docker

import (
    "fmt"
    "regexp"
    "os"
    "strings"
    "github.com/pkg/errors"
)

// This code is taken from github.com/docker/volume/mounts/windows_parser.go
// See https://github.com/moby/moby/blob/master/LICENSE for the license, Apache License 2.0 at this time.

const (
	// Spec should be in the format [source:]destination[:mode]
	//
	// Examples: c:\foo bar:d:rw
	//           c:\foo:d:\bar
	//           myname:d:
	//           d:\
	//
	// Explanation of this regex! Thanks @thaJeztah on IRC and gist for help. See
	// https://gist.github.com/thaJeztah/6185659e4978789fb2b2. A good place to
	// test is https://regex-golang.appspot.com/assets/html/index.html
	//
	// Useful link for referencing named capturing groups:
	// http://stackoverflow.com/questions/20750843/using-named-matches-from-go-regex
	//
	// There are three match groups: source, destination and mode.
	//

	// rxHostDir is the first option of a source
	rxHostDir = `(?:\\\\\?\\)?[a-z]:[\\/](?:[^\\/:*?"<>|\r\n]+[\\/]?)*`
	// rxName is the second option of a source
	rxName = `[^\\/:*?"<>|\r\n]+`

	// RXReservedNames are reserved names not possible on Windows
	rxReservedNames = `(con)|(prn)|(nul)|(aux)|(com[1-9])|(lpt[1-9])`

	// rxPipe is a named path pipe (starts with `\\.\pipe\`, possibly with / instead of \)
	rxPipe = `[/\\]{2}.[/\\]pipe[/\\][^:*?"<>|\r\n]+`
	// rxSource is the combined possibilities for a source
	rxSource = `((?P<source>((` + rxHostDir + `)|(` + rxName + `)|(` + rxPipe + `))):)?`

	// Source. Can be either a host directory, a name, or omitted:
	//  HostDir:
	//    -  Essentially using the folder solution from
	//       https://www.safaribooksonline.com/library/view/regular-expressions-cookbook/9781449327453/ch08s18.html
	//       but adding case insensitivity.
	//    -  Must be an absolute path such as c:\path
	//    -  Can include spaces such as `c:\program files`
	//    -  And then followed by a colon which is not in the capture group
	//    -  And can be optional
	//  Name:
	//    -  Must not contain invalid NTFS filename characters (https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx)
	//    -  And then followed by a colon which is not in the capture group
	//    -  And can be optional

	// rxDestination is the regex expression for the mount destination
	rxDestination = `(?P<destination>((?:\\\\\?\\)?([a-z]):((?:[\\/][^\\/:*?"<>\r\n]+)*[\\/]?))|(` + rxPipe + `))`

	// Destination (aka container path):
	//    -  Variation on hostdir but can be a drive followed by colon as well
	//    -  If a path, must be absolute. Can include spaces
	//    -  Drive cannot be c: (explicitly checked in code, not RegEx)

	// rxMode is the regex expression for the mode of the mount
	// Mode (optional):
	//    -  Hopefully self explanatory in comparison to above regex's.
	//    -  Colon is not in the capture group
	rxMode = `(:(?P<mode>(?i)ro|rw))?`
)

func errInvalidSpec(spec string) error {
	return errors.Errorf("invalid volume specification: '%s'", spec)
}

type fileInfoProvider interface {
	fileInfo(path string) (exist, isDir bool, err error)
}

type defaultFileInfoProvider struct {
}

func (defaultFileInfoProvider) fileInfo(path string) (exist, isDir bool, err error) {
	fi, err := os.Stat(path)
	if err != nil {
		if !os.IsNotExist(err) {
			return false, false, err
		}
		return false, false, nil
	}
	return true, fi.IsDir(), nil
}

var currentFileInfoProvider fileInfoProvider = defaultFileInfoProvider{}

func windowsSplitRawSpec(raw, destRegex string) ([]string, error) {
	specExp := regexp.MustCompile(`^` + rxSource + destRegex + rxMode + `$`)
	match := specExp.FindStringSubmatch(strings.ToLower(raw))

	// Must have something back
	if len(match) == 0 {
		return nil, errInvalidSpec(raw)
	}

	var split []string
	matchgroups := make(map[string]string)
	// Pull out the sub expressions from the named capture groups
	for i, name := range specExp.SubexpNames() {
		matchgroups[name] = strings.ToLower(match[i])
	}
	if source, exists := matchgroups["source"]; exists {
		if source != "" {
			split = append(split, source)
		}
	}
	if destination, exists := matchgroups["destination"]; exists {
		if destination != "" {
			split = append(split, destination)
		}
	}
	if mode, exists := matchgroups["mode"]; exists {
		if mode != "" {
			split = append(split, mode)
		}
	}
	// Fix #26329. If the destination appears to be a file, and the source is null,
	// it may be because we've fallen through the possible naming regex and hit a
	// situation where the user intention was to map a file into a container through
	// a local volume, but this is not supported by the platform.
	if matchgroups["source"] == "" && matchgroups["destination"] != "" {
		volExp := regexp.MustCompile(`^` + rxName + `$`)
		reservedNameExp := regexp.MustCompile(`^` + rxReservedNames + `$`)

		if volExp.MatchString(matchgroups["destination"]) {
			if reservedNameExp.MatchString(matchgroups["destination"]) {
				return nil, fmt.Errorf("volume name %q cannot be a reserved word for Windows filenames", matchgroups["destination"])
			}
		} else {

			exists, isDir, _ := currentFileInfoProvider.fileInfo(matchgroups["destination"])
			if exists && !isDir {
				return nil, fmt.Errorf("file '%s' cannot be mapped. Only directories can be mapped on this platform", matchgroups["destination"])

			}
		}
	}
	return split, nil
}

<commit_msg>run gofmt over the new file<commit_after>package docker

import (
	"fmt"
	"github.com/pkg/errors"
	"os"
	"regexp"
	"strings"
)

// This code is taken from github.com/docker/volume/mounts/windows_parser.go
// See https://github.com/moby/moby/blob/master/LICENSE for the license, Apache License 2.0 at this time.

const (
	// Spec should be in the format [source:]destination[:mode]
	//
	// Examples: c:\foo bar:d:rw
	//           c:\foo:d:\bar
	//           myname:d:
	//           d:\
	//
	// Explanation of this regex! Thanks @thaJeztah on IRC and gist for help. See
	// https://gist.github.com/thaJeztah/6185659e4978789fb2b2. A good place to
	// test is https://regex-golang.appspot.com/assets/html/index.html
	//
	// Useful link for referencing named capturing groups:
	// http://stackoverflow.com/questions/20750843/using-named-matches-from-go-regex
	//
	// There are three match groups: source, destination and mode.
	//

	// rxHostDir is the first option of a source
	rxHostDir = `(?:\\\\\?\\)?[a-z]:[\\/](?:[^\\/:*?"<>|\r\n]+[\\/]?)*`
	// rxName is the second option of a source
	rxName = `[^\\/:*?"<>|\r\n]+`

	// RXReservedNames are reserved names not possible on Windows
	rxReservedNames = `(con)|(prn)|(nul)|(aux)|(com[1-9])|(lpt[1-9])`

	// rxPipe is a named path pipe (starts with `\\.\pipe\`, possibly with / instead of \)
	rxPipe = `[/\\]{2}.[/\\]pipe[/\\][^:*?"<>|\r\n]+`
	// rxSource is the combined possibilities for a source
	rxSource = `((?P<source>((` + rxHostDir + `)|(` + rxName + `)|(` + rxPipe + `))):)?`

	// Source. Can be either a host directory, a name, or omitted:
	//  HostDir:
	//    -  Essentially using the folder solution from
	//       https://www.safaribooksonline.com/library/view/regular-expressions-cookbook/9781449327453/ch08s18.html
	//       but adding case insensitivity.
	//    -  Must be an absolute path such as c:\path
	//    -  Can include spaces such as `c:\program files`
	//    -  And then followed by a colon which is not in the capture group
	//    -  And can be optional
	//  Name:
	//    -  Must not contain invalid NTFS filename characters (https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx)
	//    -  And then followed by a colon which is not in the capture group
	//    -  And can be optional

	// rxDestination is the regex expression for the mount destination
	rxDestination = `(?P<destination>((?:\\\\\?\\)?([a-z]):((?:[\\/][^\\/:*?"<>\r\n]+)*[\\/]?))|(` + rxPipe + `))`

	// Destination (aka container path):
	//    -  Variation on hostdir but can be a drive followed by colon as well
	//    -  If a path, must be absolute. Can include spaces
	//    -  Drive cannot be c: (explicitly checked in code, not RegEx)

	// rxMode is the regex expression for the mode of the mount
	// Mode (optional):
	//    -  Hopefully self explanatory in comparison to above regex's.
	//    -  Colon is not in the capture group
	rxMode = `(:(?P<mode>(?i)ro|rw))?`
)

func errInvalidSpec(spec string) error {
	return errors.Errorf("invalid volume specification: '%s'", spec)
}

type fileInfoProvider interface {
	fileInfo(path string) (exist, isDir bool, err error)
}

type defaultFileInfoProvider struct {
}

func (defaultFileInfoProvider) fileInfo(path string) (exist, isDir bool, err error) {
	fi, err := os.Stat(path)
	if err != nil {
		if !os.IsNotExist(err) {
			return false, false, err
		}
		return false, false, nil
	}
	return true, fi.IsDir(), nil
}

var currentFileInfoProvider fileInfoProvider = defaultFileInfoProvider{}

func windowsSplitRawSpec(raw, destRegex string) ([]string, error) {
	specExp := regexp.MustCompile(`^` + rxSource + destRegex + rxMode + `$`)
	match := specExp.FindStringSubmatch(strings.ToLower(raw))

	// Must have something back
	if len(match) == 0 {
		return nil, errInvalidSpec(raw)
	}

	var split []string
	matchgroups := make(map[string]string)
	// Pull out the sub expressions from the named capture groups
	for i, name := range specExp.SubexpNames() {
		matchgroups[name] = strings.ToLower(match[i])
	}
	if source, exists := matchgroups["source"]; exists {
		if source != "" {
			split = append(split, source)
		}
	}
	if destination, exists := matchgroups["destination"]; exists {
		if destination != "" {
			split = append(split, destination)
		}
	}
	if mode, exists := matchgroups["mode"]; exists {
		if mode != "" {
			split = append(split, mode)
		}
	}
	// Fix #26329. If the destination appears to be a file, and the source is null,
	// it may be because we've fallen through the possible naming regex and hit a
	// situation where the user intention was to map a file into a container through
	// a local volume, but this is not supported by the platform.
	if matchgroups["source"] == "" && matchgroups["destination"] != "" {
		volExp := regexp.MustCompile(`^` + rxName + `$`)
		reservedNameExp := regexp.MustCompile(`^` + rxReservedNames + `$`)

		if volExp.MatchString(matchgroups["destination"]) {
			if reservedNameExp.MatchString(matchgroups["destination"]) {
				return nil, fmt.Errorf("volume name %q cannot be a reserved word for Windows filenames", matchgroups["destination"])
			}
		} else {

			exists, isDir, _ := currentFileInfoProvider.fileInfo(matchgroups["destination"])
			if exists && !isDir {
				return nil, fmt.Errorf("file '%s' cannot be mapped. Only directories can be mapped on this platform", matchgroups["destination"])

			}
		}
	}
	return split, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Square Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package query

// show all metrics
// show tags WHERE predicate

import (
	"testing"

	"github.com/square/metrics/assert"
)

// these queries should successfully parse,
// with a corresponding command.
var inputs = []string{
	// describes
	"describe all",
	"describe x",
	"describe cpu_usage",
	"describe inspect",
	"describe in2",
	"describe cpu_usage where key = 'value'",
	"describe cpu_usage where key = 'value\\''",
	"describe cpu_usage where key != 'value'",
	"describe cpu_usage where (key = 'value')",
	"describe cpu_usage where not (key = 'value')",
	"describe cpu_usage where not key = 'value'",
	"describe cpu_usage where (key = 'value')",
	"describe cpu_usage where key = 'value' or key = 'value'",
	"describe cpu_usage where key in ('value', 'value')",
	"describe cpu_usage where key matches 'abc'",
	"describe nodes.cpu.usage where datacenter='sjc1b' and type='idle' and host matches 'fwd'",
	// predicate parenthesis test
	"describe cpu_usage where key = 'value' and (key = 'value')",
	"describe cpu_usage where (key = 'value') and key = 'value'",
	"describe cpu_usage where (key = 'value') and (key = 'value')",
	"describe cpu_usage where (key = 'value' and key = 'value')",

	// Leading/trailing whitespace
	" describe all ",
	" describe x ",
	" select 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 from 0 to 0 ",

	// selects - spaces and keywords
	"select f( g(5) group by a,w,q) from 0 to 0",
	"select f( g(5) group by a, w, q )from 0 to 0 ",
	"select f( g(5) group by a,w, q)from 0 to 0",
	"select f( g(5) group by a,w,q)to 0 from 0",
	"select f(g(5)group by a,w,q)to 0 from 0",
	"   select f(g(5)group by a,w,q)  from  	  0 	   to     0",
	"   select( f(g(5)group by a,w,q)  )from 0 to 0",
	"select(f(g(5)group by`a`,w,q)) from 0 to 0",
	"select(f(g(5)group by`a`,w,q)) from 0 to 0",
	"select(fromx+tox+groupx+byx+selectx+describex+allx+wherex) from 0 to 0",
	// comments
	"describe all-- trailing comment",
	"describe-- trailing comment\n all",
	"describe/* inline */all",
	"describe/**/all",
	"select/**/(/**/f/**/(/**/g/**/(/**/5/**/)group/**/by/**/`a`/**/,/**/w/**/,/**/q/**/)/**/)/**/from/**/0/**/to/**/0",
}

var selects = []string{
	// All these queries are tested with and without the prefix "select"
	// selects - parenthesis
	"0 from 0 to 0",
	"(0) from 0 to 0",
	"(0) where foo = 'bar' from 0 to 0",
	// selects - numbers
	"0, 1, 2, 3, 4, 5, 6, 7, 8, 9 from 0 to 0",
	"10, 100, 1000 from 0 to 0",
	"10.1, 10.01, 10.001 from 0 to 0",
	"-10.1, -10.01, -10.001 from 0 to 0",
	"1.0e1, 1.0e2, 1.0e10, 1.0e0 from 0 to 0",
	"1.0e-5, 1.0e+5 from 0 to 0",
	// selects - trying out arithmetic
	"x from 0 to 0",
	"x-y-z from 0 to 0",
	"(x)-(y)-(z) from 0 to 0",
	"0 from 0 to 0",
	"x, y from 0 to 0",
	"1 + 2 * 3 + 4 from 0 to 0",
	"x * (y + 123), z from 0 to 0",
	// testing escaping
	"`x` from 0 to 0",
	// selects - timestamps
	"x * (y + 123), z from '2011-2-4 PTZ' to '2015-6-1 PTZ'",
	"x * (y + 123), z from 0 to 10000",
	"1 from -10m to now",
	"1 from -10M to -10m",
	// selects - function calls
	"foo(x) from 0 to 0",
	"bar(x, y) from 0 to 0",
	"baz(x, y, z+1+foo(1)) from 0 to 0",
	// selects - testing out property values
	"x from 0 to 0",
	"x from 0 to 0",
	"x from 0 to 0 resolution '10s'",
	"x from 0 to 0 resolution '10h'",
	"x from 0 to 0 resolution '300s'",
	"x from 0 to 0 resolution '17m'",
	"x from 0 to 0 sample by 'max'",
	"x from 0 to 0 sample   by 'max'",
	// selects - aggregate functions
	"scalar.max(x) from 0 to 0",
	"aggregate.max(x, y) from 0 to 0",
	"aggregate.max(x group by foo) + 3 from 0 to 0",
	// selects - where clause
	"x where y = 'z' from 0 to 0",
	// selects - per-identifier where clause
	"x + z[y = 'z'] from 0 to 0",
	"x[y = 'z'] from 0 to 0",
	// selects - complicated queries
	"aggregate.max(x[y = 'z'] group by foo) from 0 to 0",
	"aggregate.max(x[y = 'z'] collapse by foo) from 0 to 0",
	"aggregate.max(x[y = 'z'] group by foo, bar) from 0 to 0",
	"aggregate.max(x[y = 'z'] collapse by foo, bar) from 0 to 0",
	"cpu.user + cpu.kernel where host = 'apa3.sjc2b' from 0 to 0",
	"'string literal' where host = 'apa3.sjc2b' from 0 to 0",
	"timeshift( metric, '5h') where host = 'apa3.sjc2b' from 0 to 0",
	// pipe expressions
	"x | y from 0 to 0",
	"x | y + 1 from 0 to 0",
	"x | y - 1 from 0 to 0",
	"x | y * 1 from 0 to 0",
	"x | y / 1 from 0 to 0",
	"x | y(group by a) from 0 to 0",
	"x + 1 | y(group by a) from 0 to 0",
	"x | y | z + 1 from 0 to 0",
	"x|y from 0 to 0",
	"x|f + y*z from 0 to 0",
	"x|f + y|g from 0 to 0",
	"x|f + y|g(4) from 0 to 0",
	"x|f(1,2,3) + y|g(4) from 0 to 0",
	"x|f(1s,2,3y) + y|g(4mo) from 0 to 0",
	"x|f(1s,'r3r2',3y) + y|g(4mo) from 0 to 0",
	"1 + 2 | f from 0 to 0",
}

// these queries should fail with a syntax error.
var syntaxErrorQuery = []string{
	"",
	"describe ( from 0 to 0",
	"describe in",
	"describe in from 0 to 0",
	"describe invalid_regex where key matches 'ab['",
	"describe invalid_property \nwhere key matches 'ab' from 0 to 0",
	"select 'a\nac\nabc",
	"select ( from 0 to 0",
	"select ) from 0 to 0",
	"select x invalid_property 0 from 0 to 0",
	"select x sampleby 0 from 0 to 0",
	"select x sample 0 from 0 to 0",
	"select x by 0 from 0 to 0",
	"select x",
	"select x from 0",
	"select x to 0",
	"select x from 0 from 1 to 0",
	"select x from 0 to 1 to 0",
	"select x from 0 resolution '30s' resolution '25s' to 0",
	"select x from 0 from 1 sample by 'min' sample by 'min' to 0",
	"select f(3 groupby x) from 0 to 0",
	"select c group by a from 0 to 0",
	"select x[] from 0 to 0",
	"select cpu | transform.moving_average(10qq) from 0 to 0",
}

func TestParse_success(t *testing.T) {
	for _, row := range inputs {
		_, err := Parse(row)
		if err != nil {
			t.Errorf("[%s] failed to parse: %s", row, err.Error())
		}
	}

	for _, row := range selects {
		for _, prefix := range []string{"", "select "} {
			query := prefix + row
			_, err := Parse(query)
			if err != nil {
				t.Errorf("[%s] failed to parse: %s", query, err.Error())
			}
		}
	}
}

func TestParse_syntaxError(t *testing.T) {
	for _, row := range syntaxErrorQuery {
		_, err := Parse(row)
		if err == nil {
			t.Errorf("[%s] should have failed to parse", row)
		} else if _, ok := err.(SyntaxErrors); !ok {
			t.Logf("[%s] Expected SyntaxErrors, got: %s", row, err.Error())
			err.Error() // test that it does not panic.
		}
	}
}

func TestCompile(t *testing.T) {
	for _, row := range inputs {
		a := assert.New(t).Contextf(row)
		p := Parser{Buffer: row}
		p.Init()
		a.CheckError(p.Parse())
		p.Execute()
		testParserResult(a, p)
	}
}

// Helper functions
// ================

func testParserResult(a assert.Assert, p Parser) {
	a.EqInt(len(p.assertions), 0)
	if len(p.assertions) != 0 {
		for _, err := range p.assertions {
			a.Errorf("assertion error: %s", err.Error())
		}
	}
	if len(p.nodeStack) != 0 {
		for _, node := range p.nodeStack {
			a.Errorf("node error:\n%s", PrintNode(node))
		}
	}
}
<commit_msg>adding additional whitespace test<commit_after>// Copyright 2015 Square Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package query

// show all metrics
// show tags WHERE predicate

import (
	"testing"

	"github.com/square/metrics/assert"
)

// these queries should successfully parse,
// with a corresponding command.
var inputs = []string{
	// describes
	"describe all",
	"describe x",
	"describe cpu_usage",
	"describe inspect",
	"describe in2",
	"describe cpu_usage where key = 'value'",
	"describe cpu_usage where key = 'value\\''",
	"describe cpu_usage where key != 'value'",
	"describe cpu_usage where (key = 'value')",
	"describe cpu_usage where not (key = 'value')",
	"describe cpu_usage where not key = 'value'",
	"describe cpu_usage where (key = 'value')",
	"describe cpu_usage where key = 'value' or key = 'value'",
	"describe cpu_usage where key in ('value', 'value')",
	"describe cpu_usage where key matches 'abc'",
	"describe nodes.cpu.usage where datacenter='sjc1b' and type='idle' and host matches 'fwd'",
	// predicate parenthesis test
	"describe cpu_usage where key = 'value' and (key = 'value')",
	"describe cpu_usage where (key = 'value') and key = 'value'",
	"describe cpu_usage where (key = 'value') and (key = 'value')",
	"describe cpu_usage where (key = 'value' and key = 'value')",

	// Leading/trailing whitespace
	" describe all ",
	" describe x ",
	" select 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 from 0 to 0 ",

	// selects - spaces and keywords
	"select f( g(5) group by a,w,q) from 0 to 0",
	"select f( g(5) group by a, w, q )from 0 to 0 ",
	"select f( g(5) group by a,w, q)from 0 to 0",
	"select f( g(5) group by a,w,q)to 0 from 0",
	"select f(g(5)group by a,w,q)to 0 from 0",
	"   select f(g(5)group by a,w,q)  from  	  0 	   to     0",
	"   select( f(g(5)group by a,w,q)  )from 0 to 0",
	"select(f(g(5)group by`a`,w,q)) from 0 to 0",
	"select(f(g(5)group by`a`,w,q)) from 0 to 0",
	"select(fromx+tox+groupx+byx+selectx+describex+allx+wherex) from 0 to 0",
	// comments
	"describe all-- trailing comment",
	"describe-- trailing comment\n all",
	"describe/* inline */all",
	"describe/**/all",
	"select/**/(/**/f/**/(/**/g/**/(/**/5/**/)group/**/by/**/`a`/**/,/**/w/**/,/**/q/**/)/**/)/**/from/**/0/**/to/**/0",
}

var selects = []string{
	// All these queries are tested with and without the prefix "select"
	// selects - parenthesis
	"0 from 0 to 0",
	"(0) from 0 to 0",
	"(0) where foo = 'bar' from 0 to 0",
	// selects - numbers
	"0, 1, 2, 3, 4, 5, 6, 7, 8, 9 from 0 to 0",
	"10, 100, 1000 from 0 to 0",
	"10.1, 10.01, 10.001 from 0 to 0",
	"-10.1, -10.01, -10.001 from 0 to 0",
	"1.0e1, 1.0e2, 1.0e10, 1.0e0 from 0 to 0",
	"1.0e-5, 1.0e+5 from 0 to 0",
	// selects - trying out arithmetic
	"x from 0 to 0",
	"x-y-z from 0 to 0",
	"(x)-(y)-(z) from 0 to 0",
	"0 from 0 to 0",
	"x, y from 0 to 0",
	"1 + 2 * 3 + 4 from 0 to 0",
	"x * (y + 123), z from 0 to 0",
	// testing escaping
	"`x` from 0 to 0",
	// selects - timestamps
	"x * (y + 123), z from '2011-2-4 PTZ' to '2015-6-1 PTZ'",
	"x * (y + 123), z from 0 to 10000",
	"1 from -10m to now",
	"1 from -10M to -10m",
	// selects - function calls
	"foo(x) from 0 to 0",
	"bar(x, y) from 0 to 0",
	"baz(x, y, z+1+foo(1)) from 0 to 0",
	// selects - testing out property values
	"x from 0 to 0",
	"x from 0 to 0",
	"x from 0 to 0 resolution '10s'",
	"x from 0 to 0 resolution '10h'",
	"x from 0 to 0 resolution '300s'",
	"x from 0 to 0 resolution '17m'",
	"x from 0 to 0 sample by 'max'",
	"x from 0 to 0 sample   by 'max'",
	// selects - aggregate functions
	"scalar.max(x) from 0 to 0",
	"aggregate.max(x, y) from 0 to 0",
	"aggregate.max(x group by foo) + 3 from 0 to 0",
	// selects - where clause
	"x where y = 'z' from 0 to 0",
	// selects - per-identifier where clause
	"x + z[y = 'z'] from 0 to 0",
	"x[y = 'z'] from 0 to 0",
	// selects - complicated queries
	"aggregate.max(x[y = 'z'] group by foo) from 0 to 0",
	"aggregate.max(x[y = 'z'] collapse by foo) from 0 to 0",
	"aggregate.max(x[y = 'z'] group by foo, bar) from 0 to 0",
	"aggregate.max(x[y = 'z'] collapse by foo, bar) from 0 to 0",
	"cpu.user + cpu.kernel where host = 'apa3.sjc2b' from 0 to 0",
	"'string literal' where host = 'apa3.sjc2b' from 0 to 0",
	"timeshift( metric, '5h') where host = 'apa3.sjc2b' from 0 to 0",
	// pipe expressions
	"x | y from 0 to 0",
	"x | y + 1 from 0 to 0",
	"x | y - 1 from 0 to 0",
	"x | y * 1 from 0 to 0",
	"x | y / 1 from 0 to 0",
	"x | y(group by a) from 0 to 0",
	"x + 1 | y(group by a) from 0 to 0",
	"x | y | z + 1 from 0 to 0",
	"x|y from 0 to 0",
	"x|f + y*z from 0 to 0",
	"x|f + y|g from 0 to 0",
	"x|f + y|g(4) from 0 to 0",
	"x|f(1,2,3) + y|g(4) from 0 to 0",
	"x|f(1s,2,3y) + y|g(4mo) from 0 to 0",
	"x|f(1s,'r3r2',3y) + y|g(4mo) from 0 to 0",
	"1 + 2 | f from 0 to 0",
}

// these queries should fail with a syntax error.
var syntaxErrorQuery = []string{
	"",
	"\n",
	"\t",
	" ",
	"// comment only",
	"describe ( from 0 to 0",
	"describe in",
	"describe in from 0 to 0",
	"describe invalid_regex where key matches 'ab['",
	"describe invalid_property \nwhere key matches 'ab' from 0 to 0",
	"select 'a\nac\nabc",
	"select ( from 0 to 0",
	"select ) from 0 to 0",
	"select x invalid_property 0 from 0 to 0",
	"select x sampleby 0 from 0 to 0",
	"select x sample 0 from 0 to 0",
	"select x by 0 from 0 to 0",
	"select x",
	"select x from 0",
	"select x to 0",
	"select x from 0 from 1 to 0",
	"select x from 0 to 1 to 0",
	"select x from 0 resolution '30s' resolution '25s' to 0",
	"select x from 0 from 1 sample by 'min' sample by 'min' to 0",
	"select f(3 groupby x) from 0 to 0",
	"select c group by a from 0 to 0",
	"select x[] from 0 to 0",
	"select cpu | transform.moving_average(10qq) from 0 to 0",
}

func TestParse_success(t *testing.T) {
	for _, row := range inputs {
		_, err := Parse(row)
		if err != nil {
			t.Errorf("[%s] failed to parse: %s", row, err.Error())
		}
	}

	for _, row := range selects {
		for _, prefix := range []string{"", "select "} {
			query := prefix + row
			_, err := Parse(query)
			if err != nil {
				t.Errorf("[%s] failed to parse: %s", query, err.Error())
			}
		}
	}
}

func TestParse_syntaxError(t *testing.T) {
	for _, row := range syntaxErrorQuery {
		_, err := Parse(row)
		if err == nil {
			t.Errorf("[%s] should have failed to parse", row)
		} else if _, ok := err.(SyntaxErrors); !ok {
			t.Logf("[%s] Expected SyntaxErrors, got: %s", row, err.Error())
			err.Error() // test that it does not panic.
		}
	}
}

func TestCompile(t *testing.T) {
	for _, row := range inputs {
		a := assert.New(t).Contextf(row)
		p := Parser{Buffer: row}
		p.Init()
		a.CheckError(p.Parse())
		p.Execute()
		testParserResult(a, p)
	}
}

// Helper functions
// ================

func testParserResult(a assert.Assert, p Parser) {
	a.EqInt(len(p.assertions), 0)
	if len(p.assertions) != 0 {
		for _, err := range p.assertions {
			a.Errorf("assertion error: %s", err.Error())
		}
	}
	if len(p.nodeStack) != 0 {
		for _, node := range p.nodeStack {
			a.Errorf("node error:\n%s", PrintNode(node))
		}
	}
}
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2016 Red Hat, Inc.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package client

import (
	"sync"

	"github.com/skydive-project/skydive/api"
	"github.com/skydive-project/skydive/common"
	"github.com/skydive-project/skydive/etcd"
	"github.com/skydive-project/skydive/flow/ondemand"
	shttp "github.com/skydive-project/skydive/http"
	"github.com/skydive-project/skydive/logging"
	"github.com/skydive-project/skydive/topology"
	"github.com/skydive-project/skydive/topology/graph"
)

type OnDemandProbeClient struct {
	sync.RWMutex
	graph.DefaultGraphListener
	graph          *graph.Graph
	captureHandler *api.CaptureAPIHandler
	wsServer       *shttp.WSServer
	captures       map[string]*api.Capture
	watcher        api.StoppableWatcher
	elector        *etcd.EtcdMasterElector
}

func (o *OnDemandProbeClient) registerProbes(nodes []interface{}, capture *api.Capture) {
	for _, i := range nodes {
		switch i.(type) {
		case *graph.Node:
			o.graph.RLock()
			node := i.(*graph.Node)
			if _, err := node.GetFieldString("Capture/ID"); err == nil {
				o.graph.RUnlock()
				return
			}
			nodeID := node.ID
			host := node.Host()
			o.graph.RUnlock()
			o.registerProbe(nodeID, host, capture)
		case []*graph.Node:
			// case of shortestpath that return a list of nodes
			for _, node := range i.([]*graph.Node) {
				o.graph.RLock()
				if _, err := node.GetFieldString("Capture/ID"); err == nil {
					o.graph.RUnlock()
					return
				}
				nodeID := node.ID
				host := node.Host()
				o.graph.RUnlock()
				o.registerProbe(nodeID, host, capture)
			}
		}
	}
}

func (o *OnDemandProbeClient) registerProbe(id graph.Identifier, host string, capture *api.Capture) bool {
	cq := ondemand.CaptureQuery{
		NodeID:  string(id),
		Capture: *capture,
	}

	msg := shttp.NewWSMessage(ondemand.Namespace, "CaptureStart", cq)

	if !o.wsServer.SendWSMessageTo(msg, host) {
		logging.GetLogger().Errorf("Unable to send message to agent: %s", host)
		return false
	}
	return true
}

func (o *OnDemandProbeClient) unregisterProbe(node *graph.Node) bool {
	msg := shttp.NewWSMessage(ondemand.Namespace, "CaptureStop", ondemand.CaptureQuery{NodeID: string(node.ID)})

	if !o.wsServer.SendWSMessageTo(msg, node.Host()) {
		logging.GetLogger().Errorf("Unable to send message to agent: %s", node.Host())
		return false
	}

	return true
}

func (o *OnDemandProbeClient) applyGremlinExpr(query string) []interface{} {
	res, err := topology.ExecuteGremlinQuery(o.graph, query)
	if err != nil {
		logging.GetLogger().Errorf("Gremlin error: %s", err.Error())
		return nil
	}
	return res.Values()
}

func (o *OnDemandProbeClient) onNodeEvent() {
	if !o.elector.IsMaster() {
		return
	}

	for _, capture := range o.captures {
		res := o.applyGremlinExpr(capture.GremlinQuery)
		if len(res) > 0 {
			go o.registerProbes(res, capture)
		}
	}
}

func (o *OnDemandProbeClient) OnNodeAdded(n *graph.Node) {
	o.onNodeEvent()
}

func (o *OnDemandProbeClient) OnNodeUpdated(n *graph.Node) {
	o.onNodeEvent()
}

func (o *OnDemandProbeClient) OnEdgeAdded(e *graph.Edge) {
	o.onNodeEvent()
}

func (o *OnDemandProbeClient) onCaptureAdded(capture *api.Capture) {
	if !o.elector.IsMaster() {
		return
	}

	o.Lock()
	defer o.Unlock()

	o.graph.RLock()
	defer o.graph.RUnlock()

	o.captures[capture.UUID] = capture

	nodes := o.applyGremlinExpr(capture.GremlinQuery)
	if len(nodes) > 0 {
		go o.registerProbes(nodes, capture)
	}
}

func (o *OnDemandProbeClient) onCaptureDeleted(capture *api.Capture) {
	if !o.elector.IsMaster() {
		return
	}

	o.Lock()
	defer o.Unlock()

	o.graph.Lock()
	defer o.graph.Unlock()

	delete(o.captures, capture.UUID)

	res, err := topology.ExecuteGremlinQuery(o.graph, capture.GremlinQuery)
	if err != nil {
		logging.GetLogger().Errorf("Gremlin error: %s", err.Error())
		return
	}

	for _, value := range res.Values() {
		switch e := value.(type) {
		case *graph.Node:
			if !o.unregisterProbe(e) {
				logging.GetLogger().Errorf("Failed to stop capture on %s", e.ID)
			}
		case []*graph.Node:
			for _, node := range e {
				if !o.unregisterProbe(node) {
					logging.GetLogger().Errorf("Failed to stop capture on %s", node.ID)
				}
			}
		default:
			return
		}
	}
}

func (o *OnDemandProbeClient) onAPIWatcherEvent(action string, id string, resource api.APIResource) {
	logging.GetLogger().Debugf("New watcher event %s for %s", action, id)
	capture := resource.(*api.Capture)
	switch action {
	case "init", "create", "set", "update":
		o.wsServer.BroadcastWSMessage(shttp.NewWSMessage(ondemand.Namespace, "CaptureAdded", capture))
		o.onCaptureAdded(capture)
	case "expire", "delete":
		o.wsServer.BroadcastWSMessage(shttp.NewWSMessage(ondemand.Namespace, "CaptureDeleted", capture))
		o.onCaptureDeleted(capture)
	}
}

func (o *OnDemandProbeClient) Start() {
	o.elector.StartAndWait()

	o.watcher = o.captureHandler.AsyncWatch(o.onAPIWatcherEvent)
	o.graph.AddEventListener(o)
}

func (o *OnDemandProbeClient) Stop() {
	o.watcher.Stop()
	o.elector.Stop()
}

func NewOnDemandProbeClient(g *graph.Graph, ch *api.CaptureAPIHandler, w *shttp.WSServer, etcdClient *etcd.EtcdClient) *OnDemandProbeClient {
	resources := ch.Index()
	captures := make(map[string]*api.Capture)
	for _, resource := range resources {
		captures[resource.ID()] = resource.(*api.Capture)
	}

	elector := etcd.NewEtcdMasterElectorFromConfig(common.AnalyzerService, "ondemand-client", etcdClient)

	return &OnDemandProbeClient{
		graph:          g,
		captureHandler: ch,
		wsServer:       w,
		captures:       captures,
		elector:        elector,
	}
}
<commit_msg>capture: check capture allowed on client side<commit_after>/*
 * Copyright (C) 2016 Red Hat, Inc.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package client

import (
	"sync"

	"github.com/skydive-project/skydive/api"
	"github.com/skydive-project/skydive/common"
	"github.com/skydive-project/skydive/etcd"
	"github.com/skydive-project/skydive/flow/ondemand"
	shttp "github.com/skydive-project/skydive/http"
	"github.com/skydive-project/skydive/logging"
	"github.com/skydive-project/skydive/topology"
	"github.com/skydive-project/skydive/topology/graph"
)

type OnDemandProbeClient struct {
	sync.RWMutex
	graph.DefaultGraphListener
	graph          *graph.Graph
	captureHandler *api.CaptureAPIHandler
	wsServer       *shttp.WSServer
	captures       map[string]*api.Capture
	watcher        api.StoppableWatcher
	elector        *etcd.EtcdMasterElector
}

func (o *OnDemandProbeClient) registerProbes(nodes []interface{}, capture *api.Capture) {
	for _, i := range nodes {
		switch i.(type) {
		case *graph.Node:
			o.graph.RLock()
			node := i.(*graph.Node)
			if _, err := node.GetFieldString("Capture/ID"); err == nil {
				o.graph.RUnlock()
				return
			}
			tp, _ := node.GetFieldString("Type")
			if !common.IsCaptureAllowed(tp) {
				o.graph.RUnlock()
				return
			}

			nodeID := node.ID
			host := node.Host()
			o.graph.RUnlock()
			o.registerProbe(nodeID, host, capture)
		case []*graph.Node:
			// case of shortestpath that return a list of nodes
			for _, node := range i.([]*graph.Node) {
				o.graph.RLock()
				if _, err := node.GetFieldString("Capture/ID"); err == nil {
					o.graph.RUnlock()
					return
				}
				tp, _ := node.GetFieldString("Type")
				if !common.IsCaptureAllowed(tp) {
					o.graph.RUnlock()
					return
				}

				nodeID := node.ID
				host := node.Host()
				o.graph.RUnlock()
				o.registerProbe(nodeID, host, capture)
			}
		}
	}
}

func (o *OnDemandProbeClient) registerProbe(id graph.Identifier, host string, capture *api.Capture) bool {
	cq := ondemand.CaptureQuery{
		NodeID:  string(id),
		Capture: *capture,
	}

	msg := shttp.NewWSMessage(ondemand.Namespace, "CaptureStart", cq)

	if !o.wsServer.SendWSMessageTo(msg, host) {
		logging.GetLogger().Errorf("Unable to send message to agent: %s", host)
		return false
	}
	return true
}

func (o *OnDemandProbeClient) unregisterProbe(node *graph.Node) bool {
	msg := shttp.NewWSMessage(ondemand.Namespace, "CaptureStop", ondemand.CaptureQuery{NodeID: string(node.ID)})

	if !o.wsServer.SendWSMessageTo(msg, node.Host()) {
		logging.GetLogger().Errorf("Unable to send message to agent: %s", node.Host())
		return false
	}

	return true
}

func (o *OnDemandProbeClient) applyGremlinExpr(query string) []interface{} {
	res, err := topology.ExecuteGremlinQuery(o.graph, query)
	if err != nil {
		logging.GetLogger().Errorf("Gremlin error: %s", err.Error())
		return nil
	}
	return res.Values()
}

func (o *OnDemandProbeClient) onNodeEvent() {
	if !o.elector.IsMaster() {
		return
	}

	for _, capture := range o.captures {
		res := o.applyGremlinExpr(capture.GremlinQuery)
		if len(res) > 0 {
			go o.registerProbes(res, capture)
		}
	}
}

func (o *OnDemandProbeClient) OnNodeAdded(n *graph.Node) {
	o.onNodeEvent()
}

func (o *OnDemandProbeClient) OnNodeUpdated(n *graph.Node) {
	o.onNodeEvent()
}

func (o *OnDemandProbeClient) OnEdgeAdded(e *graph.Edge) {
	o.onNodeEvent()
}

func (o *OnDemandProbeClient) onCaptureAdded(capture *api.Capture) {
	if !o.elector.IsMaster() {
		return
	}

	o.Lock()
	defer o.Unlock()

	o.graph.RLock()
	defer o.graph.RUnlock()

	o.captures[capture.UUID] = capture

	nodes := o.applyGremlinExpr(capture.GremlinQuery)
	if len(nodes) > 0 {
		go o.registerProbes(nodes, capture)
	}
}

func (o *OnDemandProbeClient) onCaptureDeleted(capture *api.Capture) {
	if !o.elector.IsMaster() {
		return
	}

	o.Lock()
	defer o.Unlock()

	o.graph.Lock()
	defer o.graph.Unlock()

	delete(o.captures, capture.UUID)

	res, err := topology.ExecuteGremlinQuery(o.graph, capture.GremlinQuery)
	if err != nil {
		logging.GetLogger().Errorf("Gremlin error: %s", err.Error())
		return
	}

	for _, value := range res.Values() {
		switch e := value.(type) {
		case *graph.Node:
			if !o.unregisterProbe(e) {
				logging.GetLogger().Errorf("Failed to stop capture on %s", e.ID)
			}
		case []*graph.Node:
			for _, node := range e {
				if !o.unregisterProbe(node) {
					logging.GetLogger().Errorf("Failed to stop capture on %s", node.ID)
				}
			}
		default:
			return
		}
	}
}

func (o *OnDemandProbeClient) onAPIWatcherEvent(action string, id string, resource api.APIResource) {
	logging.GetLogger().Debugf("New watcher event %s for %s", action, id)
	capture := resource.(*api.Capture)
	switch action {
	case "init", "create", "set", "update":
		o.wsServer.BroadcastWSMessage(shttp.NewWSMessage(ondemand.Namespace, "CaptureAdded", capture))
		o.onCaptureAdded(capture)
	case "expire", "delete":
		o.wsServer.BroadcastWSMessage(shttp.NewWSMessage(ondemand.Namespace, "CaptureDeleted", capture))
		o.onCaptureDeleted(capture)
	}
}

func (o *OnDemandProbeClient) Start() {
	o.elector.StartAndWait()

	o.watcher = o.captureHandler.AsyncWatch(o.onAPIWatcherEvent)
	o.graph.AddEventListener(o)
}

func (o *OnDemandProbeClient) Stop() {
	o.watcher.Stop()
	o.elector.Stop()
}

func NewOnDemandProbeClient(g *graph.Graph, ch *api.CaptureAPIHandler, w *shttp.WSServer, etcdClient *etcd.EtcdClient) *OnDemandProbeClient {
	resources := ch.Index()
	captures := make(map[string]*api.Capture)
	for _, resource := range resources {
		captures[resource.ID()] = resource.(*api.Capture)
	}

	elector := etcd.NewEtcdMasterElectorFromConfig(common.AnalyzerService, "ondemand-client", etcdClient)

	return &OnDemandProbeClient{
		graph:          g,
		captureHandler: ch,
		wsServer:       w,
		captures:       captures,
		elector:        elector,
	}
}
<|endoftext|>


<commit_before>// +-------------------------------------------------------------------------
// | Copyright (C) 2016 Yunify, Inc.
// +-------------------------------------------------------------------------
// | Licensed under the Apache License, Version 2.0 (the "License");
// | you may not use this work except in compliance with the License.
// | You may obtain a copy of the License in the LICENSE file, or at:
// |
// | http://www.apache.org/licenses/LICENSE-2.0
// |
// | Unless required by applicable law or agreed to in writing, software
// | distributed under the License is distributed on an "AS IS" BASIS,
// | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// | See the License for the specific language governing permissions and
// | limitations under the License.
// +-------------------------------------------------------------------------

package specs

import (
	"errors"
	"strings"

	"github.com/imdario/mergo"

	"github.com/go-openapi/loads"
	"github.com/go-openapi/spec"

	"github.com/yunify/snips/capsules"
)

// Swagger holds the data that to parse swagger spec.
type Swagger struct {
	FilePath string
	Data     *capsules.Data
}

// Parse parses swagger spec to data.
func (s *Swagger) Parse(version string) error {
	switch version {
	case "v2.0":
		document, err := loads.Spec(s.FilePath)
		if err != nil {
			return err
		}

		document, err = document.Expanded()
		if err != nil {
			return err
		}

		allProperties := map[string]*capsules.Property{}
		allOperations := map[string]map[string]*capsules.Operation{}
		s.parseOperations(document.Spec(), allProperties, allOperations)

		s.loadData(document.Spec())
		s.loadService(document.Spec(), allProperties, allOperations)
		s.loadSubService(document.Spec(), allProperties, allOperations)
		s.loadCustomizedTypes(document.Spec())
	default:
		return errors.New("Swagger version not supported: " + version)
	}

	return nil
}

func (s *Swagger) parseOperations(
	swagger *spec.Swagger,
	allProperties map[string]*capsules.Property,
	allOperations map[string]map[string]*capsules.Operation) {

	parseOperation := func(uri string, method string,
		specOperation *spec.Operation, property *capsules.Property) {
		if specOperation.ID == "PostObject" {
			return
		}

		sections := []string{}

		if len(specOperation.Tags) > 0 {
			for _, subServiceName := range specOperation.Tags {
				sections = append(sections, subServiceName+"SubService")
			}
		} else {
			serviceName := swagger.Info.Title
			sections = append(sections, serviceName+"Service")
		}

		for _, sectionName := range sections {
			if allProperties[sectionName] == nil {
				allProperties[sectionName] = property
			} else {
				mergo.Merge(allProperties[sectionName], property)
			}

			operation := s.parseOperation(uri, method, specOperation, swagger)
			if allOperations[sectionName] == nil {
				allOperations[sectionName] = map[string]*capsules.Operation{}
			}
			allOperations[sectionName][specOperation.ID] = operation
		}
	}

	for requestURI, pathItem := range swagger.Paths.Paths {
		property := &capsules.Property{
			Properties: map[string]*capsules.Property{},
		}

		for _, param := range pathItem.Parameters {
			paramProperty := s.parseParameter(&param, &swagger.Parameters)
			property.Properties[paramProperty.Name] = paramProperty
		}

		if pathItem.Get != nil {
			parseOperation(requestURI, "GET", pathItem.Get, property)
		}
		if pathItem.Put != nil {
			parseOperation(requestURI, "PUT", pathItem.Put, property)
		}
		if pathItem.Post != nil {
			parseOperation(requestURI, "POST", pathItem.Post, property)
		}
		if pathItem.Delete != nil {
			parseOperation(requestURI, "DELETE", pathItem.Delete, property)
		}
		if pathItem.Options != nil {
			parseOperation(requestURI, "OPTIONS", pathItem.Options, property)
		}
		if pathItem.Head != nil {
			parseOperation(requestURI, "HEAD", pathItem.Head, property)
		}
		if pathItem.Patch != nil {
			parseOperation(requestURI, "PATCH", pathItem.Patch, property)
		}
	}
}

func (s *Swagger) loadData(swagger *spec.Swagger) {
	if s.Data == nil {
		s.Data = &capsules.Data{}
	}
	s.Data.Service = nil
	s.Data.SubServices = map[string]*capsules.SubService{}
	s.Data.CustomizedTypes = map[string]*capsules.Property{}
}

func (s *Swagger) loadService(
	swagger *spec.Swagger,
	allProperties map[string]*capsules.Property,
	allOperations map[string]map[string]*capsules.Operation) {
	serviceName := swagger.Info.Title

	property := &capsules.Property{}
	if allProperties[swagger.Info.Title] != nil {
		property = allProperties[serviceName]
	}

	s.Data.Service = &capsules.Service{
		APIVersion:  swagger.Info.Version,
		Name:        serviceName,
		BasePath:    swagger.BasePath,
		Description: swagger.Info.Description,
		Properties:  property,
		Operations:  allOperations[serviceName+"Service"],
	}

	if !strings.Contains(s.Data.Service.Name, "QingStor") {
		for _, o := range s.Data.Service.Operations {
			exchange := o.Request.Params
			o.Request.Params = o.Request.Elements
			o.Request.Elements = exchange
		}
	}
}

func (s *Swagger) loadSubService(
	swagger *spec.Swagger,
	allProperties map[string]*capsules.Property,
	allOperations map[string]map[string]*capsules.Operation) {

	for subService, operations := range allOperations {
		if strings.Contains(subService, "SubService") {
			subServiceName := strings.Replace(subService, "SubService", "", -1)
			s.Data.SubServices[subServiceName] = &capsules.SubService{
				ID:         subServiceName,
				Name:       subServiceName,
				Properties: allProperties[subService],
				Operations: operations,
			}

			if strings.Contains(s.Data.Service.Name, "QingCloud") {
				for _, o := range s.Data.SubServices[subServiceName].Operations {
					o.Request.Params, o.Request.Elements = o.Request.Elements, o.Request.Params
				}
			}
		}
	}
}

func (s *Swagger) loadCustomizedTypes(swagger *spec.Swagger) {
	for name, definition := range swagger.Definitions {
		s.Data.CustomizedTypes[name] = s.parseSchema(&definition)
		s.Data.CustomizedTypes[name].ID = name
		s.Data.CustomizedTypes[name].Name = name

		for _, schemaKey := range definition.SchemaProps.Required {
			s.Data.CustomizedTypes[name].Properties[schemaKey].IsRequired = true
		}
	}
}
<commit_msg>Fix non QingStor service not handle right in service<commit_after>// +-------------------------------------------------------------------------
// | Copyright (C) 2016 Yunify, Inc.
// +-------------------------------------------------------------------------
// | Licensed under the Apache License, Version 2.0 (the "License");
// | you may not use this work except in compliance with the License.
// | You may obtain a copy of the License in the LICENSE file, or at:
// |
// | http://www.apache.org/licenses/LICENSE-2.0
// |
// | Unless required by applicable law or agreed to in writing, software
// | distributed under the License is distributed on an "AS IS" BASIS,
// | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// | See the License for the specific language governing permissions and
// | limitations under the License.
// +-------------------------------------------------------------------------

package specs

import (
	"errors"
	"strings"

	"github.com/imdario/mergo"

	"github.com/go-openapi/loads"
	"github.com/go-openapi/spec"

	"github.com/yunify/snips/capsules"
)

// Swagger holds the data that to parse swagger spec.
type Swagger struct {
	FilePath string
	Data     *capsules.Data
}

// Parse parses swagger spec to data.
func (s *Swagger) Parse(version string) error {
	switch version {
	case "v2.0":
		document, err := loads.Spec(s.FilePath)
		if err != nil {
			return err
		}

		document, err = document.Expanded()
		if err != nil {
			return err
		}

		allProperties := map[string]*capsules.Property{}
		allOperations := map[string]map[string]*capsules.Operation{}
		s.parseOperations(document.Spec(), allProperties, allOperations)

		s.loadData(document.Spec())
		s.loadService(document.Spec(), allProperties, allOperations)
		s.loadSubService(document.Spec(), allProperties, allOperations)
		s.loadCustomizedTypes(document.Spec())
	default:
		return errors.New("Swagger version not supported: " + version)
	}

	return nil
}

func (s *Swagger) parseOperations(
	swagger *spec.Swagger,
	allProperties map[string]*capsules.Property,
	allOperations map[string]map[string]*capsules.Operation) {

	parseOperation := func(uri string, method string,
		specOperation *spec.Operation, property *capsules.Property) {
		if specOperation.ID == "PostObject" {
			return
		}

		sections := []string{}

		if len(specOperation.Tags) > 0 {
			for _, subServiceName := range specOperation.Tags {
				sections = append(sections, subServiceName+"SubService")
			}
		} else {
			serviceName := swagger.Info.Title
			sections = append(sections, serviceName+"Service")
		}

		for _, sectionName := range sections {
			if allProperties[sectionName] == nil {
				allProperties[sectionName] = property
			} else {
				mergo.Merge(allProperties[sectionName], property)
			}

			operation := s.parseOperation(uri, method, specOperation, swagger)
			if allOperations[sectionName] == nil {
				allOperations[sectionName] = map[string]*capsules.Operation{}
			}
			allOperations[sectionName][specOperation.ID] = operation
		}
	}

	for requestURI, pathItem := range swagger.Paths.Paths {
		property := &capsules.Property{
			Properties: map[string]*capsules.Property{},
		}

		for _, param := range pathItem.Parameters {
			paramProperty := s.parseParameter(&param, &swagger.Parameters)
			property.Properties[paramProperty.Name] = paramProperty
		}

		if pathItem.Get != nil {
			parseOperation(requestURI, "GET", pathItem.Get, property)
		}
		if pathItem.Put != nil {
			parseOperation(requestURI, "PUT", pathItem.Put, property)
		}
		if pathItem.Post != nil {
			parseOperation(requestURI, "POST", pathItem.Post, property)
		}
		if pathItem.Delete != nil {
			parseOperation(requestURI, "DELETE", pathItem.Delete, property)
		}
		if pathItem.Options != nil {
			parseOperation(requestURI, "OPTIONS", pathItem.Options, property)
		}
		if pathItem.Head != nil {
			parseOperation(requestURI, "HEAD", pathItem.Head, property)
		}
		if pathItem.Patch != nil {
			parseOperation(requestURI, "PATCH", pathItem.Patch, property)
		}
	}
}

func (s *Swagger) loadData(swagger *spec.Swagger) {
	if s.Data == nil {
		s.Data = &capsules.Data{}
	}
	s.Data.Service = nil
	s.Data.SubServices = map[string]*capsules.SubService{}
	s.Data.CustomizedTypes = map[string]*capsules.Property{}
}

func (s *Swagger) loadService(
	swagger *spec.Swagger,
	allProperties map[string]*capsules.Property,
	allOperations map[string]map[string]*capsules.Operation) {
	serviceName := swagger.Info.Title

	property := &capsules.Property{}
	if allProperties[swagger.Info.Title] != nil {
		property = allProperties[serviceName]
	}

	s.Data.Service = &capsules.Service{
		APIVersion:  swagger.Info.Version,
		Name:        serviceName,
		BasePath:    swagger.BasePath,
		Description: swagger.Info.Description,
		Properties:  property,
		Operations:  allOperations[serviceName+"Service"],
	}

	// Be compatible with QingCloud IaaS Services
	if strings.Contains(s.Data.Service.Name, "QingCloud") {
		for _, o := range s.Data.Service.Operations {
			o.Request.Params, o.Request.Elements = o.Request.Elements, o.Request.Params
		}
	}
}

func (s *Swagger) loadSubService(
	swagger *spec.Swagger,
	allProperties map[string]*capsules.Property,
	allOperations map[string]map[string]*capsules.Operation) {

	for subService, operations := range allOperations {
		if strings.Contains(subService, "SubService") {
			subServiceName := strings.Replace(subService, "SubService", "", -1)
			s.Data.SubServices[subServiceName] = &capsules.SubService{
				ID:         subServiceName,
				Name:       subServiceName,
				Properties: allProperties[subService],
				Operations: operations,
			}

			// Be compatible with QingCloud IaaS Services
			if strings.Contains(s.Data.Service.Name, "QingCloud") {
				for _, o := range s.Data.SubServices[subServiceName].Operations {
					o.Request.Params, o.Request.Elements = o.Request.Elements, o.Request.Params
				}
			}
		}
	}
}

func (s *Swagger) loadCustomizedTypes(swagger *spec.Swagger) {
	for name, definition := range swagger.Definitions {
		s.Data.CustomizedTypes[name] = s.parseSchema(&definition)
		s.Data.CustomizedTypes[name].ID = name
		s.Data.CustomizedTypes[name].Name = name

		for _, schemaKey := range definition.SchemaProps.Required {
			s.Data.CustomizedTypes[name].Properties[schemaKey].IsRequired = true
		}
	}
}
<|endoftext|>