<commit_before>// Package client provides an implementation of a restricted subset of kubernetes API client
package client

import (
	"bytes"
	"crypto/tls"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"regexp"
	"strings"

	"github.com/micro/go-micro/v2/logger"
	"github.com/micro/go-micro/v2/util/kubernetes/api"
)

var (
	// path to kubernetes service account token
	serviceAccountPath = "/var/run/secrets/kubernetes.io/serviceaccount"
	// ErrReadNamespace is returned when the names could not be read from service account
	ErrReadNamespace = errors.New("Could not read namespace from service account secret")
	// DefaultImage is default micro image
	DefaultImage = "micro/go-micro"
	// DefaultNamespace is the default k8s namespace
	DefaultNamespace = "default"
)

// Client ...
type client struct {
	opts *api.Options
}

// Kubernetes client
type Client interface {
	// Create creates new API resource
	Create(*Resource, ...CreateOption) error
	// Get queries API resrouces
	Get(*Resource, ...GetOption) error
	// Update patches existing API object
	Update(*Resource, ...UpdateOption) error
	// Delete deletes API resource
	Delete(*Resource, ...DeleteOption) error
	// List lists API resources
	List(*Resource, ...ListOption) error
	// Log gets log for a pod
	Log(*Resource, ...LogOption) (io.ReadCloser, error)
	// Watch for events
	Watch(*Resource, ...WatchOption) (Watcher, error)
}

// Create creates new API object
func (c *client) Create(r *Resource, opts ...CreateOption) error {
	var options CreateOptions
	for _, o := range opts {
		o(&options)
	}

	b := new(bytes.Buffer)
	if err := renderTemplate(r.Kind, b, r.Value); err != nil {
		return err
	}

	return api.NewRequest(c.opts).
		Post().
		SetHeader("Content-Type", "application/yaml").
		Namespace(options.Namespace).
		Resource(r.Kind).
		Body(b).
		Do().
		Error()
}

var (
	nameRegex = regexp.MustCompile("[^a-zA-Z0-9]+")
)

// SerializeResourceName removes all spacial chars from a string so it
// can be used as a k8s resource name
func SerializeResourceName(ns string) string {
	return nameRegex.ReplaceAllString(ns, "-")
}

// Get queries API objects and stores the result in r
func (c *client) Get(r *Resource, opts ...GetOption) error {
	options := GetOptions{
		Namespace: c.opts.Namespace,
	}
	for _, o := range opts {
		o(&options)
	}

	return api.NewRequest(c.opts).
		Get().
		Resource(r.Kind).
		Namespace(options.Namespace).
		Params(&api.Params{LabelSelector: options.Labels}).
		Do().
		Into(r.Value)
}

// Log returns logs for a pod
func (c *client) Log(r *Resource, opts ...LogOption) (io.ReadCloser, error) {
	options := LogOptions{
		Namespace: c.opts.Namespace,
	}
	for _, o := range opts {
		o(&options)
	}

	req := api.NewRequest(c.opts).
		Get().
		Resource(r.Kind).
		SubResource("log").
		Name(r.Name).
		Namespace(options.Namespace)

	if options.Params != nil {
		req.Params(&api.Params{Additional: options.Params})
	}

	resp, err := req.Raw()
	if err != nil {
		return nil, err
	}
	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		resp.Body.Close()
		return nil, errors.New(resp.Request.URL.String() + ": " + resp.Status)
	}
	return resp.Body, nil
}

// Update updates API object
func (c *client) Update(r *Resource, opts ...UpdateOption) error {
	options := UpdateOptions{
		Namespace: c.opts.Namespace,
	}
	for _, o := range opts {
		o(&options)
	}

	req := api.NewRequest(c.opts).
		Patch().
		SetHeader("Content-Type", "application/strategic-merge-patch+json").
		Resource(r.Kind).
		Name(r.Name).
		Namespace(options.Namespace)

	switch r.Kind {
	case "service":
		req.Body(r.Value.(*Service))
	case "deployment":
		req.Body(r.Value.(*Deployment))
	case "pod":
		req.Body(r.Value.(*Pod))
	default:
		return errors.New("unsupported resource")
	}

	return req.Do().Error()
}

// Delete removes API object
func (c *client) Delete(r *Resource, opts ...DeleteOption) error {
	options := DeleteOptions{
		Namespace: c.opts.Namespace,
	}
	for _, o := range opts {
		o(&options)
	}

	return api.NewRequest(c.opts).
		Delete().
		Resource(r.Kind).
		Name(r.Name).
		Namespace(options.Namespace).
		Do().
		Error()
}

// List lists API objects and stores the result in r
func (c *client) List(r *Resource, opts ...ListOption) error {
	options := ListOptions{
		Namespace: c.opts.Namespace,
	}
	for _, o := range opts {
		o(&options)
	}

	labels := map[string]string{
		"micro": "service",
	}

	return c.Get(r, GetLabels(labels), GetNamespace(options.Namespace))
}

// Watch returns an event stream
func (c *client) Watch(r *Resource, opts ...WatchOption) (Watcher, error) {
	options := WatchOptions{
		Namespace: c.opts.Namespace,
	}
	for _, o := range opts {
		o(&options)
	}

	// set the watch param
	params := &api.Params{Additional: map[string]string{
		"watch": "true",
	}}

	// get options params
	if options.Params != nil {
		for k, v := range options.Params {
			params.Additional[k] = v
		}
	}

	req := api.NewRequest(c.opts).
		Get().
		Resource(r.Kind).
		Name(r.Name).
		Namespace(options.Namespace).
		Params(params)

	return newWatcher(req)
}

// NewService returns default micro kubernetes service definition
func NewService(name, version, typ, namespace string) *Service {
	if logger.V(logger.TraceLevel, logger.DefaultLogger) {
		logger.Tracef("kubernetes default service: name: %s, version: %s", name, version)
	}

	Labels := map[string]string{
		"name":    name,
		"version": version,
		"micro":   typ,
	}

	svcName := name
	if len(version) > 0 {
		// API service object name joins name and version over "-"
		svcName = strings.Join([]string{name, version}, "-")
	}

	if len(namespace) == 0 {
		namespace = DefaultNamespace
	}

	Metadata := &Metadata{
		Name:      svcName,
		Namespace: SerializeResourceName(namespace),
		Version:   version,
		Labels:    Labels,
	}

	Spec := &ServiceSpec{
		Type:     "ClusterIP",
		Selector: Labels,
		Ports: []ServicePort{{
			"service-port", 8080, "",
		}},
	}

	return &Service{
		Metadata: Metadata,
		Spec:     Spec,
	}
}

// NewService returns default micro kubernetes deployment definition
func NewDeployment(name, version, typ, namespace string) *Deployment {
	if logger.V(logger.TraceLevel, logger.DefaultLogger) {
		logger.Tracef("kubernetes default deployment: name: %s, version: %s", name, version)
	}

	Labels := map[string]string{
		"name":    name,
		"version": version,
		"micro":   typ,
	}

	depName := name
	if len(version) > 0 {
		// API deployment object name joins name and version over "-"
		depName = strings.Join([]string{name, version}, "-")
	}

	if len(namespace) == 0 {
		namespace = DefaultNamespace
	}

	Metadata := &Metadata{
		Name:        depName,
		Namespace:   SerializeResourceName(namespace),
		Version:     version,
		Labels:      Labels,
		Annotations: map[string]string{},
	}

	// enable go modules by default
	env := EnvVar{
		Name:  "GO111MODULE",
		Value: "on",
	}

	Spec := &DeploymentSpec{
		Replicas: 1,
		Selector: &LabelSelector{
			MatchLabels: Labels,
		},
		Template: &Template{
			Metadata: Metadata,
			PodSpec: &PodSpec{
				Containers: []Container{{
					Name:    name,
					Image:   DefaultImage,
					Env:     []EnvVar{env},
					Command: []string{"go", "run", "."},
					Ports: []ContainerPort{{
						Name:          "service-port",
						ContainerPort: 8080,
					}},
				}},
			},
		},
	}

	return &Deployment{
		Metadata: Metadata,
		Spec:     Spec,
	}
}

// NewLocalClient returns a client that can be used with `kubectl proxy`
func NewLocalClient(hosts ...string) *client {
	if len(hosts) == 0 {
		hosts[0] = "http://localhost:8001"
	}
	return &client{
		opts: &api.Options{
			Client:    http.DefaultClient,
			Host:      hosts[0],
			Namespace: "default",
		},
	}
}

// NewClusterClient creates a Kubernetes client for use from within a k8s pod.
func NewClusterClient() *client {
	host := "https://" + os.Getenv("KUBERNETES_SERVICE_HOST") + ":" + os.Getenv("KUBERNETES_SERVICE_PORT")

	s, err := os.Stat(serviceAccountPath)
	if err != nil {
		logger.Fatal(err)
	}
	if s == nil || !s.IsDir() {
		logger.Fatal(errors.New("service account not found"))
	}

	token, err := ioutil.ReadFile(path.Join(serviceAccountPath, "token"))
	if err != nil {
		logger.Fatal(err)
	}
	t := string(token)

	crt, err := CertPoolFromFile(path.Join(serviceAccountPath, "ca.crt"))
	if err != nil {
		logger.Fatal(err)
	}

	c := &http.Client{
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{
				RootCAs: crt,
			},
			DisableCompression: true,
		},
	}

	return &client{
		opts: &api.Options{
			Client:      c,
			Host:        host,
			BearerToken: &t,
			Namespace:   DefaultNamespace,
		},
	}
}
<commit_msg>set namespace on create<commit_after>// Package client provides an implementation of a restricted subset of kubernetes API client
package client

import (
	"bytes"
	"crypto/tls"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"regexp"
	"strings"

	"github.com/micro/go-micro/v2/logger"
	"github.com/micro/go-micro/v2/util/kubernetes/api"
)

var (
	// path to kubernetes service account token
	serviceAccountPath = "/var/run/secrets/kubernetes.io/serviceaccount"
	// ErrReadNamespace is returned when the names could not be read from service account
	ErrReadNamespace = errors.New("Could not read namespace from service account secret")
	// DefaultImage is default micro image
	DefaultImage = "micro/go-micro"
	// DefaultNamespace is the default k8s namespace
	DefaultNamespace = "default"
)

// Client ...
type client struct {
	opts *api.Options
}

// Kubernetes client
type Client interface {
	// Create creates new API resource
	Create(*Resource, ...CreateOption) error
	// Get queries API resrouces
	Get(*Resource, ...GetOption) error
	// Update patches existing API object
	Update(*Resource, ...UpdateOption) error
	// Delete deletes API resource
	Delete(*Resource, ...DeleteOption) error
	// List lists API resources
	List(*Resource, ...ListOption) error
	// Log gets log for a pod
	Log(*Resource, ...LogOption) (io.ReadCloser, error)
	// Watch for events
	Watch(*Resource, ...WatchOption) (Watcher, error)
}

// Create creates new API object
func (c *client) Create(r *Resource, opts ...CreateOption) error {
	options := CreateOptions{
		Namespace: c.opts.Namespace,
	}
	for _, o := range opts {
		o(&options)
	}

	b := new(bytes.Buffer)
	if err := renderTemplate(r.Kind, b, r.Value); err != nil {
		return err
	}

	return api.NewRequest(c.opts).
		Post().
		SetHeader("Content-Type", "application/yaml").
		Namespace(options.Namespace).
		Resource(r.Kind).
		Body(b).
		Do().
		Error()
}

var (
	nameRegex = regexp.MustCompile("[^a-zA-Z0-9]+")
)

// SerializeResourceName removes all spacial chars from a string so it
// can be used as a k8s resource name
func SerializeResourceName(ns string) string {
	return nameRegex.ReplaceAllString(ns, "-")
}

// Get queries API objects and stores the result in r
func (c *client) Get(r *Resource, opts ...GetOption) error {
	options := GetOptions{
		Namespace: c.opts.Namespace,
	}
	for _, o := range opts {
		o(&options)
	}

	return api.NewRequest(c.opts).
		Get().
		Resource(r.Kind).
		Namespace(options.Namespace).
		Params(&api.Params{LabelSelector: options.Labels}).
		Do().
		Into(r.Value)
}

// Log returns logs for a pod
func (c *client) Log(r *Resource, opts ...LogOption) (io.ReadCloser, error) {
	options := LogOptions{
		Namespace: c.opts.Namespace,
	}
	for _, o := range opts {
		o(&options)
	}

	req := api.NewRequest(c.opts).
		Get().
		Resource(r.Kind).
		SubResource("log").
		Name(r.Name).
		Namespace(options.Namespace)

	if options.Params != nil {
		req.Params(&api.Params{Additional: options.Params})
	}

	resp, err := req.Raw()
	if err != nil {
		return nil, err
	}
	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		resp.Body.Close()
		return nil, errors.New(resp.Request.URL.String() + ": " + resp.Status)
	}
	return resp.Body, nil
}

// Update updates API object
func (c *client) Update(r *Resource, opts ...UpdateOption) error {
	options := UpdateOptions{
		Namespace: c.opts.Namespace,
	}
	for _, o := range opts {
		o(&options)
	}

	req := api.NewRequest(c.opts).
		Patch().
		SetHeader("Content-Type", "application/strategic-merge-patch+json").
		Resource(r.Kind).
		Name(r.Name).
		Namespace(options.Namespace)

	switch r.Kind {
	case "service":
		req.Body(r.Value.(*Service))
	case "deployment":
		req.Body(r.Value.(*Deployment))
	case "pod":
		req.Body(r.Value.(*Pod))
	default:
		return errors.New("unsupported resource")
	}

	return req.Do().Error()
}

// Delete removes API object
func (c *client) Delete(r *Resource, opts ...DeleteOption) error {
	options := DeleteOptions{
		Namespace: c.opts.Namespace,
	}
	for _, o := range opts {
		o(&options)
	}

	return api.NewRequest(c.opts).
		Delete().
		Resource(r.Kind).
		Name(r.Name).
		Namespace(options.Namespace).
		Do().
		Error()
}

// List lists API objects and stores the result in r
func (c *client) List(r *Resource, opts ...ListOption) error {
	options := ListOptions{
		Namespace: c.opts.Namespace,
	}
	for _, o := range opts {
		o(&options)
	}

	labels := map[string]string{
		"micro": "service",
	}

	return c.Get(r, GetLabels(labels), GetNamespace(options.Namespace))
}

// Watch returns an event stream
func (c *client) Watch(r *Resource, opts ...WatchOption) (Watcher, error) {
	options := WatchOptions{
		Namespace: c.opts.Namespace,
	}
	for _, o := range opts {
		o(&options)
	}

	// set the watch param
	params := &api.Params{Additional: map[string]string{
		"watch": "true",
	}}

	// get options params
	if options.Params != nil {
		for k, v := range options.Params {
			params.Additional[k] = v
		}
	}

	req := api.NewRequest(c.opts).
		Get().
		Resource(r.Kind).
		Name(r.Name).
		Namespace(options.Namespace).
		Params(params)

	return newWatcher(req)
}

// NewService returns default micro kubernetes service definition
func NewService(name, version, typ, namespace string) *Service {
	if logger.V(logger.TraceLevel, logger.DefaultLogger) {
		logger.Tracef("kubernetes default service: name: %s, version: %s", name, version)
	}

	Labels := map[string]string{
		"name":    name,
		"version": version,
		"micro":   typ,
	}

	svcName := name
	if len(version) > 0 {
		// API service object name joins name and version over "-"
		svcName = strings.Join([]string{name, version}, "-")
	}

	if len(namespace) == 0 {
		namespace = DefaultNamespace
	}

	Metadata := &Metadata{
		Name:      svcName,
		Namespace: SerializeResourceName(namespace),
		Version:   version,
		Labels:    Labels,
	}

	Spec := &ServiceSpec{
		Type:     "ClusterIP",
		Selector: Labels,
		Ports: []ServicePort{{
			"service-port", 8080, "",
		}},
	}

	return &Service{
		Metadata: Metadata,
		Spec:     Spec,
	}
}

// NewService returns default micro kubernetes deployment definition
func NewDeployment(name, version, typ, namespace string) *Deployment {
	if logger.V(logger.TraceLevel, logger.DefaultLogger) {
		logger.Tracef("kubernetes default deployment: name: %s, version: %s", name, version)
	}

	Labels := map[string]string{
		"name":    name,
		"version": version,
		"micro":   typ,
	}

	depName := name
	if len(version) > 0 {
		// API deployment object name joins name and version over "-"
		depName = strings.Join([]string{name, version}, "-")
	}

	if len(namespace) == 0 {
		namespace = DefaultNamespace
	}

	Metadata := &Metadata{
		Name:        depName,
		Namespace:   SerializeResourceName(namespace),
		Version:     version,
		Labels:      Labels,
		Annotations: map[string]string{},
	}

	// enable go modules by default
	env := EnvVar{
		Name:  "GO111MODULE",
		Value: "on",
	}

	Spec := &DeploymentSpec{
		Replicas: 1,
		Selector: &LabelSelector{
			MatchLabels: Labels,
		},
		Template: &Template{
			Metadata: Metadata,
			PodSpec: &PodSpec{
				Containers: []Container{{
					Name:    name,
					Image:   DefaultImage,
					Env:     []EnvVar{env},
					Command: []string{"go", "run", "."},
					Ports: []ContainerPort{{
						Name:          "service-port",
						ContainerPort: 8080,
					}},
				}},
			},
		},
	}

	return &Deployment{
		Metadata: Metadata,
		Spec:     Spec,
	}
}

// NewLocalClient returns a client that can be used with `kubectl proxy`
func NewLocalClient(hosts ...string) *client {
	if len(hosts) == 0 {
		hosts[0] = "http://localhost:8001"
	}
	return &client{
		opts: &api.Options{
			Client:    http.DefaultClient,
			Host:      hosts[0],
			Namespace: "default",
		},
	}
}

// NewClusterClient creates a Kubernetes client for use from within a k8s pod.
func NewClusterClient() *client {
	host := "https://" + os.Getenv("KUBERNETES_SERVICE_HOST") + ":" + os.Getenv("KUBERNETES_SERVICE_PORT")

	s, err := os.Stat(serviceAccountPath)
	if err != nil {
		logger.Fatal(err)
	}
	if s == nil || !s.IsDir() {
		logger.Fatal(errors.New("service account not found"))
	}

	token, err := ioutil.ReadFile(path.Join(serviceAccountPath, "token"))
	if err != nil {
		logger.Fatal(err)
	}
	t := string(token)

	crt, err := CertPoolFromFile(path.Join(serviceAccountPath, "ca.crt"))
	if err != nil {
		logger.Fatal(err)
	}

	c := &http.Client{
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{
				RootCAs: crt,
			},
			DisableCompression: true,
		},
	}

	return &client{
		opts: &api.Options{
			Client:      c,
			Host:        host,
			BearerToken: &t,
			Namespace:   DefaultNamespace,
		},
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"os"
	"regexp"
	"strings"

	dep "github.com/hashicorp/consul-template/dependency"
)

// datacentersFunc returns or accumulates datacenter dependencies.
func datacentersFunc(brain *Brain,
	used, missing map[string]dep.Dependency) func(...string) ([]string, error) {
	return func(s ...string) ([]string, error) {
		result := make([]string, 0)

		d, err := dep.ParseDatacenters(s...)
		if err != nil {
			return result, err
		}

		addDependency(used, d)

		if result, ok := brain.datacenters[d.HashCode()]; ok {
			return result, nil
		}

		addDependency(missing, d)

		return result, nil
	}
}

// fileFunc returns or accumulates file dependencies.
func fileFunc(brain *Brain,
	used, missing map[string]dep.Dependency) func(string) (string, error) {
	return func(s string) (string, error) {
		if len(s) == 0 {
			return "", nil
		}

		d, err := dep.ParseFile(s)
		if err != nil {
			return "", err
		}

		addDependency(used, d)

		if contents, ok := brain.files[d.HashCode()]; ok {
			return contents, nil
		}

		addDependency(missing, d)

		return "", nil
	}
}

// keyFunc returns or accumulates key dependencies.
func keyFunc(brain *Brain,
	used, missing map[string]dep.Dependency) func(string) (string, error) {
	return func(s string) (string, error) {
		if len(s) == 0 {
			return "", nil
		}

		d, err := dep.ParseStoreKey(s)
		if err != nil {
			return "", nil
		}

		addDependency(used, d)

		if value, ok := brain.storeKeys[d.HashCode()]; ok {
			return value, nil
		}

		addDependency(missing, d)

		return "", nil
	}
}

// lsFunc returns or accumulates keyPrefix dependencies.
func lsFunc(brain *Brain,
	used, missing map[string]dep.Dependency) func(string) ([]*dep.KeyPair, error) {
	return func(s string) ([]*dep.KeyPair, error) {
		result := make([]*dep.KeyPair, 0)

		if len(s) == 0 {
			return result, nil
		}

		d, err := dep.ParseStoreKeyPrefix(s)
		if err != nil {
			return result, err
		}

		addDependency(used, d)

		// Only return non-empty top-level keys
		if pairs, ok := brain.storeKeyPrefixes[d.HashCode()]; ok {
			for _, pair := range pairs {
				if pair.Key != "" && !strings.Contains(pair.Key, "/") {
					result = append(result, pair)
				}
			}

			return result, nil
		}

		addDependency(missing, d)

		return result, nil
	}
}

// nodesFunc returns or accumulates catalog node dependencies.
func nodesFunc(brain *Brain,
	used, missing map[string]dep.Dependency) func(...string) ([]*dep.Node, error) {
	return func(s ...string) ([]*dep.Node, error) {
		result := make([]*dep.Node, 0)

		d, err := dep.ParseCatalogNodes(s...)
		if err != nil {
			return nil, err
		}

		addDependency(used, d)

		if nodes, ok := brain.catalogNodes[d.HashCode()]; ok {
			return nodes, nil
		}

		addDependency(missing, d)

		return result, nil
	}
}

// serviceFunc returns or accumulates health service dependencies.
func serviceFunc(brain *Brain,
	used, missing map[string]dep.Dependency) func(...string) ([]*dep.HealthService, error) {
	return func(s ...string) ([]*dep.HealthService, error) {
		result := make([]*dep.HealthService, 0)

		if len(s) == 0 || s[0] == "" {
			return result, nil
		}

		d, err := dep.ParseHealthServices(s...)
		if err != nil {
			return nil, err
		}

		addDependency(used, d)

		if services, ok := brain.healthServices[d.HashCode()]; ok {
			return services, nil
		}

		addDependency(missing, d)

		return result, nil
	}
}

// servicesFunc returns or accumulates catalog services dependencies.
func servicesFunc(brain *Brain,
	used, missing map[string]dep.Dependency) func(...string) ([]*dep.CatalogService, error) {
	return func(s ...string) ([]*dep.CatalogService, error) {
		result := make([]*dep.CatalogService, 0)

		d, err := dep.ParseCatalogServices(s...)
		if err != nil {
			return nil, err
		}

		addDependency(used, d)

		if services, ok := brain.catalogServices[d.HashCode()]; ok {
			return services, nil
		}

		addDependency(missing, d)

		return result, nil
	}
}

// treeFunc returns or accumulates keyPrefix dependencies.
func treeFunc(brain *Brain,
	used, missing map[string]dep.Dependency) func(string) ([]*dep.KeyPair, error) {
	return func(s string) ([]*dep.KeyPair, error) {
		result := make([]*dep.KeyPair, 0)

		if len(s) == 0 {
			return result, nil
		}

		d, err := dep.ParseStoreKeyPrefix(s)
		if err != nil {
			return result, err
		}

		addDependency(used, d)

		// Only return non-empty top-level keys
		if pairs, ok := brain.storeKeyPrefixes[d.HashCode()]; ok {
			for _, pair := range pairs {
				parts := strings.Split(pair.Key, "/")
				if parts[len(parts)-1] != "" {
					result = append(result, pair)
				}
			}

			return result, nil
		}

		addDependency(missing, d)

		return result, nil
	}

}

// byTag is a template func that takes the provided services and
// produces a map based on Service tags.
//
// The map key is a string representing the service tag. The map value is a
// slice of Services which have the tag assigned.
func byTag(in []*dep.HealthService) (map[string][]*dep.HealthService, error) {
	m := make(map[string][]*dep.HealthService)
	for _, s := range in {
		for _, t := range s.Tags {
			m[t] = append(m[t], s)
		}
	}
	return m, nil
}

// returns the value of the environment variable set
func env(s string) (string, error) {
	return os.Getenv(s), nil
}

// parseJSON returns a structure for valid JSON
func parseJSON(s string) (interface{}, error) {
	if s == "" {
		return make([]interface{}, 0), nil
	}

	var data interface{}
	if err := json.Unmarshal([]byte(s), &data); err != nil {
		return nil, err
	}
	return data, nil
}

// toLower converts the given string (usually by a pipe) to lowercase.
func toLower(s string) (string, error) {
	return strings.ToLower(s), nil
}

// toTitle converts the given string (usually by a pipe) to titlecase.
func toTitle(s string) (string, error) {
	return strings.Title(s), nil
}

// toUpper converts the given string (usually by a pipe) to uppercase.
func toUpper(s string) (string, error) {
	return strings.ToUpper(s), nil
}

// replaceAll replaces all occurrences of a value in a string with the given
// replacement value.
func replaceAll(f, t, s string) (string, error) {
	return strings.Replace(s, f, t, -1), nil
}

// regexReplaceAll replaces all occurrences of a regular expression with
// the given replacement value.
func regexReplaceAll(re, pl, s string) (string, error) {
	compiled, err := regexp.Compile(re)
	if err != nil {
		return "", err
	}
	return compiled.ReplaceAllString(s, pl), nil
}

// addDependency adds the given Dependency to the map.
func addDependency(m map[string]dep.Dependency, d dep.Dependency) {
	if _, ok := m[d.HashCode()]; !ok {
		m[d.HashCode()] = d
	}
}
<commit_msg>Return an error when failing to parse a key dependency<commit_after>package main

import (
	"encoding/json"
	"os"
	"regexp"
	"strings"

	dep "github.com/hashicorp/consul-template/dependency"
)

// datacentersFunc returns or accumulates datacenter dependencies.
func datacentersFunc(brain *Brain,
	used, missing map[string]dep.Dependency) func(...string) ([]string, error) {
	return func(s ...string) ([]string, error) {
		result := make([]string, 0)

		d, err := dep.ParseDatacenters(s...)
		if err != nil {
			return result, err
		}

		addDependency(used, d)

		if result, ok := brain.datacenters[d.HashCode()]; ok {
			return result, nil
		}

		addDependency(missing, d)

		return result, nil
	}
}

// fileFunc returns or accumulates file dependencies.
func fileFunc(brain *Brain,
	used, missing map[string]dep.Dependency) func(string) (string, error) {
	return func(s string) (string, error) {
		if len(s) == 0 {
			return "", nil
		}

		d, err := dep.ParseFile(s)
		if err != nil {
			return "", err
		}

		addDependency(used, d)

		if contents, ok := brain.files[d.HashCode()]; ok {
			return contents, nil
		}

		addDependency(missing, d)

		return "", nil
	}
}

// keyFunc returns or accumulates key dependencies.
func keyFunc(brain *Brain,
	used, missing map[string]dep.Dependency) func(string) (string, error) {
	return func(s string) (string, error) {
		if len(s) == 0 {
			return "", nil
		}

		d, err := dep.ParseStoreKey(s)
		if err != nil {
			return "", err
		}

		addDependency(used, d)

		if value, ok := brain.storeKeys[d.HashCode()]; ok {
			return value, nil
		}

		addDependency(missing, d)

		return "", nil
	}
}

// lsFunc returns or accumulates keyPrefix dependencies.
func lsFunc(brain *Brain,
	used, missing map[string]dep.Dependency) func(string) ([]*dep.KeyPair, error) {
	return func(s string) ([]*dep.KeyPair, error) {
		result := make([]*dep.KeyPair, 0)

		if len(s) == 0 {
			return result, nil
		}

		d, err := dep.ParseStoreKeyPrefix(s)
		if err != nil {
			return result, err
		}

		addDependency(used, d)

		// Only return non-empty top-level keys
		if pairs, ok := brain.storeKeyPrefixes[d.HashCode()]; ok {
			for _, pair := range pairs {
				if pair.Key != "" && !strings.Contains(pair.Key, "/") {
					result = append(result, pair)
				}
			}

			return result, nil
		}

		addDependency(missing, d)

		return result, nil
	}
}

// nodesFunc returns or accumulates catalog node dependencies.
func nodesFunc(brain *Brain,
	used, missing map[string]dep.Dependency) func(...string) ([]*dep.Node, error) {
	return func(s ...string) ([]*dep.Node, error) {
		result := make([]*dep.Node, 0)

		d, err := dep.ParseCatalogNodes(s...)
		if err != nil {
			return nil, err
		}

		addDependency(used, d)

		if nodes, ok := brain.catalogNodes[d.HashCode()]; ok {
			return nodes, nil
		}

		addDependency(missing, d)

		return result, nil
	}
}

// serviceFunc returns or accumulates health service dependencies.
func serviceFunc(brain *Brain,
	used, missing map[string]dep.Dependency) func(...string) ([]*dep.HealthService, error) {
	return func(s ...string) ([]*dep.HealthService, error) {
		result := make([]*dep.HealthService, 0)

		if len(s) == 0 || s[0] == "" {
			return result, nil
		}

		d, err := dep.ParseHealthServices(s...)
		if err != nil {
			return nil, err
		}

		addDependency(used, d)

		if services, ok := brain.healthServices[d.HashCode()]; ok {
			return services, nil
		}

		addDependency(missing, d)

		return result, nil
	}
}

// servicesFunc returns or accumulates catalog services dependencies.
func servicesFunc(brain *Brain,
	used, missing map[string]dep.Dependency) func(...string) ([]*dep.CatalogService, error) {
	return func(s ...string) ([]*dep.CatalogService, error) {
		result := make([]*dep.CatalogService, 0)

		d, err := dep.ParseCatalogServices(s...)
		if err != nil {
			return nil, err
		}

		addDependency(used, d)

		if services, ok := brain.catalogServices[d.HashCode()]; ok {
			return services, nil
		}

		addDependency(missing, d)

		return result, nil
	}
}

// treeFunc returns or accumulates keyPrefix dependencies.
func treeFunc(brain *Brain,
	used, missing map[string]dep.Dependency) func(string) ([]*dep.KeyPair, error) {
	return func(s string) ([]*dep.KeyPair, error) {
		result := make([]*dep.KeyPair, 0)

		if len(s) == 0 {
			return result, nil
		}

		d, err := dep.ParseStoreKeyPrefix(s)
		if err != nil {
			return result, err
		}

		addDependency(used, d)

		// Only return non-empty top-level keys
		if pairs, ok := brain.storeKeyPrefixes[d.HashCode()]; ok {
			for _, pair := range pairs {
				parts := strings.Split(pair.Key, "/")
				if parts[len(parts)-1] != "" {
					result = append(result, pair)
				}
			}

			return result, nil
		}

		addDependency(missing, d)

		return result, nil
	}

}

// byTag is a template func that takes the provided services and
// produces a map based on Service tags.
//
// The map key is a string representing the service tag. The map value is a
// slice of Services which have the tag assigned.
func byTag(in []*dep.HealthService) (map[string][]*dep.HealthService, error) {
	m := make(map[string][]*dep.HealthService)
	for _, s := range in {
		for _, t := range s.Tags {
			m[t] = append(m[t], s)
		}
	}
	return m, nil
}

// returns the value of the environment variable set
func env(s string) (string, error) {
	return os.Getenv(s), nil
}

// parseJSON returns a structure for valid JSON
func parseJSON(s string) (interface{}, error) {
	if s == "" {
		return make([]interface{}, 0), nil
	}

	var data interface{}
	if err := json.Unmarshal([]byte(s), &data); err != nil {
		return nil, err
	}
	return data, nil
}

// toLower converts the given string (usually by a pipe) to lowercase.
func toLower(s string) (string, error) {
	return strings.ToLower(s), nil
}

// toTitle converts the given string (usually by a pipe) to titlecase.
func toTitle(s string) (string, error) {
	return strings.Title(s), nil
}

// toUpper converts the given string (usually by a pipe) to uppercase.
func toUpper(s string) (string, error) {
	return strings.ToUpper(s), nil
}

// replaceAll replaces all occurrences of a value in a string with the given
// replacement value.
func replaceAll(f, t, s string) (string, error) {
	return strings.Replace(s, f, t, -1), nil
}

// regexReplaceAll replaces all occurrences of a regular expression with
// the given replacement value.
func regexReplaceAll(re, pl, s string) (string, error) {
	compiled, err := regexp.Compile(re)
	if err != nil {
		return "", err
	}
	return compiled.ReplaceAllString(s, pl), nil
}

// addDependency adds the given Dependency to the map.
func addDependency(m map[string]dep.Dependency, d dep.Dependency) {
	if _, ok := m[d.HashCode()]; !ok {
		m[d.HashCode()] = d
	}
}
<|endoftext|>


<commit_before>package agent

import (
	"crypto/sha1"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/buildkite/agent/api"
	"github.com/buildkite/agent/logger"
	"github.com/buildkite/agent/pool"
	"github.com/buildkite/agent/retry"
	zglob "github.com/mattn/go-zglob"
)

const (
	ArtifactPathDelimiter = ";"
)

type ArtifactUploader struct {
	// The APIClient that will be used when uploading jobs
	APIClient *api.Client

	// The ID of the Job
	JobID string

	// The path of the uploads
	Paths string

	// Where we'll be uploading artifacts
	Destination string
}

func (a *ArtifactUploader) Upload() error {
	// Create artifact structs for all the files we need to upload
	artifacts, err := a.Collect()
	if err != nil {
		return err
	}

	if len(artifacts) == 0 {
		logger.Info("No files matched paths: %s", a.Paths)
	} else {
		logger.Info("Found %d files that match \"%s\"", len(artifacts), a.Paths)

		err := a.upload(artifacts)
		if err != nil {
			return err
		}
	}

	return nil
}

func isDir(path string) bool {
	fi, err := os.Stat(path)
	if err != nil {
		return false
	}
	return fi.IsDir()
}

func (a *ArtifactUploader) Collect() (artifacts []*api.Artifact, err error) {
	wd, err := os.Getwd()
	if err != nil {
		return nil, err
	}

	for _, globPath := range strings.Split(a.Paths, ArtifactPathDelimiter) {
		globPath = strings.TrimSpace(globPath)
		if globPath == "" {
			continue
		}

		logger.Debug("Searching for %s", globPath)

		// Resolve the globs (with * and ** in them)
		files, err := zglob.Glob(globPath)
		if err == os.ErrNotExist {
			logger.Info("No files matched glob: %s", globPath)
			continue
		} else if err != nil {
			return nil, err
		}

		// Process each glob match into an api.Artifact
		for _, file := range files {
			absolutePath, err := filepath.Abs(file)
			if err != nil {
				return nil, err
			}

			// Ignore directories, we only want files
			if isDir(absolutePath) {
				logger.Debug("Skipping directory %s", file)
				continue
			}

			// If a glob is absolute, we need to make it relative to the root so that
			// it can be combined with the download destination to make a valid path.
			// This is possibly weird and crazy, this logic dates back to
			// https://github.com/buildkite/agent/commit/8ae46d975aa60d1ae0e2cc0bff7a43d3bf960935
			// from 2014, so I'm replicating it here to avoid breaking things
			if filepath.IsAbs(globPath) {
				if runtime.GOOS == "windows" {
					wd = filepath.VolumeName(absolutePath) + "/"
				} else {
					wd = "/"
				}
			}

			path, err := filepath.Rel(wd, absolutePath)
			if err != nil {
				return nil, err
			}

			// Build an artifact object using the paths we have.
			artifact, err := a.build(path, absolutePath, globPath)
			if err != nil {
				return nil, err
			}

			artifacts = append(artifacts, artifact)
		}
	}

	return artifacts, nil
}

func (a *ArtifactUploader) build(path string, absolutePath string, globPath string) (*api.Artifact, error) {
	// Temporarily open the file to get it's size
	file, err := os.Open(absolutePath)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	// Grab it's file info (which includes it's file size)
	fileInfo, err := file.Stat()
	if err != nil {
		return nil, err
	}

	// Generate a sha1 checksum for the file
	hash := sha1.New()
	io.Copy(hash, file)
	checksum := fmt.Sprintf("%x", hash.Sum(nil))

	// Create our new artifact data structure
	artifact := &api.Artifact{
		Path:         path,
		AbsolutePath: absolutePath,
		GlobPath:     globPath,
		FileSize:     fileInfo.Size(),
		Sha1Sum:      checksum,
	}

	return artifact, nil
}

func (a *ArtifactUploader) upload(artifacts []*api.Artifact) error {
	var uploader Uploader

	// Determine what uploader to use
	if a.Destination != "" {
		if strings.HasPrefix(a.Destination, "s3://") {
			uploader = new(S3Uploader)
		} else if strings.HasPrefix(a.Destination, "gs://") {
			uploader = new(GSUploader)
		} else {
			return errors.New("Unknown upload destination: " + a.Destination)
		}
	} else {
		uploader = new(FormUploader)
	}

	// Setup the uploader
	err := uploader.Setup(a.Destination, a.APIClient.DebugHTTP)
	if err != nil {
		return err
	}

	// Set the URL's of the artifacts based on the uploader
	for _, artifact := range artifacts {
		artifact.URL = uploader.URL(artifact)
	}

	// Create the artifacts on Buildkite
	batchCreator := ArtifactBatchCreator{
		APIClient:         a.APIClient,
		JobID:             a.JobID,
		Artifacts:         artifacts,
		UploadDestination: a.Destination,
	}
	artifacts, err = batchCreator.Create()
	if err != nil {
		return err
	}

	// Prepare a concurrency pool to upload the artifacts
	p := pool.New(pool.MaxConcurrencyLimit)
	errors := []error{}
	var errorsMutex sync.Mutex

	// Create a wait group so we can make sure the uploader waits for all
	// the artifact states to upload before finishing
	var stateUploaderWaitGroup sync.WaitGroup
	stateUploaderWaitGroup.Add(1)

	// A map to keep track of artifact states and how many we've uploaded
	artifactStates := make(map[string]string)
	artifactStatesUploaded := 0
	var artifactStatesMutex sync.Mutex

	// Spin up a gourtine that'll uploading artifact statuses every few
	// seconds in batches
	go func() {
		for artifactStatesUploaded < len(artifacts) {
			statesToUpload := make(map[string]string)

			// Grab all the states we need to upload, and remove
			// them from the tracking map
			//
			// Since we mutate the artifactStates variable in
			// multiple routines, we need to lock it to make sure
			// nothing else is changing it at the same time.
			artifactStatesMutex.Lock()
			for id, state := range artifactStates {
				statesToUpload[id] = state
				delete(artifactStates, id)
			}
			artifactStatesMutex.Unlock()

			if len(statesToUpload) > 0 {
				artifactStatesUploaded += len(statesToUpload)
				for id, state := range statesToUpload {
					logger.Debug("Artifact `%s` has state `%s`", id, state)
				}

				// Update the states of the artifacts in bulk.
				err = retry.Do(func(s *retry.Stats) error {
					_, err = a.APIClient.Artifacts.Update(a.JobID, statesToUpload)
					if err != nil {
						logger.Warn("%s (%s)", err, s)
					}

					return err
				}, &retry.Config{Maximum: 10, Interval: 5 * time.Second})

				if err != nil {
					logger.Error("Error uploading artifact states: %s", err)

					// Track the error that was raised. We need to
					// aquire a lock since we mutate the errors
					// slice in mutliple routines.
					errorsMutex.Lock()
					errors = append(errors, err)
					errorsMutex.Unlock()
				}

				logger.Debug("Uploaded %d artfact states (%d/%d)", len(statesToUpload), artifactStatesUploaded, len(artifacts))
			}

			// Check again for states to upload in a few seconds
			time.Sleep(1 * time.Second)
		}

		stateUploaderWaitGroup.Done()
	}()

	for _, artifact := range artifacts {
		// Create new instance of the artifact for the goroutine
		// See: http://golang.org/doc/effective_go.html#channels
		artifact := artifact

		p.Spawn(func() {
			// Show a nice message that we're starting to upload the file
			logger.Info("Uploading artifact %s %s (%d bytes)", artifact.ID, artifact.Path, artifact.FileSize)

			// Upload the artifact and then set the state depending
			// on whether or not it passed. We'll retry the upload
			// a couple of times before giving up.
			err = retry.Do(func(s *retry.Stats) error {
				err := uploader.Upload(artifact)
				if err != nil {
					logger.Warn("%s (%s)", err, s)
				}

				return err
			}, &retry.Config{Maximum: 10, Interval: 5 * time.Second})

			var state string

			// Did the upload eventually fail?
			if err != nil {
				logger.Error("Error uploading artifact \"%s\": %s", artifact.Path, err)

				// Track the error that was raised. We need to
				// aquire a lock since we mutate the errors
				// slice in mutliple routines.
				errorsMutex.Lock()
				errors = append(errors, err)
				errorsMutex.Unlock()

				state = "error"
			} else {
				state = "finished"
			}

			// Since we mutate the artifactStates variable in
			// multiple routines, we need to lock it to make sure
			// nothing else is changing it at the same time.
			artifactStatesMutex.Lock()
			artifactStates[artifact.ID] = state
			artifactStatesMutex.Unlock()
		})
	}

	// Wait for the pool to finish
	p.Wait()

	// Wait for the statuses to finish uploading
	stateUploaderWaitGroup.Wait()

	if len(errors) > 0 {
		logger.Fatal("There were errors with uploading some of the artifacts")
	}

	return nil
}
<commit_msg>Tweak error message<commit_after>package agent

import (
	"crypto/sha1"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/buildkite/agent/api"
	"github.com/buildkite/agent/logger"
	"github.com/buildkite/agent/pool"
	"github.com/buildkite/agent/retry"
	zglob "github.com/mattn/go-zglob"
)

const (
	ArtifactPathDelimiter = ";"
)

type ArtifactUploader struct {
	// The APIClient that will be used when uploading jobs
	APIClient *api.Client

	// The ID of the Job
	JobID string

	// The path of the uploads
	Paths string

	// Where we'll be uploading artifacts
	Destination string
}

func (a *ArtifactUploader) Upload() error {
	// Create artifact structs for all the files we need to upload
	artifacts, err := a.Collect()
	if err != nil {
		return err
	}

	if len(artifacts) == 0 {
		logger.Info("No files matched paths: %s", a.Paths)
	} else {
		logger.Info("Found %d files that match \"%s\"", len(artifacts), a.Paths)

		err := a.upload(artifacts)
		if err != nil {
			return err
		}
	}

	return nil
}

func isDir(path string) bool {
	fi, err := os.Stat(path)
	if err != nil {
		return false
	}
	return fi.IsDir()
}

func (a *ArtifactUploader) Collect() (artifacts []*api.Artifact, err error) {
	wd, err := os.Getwd()
	if err != nil {
		return nil, err
	}

	for _, globPath := range strings.Split(a.Paths, ArtifactPathDelimiter) {
		globPath = strings.TrimSpace(globPath)
		if globPath == "" {
			continue
		}

		logger.Debug("Searching for %s", globPath)

		// Resolve the globs (with * and ** in them), if it's a non-globbed path and doesn't exists
		// then we will get the ErrNotExist that is handled below
		files, err := zglob.Glob(globPath)
		if err == os.ErrNotExist {
			logger.Info("File not found: %s", globPath)
			continue
		} else if err != nil {
			return nil, err
		}

		// Process each glob match into an api.Artifact
		for _, file := range files {
			absolutePath, err := filepath.Abs(file)
			if err != nil {
				return nil, err
			}

			// Ignore directories, we only want files
			if isDir(absolutePath) {
				logger.Debug("Skipping directory %s", file)
				continue
			}

			// If a glob is absolute, we need to make it relative to the root so that
			// it can be combined with the download destination to make a valid path.
			// This is possibly weird and crazy, this logic dates back to
			// https://github.com/buildkite/agent/commit/8ae46d975aa60d1ae0e2cc0bff7a43d3bf960935
			// from 2014, so I'm replicating it here to avoid breaking things
			if filepath.IsAbs(globPath) {
				if runtime.GOOS == "windows" {
					wd = filepath.VolumeName(absolutePath) + "/"
				} else {
					wd = "/"
				}
			}

			path, err := filepath.Rel(wd, absolutePath)
			if err != nil {
				return nil, err
			}

			// Build an artifact object using the paths we have.
			artifact, err := a.build(path, absolutePath, globPath)
			if err != nil {
				return nil, err
			}

			artifacts = append(artifacts, artifact)
		}
	}

	return artifacts, nil
}

func (a *ArtifactUploader) build(path string, absolutePath string, globPath string) (*api.Artifact, error) {
	// Temporarily open the file to get it's size
	file, err := os.Open(absolutePath)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	// Grab it's file info (which includes it's file size)
	fileInfo, err := file.Stat()
	if err != nil {
		return nil, err
	}

	// Generate a sha1 checksum for the file
	hash := sha1.New()
	io.Copy(hash, file)
	checksum := fmt.Sprintf("%x", hash.Sum(nil))

	// Create our new artifact data structure
	artifact := &api.Artifact{
		Path:         path,
		AbsolutePath: absolutePath,
		GlobPath:     globPath,
		FileSize:     fileInfo.Size(),
		Sha1Sum:      checksum,
	}

	return artifact, nil
}

func (a *ArtifactUploader) upload(artifacts []*api.Artifact) error {
	var uploader Uploader

	// Determine what uploader to use
	if a.Destination != "" {
		if strings.HasPrefix(a.Destination, "s3://") {
			uploader = new(S3Uploader)
		} else if strings.HasPrefix(a.Destination, "gs://") {
			uploader = new(GSUploader)
		} else {
			return errors.New("Unknown upload destination: " + a.Destination)
		}
	} else {
		uploader = new(FormUploader)
	}

	// Setup the uploader
	err := uploader.Setup(a.Destination, a.APIClient.DebugHTTP)
	if err != nil {
		return err
	}

	// Set the URL's of the artifacts based on the uploader
	for _, artifact := range artifacts {
		artifact.URL = uploader.URL(artifact)
	}

	// Create the artifacts on Buildkite
	batchCreator := ArtifactBatchCreator{
		APIClient:         a.APIClient,
		JobID:             a.JobID,
		Artifacts:         artifacts,
		UploadDestination: a.Destination,
	}
	artifacts, err = batchCreator.Create()
	if err != nil {
		return err
	}

	// Prepare a concurrency pool to upload the artifacts
	p := pool.New(pool.MaxConcurrencyLimit)
	errors := []error{}
	var errorsMutex sync.Mutex

	// Create a wait group so we can make sure the uploader waits for all
	// the artifact states to upload before finishing
	var stateUploaderWaitGroup sync.WaitGroup
	stateUploaderWaitGroup.Add(1)

	// A map to keep track of artifact states and how many we've uploaded
	artifactStates := make(map[string]string)
	artifactStatesUploaded := 0
	var artifactStatesMutex sync.Mutex

	// Spin up a gourtine that'll uploading artifact statuses every few
	// seconds in batches
	go func() {
		for artifactStatesUploaded < len(artifacts) {
			statesToUpload := make(map[string]string)

			// Grab all the states we need to upload, and remove
			// them from the tracking map
			//
			// Since we mutate the artifactStates variable in
			// multiple routines, we need to lock it to make sure
			// nothing else is changing it at the same time.
			artifactStatesMutex.Lock()
			for id, state := range artifactStates {
				statesToUpload[id] = state
				delete(artifactStates, id)
			}
			artifactStatesMutex.Unlock()

			if len(statesToUpload) > 0 {
				artifactStatesUploaded += len(statesToUpload)
				for id, state := range statesToUpload {
					logger.Debug("Artifact `%s` has state `%s`", id, state)
				}

				// Update the states of the artifacts in bulk.
				err = retry.Do(func(s *retry.Stats) error {
					_, err = a.APIClient.Artifacts.Update(a.JobID, statesToUpload)
					if err != nil {
						logger.Warn("%s (%s)", err, s)
					}

					return err
				}, &retry.Config{Maximum: 10, Interval: 5 * time.Second})

				if err != nil {
					logger.Error("Error uploading artifact states: %s", err)

					// Track the error that was raised. We need to
					// aquire a lock since we mutate the errors
					// slice in mutliple routines.
					errorsMutex.Lock()
					errors = append(errors, err)
					errorsMutex.Unlock()
				}

				logger.Debug("Uploaded %d artfact states (%d/%d)", len(statesToUpload), artifactStatesUploaded, len(artifacts))
			}

			// Check again for states to upload in a few seconds
			time.Sleep(1 * time.Second)
		}

		stateUploaderWaitGroup.Done()
	}()

	for _, artifact := range artifacts {
		// Create new instance of the artifact for the goroutine
		// See: http://golang.org/doc/effective_go.html#channels
		artifact := artifact

		p.Spawn(func() {
			// Show a nice message that we're starting to upload the file
			logger.Info("Uploading artifact %s %s (%d bytes)", artifact.ID, artifact.Path, artifact.FileSize)

			// Upload the artifact and then set the state depending
			// on whether or not it passed. We'll retry the upload
			// a couple of times before giving up.
			err = retry.Do(func(s *retry.Stats) error {
				err := uploader.Upload(artifact)
				if err != nil {
					logger.Warn("%s (%s)", err, s)
				}

				return err
			}, &retry.Config{Maximum: 10, Interval: 5 * time.Second})

			var state string

			// Did the upload eventually fail?
			if err != nil {
				logger.Error("Error uploading artifact \"%s\": %s", artifact.Path, err)

				// Track the error that was raised. We need to
				// aquire a lock since we mutate the errors
				// slice in mutliple routines.
				errorsMutex.Lock()
				errors = append(errors, err)
				errorsMutex.Unlock()

				state = "error"
			} else {
				state = "finished"
			}

			// Since we mutate the artifactStates variable in
			// multiple routines, we need to lock it to make sure
			// nothing else is changing it at the same time.
			artifactStatesMutex.Lock()
			artifactStates[artifact.ID] = state
			artifactStatesMutex.Unlock()
		})
	}

	// Wait for the pool to finish
	p.Wait()

	// Wait for the statuses to finish uploading
	stateUploaderWaitGroup.Wait()

	if len(errors) > 0 {
		logger.Fatal("There were errors with uploading some of the artifacts")
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2019 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package sqlbuilder

// CreateViewStmt 创建视图的语句
type CreateViewStmt struct {
	*ddlStmt

	selectStmt  *SelectStmt
	ViewName    string
	Columns     []string
	IsTemporary bool
	IsReplace   bool
}

// CreateViewStmtHooker CreateViewStmt.DDLSQL 的钩子函数
type CreateViewStmtHooker interface {
	CreateViewStmtHook(*CreateViewStmt) ([]string, error)
}

// CreateView 创建视图
func CreateView(e Engine, d Dialect) *CreateViewStmt {
	stmt := &CreateViewStmt{}
	stmt.ddlStmt = newDDLStmt(e, d, stmt)

	return stmt
}

// View 将当前查询语句转换为视图
func (stmt *SelectStmt) View(name string) *CreateViewStmt {
	return CreateView(stmt.Engine(), stmt.Dialect()).
		From(stmt)
}

// Reset 重置对象
func (stmt *CreateViewStmt) Reset() *CreateViewStmt {
	stmt.ViewName = ""
	stmt.selectStmt = nil
	stmt.Columns = stmt.Columns[:0]
	stmt.IsTemporary = false
	stmt.IsReplace = false

	return stmt
}

// Column 指定视图的列，如果未指定，则会直接采用 Select 中的列信息
func (stmt *CreateViewStmt) Column(col ...string) *CreateViewStmt {
	if stmt.Columns == nil {
		stmt.Columns = col
		return stmt
	}

	stmt.Columns = append(stmt.Columns, col...)
	return stmt
}

// Name 指定视图名称
func (stmt *CreateViewStmt) Name(name string) *CreateViewStmt {
	stmt.ViewName = name
	return stmt
}

// Temporary 临时视图
func (stmt *CreateViewStmt) Temporary() *CreateViewStmt {
	stmt.IsTemporary = true
	return stmt
}

// Replace 如果已经存在，则更新视图内容
func (stmt *CreateViewStmt) Replace() *CreateViewStmt {
	stmt.IsReplace = true
	return stmt
}

// SelectQuery 获取 SELECT 的查询内容内容
func (stmt *CreateViewStmt) SelectQuery() (string, error) {
	if stmt.selectStmt == nil {
		return "", ErrValueIsEmpty
	}

	return fillArgs(stmt.selectStmt)
}

// From 指定 Select 语句
//
// 在调用 DDLSQL 之前，可以继续修改 sel 的内容。
func (stmt *CreateViewStmt) From(sel *SelectStmt) *CreateViewStmt {
	stmt.selectStmt = sel
	return stmt
}

// DDLSQL 返回创建视图的 SQL 语句
func (stmt *CreateViewStmt) DDLSQL() ([]string, error) {
	if stmt.ViewName == "" {
		return nil, ErrTableIsEmpty
	}

	if hook, ok := stmt.Dialect().(CreateViewStmtHooker); ok {
		return hook.CreateViewStmtHook(stmt)
	}

	selectQuery, err := stmt.SelectQuery()
	if err != nil {
		return nil, err
	}

	builder := New("CREATE ")

	if stmt.IsReplace {
		builder.WriteString(" OR REPLACE ")
	}

	if stmt.IsTemporary {
		builder.WriteString(" TEMPORARY ")
	}

	builder.WriteString(" VIEW ").Quote(stmt.ViewName, stmt.l, stmt.r)

	if len(stmt.Columns) > 0 {
		builder.WriteBytes('(')
		for _, col := range stmt.Columns {
			builder.Quote(col, stmt.l, stmt.r).
				WriteBytes(',')
		}
		builder.TruncateLast(1).WriteBytes(')')
	}

	builder.WriteString(selectQuery)

	return []string{builder.String()}, nil
}
<commit_msg>[sqlbuilder] 添加 DropView<commit_after>// Copyright 2019 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package sqlbuilder

import "github.com/issue9/orm/v2/core"

// CreateViewStmt 创建视图的语句
type CreateViewStmt struct {
	*ddlStmt

	selectStmt  *SelectStmt
	ViewName    string
	Columns     []string
	IsTemporary bool
	IsReplace   bool
}

// CreateViewStmtHooker CreateViewStmt.DDLSQL 的钩子函数
type CreateViewStmtHooker interface {
	CreateViewStmtHook(*CreateViewStmt) ([]string, error)
}

// CreateView 创建视图
func CreateView(e core.Engine) *CreateViewStmt {
	stmt := &CreateViewStmt{}
	stmt.ddlStmt = newDDLStmt(e, stmt)

	return stmt
}

// View 将当前查询语句转换为视图
func (stmt *SelectStmt) View(name string) *CreateViewStmt {
	return CreateView(stmt.Engine()).
		From(stmt)
}

// Reset 重置对象
func (stmt *CreateViewStmt) Reset() *CreateViewStmt {
	stmt.ViewName = ""
	stmt.selectStmt = nil
	stmt.Columns = stmt.Columns[:0]
	stmt.IsTemporary = false
	stmt.IsReplace = false

	return stmt
}

// Column 指定视图的列，如果未指定，则会直接采用 Select 中的列信息
func (stmt *CreateViewStmt) Column(col ...string) *CreateViewStmt {
	if stmt.Columns == nil {
		stmt.Columns = col
		return stmt
	}

	stmt.Columns = append(stmt.Columns, col...)
	return stmt
}

// Name 指定视图名称
func (stmt *CreateViewStmt) Name(name string) *CreateViewStmt {
	stmt.ViewName = name
	return stmt
}

// Temporary 临时视图
func (stmt *CreateViewStmt) Temporary() *CreateViewStmt {
	stmt.IsTemporary = true
	return stmt
}

// Replace 如果已经存在，则更新视图内容
func (stmt *CreateViewStmt) Replace() *CreateViewStmt {
	stmt.IsReplace = true
	return stmt
}

// SelectQuery 获取 SELECT 的查询内容内容
func (stmt *CreateViewStmt) SelectQuery() (string, error) {
	if stmt.selectStmt == nil {
		return "", ErrValueIsEmpty
	}

	return fillArgs(stmt.selectStmt)
}

// From 指定 Select 语句
//
// 在调用 DDLSQL 之前，可以继续修改 sel 的内容。
func (stmt *CreateViewStmt) From(sel *SelectStmt) *CreateViewStmt {
	stmt.selectStmt = sel
	return stmt
}

// DDLSQL 返回创建视图的 SQL 语句
func (stmt *CreateViewStmt) DDLSQL() ([]string, error) {
	if stmt.ViewName == "" {
		return nil, ErrTableIsEmpty
	}

	if hook, ok := stmt.Dialect().(CreateViewStmtHooker); ok {
		return hook.CreateViewStmtHook(stmt)
	}

	selectQuery, err := stmt.SelectQuery()
	if err != nil {
		return nil, err
	}

	builder := core.NewBuilder("CREATE ")

	if stmt.IsReplace {
		builder.WriteString(" OR REPLACE ")
	}

	if stmt.IsTemporary {
		builder.WriteString(" TEMPORARY ")
	}

	builder.WriteString(" VIEW ").Quote(stmt.ViewName, stmt.l, stmt.r)

	if len(stmt.Columns) > 0 {
		builder.WriteBytes('(')
		for _, col := range stmt.Columns {
			builder.Quote(col, stmt.l, stmt.r).
				WriteBytes(',')
		}
		builder.TruncateLast(1).WriteBytes(')')
	}

	builder.WriteString(selectQuery)

	return []string{builder.String()}, nil
}

// DropViewStmt 删除视图
type DropViewStmt struct {
	*ddlStmt
	name string
}

// DropView 创建视图
func DropView(e core.Engine) *DropViewStmt {
	stmt := &DropViewStmt{}
	stmt.ddlStmt = newDDLStmt(e, stmt)

	return stmt
}

// Name 指定需要删除的视图名称
func (stmt *DropViewStmt) Name(name string) *DropViewStmt {
	stmt.name = name
	return stmt
}

// DDLSQL 返回删除视图的 SQL 语句
func (stmt *DropViewStmt) DDLSQL() ([]string, error) {
	if len(stmt.name) == 0 {
		return nil, ErrTableIsEmpty
	}

	builder := core.NewBuilder("DROP VIEW IF EXISTS ").QuoteKey(stmt.name)

	return []string{builder.String()}, nil
}

// Reset 重置对象
func (stmt *DropViewStmt) Reset() *DropViewStmt {
	stmt.name = ""

	return stmt
}
<|endoftext|>


<commit_before>package ipam

import (
	"github.com/giantswarm/microerror"
	"github.com/giantswarm/micrologger"

	"github.com/giantswarm/azure-operator/v4/pkg/locker"
)

const (
	Name                                 = "ipam"
	SubnetRange         NetworkRangeType = "subnet"
	VirtualNetworkRange NetworkRangeType = "virtual network"
)

type NetworkRangeType string

type Config struct {
	Checker            Checker
	Collector          Collector
	Locker             locker.Interface
	Logger             micrologger.Logger
	NetworkRangeGetter NetworkRangeGetter
	NetworkRangeType   NetworkRangeType
	Persister          Persister
}

// Resource finds free IP ranges:
// - AzureConfig: within an installation range to create new virtual network for the tenant cluster.
// - CAPI/CAPZ: within a virtual network to create new subnets.
type Resource struct {
	checker            Checker
	collector          Collector
	locker             locker.Interface
	logger             micrologger.Logger
	networkRangeGetter NetworkRangeGetter
	networkRangeType   NetworkRangeType
	persister          Persister
}

func New(config Config) (*Resource, error) {
	if config.Checker == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.Checker must not be empty", config)
	}
	if config.Collector == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.Collector must not be empty", config)
	}
	if config.Locker == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.Locker must not be empty", config)
	}
	if config.Logger == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.Logger must not be empty", config)
	}
	if config.NetworkRangeGetter == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.NetworkRangeGetter must not be empty", config)
	}
	if config.NetworkRangeType == "" {
		return nil, microerror.Maskf(invalidConfigError, "%T.NetworkRangeType must not be empty", config)
	}
	if config.Persister == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.Persister must not be empty", config)
	}

	r := &Resource{
		checker:            config.Checker,
		collector:          config.Collector,
		locker:             config.Locker,
		logger:             config.Logger,
		networkRangeGetter: config.NetworkRangeGetter,
		persister:          config.Persister,
	}

	return r, nil
}

func (r *Resource) Name() string {
	return Name
}
<commit_msg>Set missing field (#1009)<commit_after>package ipam

import (
	"github.com/giantswarm/microerror"
	"github.com/giantswarm/micrologger"

	"github.com/giantswarm/azure-operator/v4/pkg/locker"
)

const (
	Name                                 = "ipam"
	SubnetRange         NetworkRangeType = "subnet"
	VirtualNetworkRange NetworkRangeType = "virtual network"
)

type NetworkRangeType string

type Config struct {
	Checker            Checker
	Collector          Collector
	Locker             locker.Interface
	Logger             micrologger.Logger
	NetworkRangeGetter NetworkRangeGetter
	NetworkRangeType   NetworkRangeType
	Persister          Persister
}

// Resource finds free IP ranges:
// - AzureConfig: within an installation range to create new virtual network for the tenant cluster.
// - CAPI/CAPZ: within a virtual network to create new subnets.
type Resource struct {
	checker            Checker
	collector          Collector
	locker             locker.Interface
	logger             micrologger.Logger
	networkRangeGetter NetworkRangeGetter
	networkRangeType   NetworkRangeType
	persister          Persister
}

func New(config Config) (*Resource, error) {
	if config.Checker == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.Checker must not be empty", config)
	}
	if config.Collector == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.Collector must not be empty", config)
	}
	if config.Locker == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.Locker must not be empty", config)
	}
	if config.Logger == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.Logger must not be empty", config)
	}
	if config.NetworkRangeGetter == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.NetworkRangeGetter must not be empty", config)
	}
	if config.NetworkRangeType == "" {
		return nil, microerror.Maskf(invalidConfigError, "%T.NetworkRangeType must not be empty", config)
	}
	if config.Persister == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.Persister must not be empty", config)
	}

	r := &Resource{
		checker:            config.Checker,
		collector:          config.Collector,
		locker:             config.Locker,
		logger:             config.Logger,
		networkRangeGetter: config.NetworkRangeGetter,
		networkRangeType:   config.NetworkRangeType,
		persister:          config.Persister,
	}

	return r, nil
}

func (r *Resource) Name() string {
	return Name
}
<|endoftext|>


<commit_before>package charm

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"launchpad.net/juju-core/environs/config"
	"launchpad.net/juju-core/log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
)

// InfoResponse is sent by the charm store in response to charm-info requests.
type InfoResponse struct {
	Revision int      `json:"revision"` // Zero is valid. Can't omitempty.
	Sha256   string   `json:"sha256,omitempty"`
	Digest   string   `json:"digest,omitempty"`
	Errors   []string `json:"errors,omitempty"`
	Warnings []string `json:"warnings,omitempty"`
}

// EventResponse is sent by the charm store in response to charm-event requests.
type EventResponse struct {
	Kind     string   `json:"kind"`
	Revision int      `json:"revision"` // Zero is valid. Can't omitempty.
	Digest   string   `json:"digest,omitempty"`
	Errors   []string `json:"errors,omitempty"`
	Warnings []string `json:"warnings,omitempty"`
	Time     string   `json:"time,omitempty"`
}

// Repository respresents a collection of charms.
type Repository interface {
	Get(curl *URL) (Charm, error)
	Latest(curl *URL) (int, error)
}

// NotFoundError represents an error indicating that the requested data wasn't found.
type NotFoundError struct {
	msg string
}

func (e *NotFoundError) Error() string {
	return e.msg
}

// CharmStore is a Repository that provides access to the public juju charm store.
type CharmStore struct {
	baseURL   string
}

//var Store = &CharmStore{"https://store.juju.ubuntu.com"}
var Store = &CharmStore{"http://localhost:8080"}

// Info returns details for a charm in the charm store.
func (s *CharmStore) Info(curl *URL) (*InfoResponse, error) {
	key := curl.String()
	resp, err := http.Get(s.baseURL + "/charm-info?charms=" + url.QueryEscape(key))
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	infos := make(map[string]*InfoResponse)
	if err = json.Unmarshal(body, &infos); err != nil {
		return nil, err
	}
	info, found := infos[key]
	if !found {
		return nil, fmt.Errorf("charm: charm store returned response without charm %q", key)
	}
	if len(info.Errors) == 1 && info.Errors[0] == "entry not found" {
		return nil, &NotFoundError{fmt.Sprintf("charm not found: %s", curl)}
	}
	return info, nil
}

// Event returns details for a charm event in the charm store.
//
// If digest is empty, the latest event is returned.
func (s *CharmStore) Event(curl *URL, digest string) (*EventResponse, error) {
	key := curl.String()
	query := key
	if digest != "" {
		query += "@" + digest
	}
	resp, err := http.Get(s.baseURL + "/charm-event?charms=" + url.QueryEscape(query))
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	events := make(map[string]*EventResponse)
	if err = json.Unmarshal(body, &events); err != nil {
		return nil, err
	}
	event, found := events[key]
	if !found {
		return nil, fmt.Errorf("charm: charm store returned response without charm %q", key)
	}
	if len(event.Errors) == 1 && event.Errors[0] == "entry not found" {
		if digest == "" {
			return nil, &NotFoundError{fmt.Sprintf("charm event not found for %q", curl)}
		} else {
			return nil, &NotFoundError{fmt.Sprintf("charm event not found for %q with digest %q", curl, digest)}
		}
	}
	return event, nil
}

// revision returns the revision and SHA256 digest of the charm referenced by curl.
func (s *CharmStore) revision(curl *URL) (revision int, digest string, err error) {
	info, err := s.Info(curl)
	if err != nil {
		return 0, "", err
	}
	for _, w := range info.Warnings {
		log.Warningf("charm: charm store reports for %q: %s", curl, w)
	}
	if info.Errors != nil {
		return 0, "", fmt.Errorf("charm info errors for %q: %s", curl, strings.Join(info.Errors, "; "))
	}
	return info.Revision, info.Sha256, nil
}

// Latest returns the latest revision of the charm referenced by curl, regardless
// of the revision set on curl itself.
func (s *CharmStore) Latest(curl *URL) (int, error) {
	rev, _, err := s.revision(curl.WithRevision(-1))
	return rev, err
}

// verify returns an error unless a file exists at path with a hex-encoded
// SHA256 matching digest.
func verify(path, digest string) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()
	h := sha256.New()
	if _, err := io.Copy(h, f); err != nil {
		return err
	}
	if hex.EncodeToString(h.Sum(nil)) != digest {
		return fmt.Errorf("bad SHA256 of %q", path)
	}
	return nil
}

// Get returns the charm referenced by curl.
func (s *CharmStore) Get(curl *URL) (Charm, error) {
	cachePath := config.JujuHomePath("cache")
	if err := os.MkdirAll(cachePath, 0755); err != nil {
		return nil, err
	}
	rev, digest, err := s.revision(curl)
	if err != nil {
		return nil, err
	}
	if curl.Revision == -1 {
		curl = curl.WithRevision(rev)
	} else if curl.Revision != rev {
		return nil, fmt.Errorf("charm: store returned charm with wrong revision for %q", curl.String())
	}
	path := filepath.Join(cachePath, Quote(curl.String())+".charm")
	if verify(path, digest) != nil {
		resp, err := http.Get(s.baseURL + "/charm/" + url.QueryEscape(curl.Path()))
		if err != nil {
			return nil, err
		}
		defer resp.Body.Close()
		f, err := ioutil.TempFile(cachePath, "charm-download")
		if err != nil {
			return nil, err
		}
		dlPath := f.Name()
		_, err = io.Copy(f, resp.Body)
		if cerr := f.Close(); err == nil {
			err = cerr
		}
		if err != nil {
			os.Remove(dlPath)
			return nil, err
		}
		if err := os.Rename(dlPath, path); err != nil {
			return nil, err
		}
	}
	if err := verify(path, digest); err != nil {
		return nil, err
	}
	return ReadBundle(path)
}

// LocalRepository represents a local directory containing subdirectories
// named after an Ubuntu series, each of which contains charms targeted for
// that series. For example:
//
//   /path/to/repository/oneiric/mongodb/
//   /path/to/repository/precise/mongodb.charm
//   /path/to/repository/precise/wordpress/
type LocalRepository struct {
	Path string
}

// Latest returns the latest revision of the charm referenced by curl, regardless
// of the revision set on curl itself.
func (r *LocalRepository) Latest(curl *URL) (int, error) {
	ch, err := r.Get(curl.WithRevision(-1))
	if err != nil {
		return 0, err
	}
	return ch.Revision(), nil
}

func repoNotFound(path string) error {
	return &NotFoundError{fmt.Sprintf("no repository found at %q", path)}
}

func charmNotFound(curl *URL, repoPath string) error {
	return &NotFoundError{fmt.Sprintf("no charms found matching %q in %s", curl, repoPath)}
}

func mightBeCharm(info os.FileInfo) bool {
	if info.IsDir() {
		return !strings.HasPrefix(info.Name(), ".")
	}
	return strings.HasSuffix(info.Name(), ".charm")
}

// Get returns a charm matching curl, if one exists. If curl has a revision of
// -1, it returns the latest charm that matches curl. If multiple candidates
// satisfy the foregoing, the first one encountered will be returned.
func (r *LocalRepository) Get(curl *URL) (Charm, error) {
	if curl.Schema != "local" {
		return nil, fmt.Errorf("local repository got URL with non-local schema: %q", curl)
	}
	info, err := os.Stat(r.Path)
	if err != nil {
		if os.IsNotExist(err) {
			err = repoNotFound(r.Path)
		}
		return nil, err
	}
	if !info.IsDir() {
		return nil, repoNotFound(r.Path)
	}
	path := filepath.Join(r.Path, curl.Series)
	infos, err := ioutil.ReadDir(path)
	if err != nil {
		return nil, charmNotFound(curl, r.Path)
	}
	var latest Charm
	for _, info := range infos {
		if !mightBeCharm(info) {
			continue
		}
		chPath := filepath.Join(path, info.Name())
		if ch, err := Read(chPath); err != nil {
			log.Warningf("charm: failed to load charm at %q: %s", chPath, err)
		} else if ch.Meta().Name == curl.Name {
			if ch.Revision() == curl.Revision {
				return ch, nil
			}
			if latest == nil || ch.Revision() > latest.Revision() {
				latest = ch
			}
		}
	}
	if curl.Revision == -1 && latest != nil {
		return latest, nil
	}
	return nil, charmNotFound(curl, r.Path)
}
<commit_msg>go fmt<commit_after>package charm

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"launchpad.net/juju-core/environs/config"
	"launchpad.net/juju-core/log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
)

// InfoResponse is sent by the charm store in response to charm-info requests.
type InfoResponse struct {
	Revision int      `json:"revision"` // Zero is valid. Can't omitempty.
	Sha256   string   `json:"sha256,omitempty"`
	Digest   string   `json:"digest,omitempty"`
	Errors   []string `json:"errors,omitempty"`
	Warnings []string `json:"warnings,omitempty"`
}

// EventResponse is sent by the charm store in response to charm-event requests.
type EventResponse struct {
	Kind     string   `json:"kind"`
	Revision int      `json:"revision"` // Zero is valid. Can't omitempty.
	Digest   string   `json:"digest,omitempty"`
	Errors   []string `json:"errors,omitempty"`
	Warnings []string `json:"warnings,omitempty"`
	Time     string   `json:"time,omitempty"`
}

// Repository respresents a collection of charms.
type Repository interface {
	Get(curl *URL) (Charm, error)
	Latest(curl *URL) (int, error)
}

// NotFoundError represents an error indicating that the requested data wasn't found.
type NotFoundError struct {
	msg string
}

func (e *NotFoundError) Error() string {
	return e.msg
}

// CharmStore is a Repository that provides access to the public juju charm store.
type CharmStore struct {
	baseURL string
}

//var Store = &CharmStore{"https://store.juju.ubuntu.com"}
var Store = &CharmStore{"http://localhost:8080"}

// Info returns details for a charm in the charm store.
func (s *CharmStore) Info(curl *URL) (*InfoResponse, error) {
	key := curl.String()
	resp, err := http.Get(s.baseURL + "/charm-info?charms=" + url.QueryEscape(key))
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	infos := make(map[string]*InfoResponse)
	if err = json.Unmarshal(body, &infos); err != nil {
		return nil, err
	}
	info, found := infos[key]
	if !found {
		return nil, fmt.Errorf("charm: charm store returned response without charm %q", key)
	}
	if len(info.Errors) == 1 && info.Errors[0] == "entry not found" {
		return nil, &NotFoundError{fmt.Sprintf("charm not found: %s", curl)}
	}
	return info, nil
}

// Event returns details for a charm event in the charm store.
//
// If digest is empty, the latest event is returned.
func (s *CharmStore) Event(curl *URL, digest string) (*EventResponse, error) {
	key := curl.String()
	query := key
	if digest != "" {
		query += "@" + digest
	}
	resp, err := http.Get(s.baseURL + "/charm-event?charms=" + url.QueryEscape(query))
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	events := make(map[string]*EventResponse)
	if err = json.Unmarshal(body, &events); err != nil {
		return nil, err
	}
	event, found := events[key]
	if !found {
		return nil, fmt.Errorf("charm: charm store returned response without charm %q", key)
	}
	if len(event.Errors) == 1 && event.Errors[0] == "entry not found" {
		if digest == "" {
			return nil, &NotFoundError{fmt.Sprintf("charm event not found for %q", curl)}
		} else {
			return nil, &NotFoundError{fmt.Sprintf("charm event not found for %q with digest %q", curl, digest)}
		}
	}
	return event, nil
}

// revision returns the revision and SHA256 digest of the charm referenced by curl.
func (s *CharmStore) revision(curl *URL) (revision int, digest string, err error) {
	info, err := s.Info(curl)
	if err != nil {
		return 0, "", err
	}
	for _, w := range info.Warnings {
		log.Warningf("charm: charm store reports for %q: %s", curl, w)
	}
	if info.Errors != nil {
		return 0, "", fmt.Errorf("charm info errors for %q: %s", curl, strings.Join(info.Errors, "; "))
	}
	return info.Revision, info.Sha256, nil
}

// Latest returns the latest revision of the charm referenced by curl, regardless
// of the revision set on curl itself.
func (s *CharmStore) Latest(curl *URL) (int, error) {
	rev, _, err := s.revision(curl.WithRevision(-1))
	return rev, err
}

// verify returns an error unless a file exists at path with a hex-encoded
// SHA256 matching digest.
func verify(path, digest string) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()
	h := sha256.New()
	if _, err := io.Copy(h, f); err != nil {
		return err
	}
	if hex.EncodeToString(h.Sum(nil)) != digest {
		return fmt.Errorf("bad SHA256 of %q", path)
	}
	return nil
}

// Get returns the charm referenced by curl.
func (s *CharmStore) Get(curl *URL) (Charm, error) {
	cachePath := config.JujuHomePath("cache")
	if err := os.MkdirAll(cachePath, 0755); err != nil {
		return nil, err
	}
	rev, digest, err := s.revision(curl)
	if err != nil {
		return nil, err
	}
	if curl.Revision == -1 {
		curl = curl.WithRevision(rev)
	} else if curl.Revision != rev {
		return nil, fmt.Errorf("charm: store returned charm with wrong revision for %q", curl.String())
	}
	path := filepath.Join(cachePath, Quote(curl.String())+".charm")
	if verify(path, digest) != nil {
		resp, err := http.Get(s.baseURL + "/charm/" + url.QueryEscape(curl.Path()))
		if err != nil {
			return nil, err
		}
		defer resp.Body.Close()
		f, err := ioutil.TempFile(cachePath, "charm-download")
		if err != nil {
			return nil, err
		}
		dlPath := f.Name()
		_, err = io.Copy(f, resp.Body)
		if cerr := f.Close(); err == nil {
			err = cerr
		}
		if err != nil {
			os.Remove(dlPath)
			return nil, err
		}
		if err := os.Rename(dlPath, path); err != nil {
			return nil, err
		}
	}
	if err := verify(path, digest); err != nil {
		return nil, err
	}
	return ReadBundle(path)
}

// LocalRepository represents a local directory containing subdirectories
// named after an Ubuntu series, each of which contains charms targeted for
// that series. For example:
//
//   /path/to/repository/oneiric/mongodb/
//   /path/to/repository/precise/mongodb.charm
//   /path/to/repository/precise/wordpress/
type LocalRepository struct {
	Path string
}

// Latest returns the latest revision of the charm referenced by curl, regardless
// of the revision set on curl itself.
func (r *LocalRepository) Latest(curl *URL) (int, error) {
	ch, err := r.Get(curl.WithRevision(-1))
	if err != nil {
		return 0, err
	}
	return ch.Revision(), nil
}

func repoNotFound(path string) error {
	return &NotFoundError{fmt.Sprintf("no repository found at %q", path)}
}

func charmNotFound(curl *URL, repoPath string) error {
	return &NotFoundError{fmt.Sprintf("no charms found matching %q in %s", curl, repoPath)}
}

func mightBeCharm(info os.FileInfo) bool {
	if info.IsDir() {
		return !strings.HasPrefix(info.Name(), ".")
	}
	return strings.HasSuffix(info.Name(), ".charm")
}

// Get returns a charm matching curl, if one exists. If curl has a revision of
// -1, it returns the latest charm that matches curl. If multiple candidates
// satisfy the foregoing, the first one encountered will be returned.
func (r *LocalRepository) Get(curl *URL) (Charm, error) {
	if curl.Schema != "local" {
		return nil, fmt.Errorf("local repository got URL with non-local schema: %q", curl)
	}
	info, err := os.Stat(r.Path)
	if err != nil {
		if os.IsNotExist(err) {
			err = repoNotFound(r.Path)
		}
		return nil, err
	}
	if !info.IsDir() {
		return nil, repoNotFound(r.Path)
	}
	path := filepath.Join(r.Path, curl.Series)
	infos, err := ioutil.ReadDir(path)
	if err != nil {
		return nil, charmNotFound(curl, r.Path)
	}
	var latest Charm
	for _, info := range infos {
		if !mightBeCharm(info) {
			continue
		}
		chPath := filepath.Join(path, info.Name())
		if ch, err := Read(chPath); err != nil {
			log.Warningf("charm: failed to load charm at %q: %s", chPath, err)
		} else if ch.Meta().Name == curl.Name {
			if ch.Revision() == curl.Revision {
				return ch, nil
			}
			if latest == nil || ch.Revision() > latest.Revision() {
				latest = ch
			}
		}
	}
	if curl.Revision == -1 && latest != nil {
		return latest, nil
	}
	return nil, charmNotFound(curl, r.Path)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"

	"github.com/OpenPlatformSDN/nuage-cni/agent/server"

	"github.com/OpenPlatformSDN/nuage-cni/config"

	"flag"
	"os"
	"path"

	"github.com/golang/glog"
)

const (
	errorLogLevel = 2
)

func main() {

	Config := new(config.Config)

	Flags(Config, flag.CommandLine)
	flag.Parse()

	if len(os.Args) == 1 { // With no arguments, print default usage
		flag.PrintDefaults()
		os.Exit(0)
	}
	// Flush the logs upon exit
	defer glog.Flush()

	glog.Infof("===> Starting %s...", path.Base(os.Args[0]))

	if err := config.LoadConfig(Config); err != nil {
		glog.Fatalf("Cannot read configuration file: %s", err)

	}

	if err := server.Server(Config); err != nil {
		glog.Fatalf("Failed to start agent server: %s", err)
	}
}

////////
////////
////////

func Flags(conf *config.Config, flagSet *flag.FlagSet) {
	flagSet.StringVar(&conf.ConfigFile, "config",
		"/opt/nuage/etc/nuage-cni-config.yaml", "Nuage CNI agent server configuration file. If this file is specified, remaining arguments will be ignored")

	flagSet.StringVar(&conf.AgentConfig.ServerPort, "serverport",
		"7443", "Nuage CNI agent and client: Agent server port")

	flagSet.StringVar(&conf.AgentConfig.CaFile, "cafile",
		"/opt/nuage/etc/ca.crt", "Nuage CNI agent and client: Agent server CA certificate")

	flagSet.StringVar(&conf.AgentConfig.CertCaFile, "certcafile",
		"/opt/nuage/etc/agent-server.pem", "Nuage CNI agent and client: Agent server certificate (server + CA certificates PEM file)")

	flagSet.StringVar(&conf.AgentConfig.KeyFile, "keyfile",
		"/opt/nuage/etc/agent-server.key", "Nuage CNI agent and client: Agent server private key file")
	// Set the values for log_dir and logtostderr.  Because this happens before flag.Parse(), cli arguments will override these.
	// Also set the DefValue parameter so -help shows the new defaults.
	// XXX - Make sure "glog" package is imported at this point, otherwise this will panic
	log_dir := flagSet.Lookup("log_dir")
	log_dir.Value.Set(fmt.Sprintf("/var/log/%s", path.Base(os.Args[0])))
	log_dir.DefValue = fmt.Sprintf("/var/log/%s", path.Base(os.Args[0]))
	logtostderr := flagSet.Lookup("logtostderr")
	logtostderr.Value.Set("false")
	logtostderr.DefValue = "false"
	stderrlogthreshold := flagSet.Lookup("stderrthreshold")
	stderrlogthreshold.Value.Set("2")
	stderrlogthreshold.DefValue = "2"
	log.SetFlags(log.LstdFlags | log.Lshortfile)
}

/*

func Flags(conf *config.Config, flagSet *flag.FlagSet) {
	// Reminder
	// agentname := "agent"
	//
	flagSet.StringVar(&conf.ConfigFile, "config",
		"/opt/nuage/etc/nuage-cni-config.yaml", "Nuage CNI agent and client:  Configuration file. If this file is specified, remaining arguments (except 'orchestrator' for client)  will be ignored")

	flagSet.StringVar(&conf.Orchestrator, "orchestrator",
		"Kubernetes", "Nuage client: Container orchestrator. This must be non-empty")

	flagSet.StringVar(&conf.VrsConfig.Endpoint, "vrsendpoint",
		"/var/run/openvswitch/db.sock", "Nuage CNI client: VRS UNIX socket file")

	flagSet.StringVar(&conf.VrsConfig.Bridge, "vrsbridge",
		"alubr0", "Nuage CNI client: VRS bridge name")

	flagSet.StringVar(&conf.AgentConfig.ServerPort, "serverport",
		"7443", "Nuage CNI agent and client: Agent server port")

	flagSet.StringVar(&conf.AgentConfig.CaFile, "cafile",
		"/opt/nuage/etc/ca.crt", "Nuage CNI agent and client: Agent server CA certificate")

	flagSet.StringVar(&conf.AgentConfig.CertCaFile, "certcafile",
		"/opt/nuage/etc/agent-server.pem", "Nuage CNI agent and client: Agent server certificate (server + CA certificates PEM file)")

	flagSet.StringVar(&conf.AgentConfig.KeyFile, "keyfile",
		"/opt/nuage/etc/agent-server.key", "Nuage CNI agent and client: Agent server private key file")
	// Set the values for log_dir and logtostderr.  Because this happens before flag.Parse(), cli arguments will override these.
	// Also set the DefValue parameter so -help shows the new defaults.
	// XXX - Make sure "glog" package is imported at this point, otherwise this will panic
	log_dir := flagSet.Lookup("log_dir")
	log_dir.Value.Set(fmt.Sprintf("/var/log/%s", path.Base(os.Args[0])))
	log_dir.DefValue = fmt.Sprintf("/var/log/%s", path.Base(os.Args[0]))
	logtostderr := flagSet.Lookup("logtostderr")
	logtostderr.Value.Set("false")
	logtostderr.DefValue = "false"
	stderrlogthreshold := flagSet.Lookup("stderrthreshold")
	stderrlogthreshold.Value.Set("2")
	stderrlogthreshold.DefValue = "2"
	log.SetFlags(log.LstdFlags | log.Lshortfile)
}

*/
<commit_msg>Minor: changed flag naming<commit_after>package main

import (
	"fmt"
	"log"

	"github.com/OpenPlatformSDN/nuage-cni/agent/server"

	"github.com/OpenPlatformSDN/nuage-cni/config"

	"flag"
	"os"
	"path"

	"github.com/golang/glog"
)

const (
	errorLogLevel = 2
)

func main() {

	Config := new(config.Config)

	Flags(Config, flag.CommandLine)
	flag.Parse()

	if len(os.Args) == 1 { // With no arguments, print default usage
		flag.PrintDefaults()
		os.Exit(0)
	}
	// Flush the logs upon exit
	defer glog.Flush()

	glog.Infof("===> Starting %s...", path.Base(os.Args[0]))

	if err := config.LoadConfig(Config); err != nil {
		glog.Fatalf("Cannot read configuration file: %s", err)

	}

	if err := server.Server(Config); err != nil {
		glog.Fatalf("Failed to start agent server: %s", err)
	}
}

////////
////////
////////

func Flags(conf *config.Config, flagSet *flag.FlagSet) {
	flagSet.StringVar(&conf.ConfigFile, "config",
		"/opt/nuage/etc/nuage-cni-config.yaml", "Nuage CNI agent server configuration file. If this file is specified, remaining arguments will be ignored")

	flagSet.StringVar(&conf.AgentConfig.ServerPort, "serverport",
		"7443", "Nuage CNI agent server port")

	flagSet.StringVar(&conf.AgentConfig.CaFile, "cafile",
		"/opt/nuage/etc/ca.crt", "Nuage CNI agent server CA certificate")

	flagSet.StringVar(&conf.AgentConfig.CertCaFile, "certcafile",
		"/opt/nuage/etc/agent-server.pem", "Nuage CNI agent server certificate (server + CA certificates PEM file)")

	flagSet.StringVar(&conf.AgentConfig.KeyFile, "keyfile",
		"/opt/nuage/etc/agent-server.key", "Nuage CNI agent server private key file")
	// Set the values for log_dir and logtostderr.  Because this happens before flag.Parse(), cli arguments will override these.
	// Also set the DefValue parameter so -help shows the new defaults.
	// XXX - Make sure "glog" package is imported at this point, otherwise this will panic
	log_dir := flagSet.Lookup("log_dir")
	log_dir.Value.Set(fmt.Sprintf("/var/log/%s", path.Base(os.Args[0])))
	log_dir.DefValue = fmt.Sprintf("/var/log/%s", path.Base(os.Args[0]))
	logtostderr := flagSet.Lookup("logtostderr")
	logtostderr.Value.Set("false")
	logtostderr.DefValue = "false"
	stderrlogthreshold := flagSet.Lookup("stderrthreshold")
	stderrlogthreshold.Value.Set("2")
	stderrlogthreshold.DefValue = "2"
	log.SetFlags(log.LstdFlags | log.Lshortfile)
}

/*

func Flags(conf *config.Config, flagSet *flag.FlagSet) {
	// Reminder
	// agentname := "agent"
	//
	flagSet.StringVar(&conf.ConfigFile, "config",
		"/opt/nuage/etc/nuage-cni-config.yaml", "Nuage CNI agent and client:  Configuration file. If this file is specified, remaining arguments (except 'orchestrator' for client)  will be ignored")

	flagSet.StringVar(&conf.Orchestrator, "orchestrator",
		"Kubernetes", "Nuage client: Container orchestrator. This must be non-empty")

	flagSet.StringVar(&conf.VrsConfig.Endpoint, "vrsendpoint",
		"/var/run/openvswitch/db.sock", "Nuage CNI client: VRS UNIX socket file")

	flagSet.StringVar(&conf.VrsConfig.Bridge, "vrsbridge",
		"alubr0", "Nuage CNI client: VRS bridge name")

	flagSet.StringVar(&conf.AgentConfig.ServerPort, "serverport",
		"7443", "Nuage CNI agent and client: Agent server port")

	flagSet.StringVar(&conf.AgentConfig.CaFile, "cafile",
		"/opt/nuage/etc/ca.crt", "Nuage CNI agent and client: Agent server CA certificate")

	flagSet.StringVar(&conf.AgentConfig.CertCaFile, "certcafile",
		"/opt/nuage/etc/agent-server.pem", "Nuage CNI agent and client: Agent server certificate (server + CA certificates PEM file)")

	flagSet.StringVar(&conf.AgentConfig.KeyFile, "keyfile",
		"/opt/nuage/etc/agent-server.key", "Nuage CNI agent and client: Agent server private key file")
	// Set the values for log_dir and logtostderr.  Because this happens before flag.Parse(), cli arguments will override these.
	// Also set the DefValue parameter so -help shows the new defaults.
	// XXX - Make sure "glog" package is imported at this point, otherwise this will panic
	log_dir := flagSet.Lookup("log_dir")
	log_dir.Value.Set(fmt.Sprintf("/var/log/%s", path.Base(os.Args[0])))
	log_dir.DefValue = fmt.Sprintf("/var/log/%s", path.Base(os.Args[0]))
	logtostderr := flagSet.Lookup("logtostderr")
	logtostderr.Value.Set("false")
	logtostderr.DefValue = "false"
	stderrlogthreshold := flagSet.Lookup("stderrthreshold")
	stderrlogthreshold.Value.Set("2")
	stderrlogthreshold.DefValue = "2"
	log.SetFlags(log.LstdFlags | log.Lshortfile)
}

*/
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/rainforestapp/rainforest-cli/rainforest"
	"github.com/satori/go.uuid"
	"github.com/urfave/cli"
)

// parseError is a custom error implementing error interface for reporting RFML parsing errors.
type fileParseError struct {
	filePath   string
	parseError error
}

func (e fileParseError) Error() string {
	return fmt.Sprintf("%v:%v", e.filePath, e.parseError.Error())
}

// validateRFML is a wrapper around two other validation functions
// first one for the single file and the other for whole directory
func validateRFML(c cliContext) error {
	if path := c.Args().First(); path != "" {
		err := validateSingleRFMLFile(path)
		if err != nil {
			return cli.NewExitError(err.Error(), 1)
		}
		return nil
	}
	err := validateRFMLFilesInDirectory(c.String("test-folder"))
	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}
	return nil
}

// validateSingleRFMLFile validates RFML file syntax by
// trying to parse the file and sending any parse errors to the caller
func validateSingleRFMLFile(filePath string) error {
	if !strings.Contains(filePath, ".rfml") {
		return errors.New("RFML files should have .rfml extension")
	}
	f, err := os.Open(filePath)
	if err != nil {
		return err
	}
	defer f.Close()
	rfmlReader := rainforest.NewRFMLReader(f)
	_, err = rfmlReader.ReadAll()
	if err != nil {
		return fileParseError{filePath, err}
	}
	log.Printf("%v's syntax is valid", filePath)
	return nil
}

// validateRFMLFilesInDirectory validates RFML file syntax, embedded rfml ids,
// checks for circular dependiences and all other cool things in the specified directory
func validateRFMLFilesInDirectory(rfmlDirectory string) error {
	// first just make sure we are dealing with directory
	dirStat, err := os.Stat(rfmlDirectory)
	if err != nil {
		return err
	}
	if !dirStat.IsDir() {
		return fmt.Errorf("%v should be a directory", rfmlDirectory)
	}

	// walk through the specifed directory (also subdirs) and pick the .rfml files
	var fileList []string
	err = filepath.Walk(rfmlDirectory, func(path string, f os.FileInfo, err error) error {
		if strings.Contains(path, ".rfml") {
			fileList = append(fileList, path)
		}
		return nil
	})
	if err != nil {
		return err
	}

	// parse all of them files
	type parsedTest struct {
		filePath string
		content  *rainforest.RFTest
	}
	var validationErrors []error
	var parsedTests []parsedTest
	for _, filePath := range fileList {
		var f *os.File
		f, err = os.Open(filePath)
		if err != nil {
			return err
		}
		defer f.Close()
		rfmlReader := rainforest.NewRFMLReader(f)
		var pTest *rainforest.RFTest
		pTest, err = rfmlReader.ReadAll()
		if err != nil {
			validationErrors = append(validationErrors, fileParseError{filePath, err})
		} else {
			parsedTests = append(parsedTests, parsedTest{filePath, pTest})
		}
	}

	// check for rfml_id uniqueness
	rfmlIDToTest := make(map[string]parsedTest)
	for _, pTest := range parsedTests {
		if conflictingTest, ok := rfmlIDToTest[pTest.content.RFMLID]; ok {
			err = fmt.Errorf(" duplicate RFML id %v, also found in: %v", pTest.content.RFMLID, conflictingTest.filePath)
			validationErrors = append(validationErrors, fileParseError{pTest.filePath, err})
		} else {
			rfmlIDToTest[pTest.content.RFMLID] = pTest
		}
	}

	// check for embedded tests id validity
	// start with pulling the external test ids to validate against them as well
	if api.ClientToken != "" {
		var externalTests rainforest.TestIDMappings
		externalTests, err = api.GetRFMLIDs()
		if err != nil {
			return err
		}
		for _, externalTest := range externalTests {
			if _, ok := rfmlIDToTest[externalTest.RFMLID]; !ok {
				rfmlIDToTest[externalTest.RFMLID] = parsedTest{"external", &rainforest.RFTest{}}
			}
		}
	}
	// go through all the tests
	for _, pTest := range parsedTests {
		// and steps...
		for stepNum, step := range pTest.content.Steps {
			// then check if it's embeddedTest
			if embeddedTest, ok := step.(rainforest.RFEmbeddedTest); ok {
				// if so, check if it's rfml id exists
				if _, ok := rfmlIDToTest[embeddedTest.RFMLID]; !ok {
					if api.ClientToken != "" {
						err = fmt.Errorf("step %v - embeddedTest RFML id %v not found", stepNum+1, embeddedTest.RFMLID)
					} else {
						err = fmt.Errorf("step %v - embeddedTest RFML id %v not found. Specify token_id to check against external tests", stepNum+1, embeddedTest.RFMLID)
					}
					validationErrors = append(validationErrors, fileParseError{pTest.filePath, err})
				}
			}
		}
	}

	// TODO: validate circular dependiences probably using Tarjan's strongly connected components

	if len(validationErrors) > 0 {
		for _, err := range validationErrors {
			log.Print(err.Error())
		}
		return errors.New("Some of the files are invalid")
	}

	log.Print("All files are valid!")
	return nil
}

func newRFMLTest(c cliContext) error {
	test := rainforest.RFTest{
		RFMLID:   uuid.NewV4().String(),
		Title:    "This is a title",
		StartURI: "/lol/testing",
	}

	f, err := os.Create("testing.rfml")

	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}

	writer := rainforest.NewRFMLWriter(f)
	err = writer.WriteRFMLTest(&test)

	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}

	return nil
}
<commit_msg>add test title and test directory options to new command<commit_after>package main

import (
	"errors"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/rainforestapp/rainforest-cli/rainforest"
	"github.com/satori/go.uuid"
	"github.com/urfave/cli"
)

// parseError is a custom error implementing error interface for reporting RFML parsing errors.
type fileParseError struct {
	filePath   string
	parseError error
}

func (e fileParseError) Error() string {
	return fmt.Sprintf("%v:%v", e.filePath, e.parseError.Error())
}

// validateRFML is a wrapper around two other validation functions
// first one for the single file and the other for whole directory
func validateRFML(c cliContext) error {
	if path := c.Args().First(); path != "" {
		err := validateSingleRFMLFile(path)
		if err != nil {
			return cli.NewExitError(err.Error(), 1)
		}
		return nil
	}
	err := validateRFMLFilesInDirectory(c.String("test-folder"))
	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}
	return nil
}

// validateSingleRFMLFile validates RFML file syntax by
// trying to parse the file and sending any parse errors to the caller
func validateSingleRFMLFile(filePath string) error {
	if !strings.Contains(filePath, ".rfml") {
		return errors.New("RFML files should have .rfml extension")
	}
	f, err := os.Open(filePath)
	if err != nil {
		return err
	}
	defer f.Close()
	rfmlReader := rainforest.NewRFMLReader(f)
	_, err = rfmlReader.ReadAll()
	if err != nil {
		return fileParseError{filePath, err}
	}
	log.Printf("%v's syntax is valid", filePath)
	return nil
}

// validateRFMLFilesInDirectory validates RFML file syntax, embedded rfml ids,
// checks for circular dependiences and all other cool things in the specified directory
func validateRFMLFilesInDirectory(rfmlDirectory string) error {
	// first just make sure we are dealing with directory
	dirStat, err := os.Stat(rfmlDirectory)
	if err != nil {
		return err
	}
	if !dirStat.IsDir() {
		return fmt.Errorf("%v should be a directory", rfmlDirectory)
	}

	// walk through the specifed directory (also subdirs) and pick the .rfml files
	var fileList []string
	err = filepath.Walk(rfmlDirectory, func(path string, f os.FileInfo, err error) error {
		if strings.Contains(path, ".rfml") {
			fileList = append(fileList, path)
		}
		return nil
	})
	if err != nil {
		return err
	}

	// parse all of them files
	type parsedTest struct {
		filePath string
		content  *rainforest.RFTest
	}
	var validationErrors []error
	var parsedTests []parsedTest
	for _, filePath := range fileList {
		var f *os.File
		f, err = os.Open(filePath)
		if err != nil {
			return err
		}
		defer f.Close()
		rfmlReader := rainforest.NewRFMLReader(f)
		var pTest *rainforest.RFTest
		pTest, err = rfmlReader.ReadAll()
		if err != nil {
			validationErrors = append(validationErrors, fileParseError{filePath, err})
		} else {
			parsedTests = append(parsedTests, parsedTest{filePath, pTest})
		}
	}

	// check for rfml_id uniqueness
	rfmlIDToTest := make(map[string]parsedTest)
	for _, pTest := range parsedTests {
		if conflictingTest, ok := rfmlIDToTest[pTest.content.RFMLID]; ok {
			err = fmt.Errorf(" duplicate RFML id %v, also found in: %v", pTest.content.RFMLID, conflictingTest.filePath)
			validationErrors = append(validationErrors, fileParseError{pTest.filePath, err})
		} else {
			rfmlIDToTest[pTest.content.RFMLID] = pTest
		}
	}

	// check for embedded tests id validity
	// start with pulling the external test ids to validate against them as well
	if api.ClientToken != "" {
		var externalTests rainforest.TestIDMappings
		externalTests, err = api.GetRFMLIDs()
		if err != nil {
			return err
		}
		for _, externalTest := range externalTests {
			if _, ok := rfmlIDToTest[externalTest.RFMLID]; !ok {
				rfmlIDToTest[externalTest.RFMLID] = parsedTest{"external", &rainforest.RFTest{}}
			}
		}
	}
	// go through all the tests
	for _, pTest := range parsedTests {
		// and steps...
		for stepNum, step := range pTest.content.Steps {
			// then check if it's embeddedTest
			if embeddedTest, ok := step.(rainforest.RFEmbeddedTest); ok {
				// if so, check if it's rfml id exists
				if _, ok := rfmlIDToTest[embeddedTest.RFMLID]; !ok {
					if api.ClientToken != "" {
						err = fmt.Errorf("step %v - embeddedTest RFML id %v not found", stepNum+1, embeddedTest.RFMLID)
					} else {
						err = fmt.Errorf("step %v - embeddedTest RFML id %v not found. Specify token_id to check against external tests", stepNum+1, embeddedTest.RFMLID)
					}
					validationErrors = append(validationErrors, fileParseError{pTest.filePath, err})
				}
			}
		}
	}

	// TODO: validate circular dependiences probably using Tarjan's strongly connected components

	if len(validationErrors) > 0 {
		for _, err := range validationErrors {
			log.Print(err.Error())
		}
		return errors.New("Some of the files are invalid")
	}

	log.Print("All files are valid!")
	return nil
}

func newRFMLTest(c cliContext) error {
	testDirectory := c.String("test-folder")

	// first just make sure we are dealing with directory
	absTestDirectory, err := filepath.Abs(testDirectory)

	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}

	dirStat, err := os.Stat(absTestDirectory)
	if os.IsNotExist(err) {
		log.Printf("Creating test directory: %v", absTestDirectory)
		os.MkdirAll(absTestDirectory, os.ModePerm)
	} else {
		if !dirStat.IsDir() {
			errStr := fmt.Sprintf("%v should be a directory", absTestDirectory)
			return cli.NewExitError(errStr, 1)
		}
	}

	fileName := c.Args().First()
	title := fileName

	if fileName == "" {
		fileName = "Unnamed Test.rfml"
		title = "Unnamed Test"
	} else if strings.HasSuffix(fileName, ".rfml") {
		title = strings.TrimSuffix(title, ".rfml")
	} else {
		fileName = fileName + ".rfml"
	}

	filePath := filepath.Join(absTestDirectory, fileName)

	// Make sure that the file is unique
	basePath := strings.TrimSuffix(filePath, ".rfml")
	fileIdentifier := 0
	var identStr string
	for {
		if fileIdentifier == 0 {
			identStr = ""
		} else {
			identStr = fmt.Sprintf(" (%v)", strconv.Itoa(fileIdentifier))
		}

		testPath := basePath + identStr + ".rfml"

		_, err = os.Stat(testPath)
		if !os.IsNotExist(err) {
			fileIdentifier = fileIdentifier + 1
		} else {
			filePath = testPath
			break
		}
	}

	test := rainforest.RFTest{
		RFMLID:   uuid.NewV4().String(),
		Title:    title,
		StartURI: "/",
	}

	f, err := os.Create(filePath)

	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}

	writer := rainforest.NewRFMLWriter(f)
	err = writer.WriteRFMLTest(&test)

	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}

	return nil
}
<|endoftext|>