<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package magnum

import (
	"crypto/tls"
	"errors"
	"fmt"
	"io"
	"net/http"
	"time"

	"gopkg.in/gcfg.v1"
	netutil "k8s.io/apimachinery/pkg/util/net"
	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider/magnum/gophercloud"
	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider/magnum/gophercloud/openstack"
	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider/magnum/gophercloud/openstack/containerinfra/v1/clusters"
	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider/magnum/gophercloud/openstack/identity/v3/extensions/trusts"
	"k8s.io/autoscaler/cluster-autoscaler/config"
	"k8s.io/autoscaler/cluster-autoscaler/version"
	certutil "k8s.io/client-go/util/cert"
	klog "k8s.io/klog/v2"
)

// These Opts types are for parsing an OpenStack cloud-config file.
// The definitions are taken from cloud-provider-openstack.

// MyDuration is the encoding.TextUnmarshaler interface for time.Duration
type MyDuration struct {
	time.Duration
}

// UnmarshalText is used to convert from text to Duration
func (d *MyDuration) UnmarshalText(text []byte) error {
	res, err := time.ParseDuration(string(text))
	if err != nil {
		return err
	}
	d.Duration = res
	return nil
}

// LoadBalancerOpts have the options to talk to Neutron LBaaSV2 or Octavia
type LoadBalancerOpts struct {
	LBVersion            string     `gcfg:"lb-version"`          // overrides autodetection. Only support v2.
	UseOctavia           bool       `gcfg:"use-octavia"`         // uses Octavia V2 service catalog endpoint
	SubnetID             string     `gcfg:"subnet-id"`           // overrides autodetection.
	FloatingNetworkID    string     `gcfg:"floating-network-id"` // If specified, will create floating ip for loadbalancer, or do not create floating ip.
	LBMethod             string     `gcfg:"lb-method"`           // default to ROUND_ROBIN.
	LBProvider           string     `gcfg:"lb-provider"`
	CreateMonitor        bool       `gcfg:"create-monitor"`
	MonitorDelay         MyDuration `gcfg:"monitor-delay"`
	MonitorTimeout       MyDuration `gcfg:"monitor-timeout"`
	MonitorMaxRetries    uint       `gcfg:"monitor-max-retries"`
	ManageSecurityGroups bool       `gcfg:"manage-security-groups"`
	NodeSecurityGroupIDs []string   // Do not specify, get it automatically when enable manage-security-groups. TODO(FengyunPan): move it into cache
}

// BlockStorageOpts is used to talk to Cinder service
type BlockStorageOpts struct {
	BSVersion             string `gcfg:"bs-version"`        // overrides autodetection. v1 or v2. Defaults to auto
	TrustDevicePath       bool   `gcfg:"trust-device-path"` // See Issue #33128
	IgnoreVolumeAZ        bool   `gcfg:"ignore-volume-az"`
	NodeVolumeAttachLimit int    `gcfg:"node-volume-attach-limit"` // override volume attach limit for Cinder. Default is : 256
}

// RouterOpts is used for Neutron routes
type RouterOpts struct {
	RouterID string `gcfg:"router-id"` // required
}

// MetadataOpts is used for configuring how to talk to metadata service or config drive
type MetadataOpts struct {
	SearchOrder    string     `gcfg:"search-order"`
	RequestTimeout MyDuration `gcfg:"request-timeout"`
}

// Config is used to read and store information from the cloud configuration file
//
// Taken from kubernetes/pkg/cloudprovider/providers/openstack/openstack.go
// LoadBalancer, BlockStorage, Route, Metadata are not needed for the autoscaler,
// but are kept so that if a cloud-config file with those sections is provided
// then the parsing will not fail.
type Config struct {
	Global struct {
		AuthURL         string `gcfg:"auth-url"`
		Username        string
		UserID          string `gcfg:"user-id"`
		Password        string
		TenantID        string `gcfg:"tenant-id"`
		TenantName      string `gcfg:"tenant-name"`
		TrustID         string `gcfg:"trust-id"`
		DomainID        string `gcfg:"domain-id"`
		DomainName      string `gcfg:"domain-name"`
		Region          string
		CAFile          string `gcfg:"ca-file"`
		SecretName      string `gcfg:"secret-name"`
		SecretNamespace string `gcfg:"secret-namespace"`
	}
	LoadBalancer LoadBalancerOpts
	BlockStorage BlockStorageOpts
	Route        RouterOpts
	Metadata     MetadataOpts
}

func toAuthOptsExt(cfg Config) trusts.AuthOptsExt {
	opts := gophercloud.AuthOptions{
		IdentityEndpoint: cfg.Global.AuthURL,
		Username:         cfg.Global.Username,
		UserID:           cfg.Global.UserID,
		Password:         cfg.Global.Password,
		TenantID:         cfg.Global.TenantID,
		TenantName:       cfg.Global.TenantName,
		DomainID:         cfg.Global.DomainID,
		DomainName:       cfg.Global.DomainName,

		// Persistent service, so we need to be able to renew tokens.
		AllowReauth: true,
	}

	return trusts.AuthOptsExt{
		TrustID:            cfg.Global.TrustID,
		AuthOptionsBuilder: &opts,
	}
}

// readConfig parses an OpenStack cloud-config file from an io.Reader.
func readConfig(configReader io.Reader) (*Config, error) {
	var cfg Config
	if configReader != nil {
		if err := gcfg.ReadInto(&cfg, configReader); err != nil {
			return nil, fmt.Errorf("couldn't read cloud config: %v", err)
		}
	}
	return &cfg, nil
}

// createProviderClient creates and authenticates a gophercloud provider client.
func createProviderClient(cfg *Config, opts config.AutoscalingOptions) (*gophercloud.ProviderClient, error) {
	if opts.ClusterName == "" {
		return nil, errors.New("the cluster-name parameter must be set")
	}

	authOpts := toAuthOptsExt(*cfg)

	provider, err := openstack.NewClient(cfg.Global.AuthURL)
	if err != nil {
		return nil, fmt.Errorf("could not create openstack client: %v", err)
	}

	userAgent := gophercloud.UserAgent{}
	userAgent.Prepend(fmt.Sprintf("cluster-autoscaler/%s", version.ClusterAutoscalerVersion))
	userAgent.Prepend(fmt.Sprintf("cluster/%s", opts.ClusterName))
	provider.UserAgent = userAgent

	klog.V(5).Infof("Using user-agent %q", userAgent.Join())

	if cfg.Global.CAFile != "" {
		roots, err := certutil.NewPool(cfg.Global.CAFile)
		if err != nil {
			return nil, err
		}
		config := &tls.Config{}
		config.RootCAs = roots
		provider.HTTPClient.Transport = netutil.SetOldTransportDefaults(&http.Transport{TLSClientConfig: config})
	}

	err = openstack.AuthenticateV3(provider, authOpts, gophercloud.EndpointOpts{})
	if err != nil {
		return nil, fmt.Errorf("could not authenticate client: %v", err)
	}

	return provider, nil
}

// createClusterClient creates a gophercloud service client for communicating with Magnum.
func createClusterClient(cfg *Config, provider *gophercloud.ProviderClient, opts config.AutoscalingOptions) (*gophercloud.ServiceClient, error) {
	clusterClient, err := openstack.NewContainerInfraV1(provider, gophercloud.EndpointOpts{Type: "container-infra", Name: "magnum", Region: cfg.Global.Region})
	if err != nil {
		return nil, fmt.Errorf("could not create container-infra client: %v", err)
	}
	return clusterClient, nil
}

// checkClusterUUID replaces a cluster name with the UUID, if the name was given in the parameters
// to the cluster autoscaler.
func checkClusterUUID(provider *gophercloud.ProviderClient, clusterClient *gophercloud.ServiceClient, opts config.AutoscalingOptions) error {
	cluster, err := clusters.Get(clusterClient, opts.ClusterName).Extract()
	if err != nil {
		return fmt.Errorf("unable to access cluster %q: %v", opts.ClusterName, err)
	}

	// Prefer to use the cluster UUID if the cluster name was given in the parameters
	if opts.ClusterName != cluster.UUID {
		klog.V(2).Infof("Using cluster UUID %q instead of name %q", cluster.UUID, opts.ClusterName)
		opts.ClusterName = cluster.UUID

		// Need to remake user-agent with UUID instead of name
		userAgent := gophercloud.UserAgent{}
		userAgent.Prepend(fmt.Sprintf("cluster-autoscaler/%s", version.ClusterAutoscalerVersion))
		userAgent.Prepend(fmt.Sprintf("cluster/%s", opts.ClusterName))
		provider.UserAgent = userAgent
		klog.V(5).Infof("Using updated user-agent %q", userAgent.Join())
	}

	return nil
}

// createHeatClient creates a gophercloud service client for communicating with Heat.
func createHeatClient(cfg *Config, provider *gophercloud.ProviderClient, opts config.AutoscalingOptions) (*gophercloud.ServiceClient, error) {
	heatClient, err := openstack.NewOrchestrationV1(provider, gophercloud.EndpointOpts{Type: "orchestration", Name: "heat", Region: cfg.Global.Region})
	if err != nil {
		return nil, fmt.Errorf("could not create orchestration client: %v", err)
	}

	return heatClient, nil
}
<commit_msg>magnum: add an option to create insecure TLS connections<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package magnum

import (
	"crypto/tls"
	"errors"
	"fmt"
	"io"
	"net/http"
	"time"

	"gopkg.in/gcfg.v1"
	netutil "k8s.io/apimachinery/pkg/util/net"
	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider/magnum/gophercloud"
	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider/magnum/gophercloud/openstack"
	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider/magnum/gophercloud/openstack/containerinfra/v1/clusters"
	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider/magnum/gophercloud/openstack/identity/v3/extensions/trusts"
	"k8s.io/autoscaler/cluster-autoscaler/config"
	"k8s.io/autoscaler/cluster-autoscaler/version"
	certutil "k8s.io/client-go/util/cert"
	klog "k8s.io/klog/v2"
)

// These Opts types are for parsing an OpenStack cloud-config file.
// The definitions are taken from cloud-provider-openstack.

// MyDuration is the encoding.TextUnmarshaler interface for time.Duration
type MyDuration struct {
	time.Duration
}

// UnmarshalText is used to convert from text to Duration
func (d *MyDuration) UnmarshalText(text []byte) error {
	res, err := time.ParseDuration(string(text))
	if err != nil {
		return err
	}
	d.Duration = res
	return nil
}

// LoadBalancerOpts have the options to talk to Neutron LBaaSV2 or Octavia
type LoadBalancerOpts struct {
	LBVersion            string     `gcfg:"lb-version"`          // overrides autodetection. Only support v2.
	UseOctavia           bool       `gcfg:"use-octavia"`         // uses Octavia V2 service catalog endpoint
	SubnetID             string     `gcfg:"subnet-id"`           // overrides autodetection.
	FloatingNetworkID    string     `gcfg:"floating-network-id"` // If specified, will create floating ip for loadbalancer, or do not create floating ip.
	LBMethod             string     `gcfg:"lb-method"`           // default to ROUND_ROBIN.
	LBProvider           string     `gcfg:"lb-provider"`
	CreateMonitor        bool       `gcfg:"create-monitor"`
	MonitorDelay         MyDuration `gcfg:"monitor-delay"`
	MonitorTimeout       MyDuration `gcfg:"monitor-timeout"`
	MonitorMaxRetries    uint       `gcfg:"monitor-max-retries"`
	ManageSecurityGroups bool       `gcfg:"manage-security-groups"`
	NodeSecurityGroupIDs []string   // Do not specify, get it automatically when enable manage-security-groups. TODO(FengyunPan): move it into cache
}

// BlockStorageOpts is used to talk to Cinder service
type BlockStorageOpts struct {
	BSVersion             string `gcfg:"bs-version"`        // overrides autodetection. v1 or v2. Defaults to auto
	TrustDevicePath       bool   `gcfg:"trust-device-path"` // See Issue #33128
	IgnoreVolumeAZ        bool   `gcfg:"ignore-volume-az"`
	NodeVolumeAttachLimit int    `gcfg:"node-volume-attach-limit"` // override volume attach limit for Cinder. Default is : 256
}

// RouterOpts is used for Neutron routes
type RouterOpts struct {
	RouterID string `gcfg:"router-id"` // required
}

// MetadataOpts is used for configuring how to talk to metadata service or config drive
type MetadataOpts struct {
	SearchOrder    string     `gcfg:"search-order"`
	RequestTimeout MyDuration `gcfg:"request-timeout"`
}

// Config is used to read and store information from the cloud configuration file
//
// Taken from kubernetes/pkg/cloudprovider/providers/openstack/openstack.go
// LoadBalancer, BlockStorage, Route, Metadata are not needed for the autoscaler,
// but are kept so that if a cloud-config file with those sections is provided
// then the parsing will not fail.
type Config struct {
	Global struct {
		AuthURL         string `gcfg:"auth-url"`
		Username        string
		UserID          string `gcfg:"user-id"`
		Password        string
		TenantID        string `gcfg:"tenant-id"`
		TenantName      string `gcfg:"tenant-name"`
		TrustID         string `gcfg:"trust-id"`
		DomainID        string `gcfg:"domain-id"`
		DomainName      string `gcfg:"domain-name"`
		Region          string
		CAFile          string `gcfg:"ca-file"`
		TLSInsecure     string `gcfg:"tls-insecure"`
		SecretName      string `gcfg:"secret-name"`
		SecretNamespace string `gcfg:"secret-namespace"`
	}
	LoadBalancer LoadBalancerOpts
	BlockStorage BlockStorageOpts
	Route        RouterOpts
	Metadata     MetadataOpts
}

func toAuthOptsExt(cfg Config) trusts.AuthOptsExt {
	opts := gophercloud.AuthOptions{
		IdentityEndpoint: cfg.Global.AuthURL,
		Username:         cfg.Global.Username,
		UserID:           cfg.Global.UserID,
		Password:         cfg.Global.Password,
		TenantID:         cfg.Global.TenantID,
		TenantName:       cfg.Global.TenantName,
		DomainID:         cfg.Global.DomainID,
		DomainName:       cfg.Global.DomainName,

		// Persistent service, so we need to be able to renew tokens.
		AllowReauth: true,
	}

	return trusts.AuthOptsExt{
		TrustID:            cfg.Global.TrustID,
		AuthOptionsBuilder: &opts,
	}
}

// readConfig parses an OpenStack cloud-config file from an io.Reader.
func readConfig(configReader io.Reader) (*Config, error) {
	var cfg Config
	if configReader != nil {
		if err := gcfg.ReadInto(&cfg, configReader); err != nil {
			return nil, fmt.Errorf("couldn't read cloud config: %v", err)
		}
	}
	return &cfg, nil
}

// createProviderClient creates and authenticates a gophercloud provider client.
func createProviderClient(cfg *Config, opts config.AutoscalingOptions) (*gophercloud.ProviderClient, error) {
	if opts.ClusterName == "" {
		return nil, errors.New("the cluster-name parameter must be set")
	}

	authOpts := toAuthOptsExt(*cfg)

	provider, err := openstack.NewClient(cfg.Global.AuthURL)
	if err != nil {
		return nil, fmt.Errorf("could not create openstack client: %v", err)
	}

	userAgent := gophercloud.UserAgent{}
	userAgent.Prepend(fmt.Sprintf("cluster-autoscaler/%s", version.ClusterAutoscalerVersion))
	userAgent.Prepend(fmt.Sprintf("cluster/%s", opts.ClusterName))
	provider.UserAgent = userAgent

	klog.V(5).Infof("Using user-agent %q", userAgent.Join())

	config := &tls.Config{}
	config.InsecureSkipVerify = cfg.Global.TLSInsecure == "true"
	if cfg.Global.CAFile != "" {
		roots, err := certutil.NewPool(cfg.Global.CAFile)
		if err != nil {
			return nil, err
		}
		config.RootCAs = roots
	}
	provider.HTTPClient.Transport = netutil.SetOldTransportDefaults(&http.Transport{TLSClientConfig: config})

	err = openstack.AuthenticateV3(provider, authOpts, gophercloud.EndpointOpts{})
	if err != nil {
		return nil, fmt.Errorf("could not authenticate client: %v", err)
	}

	return provider, nil
}

// createClusterClient creates a gophercloud service client for communicating with Magnum.
func createClusterClient(cfg *Config, provider *gophercloud.ProviderClient, opts config.AutoscalingOptions) (*gophercloud.ServiceClient, error) {
	clusterClient, err := openstack.NewContainerInfraV1(provider, gophercloud.EndpointOpts{Type: "container-infra", Name: "magnum", Region: cfg.Global.Region})
	if err != nil {
		return nil, fmt.Errorf("could not create container-infra client: %v", err)
	}
	return clusterClient, nil
}

// checkClusterUUID replaces a cluster name with the UUID, if the name was given in the parameters
// to the cluster autoscaler.
func checkClusterUUID(provider *gophercloud.ProviderClient, clusterClient *gophercloud.ServiceClient, opts config.AutoscalingOptions) error {
	cluster, err := clusters.Get(clusterClient, opts.ClusterName).Extract()
	if err != nil {
		return fmt.Errorf("unable to access cluster %q: %v", opts.ClusterName, err)
	}

	// Prefer to use the cluster UUID if the cluster name was given in the parameters
	if opts.ClusterName != cluster.UUID {
		klog.V(2).Infof("Using cluster UUID %q instead of name %q", cluster.UUID, opts.ClusterName)
		opts.ClusterName = cluster.UUID

		// Need to remake user-agent with UUID instead of name
		userAgent := gophercloud.UserAgent{}
		userAgent.Prepend(fmt.Sprintf("cluster-autoscaler/%s", version.ClusterAutoscalerVersion))
		userAgent.Prepend(fmt.Sprintf("cluster/%s", opts.ClusterName))
		provider.UserAgent = userAgent
		klog.V(5).Infof("Using updated user-agent %q", userAgent.Join())
	}

	return nil
}

// createHeatClient creates a gophercloud service client for communicating with Heat.
func createHeatClient(cfg *Config, provider *gophercloud.ProviderClient, opts config.AutoscalingOptions) (*gophercloud.ServiceClient, error) {
	heatClient, err := openstack.NewOrchestrationV1(provider, gophercloud.EndpointOpts{Type: "orchestration", Name: "heat", Region: cfg.Global.Region})
	if err != nil {
		return nil, fmt.Errorf("could not create orchestration client: %v", err)
	}

	return heatClient, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 The prometheus-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package e2e

import (
	"flag"
	"log"
	"os"
	"testing"

	"k8s.io/api/core/v1"

	"github.com/coreos/prometheus-operator/pkg/k8sutil"
	operatorFramework "github.com/coreos/prometheus-operator/test/framework"
)

var framework *operatorFramework.Framework

// Basic set of e2e tests for the operator:
// - config reload (with and without external url)

func TestMain(m *testing.M) {
	kubeconfig := flag.String("kubeconfig", "", "kube config path, e.g. $HOME/.kube/config")
	opImage := flag.String("operator-image", "", "operator image, e.g. quay.io/coreos/prometheus-operator")
	ns := flag.String("namespace", "prometheus-operator-e2e-tests", "e2e test namespace")
	flag.Parse()

	var (
		err  error
		code int = 0
	)

	if framework, err = operatorFramework.New(*ns, *kubeconfig, *opImage); err != nil {
		log.Printf("failed to setup framework: %v\n", err)
		os.Exit(1)
	}

	err = k8sutil.WaitForCRDReady(framework.MonClientV1.Prometheuses(v1.NamespaceAll).List)
	if err != nil {
		log.Printf("Prometheus CRD not ready: %v\n", err)
		os.Exit(1)
	}

	err = k8sutil.WaitForCRDReady(framework.MonClientV1.ServiceMonitors(v1.NamespaceAll).List)
	if err != nil {
		log.Printf("ServiceMonitor CRD not ready: %v\n", err)
		os.Exit(1)
	}

	err = k8sutil.WaitForCRDReady(framework.MonClientV1.Alertmanagers(v1.NamespaceAll).List)
	if err != nil {
		log.Printf("Alertmanagers CRD not ready: %v\n", err)
		os.Exit(1)
	}

	defer func() {
		if err := framework.Teardown(); err != nil {
			log.Printf("failed to teardown framework: %v\n", err)
			os.Exit(1)
		}
		os.Exit(code)
	}()

	code = m.Run()
}
<commit_msg>e2e: Print Kubernetes events on test failures<commit_after>// Copyright 2016 The prometheus-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package e2e

import (
	"flag"
	"fmt"
	"log"
	"os"
	"testing"

	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/coreos/prometheus-operator/pkg/k8sutil"
	operatorFramework "github.com/coreos/prometheus-operator/test/framework"
)

var framework *operatorFramework.Framework

// Basic set of e2e tests for the operator:
// - config reload (with and without external url)

func TestMain(m *testing.M) {
	kubeconfig := flag.String("kubeconfig", "", "kube config path, e.g. $HOME/.kube/config")
	opImage := flag.String("operator-image", "", "operator image, e.g. quay.io/coreos/prometheus-operator")
	ns := flag.String("namespace", "prometheus-operator-e2e-tests", "e2e test namespace")
	flag.Parse()

	var (
		err  error
		code int = 0
	)

	if framework, err = operatorFramework.New(*ns, *kubeconfig, *opImage); err != nil {
		log.Printf("failed to setup framework: %v\n", err)
		os.Exit(1)
	}

	err = k8sutil.WaitForCRDReady(framework.MonClientV1.Prometheuses(v1.NamespaceAll).List)
	if err != nil {
		log.Printf("Prometheus CRD not ready: %v\n", err)
		os.Exit(1)
	}

	err = k8sutil.WaitForCRDReady(framework.MonClientV1.ServiceMonitors(v1.NamespaceAll).List)
	if err != nil {
		log.Printf("ServiceMonitor CRD not ready: %v\n", err)
		os.Exit(1)
	}

	err = k8sutil.WaitForCRDReady(framework.MonClientV1.Alertmanagers(v1.NamespaceAll).List)
	if err != nil {
		log.Printf("Alertmanagers CRD not ready: %v\n", err)
		os.Exit(1)
	}

	defer func() {
		if err := framework.Teardown(); err != nil {
			log.Printf("failed to teardown framework: %v\n", err)
			code = 1
		}

		if code != 0 {
			if err := printKubernetesEvents(); err != nil {
				log.Printf("failed to print events: %v", err)
			}
		}

		os.Exit(code)
	}()

	code = m.Run()
}

func printKubernetesEvents() error {
	fmt.Println("Printing Kubernetes events for debugging:")
	events, err := framework.KubeClient.CoreV1().Events("").List(metav1.ListOptions{})
	if err != nil {
		return err
	}
	if events != nil {
		for _, e := range events.Items {
			fmt.Printf("FirstTimestamp: '%v', Reason: '%v', Message: '%v'\n", e.FirstTimestamp, e.Reason, e.Message)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Serge Gebhardt. All rights reserved.
//
// Use of this source code is governed by the ISC
// license that can be found in the LICENSE file.

package acd

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestAccount_getInfo(t *testing.T) {
	r := *NewMockResponseOkString(`{ "termsOfUse": "1.0.0", "status": "ACTIVE" }`)
	c := NewMockClient(r)

	info, _, err := c.Account.GetInfo()

	assert.NoError(t, err)
	assert.Equal(t, "ACTIVE", *info.Status)
	assert.Equal(t, "1.0.0", *info.TermsOfUse)
}
<commit_msg>Test account.GetAccount<commit_after>// Copyright (c) 2015 Serge Gebhardt. All rights reserved.
//
// Use of this source code is governed by the ISC
// license that can be found in the LICENSE file.

package acd

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestAccount_getInfo(t *testing.T) {
	r := *NewMockResponseOkString(`{ "termsOfUse": "1.0.0", "status": "ACTIVE" }`)
	c := NewMockClient(r)

	info, _, err := c.Account.GetInfo()

	assert.NoError(t, err)
	assert.Equal(t, "ACTIVE", *info.Status)
	assert.Equal(t, "1.0.0", *info.TermsOfUse)
}

func TestAccount_getQuota(t *testing.T) {
	r := *NewMockResponseOkString(`
{
"quota": 5368709120,
"lastCalculated": "2014-08-13T23:01:47.479Z",
"available": 4069088896
}
	`)
	c := NewMockClient(r)

	quota, _, err := c.Account.GetQuota()

	assert.NoError(t, err)
	assert.Equal(t, "2014-08-13 23:01:47.479 +0000 UTC", quota.LastCalculated.String())
	assert.Equal(t, uint64(5368709120), *quota.Quota)
	assert.Equal(t, uint64(4069088896), *quota.Available)
}
<|endoftext|>


<commit_before>package action

import (
	"fmt"
	"sort"

	"github.com/fatih/color"
	"github.com/justwatchcom/gopass/store"
	"github.com/justwatchcom/gopass/tree/simple"
	"github.com/urfave/cli"
)

// MountRemove removes an existing mount
func (s *Action) MountRemove(c *cli.Context) error {
	if len(c.Args()) != 1 {
		return fmt.Errorf("usage: gopass mount remove [alias]")
	}
	if err := s.Store.RemoveMount(c.Args()[0]); err != nil {
		color.Yellow("Failed to remove mount: %s", err)
	}
	if err := s.Store.Config().Save(); err != nil {
		return err
	}

	color.Green("Password Store %s umounted", c.Args()[0])
	return nil
}

// MountsPrint prints all existing mounts
func (s *Action) MountsPrint(c *cli.Context) error {
	if len(s.Store.Mounts()) < 1 {
		fmt.Println("No mounts")
		return nil
	}
	root := simple.New(color.GreenString(fmt.Sprintf("gopass (%s)", s.Store.Path())))
	mounts := s.Store.Mounts()
	mps := s.Store.MountPoints()
	sort.Sort(store.ByPathLen(mps))
	for _, alias := range mps {
		path := mounts[alias]
		if err := root.AddMount(alias, path); err != nil {
			fmt.Printf("Failed to add mount: %s\n", err)
		}
	}
	fmt.Fprintln(color.Output, root.Format(0))
	return nil
}

// MountsComplete will print a list of existings mount points for bash
// completion
func (s *Action) MountsComplete(*cli.Context) {
	for alias := range s.Store.Mounts() {
		fmt.Println(alias)
	}
}

// MountAdd adds a new mount
func (s *Action) MountAdd(c *cli.Context) error {
	if len(c.Args()) != 2 {
		return fmt.Errorf("usage: gopass mount add [alias] [local path]")
	}
	keys := make([]string, 0, 1)
	if k := c.String("init"); k != "" {
		keys = append(keys, k)
	}
	if err := s.Store.AddMount(c.Args()[0], c.Args()[1], keys...); err != nil {
		return err
	}
	if err := s.Store.Config().Save(); err != nil {
		return err
	}

	color.Green("Mounted %s as %s", c.Args()[0], c.Args()[1])
	return nil
}
<commit_msg>Check for already existing entries, when add mount (#180)<commit_after>package action

import (
	"fmt"
	"sort"

	"github.com/fatih/color"
	"github.com/justwatchcom/gopass/store"
	"github.com/justwatchcom/gopass/tree/simple"
	"github.com/urfave/cli"
)

// MountRemove removes an existing mount
func (s *Action) MountRemove(c *cli.Context) error {
	if len(c.Args()) != 1 {
		return fmt.Errorf("usage: gopass mount remove [alias]")
	}
	if err := s.Store.RemoveMount(c.Args()[0]); err != nil {
		color.Yellow("Failed to remove mount: %s", err)
	}
	if err := s.Store.Config().Save(); err != nil {
		return err
	}

	color.Green("Password Store %s umounted", c.Args()[0])
	return nil
}

// MountsPrint prints all existing mounts
func (s *Action) MountsPrint(c *cli.Context) error {
	if len(s.Store.Mounts()) < 1 {
		fmt.Println("No mounts")
		return nil
	}
	root := simple.New(color.GreenString(fmt.Sprintf("gopass (%s)", s.Store.Path())))
	mounts := s.Store.Mounts()
	mps := s.Store.MountPoints()
	sort.Sort(store.ByPathLen(mps))
	for _, alias := range mps {
		path := mounts[alias]
		if err := root.AddMount(alias, path); err != nil {
			fmt.Printf("Failed to add mount: %s\n", err)
		}
	}
	fmt.Fprintln(color.Output, root.Format(0))
	return nil
}

// MountsComplete will print a list of existings mount points for bash
// completion
func (s *Action) MountsComplete(*cli.Context) {
	for alias := range s.Store.Mounts() {
		fmt.Println(alias)
	}
}

// MountAdd adds a new mount
func (s *Action) MountAdd(c *cli.Context) error {
	if len(c.Args()) != 2 {
		return fmt.Errorf("usage: gopass mount add [alias] [local path]")
	}
	keys := make([]string, 0, 1)
	if k := c.String("init"); k != "" {
		keys = append(keys, k)
	}
	alias, localPath := c.Args()[0], c.Args()[1]
	if s.Store.Exists(alias) {
		fmt.Printf(color.YellowString("WARNING: shadowing %s entry\n"), alias)
	}
	if err := s.Store.AddMount(alias, localPath, keys...); err != nil {
		return err
	}
	if err := s.Store.Config().Save(); err != nil {
		return err
	}

	color.Green("Mounted %s as %s", c.Args()[0], c.Args()[1])
	return nil
}
<|endoftext|>


<commit_before>package actions

import (
	"fmt"
	"io"
	"net/http"
	"os"
	"path"

	"github.com/agupta666/elf/utils"
)

// FileAction action represents actions which responds with the contents of a file
type FileAction struct {
	PatternHolder
	Path string
}

func (fa *FileAction) String() string {
	return fmt.Sprintf("@[Path=%s]", fa.Path)
}

// SetPattern sets the matched pattern in the action
func (fa *FileAction) SetPattern(p string) {}

// NewFileAction creates a new FileAction
func NewFileAction(p string) (*FileAction, error) {
	return &FileAction{Path: p}, nil
}

// Exec executes a file action
func (fa *FileAction) Exec(w http.ResponseWriter, r *http.Request) error {
	reader, err := os.Open(fa.Path)
	if err != nil {
		return err
	}

	emitMimeType(fa.Path, w)
	_, err = io.Copy(w, reader)
	if err != nil {
		return err
	}

	return nil
}

func emitMimeType(fname string, w http.ResponseWriter) {
	ext := path.Ext(fname)
	mimeType := utils.TypeByExtension(ext)
	w.Header().Set("Content-Type", mimeType)
}
<commit_msg>error handling in file action<commit_after>package actions

import (
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"path"

	"github.com/agupta666/elf/utils"
)

// FileAction action represents actions which responds with the contents of a file
type FileAction struct {
	PatternHolder
	Path string
}

func (fa *FileAction) String() string {
	return fmt.Sprintf("@[Path=%s]", fa.Path)
}

// SetPattern sets the matched pattern in the action
func (fa *FileAction) SetPattern(p string) {}

// NewFileAction creates a new FileAction
func NewFileAction(p string) (*FileAction, error) {
	fi, err := os.Lstat(p)

	if err != nil {
		return nil, err
	}

	if !fi.Mode().IsRegular() {
		return nil, errors.New("not a regular file")
	}

	return &FileAction{Path: p}, nil
}

// Exec executes a file action
func (fa *FileAction) Exec(w http.ResponseWriter, r *http.Request) error {
	reader, err := os.Open(fa.Path)
	if err != nil {
		return err
	}

	emitMimeType(fa.Path, w)
	_, err = io.Copy(w, reader)
	if err != nil {
		return err
	}

	return nil
}

func emitMimeType(fname string, w http.ResponseWriter) {
	ext := path.Ext(fname)
	mimeType := utils.TypeByExtension(ext)
	w.Header().Set("Content-Type", mimeType)
}
<|endoftext|>


<commit_before>package operation

type Manifest struct {
	Applications []AppManifest `yaml:"applications"`
}

type AppManifest struct {
	Name       string   `yaml:"name"`
	Buildpacks []string `yaml:"buildpacks,omitempty"`
	Command    string   `yaml:"command,omitempty"`
	DiskQuota  string   `yaml:"disk_quota,omitempty"`
	Docker     struct {
		Image    string `yaml:"image,omitempty"`
		Username string `yaml:"username,omitempty"`
	} `yaml:"docker,omitempty"`
	Env                     map[string]string   `yaml:"env,omitempty"`
	HealthCheckType         string              `yaml:"health-check-type,omitempty"`
	HealthCheckHTTPEndpoint string              `yaml:"health-check-http-endpoint,omitempty"`
	Instances               int                 `yaml:"instances,omitempty"`
	LogRateLimit            string              `yaml:"log-rate-limit,omitempty"`
	Memory                  string              `yaml:"memory,omitempty"`
	NoRoute                 bool                `yaml:"no-route,omitempty"`
	Routes                  []AppManifestRoutes `yaml:"routes,omitempty"`
	Services                []string            `yaml:"services,omitempty"`
	Stack                   string              `yaml:"stack,omitempty"`
	Timeout                 int                 `yaml:"timeout,omitempty"`
}

type AppManifestRoutes struct {
	Route string `yaml:"route,omitempty"`
}
<commit_msg>Add struct for Docker manifest<commit_after>package operation

type Manifest struct {
	Applications []AppManifest `yaml:"applications"`
}

type AppManifest struct {
	Name                    string              `yaml:"name"`
	Buildpacks              []string            `yaml:"buildpacks,omitempty"`
	Command                 string              `yaml:"command,omitempty"`
	DiskQuota               string              `yaml:"disk_quota,omitempty"`
	Docker                  *AppManifestDocker  `yaml:"docker,omitempty"`
	Env                     map[string]string   `yaml:"env,omitempty"`
	HealthCheckType         string              `yaml:"health-check-type,omitempty"`
	HealthCheckHTTPEndpoint string              `yaml:"health-check-http-endpoint,omitempty"`
	Instances               int                 `yaml:"instances,omitempty"`
	LogRateLimit            string              `yaml:"log-rate-limit,omitempty"`
	Memory                  string              `yaml:"memory,omitempty"`
	NoRoute                 bool                `yaml:"no-route,omitempty"`
	Routes                  []AppManifestRoutes `yaml:"routes,omitempty"`
	Services                []string            `yaml:"services,omitempty"`
	Stack                   string              `yaml:"stack,omitempty"`
	Timeout                 int                 `yaml:"timeout,omitempty"`
}

type AppManifestDocker struct {
	Image    string `yaml:"image,omitempty"`
	Username string `yaml:"username,omitempty"`
}

type AppManifestRoutes struct {
	Route string `yaml:"route,omitempty"`
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package oracle_test

// This file defines a test framework for oracle queries.
//
// The files beneath testdata/src/main contain Go programs containing
// query annotations of the form:
//
//   @verb id "select"
//
// where verb is the query mode (e.g. "callers"), id is a unique name
// for this query, and "select" is a regular expression matching the
// substring of the current line that is the query's input selection.
//
// The expected output for each query is provided in the accompanying
// .golden file.
//
// (Location information is not included because it's too fragile to
// display as text.  TODO(adonovan): think about how we can test its
// correctness, since it is critical information.)
//
// Run this test with:
// 	% go test code.google.com/p/go.tools/oracle -update
// to update the golden files.

// TODO(adonovan): improve coverage:
// - output of @callgraph is nondeterministic.
// - as are lists of labels.

import (
	"bytes"
	"flag"
	"fmt"
	"go/build"
	"go/parser"
	"go/token"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"testing"

	"code.google.com/p/go.tools/oracle"
)

var updateFlag = flag.Bool("update", false, "Update the golden files.")

type query struct {
	id         string         // unique id
	verb       string         // query mode, e.g. "callees"
	posn       token.Position // position of of query
	filename   string
	start, end int // selection of file to pass to oracle
}

func parseRegexp(text string) (*regexp.Regexp, error) {
	pattern, err := strconv.Unquote(text)
	if err != nil {
		return nil, fmt.Errorf("can't unquote %s", text)
	}
	return regexp.Compile(pattern)
}

// parseQueries parses and returns the queries in the named file.
func parseQueries(t *testing.T, filename string) []*query {
	filedata, err := ioutil.ReadFile(filename)
	if err != nil {
		t.Fatal(err)
	}

	// Parse the file once to discover the test queries.
	var fset token.FileSet
	f, err := parser.ParseFile(&fset, filename, filedata,
		parser.DeclarationErrors|parser.ParseComments)
	if err != nil {
		t.Fatal(err)
	}

	lines := bytes.Split(filedata, []byte("\n"))

	var queries []*query
	queriesById := make(map[string]*query)

	// Find all annotations of these forms:
	expectRe := regexp.MustCompile(`@([a-z]+)\s+(\S+)\s+(\".*)$`) // @verb id "regexp"
	for _, c := range f.Comments {
		text := strings.TrimSpace(c.Text())
		if text == "" || text[0] != '@' {
			continue
		}
		posn := fset.Position(c.Pos())

		// @verb id "regexp"
		match := expectRe.FindStringSubmatch(text)
		if match == nil {
			t.Errorf("%s: ill-formed query: %s", posn, text)
			continue
		}

		id := match[2]
		if prev, ok := queriesById[id]; ok {
			t.Errorf("%s: duplicate id %s", posn, id)
			t.Errorf("%s: previously used here", prev.posn)
			continue
		}

		selectRe, err := parseRegexp(match[3])
		if err != nil {
			t.Errorf("%s: %s", posn, err)
			continue
		}

		// Find text of the current line, sans query.
		// (Queries must be // not /**/ comments.)
		line := lines[posn.Line-1][:posn.Column-1]

		// Apply regexp to current line to find input selection.
		loc := selectRe.FindIndex(line)
		if loc == nil {
			t.Errorf("%s: selection pattern %s doesn't match line %q",
				posn, match[3], string(line))
			continue
		}

		// Assumes ASCII. TODO(adonovan): test on UTF-8.
		linestart := posn.Offset - (posn.Column - 1)

		// Compute the file offsets
		q := &query{
			id:       id,
			verb:     match[1],
			posn:     posn,
			filename: filename,
			start:    linestart + loc[0],
			end:      linestart + loc[1],
		}
		queries = append(queries, q)
		queriesById[id] = q
	}

	// Return the slice, not map, for deterministic iteration.
	return queries
}

// stripLocation removes a "file:line: " prefix.
func stripLocation(line string) string {
	if i := strings.Index(line, ": "); i >= 0 {
		line = line[i+2:]
	}
	return line
}

// doQuery poses query q to the oracle and writes its response and
// error (if any) to out.
func doQuery(out io.Writer, q *query) {
	fmt.Fprintf(out, "-------- @%s %s --------\n", q.verb, q.id)

	capture := new(bytes.Buffer) // capture standard output
	var buildContext = build.Default
	buildContext.GOPATH = "testdata"
	err := oracle.Main([]string{q.filename},
		q.verb,
		fmt.Sprintf("%s %d-%d", q.filename, q.start, q.end),
		/*PTA-log=*/ nil, capture, &buildContext)

	for _, line := range strings.Split(capture.String(), "\n") {
		fmt.Fprintf(out, "%s\n", stripLocation(line))
	}

	if err != nil {
		fmt.Fprintf(out, "Error: %s\n", stripLocation(err.Error()))
	}
}

func TestOracle(t *testing.T) {
	for _, filename := range []string{
		"testdata/src/main/calls.go",
		"testdata/src/main/describe.go",
		"testdata/src/main/freevars.go",
		"testdata/src/main/implements.go",
		"testdata/src/main/imports.go",
		"testdata/src/main/peers.go",
	} {
		queries := parseQueries(t, filename)
		golden := filename + "lden"
		got := filename + "t"
		gotfh, err := os.Create(got)
		if err != nil {
			t.Errorf("Create(%s) failed: %s", got, err)
			continue
		}
		defer gotfh.Close()

		// Run the oracle on each query, redirecting its output
		// and error (if any) to the foo.got file.
		for _, q := range queries {
			doQuery(gotfh, q)
		}

		// Compare foo.got with foo.golden.
		cmd := exec.Command("/usr/bin/diff", "-u3", golden, got) // assumes POSIX
		buf := new(bytes.Buffer)
		cmd.Stdout = buf
		if err := cmd.Run(); err != nil {
			t.Errorf("Oracle tests for %s failed: %s.\n%s\n",
				filename, err, buf)

			if *updateFlag {
				t.Logf("Updating %s...", golden)
				if err := exec.Command("/bin/cp", got, golden).Run(); err != nil {
					t.Errorf("Update failed: %s", err)
				}
			}
		}
	}
}
<commit_msg>go.tools/oracle: disable TestOracle test on windows (fixes windows build)<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package oracle_test

// This file defines a test framework for oracle queries.
//
// The files beneath testdata/src/main contain Go programs containing
// query annotations of the form:
//
//   @verb id "select"
//
// where verb is the query mode (e.g. "callers"), id is a unique name
// for this query, and "select" is a regular expression matching the
// substring of the current line that is the query's input selection.
//
// The expected output for each query is provided in the accompanying
// .golden file.
//
// (Location information is not included because it's too fragile to
// display as text.  TODO(adonovan): think about how we can test its
// correctness, since it is critical information.)
//
// Run this test with:
// 	% go test code.google.com/p/go.tools/oracle -update
// to update the golden files.

// TODO(adonovan): improve coverage:
// - output of @callgraph is nondeterministic.
// - as are lists of labels.

import (
	"bytes"
	"flag"
	"fmt"
	"go/build"
	"go/parser"
	"go/token"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"testing"

	"code.google.com/p/go.tools/oracle"
)

var updateFlag = flag.Bool("update", false, "Update the golden files.")

type query struct {
	id         string         // unique id
	verb       string         // query mode, e.g. "callees"
	posn       token.Position // position of of query
	filename   string
	start, end int // selection of file to pass to oracle
}

func parseRegexp(text string) (*regexp.Regexp, error) {
	pattern, err := strconv.Unquote(text)
	if err != nil {
		return nil, fmt.Errorf("can't unquote %s", text)
	}
	return regexp.Compile(pattern)
}

// parseQueries parses and returns the queries in the named file.
func parseQueries(t *testing.T, filename string) []*query {
	filedata, err := ioutil.ReadFile(filename)
	if err != nil {
		t.Fatal(err)
	}

	// Parse the file once to discover the test queries.
	var fset token.FileSet
	f, err := parser.ParseFile(&fset, filename, filedata,
		parser.DeclarationErrors|parser.ParseComments)
	if err != nil {
		t.Fatal(err)
	}

	lines := bytes.Split(filedata, []byte("\n"))

	var queries []*query
	queriesById := make(map[string]*query)

	// Find all annotations of these forms:
	expectRe := regexp.MustCompile(`@([a-z]+)\s+(\S+)\s+(\".*)$`) // @verb id "regexp"
	for _, c := range f.Comments {
		text := strings.TrimSpace(c.Text())
		if text == "" || text[0] != '@' {
			continue
		}
		posn := fset.Position(c.Pos())

		// @verb id "regexp"
		match := expectRe.FindStringSubmatch(text)
		if match == nil {
			t.Errorf("%s: ill-formed query: %s", posn, text)
			continue
		}

		id := match[2]
		if prev, ok := queriesById[id]; ok {
			t.Errorf("%s: duplicate id %s", posn, id)
			t.Errorf("%s: previously used here", prev.posn)
			continue
		}

		selectRe, err := parseRegexp(match[3])
		if err != nil {
			t.Errorf("%s: %s", posn, err)
			continue
		}

		// Find text of the current line, sans query.
		// (Queries must be // not /**/ comments.)
		line := lines[posn.Line-1][:posn.Column-1]

		// Apply regexp to current line to find input selection.
		loc := selectRe.FindIndex(line)
		if loc == nil {
			t.Errorf("%s: selection pattern %s doesn't match line %q",
				posn, match[3], string(line))
			continue
		}

		// Assumes ASCII. TODO(adonovan): test on UTF-8.
		linestart := posn.Offset - (posn.Column - 1)

		// Compute the file offsets
		q := &query{
			id:       id,
			verb:     match[1],
			posn:     posn,
			filename: filename,
			start:    linestart + loc[0],
			end:      linestart + loc[1],
		}
		queries = append(queries, q)
		queriesById[id] = q
	}

	// Return the slice, not map, for deterministic iteration.
	return queries
}

// stripLocation removes a "file:line: " prefix.
func stripLocation(line string) string {
	if i := strings.Index(line, ": "); i >= 0 {
		line = line[i+2:]
	}
	return line
}

// doQuery poses query q to the oracle and writes its response and
// error (if any) to out.
func doQuery(out io.Writer, q *query) {
	fmt.Fprintf(out, "-------- @%s %s --------\n", q.verb, q.id)

	capture := new(bytes.Buffer) // capture standard output
	var buildContext = build.Default
	buildContext.GOPATH = "testdata"
	err := oracle.Main([]string{q.filename},
		q.verb,
		fmt.Sprintf("%s %d-%d", q.filename, q.start, q.end),
		/*PTA-log=*/ nil, capture, &buildContext)

	for _, line := range strings.Split(capture.String(), "\n") {
		fmt.Fprintf(out, "%s\n", stripLocation(line))
	}

	if err != nil {
		fmt.Fprintf(out, "Error: %s\n", stripLocation(err.Error()))
	}
}

func TestOracle(t *testing.T) {
	switch runtime.GOOS {
	case "windows":
		t.Skipf("skipping test on %q (no /usr/bin/diff)", runtime.GOOS)
	}

	for _, filename := range []string{
		"testdata/src/main/calls.go",
		"testdata/src/main/describe.go",
		"testdata/src/main/freevars.go",
		"testdata/src/main/implements.go",
		"testdata/src/main/imports.go",
		"testdata/src/main/peers.go",
	} {
		queries := parseQueries(t, filename)
		golden := filename + "lden"
		got := filename + "t"
		gotfh, err := os.Create(got)
		if err != nil {
			t.Errorf("Create(%s) failed: %s", got, err)
			continue
		}
		defer gotfh.Close()

		// Run the oracle on each query, redirecting its output
		// and error (if any) to the foo.got file.
		for _, q := range queries {
			doQuery(gotfh, q)
		}

		// Compare foo.got with foo.golden.
		cmd := exec.Command("/usr/bin/diff", "-u3", golden, got) // assumes POSIX
		buf := new(bytes.Buffer)
		cmd.Stdout = buf
		if err := cmd.Run(); err != nil {
			t.Errorf("Oracle tests for %s failed: %s.\n%s\n",
				filename, err, buf)

			if *updateFlag {
				t.Logf("Updating %s...", golden)
				if err := exec.Command("/bin/cp", got, golden).Run(); err != nil {
					t.Errorf("Update failed: %s", err)
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package osutil

import (
	"io/ioutil"
	"os"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestExistsFile(t *testing.T) {
	assert.False(t, ExistsFile("/paht/to/notexist"))

	tmpDir := os.TempDir()
	tmp, _ := ioutil.TempFile(tmpDir, "droot_test")
	defer func() {
		tmp.Close()
		os.Remove(tmp.Name())
	}()

	assert.True(t, ExistsFile(tmp.Name()))
	assert.False(t, ExistsFile(tmpDir))
}

func TestIsSymlink(t *testing.T) {
	tmpDir := os.TempDir()
	tmp, _ := ioutil.TempFile(tmpDir, "droot_test")
	defer func() {
		tmp.Close()
		os.Remove(tmp.Name())
	}()

	assert.False(t, IsSymlink(tmp.Name()))

	os.Symlink(tmp.Name(), tmpDir+"/symlink")

	assert.True(t, IsSymlink(tmpDir+"/symlink"))
}

func TestExistsDir(t *testing.T) {
	assert.False(t, ExistsDir("/paht/to/notexist"))

	tmpDir := os.TempDir()
	tmp, _ := ioutil.TempFile(tmpDir, "droot_test")
	defer func() {
		tmp.Close()
		os.Remove(tmp.Name())
	}()

	assert.True(t, ExistsDir(tmpDir))
	assert.False(t, ExistsDir(tmp.Name()))
}

func TestIsDirEmpty(t *testing.T) {
	ok, err := IsDirEmpty("/paht/to/notexist")
	assert.False(t, ok)
	assert.Error(t, err)

	tmpDir := os.TempDir()
	os.Mkdir(tmpDir+"/empty", 0755)
	os.Mkdir(tmpDir+"/noempty", 0755)
	os.Create(tmpDir+"/noempty/test")
	defer func() {
		os.Remove(tmpDir+"/empty")
		os.RemoveAll(tmpDir+"/noempty")
	}()

	ok, err = IsDirEmpty(tmpDir+"/empty")
	assert.True(t, ok)
	assert.NoError(t, err)

	ok, err = IsDirEmpty(tmpDir+"/noempty")
	assert.False(t, ok)
	assert.NoError(t, err)
}

<commit_msg>Add TestRunCmd<commit_after>package osutil

import (
	"io/ioutil"
	"os"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestExistsFile(t *testing.T) {
	assert.False(t, ExistsFile("/paht/to/notexist"))

	tmpDir := os.TempDir()
	tmp, _ := ioutil.TempFile(tmpDir, "droot_test")
	defer func() {
		tmp.Close()
		os.Remove(tmp.Name())
	}()

	assert.True(t, ExistsFile(tmp.Name()))
	assert.False(t, ExistsFile(tmpDir))
}

func TestIsSymlink(t *testing.T) {
	tmpDir := os.TempDir()
	tmp, _ := ioutil.TempFile(tmpDir, "droot_test")
	defer func() {
		tmp.Close()
		os.Remove(tmp.Name())
	}()

	assert.False(t, IsSymlink(tmp.Name()))

	os.Symlink(tmp.Name(), tmpDir+"/symlink")

	assert.True(t, IsSymlink(tmpDir+"/symlink"))
}

func TestExistsDir(t *testing.T) {
	assert.False(t, ExistsDir("/paht/to/notexist"))

	tmpDir := os.TempDir()
	tmp, _ := ioutil.TempFile(tmpDir, "droot_test")
	defer func() {
		tmp.Close()
		os.Remove(tmp.Name())
	}()

	assert.True(t, ExistsDir(tmpDir))
	assert.False(t, ExistsDir(tmp.Name()))
}

func TestIsDirEmpty(t *testing.T) {
	ok, err := IsDirEmpty("/paht/to/notexist")
	assert.False(t, ok)
	assert.Error(t, err)

	tmpDir := os.TempDir()
	os.Mkdir(tmpDir+"/empty", 0755)
	os.Mkdir(tmpDir+"/noempty", 0755)
	os.Create(tmpDir+"/noempty/test")
	defer func() {
		os.Remove(tmpDir+"/empty")
		os.RemoveAll(tmpDir+"/noempty")
	}()

	ok, err = IsDirEmpty(tmpDir+"/empty")
	assert.True(t, ok)
	assert.NoError(t, err)

	ok, err = IsDirEmpty(tmpDir+"/noempty")
	assert.False(t, ok)
	assert.NoError(t, err)
}

func TestRunCmd(t *testing.T) {
	assert.NoError(t, RunCmd("/bin/ls"))
	assert.Error(t, RunCmd("/bin/hoge"))
}
<|endoftext|>


<commit_before>package schema

import (
	"fmt"
	"strings"

	"github.com/hashicorp/terraform/terraform"
	"github.com/mitchellh/mapstructure"
)

// DiffFieldReader reads fields out of a diff structures.
//
// It also requires access to a Reader that reads fields from the structure
// that the diff was derived from. This is usually the state. This is required
// because a diff on its own doesn't have complete data about full objects
// such as maps.
//
// The Source MUST be the data that the diff was derived from. If it isn't,
// the behavior of this struct is undefined.
//
// Reading fields from a DiffFieldReader is identical to reading from
// Source except the diff will be applied to the end result.
//
// The "Exists" field on the result will be set to true if the complete
// field exists whether its from the source, diff, or a combination of both.
// It cannot be determined whether a retrieved value is composed of
// diff elements.
type DiffFieldReader struct {
	Diff   *terraform.InstanceDiff
	Source FieldReader
	Schema map[string]*Schema
}

func (r *DiffFieldReader) ReadField(address []string) (FieldReadResult, error) {
	schemaList := addrToSchema(address, r.Schema)
	if len(schemaList) == 0 {
		return FieldReadResult{}, nil
	}

	schema := schemaList[len(schemaList)-1]
	switch schema.Type {
	case TypeBool, TypeInt, TypeFloat, TypeString:
		return r.readPrimitive(address, schema)
	case TypeList:
		return readListField(r, address, schema)
	case TypeMap:
		return r.readMap(address, schema)
	case TypeSet:
		return r.readSet(address, schema)
	case typeObject:
		return readObjectField(r, address, schema.Elem.(map[string]*Schema))
	default:
		panic(fmt.Sprintf("Unknown type: %#v", schema.Type))
	}
}

func (r *DiffFieldReader) readMap(
	address []string, schema *Schema) (FieldReadResult, error) {
	result := make(map[string]interface{})
	resultSet := false

	// First read the map from the underlying source
	source, err := r.Source.ReadField(address)
	if err != nil {
		return FieldReadResult{}, err
	}
	if source.Exists {
		result = source.Value.(map[string]interface{})
		resultSet = true
	}

	// Next, read all the elements we have in our diff, and apply
	// the diff to our result.
	prefix := strings.Join(address, ".") + "."
	for k, v := range r.Diff.Attributes {
		if !strings.HasPrefix(k, prefix) {
			continue
		}
		if strings.HasPrefix(k, prefix+"%") {
			// Ignore the count field
			continue
		}

		resultSet = true

		k = k[len(prefix):]
		if v.NewRemoved {
			delete(result, k)
			continue
		}

		result[k] = v.New
	}

	err = mapValuesToPrimitive(result, schema)
	if err != nil {
		return FieldReadResult{}, nil
	}

	var resultVal interface{}
	if resultSet {
		resultVal = result
	}

	return FieldReadResult{
		Value:  resultVal,
		Exists: resultSet,
	}, nil
}

func (r *DiffFieldReader) readPrimitive(
	address []string, schema *Schema) (FieldReadResult, error) {
	result, err := r.Source.ReadField(address)
	if err != nil {
		return FieldReadResult{}, err
	}

	attrD, ok := r.Diff.Attributes[strings.Join(address, ".")]
	if !ok {
		return result, nil
	}

	var resultVal string
	if !attrD.NewComputed {
		resultVal = attrD.New
		if attrD.NewExtra != nil {
			result.ValueProcessed = resultVal
			if err := mapstructure.WeakDecode(attrD.NewExtra, &resultVal); err != nil {
				return FieldReadResult{}, err
			}
		}
	}

	result.Computed = attrD.NewComputed
	result.Exists = true
	result.Value, err = stringToPrimitive(resultVal, false, schema)
	if err != nil {
		return FieldReadResult{}, err
	}

	return result, nil
}

func (r *DiffFieldReader) readSet(
	address []string, schema *Schema) (FieldReadResult, error) {
	prefix := strings.Join(address, ".") + "."

	// Create the set that will be our result
	set := schema.ZeroValue().(*Set)

	// Go through the map and find all the set items
	for k, d := range r.Diff.Attributes {
		if d.NewRemoved {
			// If the field is removed, we always ignore it
			continue
		}
		if !strings.HasPrefix(k, prefix) {
			continue
		}
		if strings.HasSuffix(k, "#") {
			// Ignore any count field
			continue
		}

		// Split the key, since it might be a sub-object like "idx.field"
		parts := strings.Split(k[len(prefix):], ".")
		idx := parts[0]

		raw, err := r.ReadField(append(address, idx))
		if err != nil {
			return FieldReadResult{}, err
		}
		if !raw.Exists {
			// This shouldn't happen because we just verified it does exist
			panic("missing field in set: " + k + "." + idx)
		}

		set.Add(raw.Value)
	}

	// Determine if the set "exists". It exists if there are items or if
	// the diff explicitly wanted it empty.
	exists := set.Len() > 0
	if !exists {
		// We could check if the diff value is "0" here but I think the
		// existence of "#" on its own is enough to show it existed. This
		// protects us in the future from the zero value changing from
		// "0" to "" breaking us (if that were to happen).
		if _, ok := r.Diff.Attributes[prefix+"#"]; ok {
			exists = true
		}
	}

	if !exists {
		result, err := r.Source.ReadField(address)
		if err != nil {
			return FieldReadResult{}, err
		}
		if result.Exists {
			return result, nil
		}
	}

	return FieldReadResult{
		Value:  set,
		Exists: exists,
	}, nil
}
<commit_msg>memoize DiffFieldReader.ReadField<commit_after>package schema

import (
	"fmt"
	"strings"

	"github.com/hashicorp/terraform/terraform"
	"github.com/mitchellh/mapstructure"
)

// DiffFieldReader reads fields out of a diff structures.
//
// It also requires access to a Reader that reads fields from the structure
// that the diff was derived from. This is usually the state. This is required
// because a diff on its own doesn't have complete data about full objects
// such as maps.
//
// The Source MUST be the data that the diff was derived from. If it isn't,
// the behavior of this struct is undefined.
//
// Reading fields from a DiffFieldReader is identical to reading from
// Source except the diff will be applied to the end result.
//
// The "Exists" field on the result will be set to true if the complete
// field exists whether its from the source, diff, or a combination of both.
// It cannot be determined whether a retrieved value is composed of
// diff elements.
type DiffFieldReader struct {
	Diff   *terraform.InstanceDiff
	Source FieldReader
	Schema map[string]*Schema

	// cache for memoizing ReadField calls.
	cache map[string]cachedFieldReadResult
}

type cachedFieldReadResult struct {
	val FieldReadResult
	err error
}

func (r *DiffFieldReader) ReadField(address []string) (FieldReadResult, error) {
	if r.cache == nil {
		r.cache = make(map[string]cachedFieldReadResult)
	}

	// Create the cache key by joining around a value that isn't a valid part
	// of an address. This assumes that the Source and Schema are not changed
	// for the life of this DiffFieldReader.
	cacheKey := strings.Join(address, "|")
	if cached, ok := r.cache[cacheKey]; ok {
		return cached.val, cached.err
	}

	schemaList := addrToSchema(address, r.Schema)
	if len(schemaList) == 0 {
		r.cache[cacheKey] = cachedFieldReadResult{}
		return FieldReadResult{}, nil
	}

	var res FieldReadResult
	var err error

	schema := schemaList[len(schemaList)-1]
	switch schema.Type {
	case TypeBool, TypeInt, TypeFloat, TypeString:
		res, err = r.readPrimitive(address, schema)
	case TypeList:
		res, err = readListField(r, address, schema)
	case TypeMap:
		res, err = r.readMap(address, schema)
	case TypeSet:
		res, err = r.readSet(address, schema)
	case typeObject:
		res, err = readObjectField(r, address, schema.Elem.(map[string]*Schema))
	default:
		panic(fmt.Sprintf("Unknown type: %#v", schema.Type))
	}

	r.cache[cacheKey] = cachedFieldReadResult{
		val: res,
		err: err,
	}
	return res, err
}

func (r *DiffFieldReader) readMap(
	address []string, schema *Schema) (FieldReadResult, error) {
	result := make(map[string]interface{})
	resultSet := false

	// First read the map from the underlying source
	source, err := r.Source.ReadField(address)
	if err != nil {
		return FieldReadResult{}, err
	}
	if source.Exists {
		result = source.Value.(map[string]interface{})
		resultSet = true
	}

	// Next, read all the elements we have in our diff, and apply
	// the diff to our result.
	prefix := strings.Join(address, ".") + "."
	for k, v := range r.Diff.Attributes {
		if !strings.HasPrefix(k, prefix) {
			continue
		}
		if strings.HasPrefix(k, prefix+"%") {
			// Ignore the count field
			continue
		}

		resultSet = true

		k = k[len(prefix):]
		if v.NewRemoved {
			delete(result, k)
			continue
		}

		result[k] = v.New
	}

	err = mapValuesToPrimitive(result, schema)
	if err != nil {
		return FieldReadResult{}, nil
	}

	var resultVal interface{}
	if resultSet {
		resultVal = result
	}

	return FieldReadResult{
		Value:  resultVal,
		Exists: resultSet,
	}, nil
}

func (r *DiffFieldReader) readPrimitive(
	address []string, schema *Schema) (FieldReadResult, error) {
	result, err := r.Source.ReadField(address)
	if err != nil {
		return FieldReadResult{}, err
	}

	attrD, ok := r.Diff.Attributes[strings.Join(address, ".")]
	if !ok {
		return result, nil
	}

	var resultVal string
	if !attrD.NewComputed {
		resultVal = attrD.New
		if attrD.NewExtra != nil {
			result.ValueProcessed = resultVal
			if err := mapstructure.WeakDecode(attrD.NewExtra, &resultVal); err != nil {
				return FieldReadResult{}, err
			}
		}
	}

	result.Computed = attrD.NewComputed
	result.Exists = true
	result.Value, err = stringToPrimitive(resultVal, false, schema)
	if err != nil {
		return FieldReadResult{}, err
	}

	return result, nil
}

func (r *DiffFieldReader) readSet(
	address []string, schema *Schema) (FieldReadResult, error) {
	prefix := strings.Join(address, ".") + "."

	// Create the set that will be our result
	set := schema.ZeroValue().(*Set)

	// Go through the map and find all the set items
	for k, d := range r.Diff.Attributes {
		if d.NewRemoved {
			// If the field is removed, we always ignore it
			continue
		}
		if !strings.HasPrefix(k, prefix) {
			continue
		}
		if strings.HasSuffix(k, "#") {
			// Ignore any count field
			continue
		}

		// Split the key, since it might be a sub-object like "idx.field"
		parts := strings.Split(k[len(prefix):], ".")
		idx := parts[0]

		raw, err := r.ReadField(append(address, idx))
		if err != nil {
			return FieldReadResult{}, err
		}
		if !raw.Exists {
			// This shouldn't happen because we just verified it does exist
			panic("missing field in set: " + k + "." + idx)
		}

		set.Add(raw.Value)
	}

	// Determine if the set "exists". It exists if there are items or if
	// the diff explicitly wanted it empty.
	exists := set.Len() > 0
	if !exists {
		// We could check if the diff value is "0" here but I think the
		// existence of "#" on its own is enough to show it existed. This
		// protects us in the future from the zero value changing from
		// "0" to "" breaking us (if that were to happen).
		if _, ok := r.Diff.Attributes[prefix+"#"]; ok {
			exists = true
		}
	}

	if !exists {
		result, err := r.Source.ReadField(address)
		if err != nil {
			return FieldReadResult{}, err
		}
		if result.Exists {
			return result, nil
		}
	}

	return FieldReadResult{
		Value:  set,
		Exists: exists,
	}, nil
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Delete_file implements the probe operation for deleting a file in a
// storage system.

// Package delete implements the file deletion operation with a storage system.
package delete

import (
	"context"
	"fmt"
	"math/rand"
	"time"

	"cloud.google.com/go/storage"
	"github.com/google/cloudprober/logger"
	"github.com/googleapis/google-cloud-go-testing/storage/stiface"
	"google.golang.org/api/iterator"

	probe "github.com/googleinterns/step224-2020/hermes/probe"
	m "github.com/googleinterns/step224-2020/hermes/probe/metrics"
	pb "github.com/googleinterns/step224-2020/hermes/proto"
)

// DeleteRandomFile deletes a random file in the target storage system bucket.
// It then checks that the file has been deleted by trying to get the object.
// Arguments:
//	- ctx: pass the context so this probe can be cancelled if needed.
//	- target: pass the target run information.
//	- client: pass an initialised storage client for this target system.
//	- logger: pass the logger associated with the probe calling this function.
//		-> logger can log to Google Cloud if configured to do so.
// Returns:
//	- fileID: returns the ID of the file delete OR a missing file to be created if one is found.
//	- err:
//		Status:
//		- StateJournalInconsistent: the file to be deleted does not exist in Hermes' StateJournal.
//		- FileMissing: the file to be deleted could not be found in the target bucket.
//		- BucketMissing: the target bucket on this target system was not found.
//		- ProbeFailed: there was an error during one of the API calls and the probe failed.
func DeleteRandomFile(ctx context.Context, target *probe.Target, client stiface.Client, logger *logger.Logger) (int32, error) {
	fileID := pickFileToDelete()
	return DeleteFile(ctx, fileID, target, client, logger)
}

// DeleteFile deletes the file, corresponding to the ID passed, in the target storage system bucket.
// It then checks that the file has been deleted by trying to get the object.
// Arguments:
//	- ctx: pass the context so this probe can be cancelled if needed.
//	- config: pass the HermesProbeDef config for the probe calling this function.
//	- target: pass the target run information.
//	- client: pass an initialised storage client for this target system.
//	- logger: pass the logger associated with the probe calling this function.
// Returns:
//	- fileID: returns the ID of the file delete OR a missing file to be created if one is found.
//	- err:
//		Status:
//		- StateJournalInconsistent: the file to be deleted does not exist in Hermes' StateJournal.
//		- FileMissing: the file to be deleted could not be found in the target bucket.
//		- BucketMissing: the target bucket on this target system was not found.
//		- ProbeFailed: there was an error during one of the API calls and the probe failed.
func DeleteFile(ctx context.Context, fileID int32, target *probe.Target, client stiface.Client, logger *logger.Logger) (int32, error) {
	bucket := target.Target.GetBucketName()

	// TODO(evanSpendlove): Add custom error object to return value and modify all returns.
	filename, ok := target.Journal.Filenames[fileID]
	if !ok {
		return fileID, fmt.Errorf("delete(%q, %q) failed; status StateJournalInconsistent: expected fileID %d to exist", bucket, filename, fileID)
	}

	target.Journal.Intent = &pb.Intent{
		FileOperation: pb.Intent_CREATE,
		Filename:      filename,
	}

	file := client.Bucket(bucket).Object(filename)

	start := time.Now()
	if err := file.Delete(ctx); err != nil {
		var status m.ExitStatus
		switch err {
		case storage.ErrObjectNotExist:
			status = m.FileMissing
		case storage.ErrBucketNotExist:
			status = m.BucketMissing
		default:
			status = m.ProbeFailed
		}

		target.LatencyMetrics.APICallLatency[m.APIDeleteFile][status].Metric("hermes_api_latency_s").AddFloat64(time.Now().Sub(start).Seconds())
		return fileID, fmt.Errorf("delete(%q, %q) failed; status %v: %w", bucket, filename, status, err)
	}
	target.LatencyMetrics.APICallLatency[m.APIDeleteFile][m.Success].Metric("hermes_api_latency_s").AddFloat64(time.Now().Sub(start).Seconds())

	query := &storage.Query{Prefix: filename}
	start = time.Now()
	objects := client.Bucket(bucket).Objects(ctx, query)
	target.LatencyMetrics.APICallLatency[m.APIListFiles][m.Success].Metric("hermes_api_latency_s").AddFloat64(time.Now().Sub(start).Seconds())
	for {
		obj, err := objects.Next()
		if err == iterator.Done {
			break
		}
		if obj.Name == filename {
			status := m.ProbeFailed
			return fileID, fmt.Errorf("delete(%q, %q) failed; status %v: object %v still listed after delete", bucket, filename, status, obj.Name)
		}
		if err != nil {
			status := m.BucketMissing
			return fileID, fmt.Errorf("delete(%q, %q) failed; status %v: %w", bucket, filename, status, err)
		}
	}

	// Update in-memory NIL file after delete operation.
	delete(target.Journal.Filenames, int32(fileID))

	logger.Infof("Object %v deleted in bucket %s.", file, bucket)
	return fileID, nil
}

// pickFileToDelete picks which file to delete and returns the integer ID of this file.
// Returns:
//	- ID: returns the ID of the file to be deleted.
func pickFileToDelete() int32 {
	const (
		beg                    = 10 // we can delete files staring from the file Hermes_10
		numberOfDeletableFiles = 41 // there are 41 files to delete from [Hermes_10,Hermes_50]
	)
	rand.Seed(time.Now().UnixNano())
	return int32(rand.Intn(numberOfDeletableFiles) + beg)
}
<commit_msg>Refactored constants in pickFileToDelete().<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Delete_file implements the probe operation for deleting a file in a
// storage system.

// Package delete implements the file deletion operation with a storage system.
package delete

import (
	"context"
	"fmt"
	"math/rand"
	"time"

	"cloud.google.com/go/storage"
	"github.com/google/cloudprober/logger"
	"github.com/googleapis/google-cloud-go-testing/storage/stiface"
	"google.golang.org/api/iterator"

	probe "github.com/googleinterns/step224-2020/hermes/probe"
	m "github.com/googleinterns/step224-2020/hermes/probe/metrics"
	pb "github.com/googleinterns/step224-2020/hermes/proto"
)

// DeleteRandomFile deletes a random file in the target storage system bucket.
// It then checks that the file has been deleted by trying to get the object.
// Arguments:
//	- ctx: pass the context so this probe can be cancelled if needed.
//	- target: pass the target run information.
//	- client: pass an initialised storage client for this target system.
//	- logger: pass the logger associated with the probe calling this function.
//		-> logger can log to Google Cloud if configured to do so.
// Returns:
//	- fileID: returns the ID of the file delete OR a missing file to be created if one is found.
//	- err:
//		Status:
//		- StateJournalInconsistent: the file to be deleted does not exist in Hermes' StateJournal.
//		- FileMissing: the file to be deleted could not be found in the target bucket.
//		- BucketMissing: the target bucket on this target system was not found.
//		- ProbeFailed: there was an error during one of the API calls and the probe failed.
func DeleteRandomFile(ctx context.Context, target *probe.Target, client stiface.Client, logger *logger.Logger) (int32, error) {
	fileID := pickFileToDelete()
	return DeleteFile(ctx, fileID, target, client, logger)
}

// DeleteFile deletes the file, corresponding to the ID passed, in the target storage system bucket.
// It then checks that the file has been deleted by trying to get the object.
// Arguments:
//	- ctx: pass the context so this probe can be cancelled if needed.
//	- config: pass the HermesProbeDef config for the probe calling this function.
//	- target: pass the target run information.
//	- client: pass an initialised storage client for this target system.
//	- logger: pass the logger associated with the probe calling this function.
// Returns:
//	- fileID: returns the ID of the file delete OR a missing file to be created if one is found.
//	- err:
//		Status:
//		- StateJournalInconsistent: the file to be deleted does not exist in Hermes' StateJournal.
//		- FileMissing: the file to be deleted could not be found in the target bucket.
//		- BucketMissing: the target bucket on this target system was not found.
//		- ProbeFailed: there was an error during one of the API calls and the probe failed.
func DeleteFile(ctx context.Context, fileID int32, target *probe.Target, client stiface.Client, logger *logger.Logger) (int32, error) {
	bucket := target.Target.GetBucketName()

	// TODO(evanSpendlove): Add custom error object to return value and modify all returns.
	filename, ok := target.Journal.Filenames[fileID]
	if !ok {
		return fileID, fmt.Errorf("delete(%q, %q) failed; status StateJournalInconsistent: expected fileID %d to exist", bucket, filename, fileID)
	}

	target.Journal.Intent = &pb.Intent{
		FileOperation: pb.Intent_CREATE,
		Filename:      filename,
	}

	file := client.Bucket(bucket).Object(filename)

	start := time.Now()
	if err := file.Delete(ctx); err != nil {
		var status m.ExitStatus
		switch err {
		case storage.ErrObjectNotExist:
			status = m.FileMissing
		case storage.ErrBucketNotExist:
			status = m.BucketMissing
		default:
			status = m.ProbeFailed
		}

		target.LatencyMetrics.APICallLatency[m.APIDeleteFile][status].Metric("hermes_api_latency_s").AddFloat64(time.Now().Sub(start).Seconds())
		return fileID, fmt.Errorf("delete(%q, %q) failed; status %v: %w", bucket, filename, status, err)
	}
	target.LatencyMetrics.APICallLatency[m.APIDeleteFile][m.Success].Metric("hermes_api_latency_s").AddFloat64(time.Now().Sub(start).Seconds())

	query := &storage.Query{Prefix: filename}
	start = time.Now()
	objects := client.Bucket(bucket).Objects(ctx, query)
	target.LatencyMetrics.APICallLatency[m.APIListFiles][m.Success].Metric("hermes_api_latency_s").AddFloat64(time.Now().Sub(start).Seconds())
	for {
		obj, err := objects.Next()
		if err == iterator.Done {
			break
		}
		if obj.Name == filename {
			status := m.ProbeFailed
			return fileID, fmt.Errorf("delete(%q, %q) failed; status %v: object %v still listed after delete", bucket, filename, status, obj.Name)
		}
		if err != nil {
			status := m.BucketMissing
			return fileID, fmt.Errorf("delete(%q, %q) failed; status %v: %w", bucket, filename, status, err)
		}
	}

	// Update in-memory NIL file after delete operation.
	delete(target.Journal.Filenames, int32(fileID))

	logger.Infof("Object %v deleted in bucket %s.", file, bucket)
	return fileID, nil
}

// pickFileToDelete picks which file to delete and returns the integer ID of this file.
// Returns:
//	- ID: returns the ID of the file to be deleted.
func pickFileToDelete() int32 {
	const (
		begin                  = 11 // we can delete files staring from the file Hermes_11
		numberOfDeletableFiles = 40 // there are 40 files to delete from [Hermes_11,Hermes_50]
	)
	rand.Seed(time.Now().UnixNano())
	return int32(rand.Intn(numberOfDeletableFiles) + being)
}
<|endoftext|>


<commit_before>package ams

import (
	"context"
	"fmt"
	"html"
	"net/http"

	"github.com/pkg/errors"
	"time"
)

type MetaData struct {
	URI string `json:"uri"`
}

type MediaAsset struct {
	MetaData MetaData `json:"__metadata"`
}

const (
	taskBodyTemplate = "<?xml version=\"1.0\" encoding=\"utf-8\"?><taskBody><inputAsset>JobInputAsset(0)</inputAsset><outputAsset assetName=\"%s\">JobOutputAsset(0)</outputAsset></taskBody>"
)

const (
	JobQueued = iota
	JobScheduled
	JobProcessing
	JobFinished
	JobError
	JobCanceled
	JobCanceling
)

func NewMediaAsset(uri string) MediaAsset {
	return MediaAsset{
		MetaData: MetaData{
			URI: uri,
		},
	}
}

type Task struct {
	Name             string `json:"Name"`
	Configuration    string `json:"Configuration"`
	MediaProcessorID string `json:"MediaProcessorId"`
	TaskBody         string `json:"TaskBody"`
}

type Job struct {
	ID              string  `json:"Id"`
	Name            string  `json:"Name"`
	StartTime       string  `json:"StartTime"`
	EndTime         string  `json:"EndTime"`
	LastModified    string  `json:"LastModified"`
	Priority        int     `json:"Priority"`
	RunningDuration float64 `json:"RunningDuration"`
	State           int     `json:"State"`
}

func (j *Job) toResource() string {
	return fmt.Sprintf("Jobs('%s')", j.ID)
}

func (c *Client) EncodeAssetWithContext(ctx context.Context, mediaProcessorID, configuration string, asset *Asset) (*Job, error) {
	outAssetName := fmt.Sprintf("[ENCODED]%s", asset.Name)

	params := map[string]interface{}{
		"Name": fmt.Sprintf("EncodeJob - %s", asset.ID),
		"InputMediaAssets": []MediaAsset{
			NewMediaAsset(c.buildAssetURI(asset)),
		},
		"Tasks": []Task{
			Task{
				Name:             fmt.Sprintf("task-%s", outAssetName),
				Configuration:    configuration,
				MediaProcessorID: mediaProcessorID,
				TaskBody:         buildTaskBody(outAssetName),
			},
		},
	}
	body, err := encodeParams(params)
	if err != nil {
		return nil, err
	}
	req, err := c.newRequest(ctx, http.MethodPost, "Jobs", body)
	req.Header.Set("Content-Type", "application/json;odata=verbose")
	req.Header.Set("Accept", "application/json;odata=verbose")

	var out struct {
		Data Job `json:"d"`
	}
	if err := c.do(req, http.StatusCreated, &out); err != nil {
		return nil, err
	}
	return &out.Data, nil
}

func (c *Client) GetOutputMediaAssetsWithContext(ctx context.Context, job *Job) ([]Asset, error) {
	endpoint := job.toResource() + "/OutputMediaAssets"
	req, err := c.newRequest(ctx, "GET", endpoint, nil)
	if err != nil {
		return nil, errors.Wrap(err, "get output media assets request build failed")
	}
	var out struct {
		Assets []Asset `json:"value"`
	}
	if err := c.do(req, http.StatusOK, &out); err != nil {
		return nil, errors.Wrap(err, "get output media assets request failed")
	}
	return out.Assets, nil
}

func (c *Client) GetJobWithContext(ctx context.Context, jobID string) (*Job, error) {
	req, err := c.newRequest(ctx, http.MethodGet, fmt.Sprintf("Jobs('%s')", jobID), nil)
	if err != nil {
		return nil, err
	}
	var out Job
	if err := c.do(req, http.StatusOK, &out); err != nil {
		return nil, err
	}
	return &out, nil
}

func (c *Client) WaitJobWithContext(ctx context.Context, job *Job) error {
	for {
		current, err := c.GetJobWithContext(ctx, job.ID)
		if err != nil {
			return err
		}

		if current.State == JobError {
			return errors.New("job failed")
		}
		if current.State == JobCanceled {
			return errors.New("job canceled")
		}
		if current.State == JobFinished {
			return nil
		}
		time.Sleep(8 * time.Second)
	}
}

func buildTaskBody(assetName string) string {
	return fmt.Sprintf(taskBodyTemplate, html.EscapeString(assetName))
}
<commit_msg>style: following goimports<commit_after>package ams

import (
	"context"
	"fmt"
	"html"
	"net/http"
	"time"

	"github.com/pkg/errors"
)

type MetaData struct {
	URI string `json:"uri"`
}

type MediaAsset struct {
	MetaData MetaData `json:"__metadata"`
}

const (
	taskBodyTemplate = "<?xml version=\"1.0\" encoding=\"utf-8\"?><taskBody><inputAsset>JobInputAsset(0)</inputAsset><outputAsset assetName=\"%s\">JobOutputAsset(0)</outputAsset></taskBody>"
)

const (
	JobQueued = iota
	JobScheduled
	JobProcessing
	JobFinished
	JobError
	JobCanceled
	JobCanceling
)

func NewMediaAsset(uri string) MediaAsset {
	return MediaAsset{
		MetaData: MetaData{
			URI: uri,
		},
	}
}

type Task struct {
	Name             string `json:"Name"`
	Configuration    string `json:"Configuration"`
	MediaProcessorID string `json:"MediaProcessorId"`
	TaskBody         string `json:"TaskBody"`
}

type Job struct {
	ID              string  `json:"Id"`
	Name            string  `json:"Name"`
	StartTime       string  `json:"StartTime"`
	EndTime         string  `json:"EndTime"`
	LastModified    string  `json:"LastModified"`
	Priority        int     `json:"Priority"`
	RunningDuration float64 `json:"RunningDuration"`
	State           int     `json:"State"`
}

func (j *Job) toResource() string {
	return fmt.Sprintf("Jobs('%s')", j.ID)
}

func (c *Client) EncodeAssetWithContext(ctx context.Context, mediaProcessorID, configuration string, asset *Asset) (*Job, error) {
	outAssetName := fmt.Sprintf("[ENCODED]%s", asset.Name)

	params := map[string]interface{}{
		"Name": fmt.Sprintf("EncodeJob - %s", asset.ID),
		"InputMediaAssets": []MediaAsset{
			NewMediaAsset(c.buildAssetURI(asset)),
		},
		"Tasks": []Task{
			Task{
				Name:             fmt.Sprintf("task-%s", outAssetName),
				Configuration:    configuration,
				MediaProcessorID: mediaProcessorID,
				TaskBody:         buildTaskBody(outAssetName),
			},
		},
	}
	body, err := encodeParams(params)
	if err != nil {
		return nil, err
	}
	req, err := c.newRequest(ctx, http.MethodPost, "Jobs", body)
	req.Header.Set("Content-Type", "application/json;odata=verbose")
	req.Header.Set("Accept", "application/json;odata=verbose")

	var out struct {
		Data Job `json:"d"`
	}
	if err := c.do(req, http.StatusCreated, &out); err != nil {
		return nil, err
	}
	return &out.Data, nil
}

func (c *Client) GetOutputMediaAssetsWithContext(ctx context.Context, job *Job) ([]Asset, error) {
	endpoint := job.toResource() + "/OutputMediaAssets"
	req, err := c.newRequest(ctx, "GET", endpoint, nil)
	if err != nil {
		return nil, errors.Wrap(err, "get output media assets request build failed")
	}
	var out struct {
		Assets []Asset `json:"value"`
	}
	if err := c.do(req, http.StatusOK, &out); err != nil {
		return nil, errors.Wrap(err, "get output media assets request failed")
	}
	return out.Assets, nil
}

func (c *Client) GetJobWithContext(ctx context.Context, jobID string) (*Job, error) {
	req, err := c.newRequest(ctx, http.MethodGet, fmt.Sprintf("Jobs('%s')", jobID), nil)
	if err != nil {
		return nil, err
	}
	var out Job
	if err := c.do(req, http.StatusOK, &out); err != nil {
		return nil, err
	}
	return &out, nil
}

func (c *Client) WaitJobWithContext(ctx context.Context, job *Job) error {
	for {
		current, err := c.GetJobWithContext(ctx, job.ID)
		if err != nil {
			return err
		}

		if current.State == JobError {
			return errors.New("job failed")
		}
		if current.State == JobCanceled {
			return errors.New("job canceled")
		}
		if current.State == JobFinished {
			return nil
		}
		time.Sleep(8 * time.Second)
	}
}

func buildTaskBody(assetName string) string {
	return fmt.Sprintf(taskBodyTemplate, html.EscapeString(assetName))
}
<|endoftext|>


<commit_before>package apnsapi

import (
	"bytes"
	"crypto"
	"crypto/ecdsa"
	"crypto/rand"
	"encoding/base64"
	"encoding/json"
	"io"
	"time"
)

type jwtHeader struct {
	Alg string `json:"alg"`
	Kid string `json:"kid"`
}

type jwtClaim struct {
	Iss string `json:"iss"`
	Iat int64  `json:"iat"`
}

func CreateToken(key *ecdsa.PrivateKey, kid string, teamID string) (string, error) {
	header := jwtHeader{
		Alg: "ES256",
		Kid: kid,
	}
	claim := jwtClaim{
		Iss: teamID,
		Iat: time.Now().Unix(),
	}
	var b bytes.Buffer

	headerJson, err := json.Marshal(&header)
	if err != nil {
		return "", err
	}
	if err := base64Encode(&b, headerJson); err != nil {
		return "", err
	}
	b.WriteString(".")

	claimJson, err := json.Marshal(&claim)
	if err != nil {
		return "", err
	}
	if err := base64Encode(&b, claimJson); err != nil {
		return "", err
	}

	h := crypto.SHA256.New()
	h.Write(b.Bytes())
	msg := h.Sum(nil)
	sig, err := key.Sign(rand.Reader, msg, crypto.SHA256)
	if err != nil {
		return "", err
	}

	b.WriteString(".")
	if err := base64Encode(&b, sig); err != nil {
		return "", err
	}

	return b.String(), nil
}

func base64Encode(w io.Writer, byt []byte) error {
	enc := base64.NewEncoder(base64.RawURLEncoding, w)
	enc.Write(byt)
	return enc.Close()
}
<commit_msg>use ecdsa.Sign directly<commit_after>package apnsapi

import (
	"bytes"
	"crypto"
	"crypto/ecdsa"
	"crypto/rand"
	"encoding/asn1"
	"encoding/base64"
	"encoding/json"
	"io"
	"math/big"
	"time"
)

type jwtHeader struct {
	Alg string `json:"alg"`
	Kid string `json:"kid"`
}

type jwtClaim struct {
	Iss string `json:"iss"`
	Iat int64  `json:"iat"`
}

type ecdsaSignature struct {
	R, S *big.Int
}

func CreateToken(key *ecdsa.PrivateKey, kid string, teamID string) (string, error) {
	header := jwtHeader{
		Alg: "ES256",
		Kid: kid,
	}
	claim := jwtClaim{
		Iss: teamID,
		Iat: time.Now().Unix(),
	}
	var b bytes.Buffer

	headerJson, err := json.Marshal(&header)
	if err != nil {
		return "", err
	}
	if err := base64Encode(&b, headerJson); err != nil {
		return "", err
	}
	b.WriteString(".")

	claimJson, err := json.Marshal(&claim)
	if err != nil {
		return "", err
	}
	if err := base64Encode(&b, claimJson); err != nil {
		return "", err
	}

	h := crypto.SHA256.New()
	h.Write(b.Bytes())
	msg := h.Sum(nil)

	r, s, err := ecdsa.Sign(rand.Reader, key, msg)
	if err != nil {
		return "", err
	}

	sig, err := asn1.Marshal(ecdsaSignature{r, s})
	if err != nil {
		return "", err
	}

	b.WriteString(".")
	if err := base64Encode(&b, sig); err != nil {
		return "", err
	}

	return b.String(), nil
}

func base64Encode(w io.Writer, byt []byte) error {
	enc := base64.NewEncoder(base64.RawURLEncoding, w)
	enc.Write(byt)
	return enc.Close()
}
<|endoftext|>


<commit_before>// Copyright ©2015 The corridor Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.package main

package corridor

import (
	"math"

	"github.com/gonum/matrix/mat64"
)

// compute euclidean distance for a pair of subscript indices
func Distance(aSubs, bSubs []int) (dist float64) {

	// initialize power variable
	var pow float64 = 2.0

	// initialize output variable
	var output float64

	// compute distance
	output = math.Sqrt(math.Pow(float64(aSubs[1]-aSubs[0]), pow) + math.Pow(float64(bSubs[0]-bSubs[1]), pow))

	// return final output

	return output
}

// THIS CODE IS UNFINISHED AND NOT WORKING PROPERLY...NEEDS WORK...
func Bresenham(aSubs, bSubs []int, searchDomain *mat64.Dense) (lineSubs [][]int) {

	var x0 = aSubs[0]
	var x1 = bSubs[0]
	var y0 = aSubs[1]
	var y1 = bSubs[1]

	dx := x1 - x0

	if dx < 0 {
		dx = -dx
	}

	dy := y1 - y0

	if dy < 0 {
		dy = -dy
	}

	var sx, sy int

	if x0 < x1 {
		sx = 1
	} else {
		sx = -1
	}

	if y0 < y1 {
		sy = 1
	} else {
		sy = -1
	}

	err := dx - dy
	rows, cols := searchDomain.Dims()
	maxLen := rows * cols
	output := make([][]int, 1, maxLen)
	output[0] = make([]int, 2)
	val := make([]int, 2)

	for {
		if x0 == x1 && y0 == y1 {
			break
		}
		e2 := 2 * err
		if e2 > -dy {
			err -= dy
			x0 += sx
		}
		if e2 < dx {
			err += dx
			y0 += sy
		}
		val[0] = x0
		val[1] = y0
		output = append(output, val)
	}

	return output
}

// fitness function to generate the total fitness and individual
// fitness values for a given input set of subscripts
// corresponding to a single individual
func Fitness(subs [][]int, obj *mat64.Dense) (fitnessValues []float64, totalFitness float64) {

	// get individual length
	indSize := len(subs)

	// initialize fitness values and total fitness
	fitVal := make([]float64, indSize)
	var totFit float64 = 0.0

	// evaluate individual fitness according to input objective
	for i := 0; i < indSize; i++ {
		curFit := obj.At(subs[i][0], subs[i][1])
		fitVal[i] = curFit
		totFit = totFit + curFit
	}

	// return outputs
	return fitVal, totFit

}
<commit_msg>Bresenham's algorithm now working. Implemented single distance and minimum distance functions for use in establishing the covariance values for sigma dynamically during the directed walk procedure.<commit_after>// Copyright ©2015 The corridor Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.package main

package corridor

import (
	"math"
	"sort"

	"github.com/gonum/matrix/mat64"
)

// compute euclidean distance for a pair of subscript indices
func Distance(aSubs, bSubs []int) (dist float64) {

	// initialize variables
	var x0 float64 = float64(aSubs[0])
	var x1 float64 = float64(bSubs[0])
	var y0 float64 = float64(aSubs[1])
	var y1 float64 = float64(bSubs[1])
	var pow float64 = 2.0
	var dx float64 = x1 - x0
	var dy float64 = y1 - y0

	// compute distance
	var output float64 = math.Sqrt(math.Pow(dx, pow) + math.Pow(dy, pow))

	// return final output
	return output
}

func MinDistance(aSubs []int, lineSubs [][]int) (minDist float64) {

	// initialize variables
	maxLen := len(lineSubs)
	distVec := make([]float64, maxLen)

	// loop through and compute distances
	for i := 0; i < maxLen; i++ {
		distVec[i] = Distance(aSubs, lineSubs[i])
	}

	// sort distances
	sort.Float64s(distVec)

	// get final output
	output := distVec[0]

	// return final output
	return output
}

// bresenham generates the list of subscript indices corresponding to the
// euclidean shortest paths connecting two subscript pairs in discrete space
func Bresenham(aSubs, bSubs []int) (lineSubs [][]int) {

	// initialize variables
	var x0 int = aSubs[0]
	var x1 int = bSubs[0]
	var y0 int = aSubs[1]
	var y1 int = bSubs[1]

	// check row differential
	dx := x1 - x0
	if dx < 0 {
		dx = -dx
	}

	// check column differential
	dy := y1 - y0

	if dy < 0 {
		dy = -dy
	}

	// initialize stride variables
	var sx, sy int

	// set row stride direction
	if x0 < x1 {
		sx = 1
	} else {
		sx = -1
	}

	// set column stride direction
	if y0 < y1 {
		sy = 1
	} else {
		sy = -1
	}

	// calculate error component
	err := dx - dy

	// initialize output 2D slice vector
	dist := math.Ceil(Distance(aSubs, bSubs))
	maxLen := int(dist)
	output := make([][]int, 0, maxLen)

	// loop through and generate subscripts
	for {
		var val = []int{x0, y0}
		output = append(output, val)
		if x0 == x1 && y0 == y1 {
			break
		}
		e2 := 2 * err
		if e2 > -dy {
			err -= dy
			x0 += sx
		}
		if e2 < dx {
			err += dx
			y0 += sy
		}
	}

	// return final output
	return output
}

// fitness function to generate the total fitness and individual
// fitness values for a given input set of subscripts
// corresponding to a single individual
func Fitness(subs [][]int, obj *mat64.Dense) (fitnessValues []float64, totalFitness float64) {

	// get individual length
	indSize := len(subs)

	// initialize fitness values and total fitness
	fitVal := make([]float64, indSize)
	var totFit float64 = 0.0

	// evaluate individual fitness according to input objective
	for i := 0; i < indSize; i++ {
		curFit := obj.At(subs[i][0], subs[i][1])
		fitVal[i] = curFit
		totFit = totFit + curFit
	}

	// return outputs
	return fitVal, totFit

}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"context"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/buger/jsonparser"
	"github.com/gorilla/websocket"
	"github.com/joho/godotenv"
	"github.com/syhlion/requestwork.v2"
	"github.com/urfave/cli"
)

var (
	name     string
	version  string
	cmdStart = cli.Command{
		Name:   "start",
		Usage:  "connect ws cli",
		Action: start,
		Flags: []cli.Flag{
			cli.StringFlag{
				Name: "env-file",
			},
		},
	}
	wg        sync.WaitGroup
	listen_wg sync.WaitGroup
)

func start(c *cli.Context) {
	if c.String("env-file") != "" {
		envfile := c.String("env-file")
		//flag.Parse()
		err := godotenv.Load(envfile)
		if err != nil {
			log.Fatal(err)
		}
	}
	ws_api := os.Getenv("GUSHER-CONN-TEST_WS_API")
	if ws_api == "" {
		log.Fatal("empty env GUSHER-CONN-TEST_WS_API")
	}
	push_api := os.Getenv("GUSHER-CONN-TEST_PUSH_API")
	if push_api == "" {
		log.Fatal("empty env GUSHER-CONN-TEST_PUSH_API")
	}
	login_msg := os.Getenv("GUSHER-CONN-TEST_LOGIN_MESSAGE")
	if login_msg == "" {
		log.Fatal("empty env GUSHER-CONN-TEST_LOGIN_MESSAGE")
	}
	sub_msg := os.Getenv("GUSHER-CONN-TEST_SUBSCRIBE_MESSAGE")
	if sub_msg == "" {
		log.Fatal("empty env GUSHER-CONN-TEST_SUBSCRIBE_MESSAGE")
	}
	sub_resp := os.Getenv("GUSHER-CONN-TEST_SUBSCRIBE_RESPONSE")
	if sub_resp == "" {
		log.Fatal("empty env GUSHER-CONN-TEST_SUBSCRIBE_RESPONSE")
	}
	push_msg := os.Getenv("GUSHER-CONN-TEST_PUSH_MESSAGE")
	if push_msg == "" {
		log.Fatal("empty env GUSHER-CONN-TEST_PUSH_MESSAGE")
	}
	connections := os.Getenv("GUSHER-CONN-TEST_CONNECTIONS")
	if connections == "" {
		log.Fatal("empty env GUSHER-CONN-TEST_CONNECTIONS")
	}
	conn_total, err := strconv.Atoi(connections)
	if err != nil {
		log.Fatal(err)
	}
	wsurl, err := url.Parse(ws_api)
	if err != nil {
		log.Fatal(err)
	}
	pushurl, err := url.Parse(push_api)
	if err != nil {
		log.Fatal(err)
	}
	wsHeaders := http.Header{
		"Origin":                   {wsurl.String()},
		"Sec-WebSocket-Extensions": {"permessage-deflate; client_max_window_bits, x-webkit-deflate-frame"},
	}
	conns := make([]*websocket.Conn, 0)
	for i := 0; i < conn_total; i++ {
		wg.Add(1)
		rawConn, err := net.Dial("tcp", wsurl.Host)
		if err != nil {
			log.Fatal(err)
			wg.Done()
			continue
		}

		conn, _, err := websocket.NewClient(rawConn, wsurl, wsHeaders, 1024, 1024)
		if err != nil {
			rawConn.Close()
			wg.Done()
			log.Fatal(err)
			continue
		}
		err = conn.WriteMessage(websocket.TextMessage, []byte(login_msg))
		if err != nil {
			rawConn.Close()
			conn.Close()
			wg.Done()
			log.Warn(err)
			continue
		}
		conns = append(conns, conn)
	}
	for i, conn := range conns {
		listen_wg.Add(1)
		go func(i int, conn *websocket.Conn) {
			for {
				_, d, err := conn.ReadMessage()
				if err != nil {
					log.Fatal(err)
					listen_wg.Done()
					wg.Done()
					return
				}

				if string(d) == sub_resp {
					listen_wg.Done()
				}
				log.Println(i, " slave repsonse message", string(d))
				data, _ := jsonparser.GetString(d, "data")
				if data == push_msg {
					wg.Done()
					return
				}
			}
		}(i, conn)
		err = conn.WriteMessage(websocket.TextMessage, []byte(sub_msg))
		if err != nil {
			conn.Close()
			continue
		}
	}

	listen_wg.Wait()
	//push start
	work := requestwork.New(5)
	v := url.Values{}

	v.Add("data", push_msg)
	req, err := http.NewRequest("POST", pushurl.String(), bytes.NewBufferString(v.Encode()))
	if err != nil {
		log.Fatal(err)
	}
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	req.Header.Add("Content-Length", strconv.Itoa(len(v.Encode())))

	if err != nil {
		log.Fatal(err)
	}
	var firstTime time.Time
	ctx, _ := context.WithTimeout(context.Background(), 30*time.Second)
	err = work.Execute(ctx, req, func(resp *http.Response, e error) (err error) {
		if e != nil {
			return
		}
		defer resp.Body.Close()
		b, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		log.Println("master response", string(b))
		firstTime = time.Now()
		return
	})
	log.Println("Waiting...")
	wg.Wait()
	log.Println("Sucess")
	t := time.Now().Sub(firstTime)
	log.Printf("Total Use time:%s", t)

	return
}

func main() {
	gusher := cli.NewApp()
	gusher.Name = name
	gusher.Version = version
	gusher.Commands = []cli.Command{
		cmdStart,
	}
	gusher.Compiled = time.Now()
	gusher.Run(os.Args)
}
<commit_msg>add connect time<commit_after>package main

import (
	"bytes"
	"context"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/buger/jsonparser"
	"github.com/gorilla/websocket"
	"github.com/joho/godotenv"
	"github.com/syhlion/requestwork.v2"
	"github.com/urfave/cli"
)

var (
	name     string
	version  string
	cmdStart = cli.Command{
		Name:   "start",
		Usage:  "connect ws cli",
		Action: start,
		Flags: []cli.Flag{
			cli.StringFlag{
				Name: "env-file",
			},
		},
	}
	wg        sync.WaitGroup
	listen_wg sync.WaitGroup
)

func start(c *cli.Context) {
	if c.String("env-file") != "" {
		envfile := c.String("env-file")
		//flag.Parse()
		err := godotenv.Load(envfile)
		if err != nil {
			log.Fatal(err)
		}
	}
	ws_api := os.Getenv("GUSHER-CONN-TEST_WS_API")
	if ws_api == "" {
		log.Fatal("empty env GUSHER-CONN-TEST_WS_API")
	}
	push_api := os.Getenv("GUSHER-CONN-TEST_PUSH_API")
	if push_api == "" {
		log.Fatal("empty env GUSHER-CONN-TEST_PUSH_API")
	}
	login_msg := os.Getenv("GUSHER-CONN-TEST_LOGIN_MESSAGE")
	if login_msg == "" {
		log.Fatal("empty env GUSHER-CONN-TEST_LOGIN_MESSAGE")
	}
	sub_msg := os.Getenv("GUSHER-CONN-TEST_SUBSCRIBE_MESSAGE")
	if sub_msg == "" {
		log.Fatal("empty env GUSHER-CONN-TEST_SUBSCRIBE_MESSAGE")
	}
	sub_resp := os.Getenv("GUSHER-CONN-TEST_SUBSCRIBE_RESPONSE")
	if sub_resp == "" {
		log.Fatal("empty env GUSHER-CONN-TEST_SUBSCRIBE_RESPONSE")
	}
	push_msg := os.Getenv("GUSHER-CONN-TEST_PUSH_MESSAGE")
	if push_msg == "" {
		log.Fatal("empty env GUSHER-CONN-TEST_PUSH_MESSAGE")
	}
	connections := os.Getenv("GUSHER-CONN-TEST_CONNECTIONS")
	if connections == "" {
		log.Fatal("empty env GUSHER-CONN-TEST_CONNECTIONS")
	}
	conn_total, err := strconv.Atoi(connections)
	if err != nil {
		log.Fatal(err)
	}
	wsurl, err := url.Parse(ws_api)
	if err != nil {
		log.Fatal(err)
	}
	pushurl, err := url.Parse(push_api)
	if err != nil {
		log.Fatal(err)
	}
	wsHeaders := http.Header{
		"Origin":                   {wsurl.String()},
		"Sec-WebSocket-Extensions": {"permessage-deflate; client_max_window_bits, x-webkit-deflate-frame"},
	}
	conns := make([]*websocket.Conn, 0)
	listenStart := time.Now()
	for i := 0; i < conn_total; i++ {
		wg.Add(1)
		rawConn, err := net.Dial("tcp", wsurl.Host)
		if err != nil {
			log.Fatal(err)
			wg.Done()
			continue
		}

		conn, _, err := websocket.NewClient(rawConn, wsurl, wsHeaders, 1024, 1024)
		if err != nil {
			rawConn.Close()
			wg.Done()
			log.Fatal(err)
			continue
		}
		err = conn.WriteMessage(websocket.TextMessage, []byte(login_msg))
		if err != nil {
			rawConn.Close()
			conn.Close()
			wg.Done()
			log.Warn(err)
			continue
		}
		conns = append(conns, conn)
	}
	for i, conn := range conns {
		listen_wg.Add(1)
		go func(i int, conn *websocket.Conn) {
			for {
				_, d, err := conn.ReadMessage()
				if err != nil {
					log.Fatal(err)
					listen_wg.Done()
					wg.Done()
					return
				}

				if string(d) == sub_resp {
					listen_wg.Done()
				}
				log.Println(i, " slave repsonse message", string(d))
				data, _ := jsonparser.GetString(d, "data")
				if data == push_msg {
					wg.Done()
					return
				}
			}
		}(i, conn)
		err = conn.WriteMessage(websocket.TextMessage, []byte(sub_msg))
		if err != nil {
			conn.Close()
			continue
		}
	}

	listen_wg.Wait()
	listenTime := time.Now().Sub(listenStart)
	//push start
	work := requestwork.New(5)
	v := url.Values{}

	v.Add("data", push_msg)
	req, err := http.NewRequest("POST", pushurl.String(), bytes.NewBufferString(v.Encode()))
	if err != nil {
		log.Fatal(err)
	}
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	req.Header.Add("Content-Length", strconv.Itoa(len(v.Encode())))

	if err != nil {
		log.Fatal(err)
	}
	var pushStart time.Time
	ctx, _ := context.WithTimeout(context.Background(), 30*time.Second)
	err = work.Execute(ctx, req, func(resp *http.Response, e error) (err error) {
		if e != nil {
			return
		}
		defer resp.Body.Close()
		b, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		log.Println("master response", string(b))
		pushStart = time.Now()
		return
	})
	log.Println("Waiting...")
	wg.Wait()
	log.Println("Sucess")
	t := time.Now().Sub(pushStart)
	log.Printf("All Client receive msg time:%s, All Client connect time:%s", t, listenTime)

	return
}

func main() {
	gusher := cli.NewApp()
	gusher.Name = name
	gusher.Version = version
	gusher.Commands = []cli.Command{
		cmdStart,
	}
	gusher.Compiled = time.Now()
	gusher.Run(os.Args)
}
<|endoftext|>