<commit_before>package main

import (
	"fmt"
	"net/url"
	"time"

	cfclient "github.com/cloudfoundry-community/go-cfclient"
)

func QuotaGauge(c *Client, interval time.Duration) MetricReadCloser {
	return NewMetricPoller(interval, func(w MetricWriter) error {
		orgs, err := c.cf.ListOrgs()
		if err != nil {
			return err
		}

		reservedMemory := 0
		reservedServices := 0
		allocatedMemory := 0
		allocatedServices := 0
		reservedRoutes := 0

		for _, org := range orgs {
			quota, err := org.Quota()
			if err != nil {
				return err
			}
			reservedMemory += quota.MemoryLimit
			reservedServices += quota.TotalServices
			reservedRoutes += quota.TotalRoutes
		}

		apps, err := c.cf.ListApps()
		if err != nil {
			return err
		}
		for _, app := range apps {
			allocatedMemory += (app.Memory * app.Instances)
		}

		allocatedServices, err = c.CountServiceInstances()
		if err != nil {
			return err
		}

		return w.WriteMetrics([]Metric{
			{
				Kind:  Gauge,
				Time:  time.Now(),
				Name:  "quota.services.reserved", // number of services reserved by quotas
				Value: float64(reservedServices),
			},
			{
				Kind:  Gauge,
				Time:  time.Now(),
				Name:  "quota.services.allocated", // number of services in use
				Value: float64(allocatedServices),
			},
			{
				Kind:  Gauge,
				Time:  time.Now(),
				Name:  "quota.memory.reserved", // memory reserved by org quotas
				Value: float64(reservedMemory),
			},
			{
				Kind:  Gauge,
				Time:  time.Now(),
				Name:  "quota.memory.allocated", // memory allocated to apps
				Value: float64(allocatedMemory),
			},
			{
				Kind:  Gauge,
				Time:  time.Now(),
				Name:  "quota.routes.reserved", // number of routes reserved
				Value: float64(reservedRoutes),
			},
		})
	})
}

func UserCountGauge(c *Client, interval time.Duration) MetricReadCloser {
	return NewMetricPoller(interval, func(w MetricWriter) error {
		// global auditor role cannot use /v2/users
		// so we have to fetch users from each org
		orgs, err := c.cf.ListOrgs()
		if err != nil {
			return err
		}

		userGuids := map[string]bool{}

		for _, org := range orgs {
			users, err := c.OrgUsers(org.Guid)
			if err != nil {
				return err
			}
			for _, u := range users {
				userGuids[u.Guid] = true
			}
		}

		return w.WriteMetrics([]Metric{
			{
				Kind:  Gauge,
				Time:  time.Now(),
				Name:  "users.count",
				Value: float64(len(userGuids)),
			},
		})
	})
}

func AppCountGauge(c *Client, interval time.Duration) MetricReadCloser {
	return NewMetricPoller(interval, func(w MetricWriter) error {
		apps, err := c.cf.ListApps()
		if err != nil {
			return err
		}

		// Number of relevant apps in
		// - APP_STATE: string of whether each app is "started" or "stopped"
		// - ORG_IS_TRIAL: boolean of whether each app is owned by a trial organisation
		// counters[APP_STATE][ORG_IS_TRIAL]
		counters := map[string]map[bool]int{
			"started": map[bool]int{},
			"stopped": map[bool]int{},
		}
		for _, app := range apps {
			org_quota, err := findOrgQuotaFromSpaceGUID(c, app.SpaceGuid)
			if err != nil {
				continue
			}
			org_is_trial := isOrgQuotaTrial(org_quota)
			if app.State == "STARTED" {
				counters["started"][org_is_trial]++
			}
			if app.State == "STOPPED" {
				counters["stopped"][org_is_trial]++
			}
		}

		metrics := []Metric{}
		for state, count_by_trial := range counters {
			for org_is_trial, count := range count_by_trial {
				metrics = append(metrics, Metric{
					Kind:  Gauge,
					Time:  time.Now(),
					Name:  "apps.count",
					Value: float64(count),
					Tags: []string{
						"state:" + state,
						fmt.Sprintf("trial_org:%t", org_is_trial),
					},
				})
			}
		}
		return w.WriteMetrics(metrics)
	})
}

func ServiceCountGauge(c *Client, interval time.Duration) MetricReadCloser {
	return NewMetricPoller(interval, func(w MetricWriter) error {
		serviceInstances, err := c.cf.ListServiceInstances()
		if err != nil {
			return err
		}
		services, err := c.cf.ListServices()
		if err != nil {
			return nil
		}
		service_plans, err := c.cf.ListServicePlans()
		if err != nil {
			return nil
		}

		// Number of relevant service instances in
		// - ORG_IS_TRIAL: boolean of whether each app is owned by a trial organisation
		// - SERVICE_PLAN_IS_FREE: whether the instance's service plan is free
		// - NAME_OF_SERVICE: e.g., "mysql" or "postgres"
		// counters[ORG_IS_TRIAL][SERVICE_PLAN_IS_FREE][NAME_OF_SERVICE]
		counters := map[bool]map[bool]map[string]int{
			true: map[bool]map[string]int{
				true:  map[string]int{},
				false: map[string]int{},
			},
			false: map[bool]map[string]int{
				true:  map[string]int{},
				false: map[string]int{},
			},
		}
		for _, instance := range serviceInstances {
			service := findService(services, instance.ServiceGuid)
			if service == nil || service.Label == "" {
				continue
			}
			service_plan := findServicePlan(service_plans, instance.ServicePlanGuid)
			if service_plan == nil {
				continue
			}
			org_quota, err := findOrgQuotaFromSpaceGUID(c, instance.SpaceGuid)
			if err != nil {
				continue
			}
			org_is_trial := isOrgQuotaTrial(org_quota)
			service_plan_is_free := isServicePlanFree(service_plan)
			counters[org_is_trial][service_plan_is_free][service.Label]++
		}

		metrics := []Metric{}
		for org_is_trial, x := range counters {
			for service_plan_is_free, y := range x {
				for service_label, count := range y {
					metrics = append(metrics, Metric{
						Kind:  Gauge,
						Time:  time.Now(),
						Name:  "services.provisioned",
						Value: float64(count),
						Tags: []string{
							"type:" + service_label,
							fmt.Sprintf("trial_org:%t", org_is_trial),
							fmt.Sprintf("free_service:%t", service_plan_is_free),
						},
					})
				}
			}
		}
		return w.WriteMetrics(metrics)
	})
}

func OrgCountGauge(c *Client, interval time.Duration) MetricReadCloser {
	return NewMetricPoller(interval, func(w MetricWriter) error {
		orgs, err := c.cf.ListOrgs()
		if err != nil {
			return err
		}
		counters := map[string]int{}
		for _, org := range orgs {
			quota, err := org.Quota()
			if err != nil {
				continue
			}
			counters[quota.Name]++
		}
		metrics := []Metric{}
		for name, count := range counters {
			metrics = append(metrics, Metric{
				Kind:  Gauge,
				Time:  time.Now(),
				Name:  "orgs.count",
				Value: float64(count),
				Tags:  []string{"quota:" + name},
			})
		}
		return w.WriteMetrics(metrics)
	})
}

func SpaceCountGauge(c *Client, interval time.Duration) MetricReadCloser {
	return NewMetricPoller(interval, func(w MetricWriter) error {
		spaces, err := c.cf.ListSpaces()
		if err != nil {
			return err
		}
		return w.WriteMetrics([]Metric{
			{
				Kind:  Gauge,
				Time:  time.Now(),
				Name:  "spaces.count",
				Value: float64(len(spaces)),
			},
		})
	})
}

func EventCountGauge(c *Client, eventType string, interval time.Duration) MetricReadCloser {
	return NewMetricPoller(interval, func(w MetricWriter) error {
		u, err := url.Parse("/v2/events")
		if err != nil {
			return err
		}
		maxAge := time.Now().Add(-1 * interval)
		q := u.Query()
		q.Set("order-direction", "desc")
		q.Set("results-per-page", "100")
		q.Add("q", "type:"+eventType)
		q.Add("q", "timestamp>"+maxAge.Format(time.RFC3339Nano))
		u.RawQuery = q.Encode()
		batchUrl := u.String()
		gauge := Metric{
			Time: time.Now(),
			Kind: Gauge,
			Name: "events." + eventType,
		}
		for batchUrl != "" {
			var batch struct {
				NextUrl   string          `json:"next_url"`
				Resources []AppUsageEvent `json:"resources"`
			}
			if err := c.get(batchUrl, &batch); err != nil {
				return err
			}
			for _, res := range batch.Resources {
				if res.MetaData.CreatedAt.Before(maxAge) {
					break
				}
				gauge.Value += 1
			}
			batchUrl = batch.NextUrl
		}
		return w.WriteMetrics([]Metric{gauge})
	})
}

func findService(services []cfclient.Service, guid string) *cfclient.Service {
	for _, service := range services {
		if service.Guid == guid {
			return &service
		}
	}
	return nil
}

func findServicePlan(service_plans []cfclient.ServicePlan, guid string) *cfclient.ServicePlan {
	for _, service_plan := range service_plans {
		if service_plan.Guid == guid {
			return &service_plan
		}
	}
	return nil
}

func findOrgQuotaFromSpaceGUID(c *Client, guid string) (*cfclient.OrgQuota, error) {
	space, err := c.cf.GetSpaceByGuid(guid)
	if err != nil {
		return nil, err
	}
	org, err := space.Org()
	if err != nil {
		return nil, err
	}
	org_quota, err := org.Quota()
	if err != nil {
		return nil, err
	}
	return org_quota, nil
}

// Determine if an organisation is on a trial plan.
func isOrgQuotaTrial(quota *cfclient.OrgQuota) bool {
	return quota.Name == "default"
}

// Determine if a service plan is free.
func isServicePlanFree(plan *cfclient.ServicePlan) bool {
	return plan.Name == "Free"
}
<commit_msg>fixup! Tag app and service metrics with free/paid status<commit_after>package main

import (
	"fmt"
	"log"
	"net/url"
	"time"

	cfclient "github.com/cloudfoundry-community/go-cfclient"
)

func QuotaGauge(c *Client, interval time.Duration) MetricReadCloser {
	return NewMetricPoller(interval, func(w MetricWriter) error {
		orgs, err := c.cf.ListOrgs()
		if err != nil {
			return err
		}

		reservedMemory := 0
		reservedServices := 0
		allocatedMemory := 0
		allocatedServices := 0
		reservedRoutes := 0

		for _, org := range orgs {
			quota, err := org.Quota()
			if err != nil {
				return err
			}
			reservedMemory += quota.MemoryLimit
			reservedServices += quota.TotalServices
			reservedRoutes += quota.TotalRoutes
		}

		apps, err := c.cf.ListApps()
		if err != nil {
			return err
		}
		for _, app := range apps {
			allocatedMemory += (app.Memory * app.Instances)
		}

		allocatedServices, err = c.CountServiceInstances()
		if err != nil {
			return err
		}

		return w.WriteMetrics([]Metric{
			{
				Kind:  Gauge,
				Time:  time.Now(),
				Name:  "quota.services.reserved", // number of services reserved by quotas
				Value: float64(reservedServices),
			},
			{
				Kind:  Gauge,
				Time:  time.Now(),
				Name:  "quota.services.allocated", // number of services in use
				Value: float64(allocatedServices),
			},
			{
				Kind:  Gauge,
				Time:  time.Now(),
				Name:  "quota.memory.reserved", // memory reserved by org quotas
				Value: float64(reservedMemory),
			},
			{
				Kind:  Gauge,
				Time:  time.Now(),
				Name:  "quota.memory.allocated", // memory allocated to apps
				Value: float64(allocatedMemory),
			},
			{
				Kind:  Gauge,
				Time:  time.Now(),
				Name:  "quota.routes.reserved", // number of routes reserved
				Value: float64(reservedRoutes),
			},
		})
	})
}

func UserCountGauge(c *Client, interval time.Duration) MetricReadCloser {
	return NewMetricPoller(interval, func(w MetricWriter) error {
		// global auditor role cannot use /v2/users
		// so we have to fetch users from each org
		orgs, err := c.cf.ListOrgs()
		if err != nil {
			return err
		}

		userGuids := map[string]bool{}

		for _, org := range orgs {
			users, err := c.OrgUsers(org.Guid)
			if err != nil {
				return err
			}
			for _, u := range users {
				userGuids[u.Guid] = true
			}
		}

		return w.WriteMetrics([]Metric{
			{
				Kind:  Gauge,
				Time:  time.Now(),
				Name:  "users.count",
				Value: float64(len(userGuids)),
			},
		})
	})
}

func AppCountGauge(c *Client, interval time.Duration) MetricReadCloser {
	return NewMetricPoller(interval, func(w MetricWriter) error {
		apps, err := c.cf.ListApps()
		if err != nil {
			return err
		}

		// Number of relevant apps in
		// - APP_STATE: string of whether each app is "started" or "stopped"
		// - ORG_IS_TRIAL: boolean of whether each app is owned by a trial organisation
		// counters[APP_STATE][ORG_IS_TRIAL]
		counters := map[string]map[bool]int{
			"started": map[bool]int{},
			"stopped": map[bool]int{},
		}
		for _, app := range apps {
			org_quota, err := findOrgQuotaFromSpaceGUID(c, app.SpaceGuid)
			if err != nil {
				log.Printf("Error finding org quota for space %s for app %s: %s\n", app.SpaceGuid, app.Guid, err)
				continue
			}
			org_is_trial := isOrgQuotaTrial(org_quota)
			if app.State == "STARTED" {
				counters["started"][org_is_trial]++
			}
			if app.State == "STOPPED" {
				counters["stopped"][org_is_trial]++
			}
		}

		metrics := []Metric{}
		for state, count_by_trial := range counters {
			for org_is_trial, count := range count_by_trial {
				metrics = append(metrics, Metric{
					Kind:  Gauge,
					Time:  time.Now(),
					Name:  "apps.count",
					Value: float64(count),
					Tags: []string{
						"state:" + state,
						fmt.Sprintf("trial_org:%t", org_is_trial),
					},
				})
			}
		}
		return w.WriteMetrics(metrics)
	})
}

func ServiceCountGauge(c *Client, interval time.Duration) MetricReadCloser {
	return NewMetricPoller(interval, func(w MetricWriter) error {
		serviceInstances, err := c.cf.ListServiceInstances()
		if err != nil {
			return err
		}
		services, err := c.cf.ListServices()
		if err != nil {
			return nil
		}
		service_plans, err := c.cf.ListServicePlans()
		if err != nil {
			return nil
		}

		// Number of relevant service instances in
		// - ORG_IS_TRIAL: boolean of whether each app is owned by a trial organisation
		// - SERVICE_PLAN_IS_FREE: whether the instance's service plan is free
		// - NAME_OF_SERVICE: e.g., "mysql" or "postgres"
		// counters[ORG_IS_TRIAL][SERVICE_PLAN_IS_FREE][NAME_OF_SERVICE]
		counters := map[bool]map[bool]map[string]int{
			true: map[bool]map[string]int{
				true:  map[string]int{},
				false: map[string]int{},
			},
			false: map[bool]map[string]int{
				true:  map[string]int{},
				false: map[string]int{},
			},
		}
		for _, instance := range serviceInstances {
			service := findService(services, instance.ServiceGuid)
			if service == nil {
				log.Printf("Service was not found for service instance %s\n", instance.Guid)
				continue
			}
			if service.Label == "" {
				log.Printf("Service label was empty for service %s and service instance %s\n", service.Guid, instance.Guid)
				continue
			}
			service_plan := findServicePlan(service_plans, instance.ServicePlanGuid)
			if service_plan == nil {
				log.Printf("Error finding service plan for service instance %s: %s\n", instance.Guid, err)
				continue
			}
			org_quota, err := findOrgQuotaFromSpaceGUID(c, instance.SpaceGuid)
			if err != nil {
				log.Printf("Error finding org quota for space %s for service instance %s: %s\n", instance.SpaceGuid, instance.Guid, err)
				continue
			}
			org_is_trial := isOrgQuotaTrial(org_quota)
			service_plan_is_free := isServicePlanFree(service_plan)
			counters[org_is_trial][service_plan_is_free][service.Label]++
		}

		metrics := []Metric{}
		for org_is_trial, x := range counters {
			for service_plan_is_free, y := range x {
				for service_label, count := range y {
					metrics = append(metrics, Metric{
						Kind:  Gauge,
						Time:  time.Now(),
						Name:  "services.provisioned",
						Value: float64(count),
						Tags: []string{
							"type:" + service_label,
							fmt.Sprintf("trial_org:%t", org_is_trial),
							fmt.Sprintf("free_service:%t", service_plan_is_free),
						},
					})
				}
			}
		}
		return w.WriteMetrics(metrics)
	})
}

func OrgCountGauge(c *Client, interval time.Duration) MetricReadCloser {
	return NewMetricPoller(interval, func(w MetricWriter) error {
		orgs, err := c.cf.ListOrgs()
		if err != nil {
			return err
		}
		counters := map[string]int{}
		for _, org := range orgs {
			quota, err := org.Quota()
			if err != nil {
				log.Printf("Error finding org quota for org %s: %s\n", org.Guid, err)
				continue
			}
			counters[quota.Name]++
		}
		metrics := []Metric{}
		for name, count := range counters {
			metrics = append(metrics, Metric{
				Kind:  Gauge,
				Time:  time.Now(),
				Name:  "orgs.count",
				Value: float64(count),
				Tags:  []string{"quota:" + name},
			})
		}
		return w.WriteMetrics(metrics)
	})
}

func SpaceCountGauge(c *Client, interval time.Duration) MetricReadCloser {
	return NewMetricPoller(interval, func(w MetricWriter) error {
		spaces, err := c.cf.ListSpaces()
		if err != nil {
			return err
		}
		return w.WriteMetrics([]Metric{
			{
				Kind:  Gauge,
				Time:  time.Now(),
				Name:  "spaces.count",
				Value: float64(len(spaces)),
			},
		})
	})
}

func EventCountGauge(c *Client, eventType string, interval time.Duration) MetricReadCloser {
	return NewMetricPoller(interval, func(w MetricWriter) error {
		u, err := url.Parse("/v2/events")
		if err != nil {
			return err
		}
		maxAge := time.Now().Add(-1 * interval)
		q := u.Query()
		q.Set("order-direction", "desc")
		q.Set("results-per-page", "100")
		q.Add("q", "type:"+eventType)
		q.Add("q", "timestamp>"+maxAge.Format(time.RFC3339Nano))
		u.RawQuery = q.Encode()
		batchUrl := u.String()
		gauge := Metric{
			Time: time.Now(),
			Kind: Gauge,
			Name: "events." + eventType,
		}
		for batchUrl != "" {
			var batch struct {
				NextUrl   string          `json:"next_url"`
				Resources []AppUsageEvent `json:"resources"`
			}
			if err := c.get(batchUrl, &batch); err != nil {
				return err
			}
			for _, res := range batch.Resources {
				if res.MetaData.CreatedAt.Before(maxAge) {
					break
				}
				gauge.Value += 1
			}
			batchUrl = batch.NextUrl
		}
		return w.WriteMetrics([]Metric{gauge})
	})
}

func findService(services []cfclient.Service, guid string) *cfclient.Service {
	for _, service := range services {
		if service.Guid == guid {
			return &service
		}
	}
	return nil
}

func findServicePlan(service_plans []cfclient.ServicePlan, guid string) *cfclient.ServicePlan {
	for _, service_plan := range service_plans {
		if service_plan.Guid == guid {
			return &service_plan
		}
	}
	return nil
}

func findOrgQuotaFromSpaceGUID(c *Client, guid string) (*cfclient.OrgQuota, error) {
	space, err := c.cf.GetSpaceByGuid(guid)
	if err != nil {
		return nil, err
	}
	org, err := space.Org()
	if err != nil {
		return nil, err
	}
	org_quota, err := org.Quota()
	if err != nil {
		return nil, err
	}
	return org_quota, nil
}

// Determine if an organisation is on a trial plan.
func isOrgQuotaTrial(quota *cfclient.OrgQuota) bool {
	return quota.Name == "default"
}

// Determine if a service plan is free.
func isServicePlanFree(plan *cfclient.ServicePlan) bool {
	return plan.Name == "Free"
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/voxelbrain/goptions"
	"io"
	"launchpad.net/goamz/aws"
	"launchpad.net/goamz/s3"
	"log"
	"mime"
	"os"
	"path/filepath"
	"sync"
)

const (
	VERSION = "1.0"
)

type Item struct {
	Prefix string
	Path   string
	os.FileInfo
}

var (
	options = struct {
		AccessKey   string `goptions:"-k, --access-key, obligatory, description='AWS Access Key ID'"`
		SecretKey   string `goptions:"-s, --secret-key, obligatory, description='AWS Secret Access Key'"`
		Region      string `goptions:"-r, --region, description='API Region name (default: us-west-1)'"`
		Bucket      string `goptions:"-b, --bucket, obligatory, description='Bucket to push to'"`
		Concurrency int    `goptions:"-c, --concurrency, description='Number of coroutines (default: 10)'"`
		goptions.Remainder
		goptions.Verbs
		Put struct {
			Prefix string `goptions:"-p, --prefix, description='Prefix to prepend to the items'"`
		} `goptions:"put"`
		Get struct {
			Prefix string `goptions:"-p, --prefix, description='Only get items starting with prefix'"`
		} `goptions:"get"`
	}{
		Concurrency: 10,
		Region:      aws.USWest.Name,
	}
)

func init() {
	err := goptions.Parse(&options)
	if err != nil || len(options.Remainder) <= 0 || len(options.Verbs) <= 0 {
		if err != goptions.ErrHelpRequest && err != nil {
			log.Printf("Error: %s", err)
		}
		goptions.PrintHelp()
		os.Exit(1)
	}

}

func main() {
	auth := aws.Auth{
		AccessKey: options.AccessKey,
		SecretKey: options.SecretKey,
	}

	region, ok := aws.Regions[options.Region]
	if !ok {
		log.Fatalf("Invalid region name %s", options.Region)
	}

	s3i := s3.New(auth, region)
	bucket := s3i.Bucket(options.Bucket)

	switch options.Verbs {
	case "put":
		c := listLocalFiles(options.Remainder...)
		putFiles(bucket, c)
	case "get":
		c := listBucketFiles(bucket)
		getFiles(bucket, c)
	}
}

func listLocalFiles(path ...string) <-chan *Item {
	c := make(chan *Item)
	go func() {
		for _, prefix := range options.Remainder {
			newprefix, err := filepath.Abs(prefix)
			if err != nil {
				log.Printf("Path %s could not be made absolute: %s. Skipping...", prefix, err)
				continue
			}
			log.Printf("Traversing %s...", newprefix)
			filepath.Walk(newprefix, func(path string, info os.FileInfo, err error) error {
				if info.IsDir() {
					return nil
				}
				c <- &Item{
					Prefix:   newprefix,
					Path:     path,
					FileInfo: info,
				}
				return nil
			})
		}
		close(c)
	}()
	return c
}

func listBucketFiles(bucket *s3.Bucket) <-chan *Item {
	c := make(chan *Item)
	go func() {
		resp, err := bucket.List(options.Get.Prefix, "", "", 1000000)
		if err != nil {
			log.Printf("Could not list items in bucket: %s", err)
		}
		for _, item := range resp.Contents {
			c <- &Item{
				Prefix:   options.Get.Prefix,
				Path:     item.Key,
				FileInfo: nil,
			}
		}
		close(c)
	}()
	return c
}

func putFiles(bucket *s3.Bucket, c <-chan *Item) {
	var wg sync.WaitGroup
	wg.Add(options.Concurrency)
	for i := 0; i < options.Concurrency; i++ {
		go func() {
			for item := range c {
				func() {
					f, err := os.Open(item.Path)
					if err != nil {
						log.Printf("Pushing %s failed: %s", item.Path, err)
					}
					defer f.Close()

					path := item.Path[len(item.Prefix)+1:]
					err = bucket.PutReader(options.Put.Prefix+path, f, item.FileInfo.Size(), mime.TypeByExtension(filepath.Ext(item.Path)), s3.BucketOwnerFull)
					if err != nil {
						log.Printf("Uploading %s failed: %s", path, err)
						return
					}
					log.Printf("Uploading %s done", path)
				}()
			}
			wg.Done()
		}()
	}
	wg.Wait()
}

func getFiles(bucket *s3.Bucket, c <-chan *Item) {
	var wg sync.WaitGroup
	wg.Add(options.Concurrency)
	for i := 0; i < options.Concurrency; i++ {
		go func() {
			for item := range c {
				func() {
					itempath := item.Path[len(item.Prefix):]
					dirname, fname := filepath.Split(itempath)
					dirname = filepath.Join(options.Remainder[0], dirname)

					os.MkdirAll(dirname, os.FileMode(0755))
					f, err := os.Create(filepath.Join(dirname, fname))
					if err != nil {
						log.Printf("Opening %s failed: %s", item.Path, err)
					}
					defer f.Close()

					rc, err := bucket.GetReader(item.Path)
					if err != nil {
						log.Printf("Downloading %s failed: %s", item.Path, err)
						return
					}
					defer rc.Close()
					io.Copy(f, rc)
					log.Printf("Downloading %s done", item.Path)
				}()
			}
			wg.Done()
		}()
	}
	wg.Wait()
}
<commit_msg>Support more than 1000 files in a bucket<commit_after>package main

import (
	"github.com/voxelbrain/goptions"
	"io"
	"launchpad.net/goamz/aws"
	"launchpad.net/goamz/s3"
	"log"
	"mime"
	"os"
	"path/filepath"
	"sync"
)

const (
	VERSION = "1.0"
)

type Item struct {
	Prefix string
	Path   string
	os.FileInfo
}

var (
	options = struct {
		AccessKey   string `goptions:"-k, --access-key, obligatory, description='AWS Access Key ID'"`
		SecretKey   string `goptions:"-s, --secret-key, obligatory, description='AWS Secret Access Key'"`
		Region      string `goptions:"-r, --region, description='API Region name (default: us-west-1)'"`
		Bucket      string `goptions:"-b, --bucket, obligatory, description='Bucket to push to'"`
		Concurrency int    `goptions:"-c, --concurrency, description='Number of coroutines (default: 10)'"`
		goptions.Remainder
		goptions.Verbs
		Put struct {
			Prefix string `goptions:"-p, --prefix, description='Prefix to prepend to the items'"`
		} `goptions:"put"`
		Get struct {
			Prefix string `goptions:"-p, --prefix, description='Only get items starting with prefix'"`
		} `goptions:"get"`
	}{
		Concurrency: 10,
		Region:      aws.USWest.Name,
	}
)

func init() {
	err := goptions.Parse(&options)
	if err != nil || len(options.Remainder) <= 0 || len(options.Verbs) <= 0 {
		if err != goptions.ErrHelpRequest && err != nil {
			log.Printf("Error: %s", err)
		}
		goptions.PrintHelp()
		os.Exit(1)
	}

}

func main() {
	auth := aws.Auth{
		AccessKey: options.AccessKey,
		SecretKey: options.SecretKey,
	}

	region, ok := aws.Regions[options.Region]
	if !ok {
		log.Fatalf("Invalid region name %s", options.Region)
	}

	s3i := s3.New(auth, region)
	bucket := s3i.Bucket(options.Bucket)

	switch options.Verbs {
	case "put":
		c := listLocalFiles(options.Remainder...)
		putFiles(bucket, c)
	case "get":
		c := listBucketFiles(bucket)
		getFiles(bucket, c)
	}
}

func listLocalFiles(path ...string) <-chan *Item {
	c := make(chan *Item)
	go func() {
		for _, prefix := range options.Remainder {
			newprefix, err := filepath.Abs(prefix)
			if err != nil {
				log.Printf("Path %s could not be made absolute: %s. Skipping...", prefix, err)
				continue
			}
			log.Printf("Traversing %s...", newprefix)
			filepath.Walk(newprefix, func(path string, info os.FileInfo, err error) error {
				if info.IsDir() {
					return nil
				}
				c <- &Item{
					Prefix:   newprefix,
					Path:     path,
					FileInfo: info,
				}
				return nil
			})
		}
		close(c)
	}()
	return c
}

func listBucketFiles(bucket *s3.Bucket) <-chan *Item {
	c := make(chan *Item)
	go func() {
		marker := ""
		for {
			resp, err := bucket.List(options.Get.Prefix, "", marker, 1000)
			if err != nil {
				log.Printf("Could not list items in bucket: %s", err)
			}
			for _, item := range resp.Contents {
				c <- &Item{
					Prefix:   options.Get.Prefix,
					Path:     item.Key,
					FileInfo: nil,
				}
				marker = item.Key
			}
			if !resp.IsTruncated {
				break
			}
		}
		close(c)
	}()
	return c
}

func putFiles(bucket *s3.Bucket, c <-chan *Item) {
	var wg sync.WaitGroup
	wg.Add(options.Concurrency)
	for i := 0; i < options.Concurrency; i++ {
		go func() {
			for item := range c {
				func() {
					f, err := os.Open(item.Path)
					if err != nil {
						log.Printf("Pushing %s failed: %s", item.Path, err)
					}
					defer f.Close()

					path := item.Path[len(item.Prefix)+1:]
					err = bucket.PutReader(options.Put.Prefix+path, f, item.FileInfo.Size(), mime.TypeByExtension(filepath.Ext(item.Path)), s3.BucketOwnerFull)
					if err != nil {
						log.Printf("Uploading %s failed: %s", path, err)
						return
					}
					log.Printf("Uploading %s done", path)
				}()
			}
			wg.Done()
		}()
	}
	wg.Wait()
}

func getFiles(bucket *s3.Bucket, c <-chan *Item) {
	var wg sync.WaitGroup
	wg.Add(options.Concurrency)
	for i := 0; i < options.Concurrency; i++ {
		go func() {
			for item := range c {
				func() {
					itempath := item.Path[len(item.Prefix):]
					dirname, fname := filepath.Split(itempath)
					dirname = filepath.Join(options.Remainder[0], dirname)

					os.MkdirAll(dirname, os.FileMode(0755))
					f, err := os.Create(filepath.Join(dirname, fname))
					if err != nil {
						log.Printf("Opening %s failed: %s", item.Path, err)
					}
					defer f.Close()

					rc, err := bucket.GetReader(item.Path)
					if err != nil {
						log.Printf("Downloading %s failed: %s", item.Path, err)
						return
					}
					defer rc.Close()
					io.Copy(f, rc)
					log.Printf("Downloading %s done", item.Path)
				}()
			}
			wg.Done()
		}()
	}
	wg.Wait()
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"os"
	"os/signal"
	"strings"
	"syscall"

	"golang.org/x/net/context"

	"github.com/pkg/errors"
	"github.com/urfave/cli"

	"github.com/currantlabs/ble"
	"github.com/currantlabs/ble/examples/lib"
	"github.com/currantlabs/ble/examples/lib/dev"
	"github.com/currantlabs/ble/linux"
)

var curr struct {
	device  ble.Device
	client  ble.Client
	uuid    ble.UUID
	addr    ble.Addr
	profile *ble.Profile
}

var (
	errNotConnected = fmt.Errorf("not connected")
	errNoProfile    = fmt.Errorf("no profile")
	errNoUUID       = fmt.Errorf("no UUID")
	errInvalidUUID  = fmt.Errorf("invalid UUID")
)

func main() {
	app := cli.NewApp()

	app.Name = "blesh"
	app.Usage = "A CLI tool for ble"
	app.Version = "0.0.1"
	app.Action = cli.ShowAppHelp
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "device",
			Value: "default",
			Usage: "implementation of ble (default / bled)",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:    "status",
			Aliases: []string{"st"},
			Usage:   "Display current status",
			Before:  setup,
			Action:  cmdStatus,
		},
		{
			Name:    "adv",
			Aliases: []string{"a"},
			Usage:   "Advertise name, UUIDs, iBeacon (TODO)",
			Before:  setup,
			Action:  cmdAdv,
			Flags:   []cli.Flag{flgTimeout, flgName},
		},
		{
			Name:    "serve",
			Aliases: []string{"sv"},
			Usage:   "Start the GATT Server",
			Before:  setup,
			Action:  cmdServe,
			Flags:   []cli.Flag{flgTimeout, flgName},
		},
		{
			Name:    "scan",
			Aliases: []string{"s"},
			Usage:   "Scan surrounding with specified filter",
			Before:  setup,
			Action:  cmdScan,
			Flags:   []cli.Flag{flgTimeout, flgName, flgAddr, flgAllowDup},
		},
		{
			Name:    "connect",
			Aliases: []string{"c"},
			Usage:   "Connect to a peripheral device",
			Before:  setup,
			Action:  cmdConnect,
			Flags:   []cli.Flag{flgTimeout, flgName, flgAddr},
		},
		{
			Name:    "disconnect",
			Aliases: []string{"x"},
			Usage:   "Disconnect a connected peripheral device",
			Before:  setup,
			Action:  cmdDisconnect,
		},
		{
			Name:    "discover",
			Aliases: []string{"d"},
			Usage:   "Discover profile on connected device",
			Before:  setup,
			Action:  cmdDiscover,
			Flags:   []cli.Flag{flgTimeout, flgName, flgAddr},
		},
		{
			Name:    "explore",
			Aliases: []string{"e"},
			Usage:   "Display discovered profile",
			Before:  setup,
			Action:  cmdExplore,
			Flags:   []cli.Flag{flgTimeout, flgName, flgAddr},
		},
		{
			Name:    "read",
			Aliases: []string{"r"},
			Usage:   "Read value from a characteristic or descriptor",
			Before:  setup,
			Action:  cmdRead,
			Flags:   []cli.Flag{flgUUID, flgTimeout, flgName, flgAddr},
		},
		{
			Name:    "write",
			Aliases: []string{"w"},
			Usage:   "Write value to a characteristic or descriptor",
			Before:  setup,
			Action:  cmdWrite,
			Flags:   []cli.Flag{flgUUID, flgTimeout, flgName, flgAddr},
		},
		{
			Name:   "sub",
			Usage:  "Subscribe to notification (or indication)",
			Before: setup,
			Action: cmdSub,
			Flags:  []cli.Flag{flgUUID, flgInd, flgTimeout, flgName, flgAddr},
		},
		{
			Name:   "unsub",
			Usage:  "Unsubscribe to notification (or indication)",
			Before: setup,
			Action: cmdUnsub,
			Flags:  []cli.Flag{flgUUID, flgInd},
		},
		{
			Name:    "shell",
			Aliases: []string{"sh"},
			Usage:   "Enter interactive mode",
			Before:  setup,
			Action:  func(c *cli.Context) { cmdShell(app) },
		},
	}

	// app.Before = setup
	app.Run(os.Args)
}

func setup(c *cli.Context) error {
	if curr.device != nil {
		return nil
	}
	fmt.Printf("Initializing device ...\n")
	d, err := dev.NewDevice("device")
	if err != nil {
		return errors.Wrap(err, "can't new device")
	}
	ble.SetDefaultDevice(d)
	curr.device = d

	// Optinal. Demostrate changing HCI parameters on Linux.
	if dev, ok := d.(*linux.Device); ok {
		return errors.Wrap(updateLinuxParam(dev), "can't update hci parameters")
	}

	return nil
}
func cmdStatus(c *cli.Context) error {
	m := map[bool]string{true: "yes", false: "no"}
	fmt.Printf("Current status:\n")
	fmt.Printf("  Initialized: %s\n", m[curr.device != nil])

	if curr.addr != nil {
		fmt.Printf("  Address:     %s\n", curr.addr)
	} else {
		fmt.Printf("  Address:\n")
	}

	if curr.client != nil {
		fmt.Printf("  Connected:   %s\n", curr.client.Address())
	} else {
		fmt.Printf("  Connected:\n")
	}

	fmt.Printf("  Profile:\n")
	if curr.profile != nil {
		fmt.Printf("\n")
		explore(curr.client, curr.profile)
	}

	if curr.uuid != nil {
		fmt.Printf("  UUID:       %s\n", curr.uuid)
	} else {
		fmt.Printf("  UUID:\n")
	}

	return nil
}

func cmdAdv(c *cli.Context) error {
	fmt.Printf("Advertising for %s...\n", c.Duration("tmo"))
	ctx := ble.WithSigHandler(context.WithTimeout(context.Background(), c.Duration("tmo")))
	return chkErr(ble.AdvertiseNameAndServices(ctx, "Gopher"))
}

func cmdScan(c *cli.Context) error {
	fmt.Printf("Scanning for %s...\n", c.Duration("tmo"))
	ctx := ble.WithSigHandler(context.WithTimeout(context.Background(), c.Duration("tmo")))
	return chkErr(ble.Scan(ctx, c.Bool("dup"), advHandler, filter(c)))
}

func cmdServe(c *cli.Context) error {
	testSvc := ble.NewService(lib.TestSvcUUID)
	testSvc.AddCharacteristic(lib.NewCountChar())
	testSvc.AddCharacteristic(lib.NewEchoChar())

	if err := ble.AddService(testSvc); err != nil {
		return errors.Wrap(err, "can't add service")
	}

	fmt.Printf("Serving GATT Server for %s...\n", c.Duration("tmo"))
	ctx := ble.WithSigHandler(context.WithTimeout(context.Background(), c.Duration("tmo")))
	return chkErr(ble.AdvertiseNameAndServices(ctx, "Gopher", testSvc.UUID))
}

func cmdConnect(c *cli.Context) error {
	curr.client = nil

	var cln ble.Client
	var err error

	fmt.Printf("Connecting...\n")
	ctx := ble.WithSigHandler(context.WithTimeout(context.Background(), c.Duration("tmo")))
	if filter(c) != nil {
		if cln, err = ble.Connect(ctx, filter(c)); err == nil {
			curr.addr = cln.Address()
		}
	} else if curr.addr != nil {
		cln, err = ble.Dial(ctx, curr.addr)
	} else {
		return fmt.Errorf("no filter specified, and cached peripheral address")
	}
	if err == nil {
		curr.client = cln
	}
	return err
}

func cmdDisconnect(c *cli.Context) error {
	if curr.client == nil {
		return errNotConnected
	}
	defer func() {
		curr.client = nil
		curr.profile = nil
	}()

	fmt.Printf("Disconnecting [ %s ]... (this might take up to few seconds on OS X)\n", curr.client.Address())
	return curr.client.CancelConnection()
}

func cmdDiscover(c *cli.Context) error {
	curr.profile = nil
	if curr.client == nil {
		if err := cmdConnect(c); err != nil {
			return errors.Wrap(err, "can't connect")
		}
	}

	fmt.Printf("Discovering profile...\n")
	p, err := curr.client.DiscoverProfile(true)
	if err != nil {
		return errors.Wrap(err, "can't discover profile")
	}

	curr.profile = p
	return nil
}

func cmdExplore(c *cli.Context) error {
	if curr.client == nil {
		if err := cmdConnect(c); err != nil {
			return errors.Wrap(err, "can't connect")
		}
	}
	if curr.profile == nil {
		if err := cmdDiscover(c); err != nil {
			return errors.Wrap(err, "can't discover profile")
		}
	}
	return explore(curr.client, curr.profile)
}

func cmdRead(c *cli.Context) error {
	if err := doGetUUID(c); err != nil {
		return err
	}
	if err := doConnect(c); err != nil {
		return err
	}
	if err := doDiscover(c); err != nil {
		return err
	}
	if u := curr.profile.Find(ble.NewCharacteristic(curr.uuid)); u != nil {
		b, err := curr.client.ReadCharacteristic(u.(*ble.Characteristic))
		if err != nil {
			return errors.Wrap(err, "can't read characteristic")
		}
		fmt.Printf("    Value         %x | %q\n", b, b)
		return nil
	}
	if u := curr.profile.Find(ble.NewDescriptor(curr.uuid)); u != nil {
		b, err := curr.client.ReadDescriptor(u.(*ble.Descriptor))
		if err != nil {
			return errors.Wrap(err, "can't read descriptor")
		}
		fmt.Printf("    Value         %x | %q\n", b, b)
		return nil
	}
	return errNoUUID
}

func cmdWrite(c *cli.Context) error {
	if err := doGetUUID(c); err != nil {
		return err
	}
	if err := doConnect(c); err != nil {
		return err
	}
	if err := doDiscover(c); err != nil {
		return err
	}
	if u := curr.profile.Find(ble.NewCharacteristic(curr.uuid)); u != nil {
		err := curr.client.WriteCharacteristic(u.(*ble.Characteristic), []byte("hello"), true)
		return errors.Wrap(err, "can't write characteristic")
	}
	if u := curr.profile.Find(ble.NewDescriptor(curr.uuid)); u != nil {
		err := curr.client.WriteDescriptor(u.(*ble.Descriptor), []byte("fixme"))
		return errors.Wrap(err, "can't write descriptor")
	}
	return errNoUUID
}

func cmdSub(c *cli.Context) error {
	if err := doGetUUID(c); err != nil {
		return err
	}
	if err := doConnect(c); err != nil {
		return err
	}
	// NotificationHandler
	h := func(req []byte) { fmt.Printf("notified: %x | %q\n", req, req) }
	if u := curr.profile.Find(ble.NewCharacteristic(curr.uuid)); u != nil {
		err := curr.client.Subscribe(u.(*ble.Characteristic), c.Bool("ind"), h)
		return errors.Wrap(err, "can't subscribe to characteristic")
	}
	return errNoUUID
}

func cmdUnsub(c *cli.Context) error {
	if err := doGetUUID(c); err != nil {
		return err
	}
	if err := doConnect(c); err != nil {
		return err
	}
	if u := curr.profile.Find(ble.NewCharacteristic(curr.uuid)); u != nil {
		err := curr.client.Unsubscribe(u.(*ble.Characteristic), c.Bool("ind"))
		return errors.Wrap(err, "can't unsubscribe to characteristic")
	}
	return errNoUUID
}

func cmdShell(app *cli.App) {
	cli.OsExiter = func(c int) {}
	reader := bufio.NewReader(os.Stdin)
	sigs := make(chan os.Signal, 1)
	go func() {
		for range sigs {
			fmt.Printf("\n(type quit or q to exit)\n\nblesh >")
		}
	}()
	defer close(sigs)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
	for {
		fmt.Print("blesh > ")
		text, _ := reader.ReadString('\n')
		text = strings.TrimSpace(text)
		if text == "" {
			continue
		}
		if text == "quit" || text == "q" {
			break
		}
		app.Run(append(os.Args[1:], strings.Split(text, " ")...))
	}
	signal.Stop(sigs)
}
<commit_msg>blesh: discover profile before sub<commit_after>package main

import (
	"bufio"
	"fmt"
	"os"
	"os/signal"
	"strings"
	"syscall"

	"golang.org/x/net/context"

	"github.com/pkg/errors"
	"github.com/urfave/cli"

	"github.com/currantlabs/ble"
	"github.com/currantlabs/ble/examples/lib"
	"github.com/currantlabs/ble/examples/lib/dev"
	"github.com/currantlabs/ble/linux"
)

var curr struct {
	device  ble.Device
	client  ble.Client
	uuid    ble.UUID
	addr    ble.Addr
	profile *ble.Profile
}

var (
	errNotConnected = fmt.Errorf("not connected")
	errNoProfile    = fmt.Errorf("no profile")
	errNoUUID       = fmt.Errorf("no UUID")
	errInvalidUUID  = fmt.Errorf("invalid UUID")
)

func main() {
	app := cli.NewApp()

	app.Name = "blesh"
	app.Usage = "A CLI tool for ble"
	app.Version = "0.0.1"
	app.Action = cli.ShowAppHelp
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "device",
			Value: "default",
			Usage: "implementation of ble (default / bled)",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:    "status",
			Aliases: []string{"st"},
			Usage:   "Display current status",
			Before:  setup,
			Action:  cmdStatus,
		},
		{
			Name:    "adv",
			Aliases: []string{"a"},
			Usage:   "Advertise name, UUIDs, iBeacon (TODO)",
			Before:  setup,
			Action:  cmdAdv,
			Flags:   []cli.Flag{flgTimeout, flgName},
		},
		{
			Name:    "serve",
			Aliases: []string{"sv"},
			Usage:   "Start the GATT Server",
			Before:  setup,
			Action:  cmdServe,
			Flags:   []cli.Flag{flgTimeout, flgName},
		},
		{
			Name:    "scan",
			Aliases: []string{"s"},
			Usage:   "Scan surrounding with specified filter",
			Before:  setup,
			Action:  cmdScan,
			Flags:   []cli.Flag{flgTimeout, flgName, flgAddr, flgAllowDup},
		},
		{
			Name:    "connect",
			Aliases: []string{"c"},
			Usage:   "Connect to a peripheral device",
			Before:  setup,
			Action:  cmdConnect,
			Flags:   []cli.Flag{flgTimeout, flgName, flgAddr},
		},
		{
			Name:    "disconnect",
			Aliases: []string{"x"},
			Usage:   "Disconnect a connected peripheral device",
			Before:  setup,
			Action:  cmdDisconnect,
		},
		{
			Name:    "discover",
			Aliases: []string{"d"},
			Usage:   "Discover profile on connected device",
			Before:  setup,
			Action:  cmdDiscover,
			Flags:   []cli.Flag{flgTimeout, flgName, flgAddr},
		},
		{
			Name:    "explore",
			Aliases: []string{"e"},
			Usage:   "Display discovered profile",
			Before:  setup,
			Action:  cmdExplore,
			Flags:   []cli.Flag{flgTimeout, flgName, flgAddr},
		},
		{
			Name:    "read",
			Aliases: []string{"r"},
			Usage:   "Read value from a characteristic or descriptor",
			Before:  setup,
			Action:  cmdRead,
			Flags:   []cli.Flag{flgUUID, flgTimeout, flgName, flgAddr},
		},
		{
			Name:    "write",
			Aliases: []string{"w"},
			Usage:   "Write value to a characteristic or descriptor",
			Before:  setup,
			Action:  cmdWrite,
			Flags:   []cli.Flag{flgUUID, flgTimeout, flgName, flgAddr},
		},
		{
			Name:   "sub",
			Usage:  "Subscribe to notification (or indication)",
			Before: setup,
			Action: cmdSub,
			Flags:  []cli.Flag{flgUUID, flgInd, flgTimeout, flgName, flgAddr},
		},
		{
			Name:   "unsub",
			Usage:  "Unsubscribe to notification (or indication)",
			Before: setup,
			Action: cmdUnsub,
			Flags:  []cli.Flag{flgUUID, flgInd},
		},
		{
			Name:    "shell",
			Aliases: []string{"sh"},
			Usage:   "Enter interactive mode",
			Before:  setup,
			Action:  func(c *cli.Context) { cmdShell(app) },
		},
	}

	// app.Before = setup
	app.Run(os.Args)
}

func setup(c *cli.Context) error {
	if curr.device != nil {
		return nil
	}
	fmt.Printf("Initializing device ...\n")
	d, err := dev.NewDevice("device")
	if err != nil {
		return errors.Wrap(err, "can't new device")
	}
	ble.SetDefaultDevice(d)
	curr.device = d

	// Optinal. Demostrate changing HCI parameters on Linux.
	if dev, ok := d.(*linux.Device); ok {
		return errors.Wrap(updateLinuxParam(dev), "can't update hci parameters")
	}

	return nil
}
func cmdStatus(c *cli.Context) error {
	m := map[bool]string{true: "yes", false: "no"}
	fmt.Printf("Current status:\n")
	fmt.Printf("  Initialized: %s\n", m[curr.device != nil])

	if curr.addr != nil {
		fmt.Printf("  Address:     %s\n", curr.addr)
	} else {
		fmt.Printf("  Address:\n")
	}

	if curr.client != nil {
		fmt.Printf("  Connected:   %s\n", curr.client.Address())
	} else {
		fmt.Printf("  Connected:\n")
	}

	fmt.Printf("  Profile:\n")
	if curr.profile != nil {
		fmt.Printf("\n")
		explore(curr.client, curr.profile)
	}

	if curr.uuid != nil {
		fmt.Printf("  UUID:       %s\n", curr.uuid)
	} else {
		fmt.Printf("  UUID:\n")
	}

	return nil
}

func cmdAdv(c *cli.Context) error {
	fmt.Printf("Advertising for %s...\n", c.Duration("tmo"))
	ctx := ble.WithSigHandler(context.WithTimeout(context.Background(), c.Duration("tmo")))
	return chkErr(ble.AdvertiseNameAndServices(ctx, "Gopher"))
}

func cmdScan(c *cli.Context) error {
	fmt.Printf("Scanning for %s...\n", c.Duration("tmo"))
	ctx := ble.WithSigHandler(context.WithTimeout(context.Background(), c.Duration("tmo")))
	return chkErr(ble.Scan(ctx, c.Bool("dup"), advHandler, filter(c)))
}

func cmdServe(c *cli.Context) error {
	testSvc := ble.NewService(lib.TestSvcUUID)
	testSvc.AddCharacteristic(lib.NewCountChar())
	testSvc.AddCharacteristic(lib.NewEchoChar())

	if err := ble.AddService(testSvc); err != nil {
		return errors.Wrap(err, "can't add service")
	}

	fmt.Printf("Serving GATT Server for %s...\n", c.Duration("tmo"))
	ctx := ble.WithSigHandler(context.WithTimeout(context.Background(), c.Duration("tmo")))
	return chkErr(ble.AdvertiseNameAndServices(ctx, "Gopher", testSvc.UUID))
}

func cmdConnect(c *cli.Context) error {
	curr.client = nil

	var cln ble.Client
	var err error

	fmt.Printf("Connecting...\n")
	ctx := ble.WithSigHandler(context.WithTimeout(context.Background(), c.Duration("tmo")))
	if filter(c) != nil {
		if cln, err = ble.Connect(ctx, filter(c)); err == nil {
			curr.addr = cln.Address()
		}
	} else if curr.addr != nil {
		cln, err = ble.Dial(ctx, curr.addr)
	} else {
		return fmt.Errorf("no filter specified, and cached peripheral address")
	}
	if err == nil {
		curr.client = cln
	}
	return err
}

func cmdDisconnect(c *cli.Context) error {
	if curr.client == nil {
		return errNotConnected
	}
	defer func() {
		curr.client = nil
		curr.profile = nil
	}()

	fmt.Printf("Disconnecting [ %s ]... (this might take up to few seconds on OS X)\n", curr.client.Address())
	return curr.client.CancelConnection()
}

func cmdDiscover(c *cli.Context) error {
	curr.profile = nil
	if curr.client == nil {
		if err := cmdConnect(c); err != nil {
			return errors.Wrap(err, "can't connect")
		}
	}

	fmt.Printf("Discovering profile...\n")
	p, err := curr.client.DiscoverProfile(true)
	if err != nil {
		return errors.Wrap(err, "can't discover profile")
	}

	curr.profile = p
	return nil
}

func cmdExplore(c *cli.Context) error {
	if curr.client == nil {
		if err := cmdConnect(c); err != nil {
			return errors.Wrap(err, "can't connect")
		}
	}
	if curr.profile == nil {
		if err := cmdDiscover(c); err != nil {
			return errors.Wrap(err, "can't discover profile")
		}
	}
	return explore(curr.client, curr.profile)
}

func cmdRead(c *cli.Context) error {
	if err := doGetUUID(c); err != nil {
		return err
	}
	if err := doConnect(c); err != nil {
		return err
	}
	if err := doDiscover(c); err != nil {
		return err
	}
	if u := curr.profile.Find(ble.NewCharacteristic(curr.uuid)); u != nil {
		b, err := curr.client.ReadCharacteristic(u.(*ble.Characteristic))
		if err != nil {
			return errors.Wrap(err, "can't read characteristic")
		}
		fmt.Printf("    Value         %x | %q\n", b, b)
		return nil
	}
	if u := curr.profile.Find(ble.NewDescriptor(curr.uuid)); u != nil {
		b, err := curr.client.ReadDescriptor(u.(*ble.Descriptor))
		if err != nil {
			return errors.Wrap(err, "can't read descriptor")
		}
		fmt.Printf("    Value         %x | %q\n", b, b)
		return nil
	}
	return errNoUUID
}

func cmdWrite(c *cli.Context) error {
	if err := doGetUUID(c); err != nil {
		return err
	}
	if err := doConnect(c); err != nil {
		return err
	}
	if err := doDiscover(c); err != nil {
		return err
	}
	if u := curr.profile.Find(ble.NewCharacteristic(curr.uuid)); u != nil {
		err := curr.client.WriteCharacteristic(u.(*ble.Characteristic), []byte("hello"), true)
		return errors.Wrap(err, "can't write characteristic")
	}
	if u := curr.profile.Find(ble.NewDescriptor(curr.uuid)); u != nil {
		err := curr.client.WriteDescriptor(u.(*ble.Descriptor), []byte("fixme"))
		return errors.Wrap(err, "can't write descriptor")
	}
	return errNoUUID
}

func cmdSub(c *cli.Context) error {
	if err := doGetUUID(c); err != nil {
		return err
	}
	if err := doConnect(c); err != nil {
		return err
	}
	if err := doDiscover(c); err != nil {
		return err
	}
	// NotificationHandler
	h := func(req []byte) { fmt.Printf("notified: %x | %q\n", req, req) }
	if u := curr.profile.Find(ble.NewCharacteristic(curr.uuid)); u != nil {
		err := curr.client.Subscribe(u.(*ble.Characteristic), c.Bool("ind"), h)
		return errors.Wrap(err, "can't subscribe to characteristic")
	}
	return errNoUUID
}

func cmdUnsub(c *cli.Context) error {
	if err := doGetUUID(c); err != nil {
		return err
	}
	if err := doConnect(c); err != nil {
		return err
	}
	if u := curr.profile.Find(ble.NewCharacteristic(curr.uuid)); u != nil {
		err := curr.client.Unsubscribe(u.(*ble.Characteristic), c.Bool("ind"))
		return errors.Wrap(err, "can't unsubscribe to characteristic")
	}
	return errNoUUID
}

func cmdShell(app *cli.App) {
	cli.OsExiter = func(c int) {}
	reader := bufio.NewReader(os.Stdin)
	sigs := make(chan os.Signal, 1)
	go func() {
		for range sigs {
			fmt.Printf("\n(type quit or q to exit)\n\nblesh >")
		}
	}()
	defer close(sigs)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
	for {
		fmt.Print("blesh > ")
		text, _ := reader.ReadString('\n')
		text = strings.TrimSpace(text)
		if text == "" {
			continue
		}
		if text == "quit" || text == "q" {
			break
		}
		app.Run(append(os.Args[1:], strings.Split(text, " ")...))
	}
	signal.Stop(sigs)
}
<|endoftext|>


<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package controller

import (
	"sync"
	"time"

	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/labels"

	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"

	"kubevirt.io/kubevirt/pkg/kubecli"

	kubev1 "kubevirt.io/kubevirt/pkg/api/v1"
	"kubevirt.io/kubevirt/pkg/log"
)

type newSharedInformer func() cache.SharedIndexInformer

type KubeInformerFactory interface {
	// Starts any informers that have not been started yet
	// This function is thread safe and idempotent
	Start(stopCh <-chan struct{})

	// Watches for vm objects
	VM() cache.SharedIndexInformer
	// Watches for migration objects
	Migration() cache.SharedIndexInformer

	VMReplicaSet() cache.SharedIndexInformer
	// Watches for pods related only to kubevirt
	KubeVirtPod() cache.SharedIndexInformer
}

type kubeInformerFactory struct {
	restClient    *rest.RESTClient
	clientSet     kubecli.KubevirtClient
	lock          sync.Mutex
	defaultResync time.Duration

	informers        map[string]cache.SharedIndexInformer
	startedInformers map[string]bool
}

func NewKubeInformerFactory(restClient *rest.RESTClient, clientSet kubecli.KubevirtClient) KubeInformerFactory {
	return &kubeInformerFactory{
		restClient:       restClient,
		clientSet:        clientSet,
		defaultResync:    0,
		informers:        make(map[string]cache.SharedIndexInformer),
		startedInformers: make(map[string]bool),
	}
}

// Start can be called from multiple controllers in different go routines safely.
// Only informers that have not started are triggered by this function.
// Multiple calls to this function are idempotent.
func (f *kubeInformerFactory) Start(stopCh <-chan struct{}) {
	f.lock.Lock()
	defer f.lock.Unlock()

	for name, informer := range f.informers {
		if f.startedInformers[name] {
			// skip informers that have already started.
			log.Log.Infof("SKIPPING informer %s", name)
			continue
		}
		log.Log.Infof("STARTING informer %s", name)
		go informer.Run(stopCh)
		f.startedInformers[name] = true
	}
}

// internal function used to retrieve an already created informer
// or create a new informer if one does not already exist.
// Thread safe
func (f *kubeInformerFactory) getInformer(key string, newFunc newSharedInformer) cache.SharedIndexInformer {
	f.lock.Lock()
	defer f.lock.Unlock()

	informer, exists := f.informers[key]
	if exists {
		return informer
	}
	informer = newFunc()
	f.informers[key] = informer

	return informer
}

func (f *kubeInformerFactory) VM() cache.SharedIndexInformer {
	return f.getInformer("vmInformer", func() cache.SharedIndexInformer {
		lw := cache.NewListWatchFromClient(f.restClient, "virtualmachines", k8sv1.NamespaceAll, fields.Everything())
		return cache.NewSharedIndexInformer(lw, &kubev1.VirtualMachine{}, f.defaultResync, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
	})
}

func (f *kubeInformerFactory) Migration() cache.SharedIndexInformer {
	return f.getInformer("migrationInformer", func() cache.SharedIndexInformer {
		lw := cache.NewListWatchFromClient(f.restClient, "migrations", k8sv1.NamespaceAll, fields.Everything())
		return cache.NewSharedIndexInformer(lw, &kubev1.Migration{}, f.defaultResync, cache.Indexers{})
	})
}

func (f *kubeInformerFactory) VMReplicaSet() cache.SharedIndexInformer {
	return f.getInformer("vmrsInformer", func() cache.SharedIndexInformer {
		lw := cache.NewListWatchFromClient(f.restClient, "virtualmachinereplicasets", k8sv1.NamespaceAll, fields.Everything())
		return cache.NewSharedIndexInformer(lw, &kubev1.VirtualMachineReplicaSet{}, f.defaultResync, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
	})
}

func (f *kubeInformerFactory) KubeVirtPod() cache.SharedIndexInformer {
	return f.getInformer("kubeVirtPodInformer", func() cache.SharedIndexInformer {
		// Watch all pods with the kubevirt app label
		labelSelector, err := labels.Parse(kubev1.AppLabel)
		if err != nil {
			panic(err)
		}

		lw := NewListWatchFromClient(f.clientSet.CoreV1().RESTClient(), "pods", k8sv1.NamespaceAll, fields.Everything(), labelSelector)
		return cache.NewSharedIndexInformer(lw, &k8sv1.Pod{}, f.defaultResync, cache.Indexers{})
	})
}
<commit_msg>Set informer resync period in an equivalent fashion like k8s<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package controller

import (
	"math/rand"
	"sync"
	"time"

	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/labels"

	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"

	kubev1 "kubevirt.io/kubevirt/pkg/api/v1"
	"kubevirt.io/kubevirt/pkg/kubecli"
	"kubevirt.io/kubevirt/pkg/log"
)

type newSharedInformer func() cache.SharedIndexInformer

type KubeInformerFactory interface {
	// Starts any informers that have not been started yet
	// This function is thread safe and idempotent
	Start(stopCh <-chan struct{})

	// Watches for vm objects
	VM() cache.SharedIndexInformer
	// Watches for migration objects
	Migration() cache.SharedIndexInformer

	VMReplicaSet() cache.SharedIndexInformer
	// Watches for pods related only to kubevirt
	KubeVirtPod() cache.SharedIndexInformer
}

type kubeInformerFactory struct {
	restClient    *rest.RESTClient
	clientSet     kubecli.KubevirtClient
	lock          sync.Mutex
	defaultResync time.Duration

	informers        map[string]cache.SharedIndexInformer
	startedInformers map[string]bool
}

func NewKubeInformerFactory(restClient *rest.RESTClient, clientSet kubecli.KubevirtClient) KubeInformerFactory {
	return &kubeInformerFactory{
		restClient: restClient,
		clientSet:  clientSet,
		// Resulting resync period will be between 12 and 24 hours, like the default for k8s
		defaultResync:    resyncPeriod(12 * time.Hour),
		informers:        make(map[string]cache.SharedIndexInformer),
		startedInformers: make(map[string]bool),
	}
}

// Start can be called from multiple controllers in different go routines safely.
// Only informers that have not started are triggered by this function.
// Multiple calls to this function are idempotent.
func (f *kubeInformerFactory) Start(stopCh <-chan struct{}) {
	f.lock.Lock()
	defer f.lock.Unlock()

	for name, informer := range f.informers {
		if f.startedInformers[name] {
			// skip informers that have already started.
			log.Log.Infof("SKIPPING informer %s", name)
			continue
		}
		log.Log.Infof("STARTING informer %s", name)
		go informer.Run(stopCh)
		f.startedInformers[name] = true
	}
}

// internal function used to retrieve an already created informer
// or create a new informer if one does not already exist.
// Thread safe
func (f *kubeInformerFactory) getInformer(key string, newFunc newSharedInformer) cache.SharedIndexInformer {
	f.lock.Lock()
	defer f.lock.Unlock()

	informer, exists := f.informers[key]
	if exists {
		return informer
	}
	informer = newFunc()
	f.informers[key] = informer

	return informer
}

func (f *kubeInformerFactory) VM() cache.SharedIndexInformer {
	return f.getInformer("vmInformer", func() cache.SharedIndexInformer {
		lw := cache.NewListWatchFromClient(f.restClient, "virtualmachines", k8sv1.NamespaceAll, fields.Everything())
		return cache.NewSharedIndexInformer(lw, &kubev1.VirtualMachine{}, f.defaultResync, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
	})
}

func (f *kubeInformerFactory) Migration() cache.SharedIndexInformer {
	return f.getInformer("migrationInformer", func() cache.SharedIndexInformer {
		lw := cache.NewListWatchFromClient(f.restClient, "migrations", k8sv1.NamespaceAll, fields.Everything())
		return cache.NewSharedIndexInformer(lw, &kubev1.Migration{}, f.defaultResync, cache.Indexers{})
	})
}

func (f *kubeInformerFactory) VMReplicaSet() cache.SharedIndexInformer {
	return f.getInformer("vmrsInformer", func() cache.SharedIndexInformer {
		lw := cache.NewListWatchFromClient(f.restClient, "virtualmachinereplicasets", k8sv1.NamespaceAll, fields.Everything())
		return cache.NewSharedIndexInformer(lw, &kubev1.VirtualMachineReplicaSet{}, f.defaultResync, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
	})
}

func (f *kubeInformerFactory) KubeVirtPod() cache.SharedIndexInformer {
	return f.getInformer("kubeVirtPodInformer", func() cache.SharedIndexInformer {
		// Watch all pods with the kubevirt app label
		labelSelector, err := labels.Parse(kubev1.AppLabel)
		if err != nil {
			panic(err)
		}

		lw := NewListWatchFromClient(f.clientSet.CoreV1().RESTClient(), "pods", k8sv1.NamespaceAll, fields.Everything(), labelSelector)
		return cache.NewSharedIndexInformer(lw, &k8sv1.Pod{}, f.defaultResync, cache.Indexers{})
	})
}

// resyncPeriod computes the time interval a shared informer waits before resyncing with the api server
func resyncPeriod(minResyncPeriod time.Duration) time.Duration {
	factor := rand.Float64() + 1
	return time.Duration(float64(minResyncPeriod.Nanoseconds()) * factor)
}
<|endoftext|>


<commit_before>// Copyright © 2018 Banzai Cloud
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tls

import (
	"crypto/tls"
	"crypto/x509"
	"io/ioutil"
	"net/http"
	"os"
	"testing"
)

const (
	ServerKey  = "server.key"
	ServerCert = "server.crt"
)

func helloServer(w http.ResponseWriter, _ *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	w.Write([]byte("This is an example TLS server.\n"))
}

func TestGenerateTLS(t *testing.T) {
	cc, err := GenerateTLS("localhost", "1h")
	if err != nil {
		t.Fatal(err)
	}

	err = ioutil.WriteFile(ServerKey, []byte(cc.ServerKey), 0600)
	if err != nil {
		t.Fatal(err)
	}
	err = ioutil.WriteFile(ServerCert, []byte(cc.ServerCert), 0600)
	if err != nil {
		t.Fatal(err)
	}

	defer os.Remove(ServerKey)
	defer os.Remove(ServerCert)

	// Load CA cert
	caCertPool := x509.NewCertPool()
	caCertPool.AppendCertsFromPEM([]byte(cc.CACert))

	tlsConfig := &tls.Config{
		ClientAuth:               tls.RequireAndVerifyClientCert,
		ClientCAs:                caCertPool,
		PreferServerCipherSuites: true,
		MinVersion:               tls.VersionTLS12,
	}

	tlsConfig.BuildNameToCertificate()

	server := &http.Server{
		Addr:      ":8443",
		TLSConfig: tlsConfig,
	}

	http.HandleFunc("/", helloServer)

	go server.ListenAndServeTLS(ServerCert, ServerKey)

	// Load client cert
	cert, err := tls.X509KeyPair([]byte(cc.ClientCert), []byte(cc.ClientKey))
	if err != nil {
		t.Fatal(err)
	}

	// Setup HTTPS client
	clientTLSConfig := &tls.Config{
		Certificates: []tls.Certificate{cert},
		RootCAs:      caCertPool,
	}
	clientTLSConfig.BuildNameToCertificate()
	transport := &http.Transport{TLSClientConfig: clientTLSConfig}
	client := &http.Client{Transport: transport}

	req, err := http.NewRequest("GET", "https://localhost:8443/", nil)
	if err != nil {
		t.Fatal(err)
	}

	_, err = client.Do(req)
	if err != nil {
		t.Fatal(err)
	}

	server.Close()
}
<commit_msg>Improve TLS integration test<commit_after>// Copyright © 2018 Banzai Cloud
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tls

import (
	"crypto/tls"
	"crypto/x509"
	"net/http"
	"testing"
)

func TestGenerateTLS(t *testing.T) {
	cc, err := GenerateTLS("localhost", "1h")
	if err != nil {
		t.Fatal(err)
	}

	// Load CA cert
	caCertPool := x509.NewCertPool()
	caCertPool.AppendCertsFromPEM([]byte(cc.CACert))

	serverCert, err := tls.X509KeyPair([]byte(cc.ServerCert), []byte(cc.ServerKey))
	if err != nil {
		t.Fatal(err)
	}

	tlsConfig := &tls.Config{
		ClientAuth:               tls.RequireAndVerifyClientCert,
		ClientCAs:                caCertPool,
		PreferServerCipherSuites: true,
		MinVersion:               tls.VersionTLS12,
		Certificates:             []tls.Certificate{serverCert},
	}

	tlsConfig.BuildNameToCertificate()

	ln, err := tls.Listen("tcp", "127.0.0.1:8443", tlsConfig)
	if err != nil {
		t.Fatal(err)
	}

	server := &http.Server{
		Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
			w.Header().Set("Content-Type", "text/plain")
			w.Write([]byte("This is an example TLS server.\n"))
		}),
		TLSConfig: tlsConfig,
	}

	go server.Serve(ln)

	// Load client cert
	clientCert, err := tls.X509KeyPair([]byte(cc.ClientCert), []byte(cc.ClientKey))
	if err != nil {
		t.Fatal(err)
	}

	// Setup HTTPS client
	clientTLSConfig := &tls.Config{
		Certificates: []tls.Certificate{clientCert},
		RootCAs:      caCertPool,
	}
	clientTLSConfig.BuildNameToCertificate()
	transport := &http.Transport{TLSClientConfig: clientTLSConfig}
	client := &http.Client{Transport: transport}

	req, err := http.NewRequest("GET", "https://localhost:8443/", nil)
	if err != nil {
		t.Fatal(err)
	}

	_, err = client.Do(req)
	if err != nil {
		t.Fatal(err)
	}

	server.Close()
}
<|endoftext|>


<commit_before>/*
Copyright 2018 Gravitational, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package s3sessions

import (
	"context"
	"fmt"
	"io"
	"net/url"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/gravitational/teleport"
	"github.com/gravitational/teleport/lib/session"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	awssession "github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/gravitational/trace"
	log "github.com/sirupsen/logrus"
)

// Config is handler configuration
type Config struct {
	// Bucket is S3 bucket name
	Bucket string
	// Region is S3 bucket region
	Region string
	// Path is an optional bucket path
	Path string
	// Host is an optional third party S3 compatible endpoint
	Endpoint string
	// Insecure is an optional switch to opt out of https connections
	Insecure bool
	//DisableServerSideEncryption is an optional switch to opt out of SSE in case the provider does not support it
	DisableServerSideEncryption bool
	// Session is an optional existing AWS client session
	Session *awssession.Session
	// Credentials if supplied are used in tests
	Credentials *credentials.Credentials
}

// SetFromURL sets values on the Config from the supplied URI
func (s *Config) SetFromURL(in *url.URL, inRegion string) error {
	region := inRegion
	if uriRegion := in.Query().Get(teleport.Region); uriRegion != "" {
		region = uriRegion
	}
	if endpoint := in.Query().Get(teleport.Endpoint); endpoint != "" {
		s.Endpoint = endpoint
	}
	if val := in.Query().Get(teleport.Insecure); val != "" {
		insecure, err := strconv.ParseBool(val)
		if err != nil {
			return trace.BadParameter("failed to parse URI %q flag %q - %q, supported values are 'true' or 'false'", in.String(), teleport.Insecure, val)
		}
		s.Insecure = insecure
	}
	if val := in.Query().Get(teleport.DisableServerSideEncryption); val != "" {
		disableServerSideEncryption, err := strconv.ParseBool(val)
		if err != nil {
			return trace.BadParameter("failed to parse URI %q flag %q - %q, supported values are 'true' or 'false'", in.String(), teleport.DisableServerSideEncryption, val)
		}
		s.DisableServerSideEncryption = disableServerSideEncryption
	}
	s.Region = region
	s.Bucket = in.Host
	s.Path = in.Path
	return nil
}

// CheckAndSetDefaults checks and sets defaults
func (s *Config) CheckAndSetDefaults() error {
	if s.Bucket == "" {
		return trace.BadParameter("missing parameter Bucket")
	}
	if s.Session == nil {
		// create an AWS session using default SDK behavior, i.e. it will interpret
		// the environment and ~/.aws directory just like an AWS CLI tool would:
		sess, err := awssession.NewSessionWithOptions(awssession.Options{
			SharedConfigState: awssession.SharedConfigEnable,
		})
		if err != nil {
			return trace.Wrap(err)
		}
		// override the default environment (region + Host + credentials) with the values
		// from the YAML file:
		if s.Region != "" {
			sess.Config.Region = aws.String(s.Region)
		}
		if s.Endpoint != "" {
			sess.Config.Endpoint = aws.String(s.Endpoint)
			sess.Config.S3ForcePathStyle = aws.Bool(true)
		}
		if s.Insecure {
			sess.Config.DisableSSL = aws.Bool(s.Insecure)
		}
		if s.Credentials != nil {
			sess.Config.Credentials = s.Credentials
		}
		s.Session = sess
	}
	return nil
}

// NewHandler returns new S3 uploader
func NewHandler(cfg Config) (*Handler, error) {
	if err := cfg.CheckAndSetDefaults(); err != nil {
		return nil, trace.Wrap(err)
	}

	h := &Handler{
		Entry: log.WithFields(log.Fields{
			trace.Component: teleport.Component(teleport.SchemeS3),
		}),
		Config:     cfg,
		uploader:   s3manager.NewUploader(cfg.Session),
		downloader: s3manager.NewDownloader(cfg.Session),
		client:     s3.New(cfg.Session),
	}
	start := time.Now()
	h.Infof("Setting up bucket %q, sessions path %q in region %q.", h.Bucket, h.Path, h.Region)
	if err := h.ensureBucket(); err != nil {
		return nil, trace.Wrap(err)
	}
	h.WithFields(log.Fields{"duration": time.Now().Sub(start)}).Infof("Setup bucket %q completed.", h.Bucket)
	return h, nil
}

// Handler handles upload and downloads to S3 object storage
type Handler struct {
	// Config is handler configuration
	Config
	// Entry is a logging entry
	*log.Entry
	uploader   *s3manager.Uploader
	downloader *s3manager.Downloader
	client     *s3.S3
}

// Closer releases connection and resources associated with log if any
func (l *Handler) Close() error {
	return nil
}

// Upload uploads object to S3 bucket, reads the contents of the object from reader
// and returns the target S3 bucket path in case of successful upload.
func (l *Handler) Upload(ctx context.Context, sessionID session.ID, reader io.Reader) (string, error) {
	path := l.path(sessionID)
	_, err := l.uploader.UploadWithContext(ctx, &s3manager.UploadInput{
		Bucket:               aws.String(l.Bucket),
		Key:                  aws.String(path),
		Body:                 reader,
		ServerSideEncryption: aws.String(s3.ServerSideEncryptionAwsKms),
	})
	if err != nil {
		return "", ConvertS3Error(err)
	}
	return fmt.Sprintf("%v://%v/%v", teleport.SchemeS3, l.Bucket, path), nil
}

// Download downloads recorded session from S3 bucket and writes the results
// into writer return trace.NotFound error is object is not found.
func (l *Handler) Download(ctx context.Context, sessionID session.ID, writer io.WriterAt) error {
	// Get the oldest version of this object. This has to be done because S3
	// allows overwriting objects in a bucket. To prevent corruption of recording
	// data, get all versions and always return the first.
	versionID, err := l.getOldestVersion(l.Bucket, l.path(sessionID))
	if err != nil {
		return trace.Wrap(err)
	}

	l.Debugf("Downloading %v/%v [%v].", l.Bucket, l.path(sessionID), versionID)

	written, err := l.downloader.DownloadWithContext(ctx, writer, &s3.GetObjectInput{
		Bucket:    aws.String(l.Bucket),
		Key:       aws.String(l.path(sessionID)),
		VersionId: aws.String(versionID),
	})
	if err != nil {
		return ConvertS3Error(err)
	}
	if written == 0 {
		return trace.NotFound("recording for %v is not found", sessionID)
	}
	return nil
}

// versionID is used to store versions of a key to allow sorting by timestamp.
type versionID struct {
	// ID is the version ID.
	ID string

	// Timestamp is the last time the object was modified.
	Timestamp time.Time
}

// getOldestVersion returns the oldest version of the object.
func (l *Handler) getOldestVersion(bucket string, prefix string) (string, error) {
	var versions []versionID

	// Get all versions of this object.
	err := l.client.ListObjectVersionsPages(&s3.ListObjectVersionsInput{
		Bucket: aws.String(bucket),
		Prefix: aws.String(prefix),
	}, func(page *s3.ListObjectVersionsOutput, lastPage bool) bool {
		for _, v := range page.Versions {
			versions = append(versions, versionID{
				ID:        *v.VersionId,
				Timestamp: *v.LastModified,
			})
		}

		// Returning false stops iteration, stop iteration upon last page.
		return !lastPage
	})
	if err != nil {
		return "", ConvertS3Error(err)
	}
	if len(versions) == 0 {
		return "", trace.NotFound("%v/%v not found", bucket, prefix)
	}

	// Sort the versions slice so the first entry is the oldest and return it.
	sort.Slice(versions, func(i int, j int) bool {
		return versions[i].Timestamp.Before(versions[j].Timestamp)
	})
	return versions[0].ID, nil
}

// delete bucket deletes bucket and all it's contents and is used in tests
func (h *Handler) deleteBucket() error {
	// first, list and delete all the objects in the bucket
	out, err := h.client.ListObjectVersions(&s3.ListObjectVersionsInput{
		Bucket: aws.String(h.Bucket),
	})
	if err != nil {
		return ConvertS3Error(err)
	}
	for _, ver := range out.Versions {
		_, err := h.client.DeleteObject(&s3.DeleteObjectInput{
			Bucket:    aws.String(h.Bucket),
			Key:       ver.Key,
			VersionId: ver.VersionId,
		})
		if err != nil {
			return ConvertS3Error(err)
		}
	}
	_, err = h.client.DeleteBucket(&s3.DeleteBucketInput{
		Bucket: aws.String(h.Bucket),
	})
	return ConvertS3Error(err)
}

func (l *Handler) path(sessionID session.ID) string {
	if l.Path == "" {
		return string(sessionID) + ".tar"
	}
	return strings.TrimPrefix(filepath.Join(l.Path, string(sessionID)+".tar"), "/")
}

// ensureBucket makes sure bucket exists, and if it does not, creates it
func (h *Handler) ensureBucket() error {
	_, err := h.client.HeadBucket(&s3.HeadBucketInput{
		Bucket: aws.String(h.Bucket),
	})
	err = ConvertS3Error(err)
	// assumes that bucket is administered by other entity
	if err == nil {
		return nil
	}
	if !trace.IsNotFound(err) {
		return trace.Wrap(err)
	}
	input := &s3.CreateBucketInput{
		Bucket: aws.String(h.Bucket),
		ACL:    aws.String("private"),
	}
	_, err = h.client.CreateBucket(input)
	err = ConvertS3Error(err, "bucket %v already exists", aws.String(h.Bucket))
	if err != nil {
		if !trace.IsAlreadyExists(err) {
			return trace.Wrap(err)
		}
		// if this client has not created the bucket, don't reconfigure it
		return nil
	}

	// Turn on versioning.
	ver := &s3.PutBucketVersioningInput{
		Bucket: aws.String(h.Bucket),
		VersioningConfiguration: &s3.VersioningConfiguration{
			Status: aws.String("Enabled"),
		},
	}
	_, err = h.client.PutBucketVersioning(ver)
	err = ConvertS3Error(err, "failed to set versioning state for bucket %q", h.Bucket)
	if err != nil {
		return trace.Wrap(err)
	}

	// Turn on server-side encryption for the bucket.
	if !h.DisableServerSideEncryption {
		_, err = h.client.PutBucketEncryption(&s3.PutBucketEncryptionInput{
			Bucket: aws.String(h.Bucket),
			ServerSideEncryptionConfiguration: &s3.ServerSideEncryptionConfiguration{
				Rules: []*s3.ServerSideEncryptionRule{&s3.ServerSideEncryptionRule{
					ApplyServerSideEncryptionByDefault: &s3.ServerSideEncryptionByDefault{
						SSEAlgorithm: aws.String(s3.ServerSideEncryptionAwsKms),
					},
				}},
			},
		})
		err = ConvertS3Error(err, "failed to set versioning state for bucket %q", h.Bucket)
		if err != nil {
			return trace.Wrap(err)
		}
	}
	return nil
}

// ConvertS3Error wraps S3 error and returns trace equivalent
func ConvertS3Error(err error, args ...interface{}) error {
	if err == nil {
		return nil
	}
	if aerr, ok := err.(awserr.Error); ok {
		switch aerr.Code() {
		case s3.ErrCodeNoSuchKey, s3.ErrCodeNoSuchBucket, s3.ErrCodeNoSuchUpload, "NotFound":
			return trace.NotFound(aerr.Error(), args...)
		case s3.ErrCodeBucketAlreadyExists, s3.ErrCodeBucketAlreadyOwnedByYou:
			return trace.AlreadyExists(aerr.Error(), args...)
		default:
			return trace.BadParameter(aerr.Error(), args...)
		}
	}
	return err
}
<commit_msg>Fix S3 third party upload handling (gravitational#3322)<commit_after>/*
Copyright 2018 Gravitational, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package s3sessions

import (
	"context"
	"fmt"
	"io"
	"net/url"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/gravitational/teleport"
	"github.com/gravitational/teleport/lib/session"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	awssession "github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/gravitational/trace"
	log "github.com/sirupsen/logrus"
)

// Config is handler configuration
type Config struct {
	// Bucket is S3 bucket name
	Bucket string
	// Region is S3 bucket region
	Region string
	// Path is an optional bucket path
	Path string
	// Host is an optional third party S3 compatible endpoint
	Endpoint string
	// Insecure is an optional switch to opt out of https connections
	Insecure bool
	//DisableServerSideEncryption is an optional switch to opt out of SSE in case the provider does not support it
	DisableServerSideEncryption bool
	// Session is an optional existing AWS client session
	Session *awssession.Session
	// Credentials if supplied are used in tests
	Credentials *credentials.Credentials
}

// SetFromURL sets values on the Config from the supplied URI
func (s *Config) SetFromURL(in *url.URL, inRegion string) error {
	region := inRegion
	if uriRegion := in.Query().Get(teleport.Region); uriRegion != "" {
		region = uriRegion
	}
	if endpoint := in.Query().Get(teleport.Endpoint); endpoint != "" {
		s.Endpoint = endpoint
	}
	if val := in.Query().Get(teleport.Insecure); val != "" {
		insecure, err := strconv.ParseBool(val)
		if err != nil {
			return trace.BadParameter("failed to parse URI %q flag %q - %q, supported values are 'true' or 'false'", in.String(), teleport.Insecure, val)
		}
		s.Insecure = insecure
	}
	if val := in.Query().Get(teleport.DisableServerSideEncryption); val != "" {
		disableServerSideEncryption, err := strconv.ParseBool(val)
		if err != nil {
			return trace.BadParameter("failed to parse URI %q flag %q - %q, supported values are 'true' or 'false'", in.String(), teleport.DisableServerSideEncryption, val)
		}
		s.DisableServerSideEncryption = disableServerSideEncryption
	}
	s.Region = region
	s.Bucket = in.Host
	s.Path = in.Path
	return nil
}

// CheckAndSetDefaults checks and sets defaults
func (s *Config) CheckAndSetDefaults() error {
	if s.Bucket == "" {
		return trace.BadParameter("missing parameter Bucket")
	}
	if s.Session == nil {
		// create an AWS session using default SDK behavior, i.e. it will interpret
		// the environment and ~/.aws directory just like an AWS CLI tool would:
		sess, err := awssession.NewSessionWithOptions(awssession.Options{
			SharedConfigState: awssession.SharedConfigEnable,
		})
		if err != nil {
			return trace.Wrap(err)
		}
		// override the default environment (region + Host + credentials) with the values
		// from the YAML file:
		if s.Region != "" {
			sess.Config.Region = aws.String(s.Region)
		}
		if s.Endpoint != "" {
			sess.Config.Endpoint = aws.String(s.Endpoint)
			sess.Config.S3ForcePathStyle = aws.Bool(true)
		}
		if s.Insecure {
			sess.Config.DisableSSL = aws.Bool(s.Insecure)
		}
		if s.Credentials != nil {
			sess.Config.Credentials = s.Credentials
		}
		s.Session = sess
	}
	return nil
}

// NewHandler returns new S3 uploader
func NewHandler(cfg Config) (*Handler, error) {
	if err := cfg.CheckAndSetDefaults(); err != nil {
		return nil, trace.Wrap(err)
	}

	h := &Handler{
		Entry: log.WithFields(log.Fields{
			trace.Component: teleport.Component(teleport.SchemeS3),
		}),
		Config:     cfg,
		uploader:   s3manager.NewUploader(cfg.Session),
		downloader: s3manager.NewDownloader(cfg.Session),
		client:     s3.New(cfg.Session),
	}
	start := time.Now()
	h.Infof("Setting up bucket %q, sessions path %q in region %q.", h.Bucket, h.Path, h.Region)
	if err := h.ensureBucket(); err != nil {
		return nil, trace.Wrap(err)
	}
	h.WithFields(log.Fields{"duration": time.Now().Sub(start)}).Infof("Setup bucket %q completed.", h.Bucket)
	return h, nil
}

// Handler handles upload and downloads to S3 object storage
type Handler struct {
	// Config is handler configuration
	Config
	// Entry is a logging entry
	*log.Entry
	uploader   *s3manager.Uploader
	downloader *s3manager.Downloader
	client     *s3.S3
}

// Closer releases connection and resources associated with log if any
func (l *Handler) Close() error {
	return nil
}

// Upload uploads object to S3 bucket, reads the contents of the object from reader
// and returns the target S3 bucket path in case of successful upload.
func (l *Handler) Upload(ctx context.Context, sessionID session.ID, reader io.Reader) (string, error) {
	var err error
	path := l.path(sessionID)
	if l.Config.DisableServerSideEncryption {
		_, err = l.uploader.UploadWithContext(ctx, &s3manager.UploadInput{
			Bucket:               aws.String(l.Bucket),
			Key:                  aws.String(path),
			Body:                 reader,
		})
	} else {
		_, err = l.uploader.UploadWithContext(ctx, &s3manager.UploadInput{
			Bucket:               aws.String(l.Bucket),
			Key:                  aws.String(path),
			Body:                 reader,
			ServerSideEncryption: aws.String(s3.ServerSideEncryptionAwsKms),
		})
	}
	if err != nil {
		return "", ConvertS3Error(err)
	}
	return fmt.Sprintf("%v://%v/%v", teleport.SchemeS3, l.Bucket, path), nil
}

// Download downloads recorded session from S3 bucket and writes the results
// into writer return trace.NotFound error is object is not found.
func (l *Handler) Download(ctx context.Context, sessionID session.ID, writer io.WriterAt) error {
	// Get the oldest version of this object. This has to be done because S3
	// allows overwriting objects in a bucket. To prevent corruption of recording
	// data, get all versions and always return the first.
	versionID, err := l.getOldestVersion(l.Bucket, l.path(sessionID))
	if err != nil {
		return trace.Wrap(err)
	}

	l.Debugf("Downloading %v/%v [%v].", l.Bucket, l.path(sessionID), versionID)

	written, err := l.downloader.DownloadWithContext(ctx, writer, &s3.GetObjectInput{
		Bucket:    aws.String(l.Bucket),
		Key:       aws.String(l.path(sessionID)),
		VersionId: aws.String(versionID),
	})
	if err != nil {
		return ConvertS3Error(err)
	}
	if written == 0 {
		return trace.NotFound("recording for %v is not found", sessionID)
	}
	return nil
}

// versionID is used to store versions of a key to allow sorting by timestamp.
type versionID struct {
	// ID is the version ID.
	ID string

	// Timestamp is the last time the object was modified.
	Timestamp time.Time
}

// getOldestVersion returns the oldest version of the object.
func (l *Handler) getOldestVersion(bucket string, prefix string) (string, error) {
	var versions []versionID

	// Get all versions of this object.
	err := l.client.ListObjectVersionsPages(&s3.ListObjectVersionsInput{
		Bucket: aws.String(bucket),
		Prefix: aws.String(prefix),
	}, func(page *s3.ListObjectVersionsOutput, lastPage bool) bool {
		for _, v := range page.Versions {
			versions = append(versions, versionID{
				ID:        *v.VersionId,
				Timestamp: *v.LastModified,
			})
		}

		// Returning false stops iteration, stop iteration upon last page.
		return !lastPage
	})
	if err != nil {
		return "", ConvertS3Error(err)
	}
	if len(versions) == 0 {
		return "", trace.NotFound("%v/%v not found", bucket, prefix)
	}

	// Sort the versions slice so the first entry is the oldest and return it.
	sort.Slice(versions, func(i int, j int) bool {
		return versions[i].Timestamp.Before(versions[j].Timestamp)
	})
	return versions[0].ID, nil
}

// delete bucket deletes bucket and all it's contents and is used in tests
func (h *Handler) deleteBucket() error {
	// first, list and delete all the objects in the bucket
	out, err := h.client.ListObjectVersions(&s3.ListObjectVersionsInput{
		Bucket: aws.String(h.Bucket),
	})
	if err != nil {
		return ConvertS3Error(err)
	}
	for _, ver := range out.Versions {
		_, err := h.client.DeleteObject(&s3.DeleteObjectInput{
			Bucket:    aws.String(h.Bucket),
			Key:       ver.Key,
			VersionId: ver.VersionId,
		})
		if err != nil {
			return ConvertS3Error(err)
		}
	}
	_, err = h.client.DeleteBucket(&s3.DeleteBucketInput{
		Bucket: aws.String(h.Bucket),
	})
	return ConvertS3Error(err)
}

func (l *Handler) path(sessionID session.ID) string {
	if l.Path == "" {
		return string(sessionID) + ".tar"
	}
	return strings.TrimPrefix(filepath.Join(l.Path, string(sessionID)+".tar"), "/")
}

// ensureBucket makes sure bucket exists, and if it does not, creates it
func (h *Handler) ensureBucket() error {
	_, err := h.client.HeadBucket(&s3.HeadBucketInput{
		Bucket: aws.String(h.Bucket),
	})
	err = ConvertS3Error(err)
	// assumes that bucket is administered by other entity
	if err == nil {
		return nil
	}
	if !trace.IsNotFound(err) {
		return trace.Wrap(err)
	}
	input := &s3.CreateBucketInput{
		Bucket: aws.String(h.Bucket),
		ACL:    aws.String("private"),
	}
	_, err = h.client.CreateBucket(input)
	err = ConvertS3Error(err, "bucket %v already exists", aws.String(h.Bucket))
	if err != nil {
		if !trace.IsAlreadyExists(err) {
			return trace.Wrap(err)
		}
		// if this client has not created the bucket, don't reconfigure it
		return nil
	}

	// Turn on versioning.
	ver := &s3.PutBucketVersioningInput{
		Bucket: aws.String(h.Bucket),
		VersioningConfiguration: &s3.VersioningConfiguration{
			Status: aws.String("Enabled"),
		},
	}
	_, err = h.client.PutBucketVersioning(ver)
	err = ConvertS3Error(err, "failed to set versioning state for bucket %q", h.Bucket)
	if err != nil {
		return trace.Wrap(err)
	}

	// Turn on server-side encryption for the bucket.
	if !h.DisableServerSideEncryption {
		_, err = h.client.PutBucketEncryption(&s3.PutBucketEncryptionInput{
			Bucket: aws.String(h.Bucket),
			ServerSideEncryptionConfiguration: &s3.ServerSideEncryptionConfiguration{
				Rules: []*s3.ServerSideEncryptionRule{&s3.ServerSideEncryptionRule{
					ApplyServerSideEncryptionByDefault: &s3.ServerSideEncryptionByDefault{
						SSEAlgorithm: aws.String(s3.ServerSideEncryptionAwsKms),
					},
				}},
			},
		})
		err = ConvertS3Error(err, "failed to set versioning state for bucket %q", h.Bucket)
		if err != nil {
			return trace.Wrap(err)
		}
	}
	return nil
}

// ConvertS3Error wraps S3 error and returns trace equivalent
func ConvertS3Error(err error, args ...interface{}) error {
	if err == nil {
		return nil
	}
	if aerr, ok := err.(awserr.Error); ok {
		switch aerr.Code() {
		case s3.ErrCodeNoSuchKey, s3.ErrCodeNoSuchBucket, s3.ErrCodeNoSuchUpload, "NotFound":
			return trace.NotFound(aerr.Error(), args...)
		case s3.ErrCodeBucketAlreadyExists, s3.ErrCodeBucketAlreadyOwnedByYou:
			return trace.AlreadyExists(aerr.Error(), args...)
		default:
			return trace.BadParameter(aerr.Error(), args...)
		}
	}
	return err
}
<|endoftext|>


<commit_before>//
// Last.Backend LLC CONFIDENTIAL
// __________________
//
// [2014] - [2019] Last.Backend LLC
// All Rights Reserved.
//
// NOTICE:  All information contained herein is, and remains
// the property of Last.Backend LLC and its suppliers,
// if any.  The intellectual and technical concepts contained
// herein are proprietary to Last.Backend LLC
// and its suppliers and may be covered by Russian Federation and Foreign Patents,
// patents in process, and are protected by trade secret or copyright law.
// Dissemination of this information or reproduction of this material
// is strictly forbidden unless prior written permission is obtained
// from Last.Backend LLC.
//

package converter

import (
	"encoding/base64"
	"errors"
	"fmt"
	"reflect"
	"regexp"
	"strconv"
	"strings"
)

type source struct {
	Resource string
	Hub      string
	Repo     string
	Owner    string
	Vendor   string
	Branch   string
}

func StringToInt64(s string) int64 {
	i, _ := strconv.ParseInt(s, 10, 64)
	return i
}

func StringToInt(s string) int {
	i, _ := strconv.Atoi(s)
	return i
}

func IntToString(i int) string {
	return strconv.Itoa(i)
}

func StringToBool(s string) bool {
	s = strings.ToLower(s)
	if s == "true" || s == "1" || s == "t" {
		return true
	}
	return false
}

func Int64ToInt(i int64) int {
	return StringToInt(strconv.FormatInt(i, 10))
}

func DecodeBase64(s string) string {
	buf, _ := base64.StdEncoding.DecodeString(s)
	return string(buf)
}

// Parse incoming string git url in source type
// Ex:
// 	* https://github.com/lastbackend/lastbackend.git
// 	* git@github.com:lastbackend/lastbackend.git
func GitUrlParse(url string) (*source, error) {

	var match []string = regexp.MustCompile(`^(?:ssh|git|http(?:s)?)(?:@|:\/\/(?:.+@)?)((\w+)\.\w+)(?:\/|:)(.+)(?:\/)(.+)(?:\..+)$`).FindStringSubmatch(url)

	if len(match) < 5 {
		return nil, errors.New("can't parse url")
	}

	return &source{
		Resource: match[0],
		Hub:      match[1],
		Vendor:   match[2],
		Owner:    match[3],
		Repo:     match[4],
		Branch:   "master",
	}, nil

}

func DockerNamespaceParse(namespace string) (*source, error) {

	var parsingNamespace *source = new(source)
	parsingNamespace.Vendor = "dockerhub"

	splitStr := strings.Split(namespace, "/")
	switch len(splitStr) {
	case 1:
		parsingNamespace.Repo = splitStr[0]
		return parsingNamespace, nil
	case 2:
		parsingNamespace.Owner = splitStr[0]
	case 3:
		parsingNamespace.Hub = splitStr[0]
		parsingNamespace.Owner = splitStr[1]
	default:
		return nil, errors.New("can't parse url")
	}
	repoAndTag := strings.Split(splitStr[len(splitStr)-1], ":")
	parsingNamespace.Repo = repoAndTag[0]
	if len(repoAndTag) == 2 {
		parsingNamespace.Branch = repoAndTag[1]
	}

	return parsingNamespace, nil

}

func EnforcePtr(obj interface{}) (reflect.Value, error) {
	v := reflect.ValueOf(obj)
	if v.Kind() != reflect.Ptr {
		if v.Kind() == reflect.Invalid {
			return reflect.Value{}, fmt.Errorf("expected pointer, but got invalid kind")
		}
		return reflect.Value{}, fmt.Errorf("expected pointer, but got %v type", v.Type())
	}
	if v.IsNil() {
		return reflect.Value{}, fmt.Errorf("expected pointer, but got nil")
	}
	return v.Elem(), nil
}

func StringToFloat(s string) float64 {
	i, _ := strconv.ParseFloat(s, 64)
	return i
}
<commit_msg>update utils<commit_after>//
// Last.Backend LLC CONFIDENTIAL
// __________________
//
// [2014] - [2019] Last.Backend LLC
// All Rights Reserved.
//
// NOTICE:  All information contained herein is, and remains
// the property of Last.Backend LLC and its suppliers,
// if any.  The intellectual and technical concepts contained
// herein are proprietary to Last.Backend LLC
// and its suppliers and may be covered by Russian Federation and Foreign Patents,
// patents in process, and are protected by trade secret or copyright law.
// Dissemination of this information or reproduction of this material
// is strictly forbidden unless prior written permission is obtained
// from Last.Backend LLC.
//

package converter

import (
	"encoding/base64"
	"errors"
	"fmt"
	"reflect"
	"regexp"
	"strconv"
	"strings"
)

type source struct {
	Resource string
	Hub      string
	Repo     string
	Owner    string
	Vendor   string
	Branch   string
}

func StringToInt64(s string) int64 {
	i, _ := strconv.ParseInt(s, 10, 64)
	return i
}

func StringToInt(s string) int {
	i, _ := strconv.Atoi(s)
	return i
}

func IntToString(i int) string {
	return strconv.Itoa(i)
}

func StringToBool(s string) bool {
	s = strings.ToLower(s)
	if s == "true" || s == "1" || s == "t" {
		return true
	}
	return false
}

func ParseBool(str string) (bool, error) {
	switch str {
	case "":
		return false, nil
	case "1", "t", "T", "true", "TRUE", "True":
		return true, nil
	case "0", "f", "F", "false", "FALSE", "False":
		return false, nil
	}
	return false, errors.New(fmt.Sprintf("parse bool string: %s", str))
}

func Int64ToInt(i int64) int {
	return StringToInt(strconv.FormatInt(i, 10))
}

func DecodeBase64(s string) string {
	buf, _ := base64.StdEncoding.DecodeString(s)
	return string(buf)
}

// Parse incoming string git url in source type
// Ex:
// 	* https://github.com/lastbackend/lastbackend.git
// 	* git@github.com:lastbackend/lastbackend.git
func GitUrlParse(url string) (*source, error) {

	var match []string = regexp.MustCompile(`^(?:ssh|git|http(?:s)?)(?:@|:\/\/(?:.+@)?)((\w+)\.\w+)(?:\/|:)(.+)(?:\/)(.+)(?:\..+)$`).FindStringSubmatch(url)

	if len(match) < 5 {
		return nil, errors.New("can't parse url")
	}

	return &source{
		Resource: match[0],
		Hub:      match[1],
		Vendor:   match[2],
		Owner:    match[3],
		Repo:     match[4],
		Branch:   "master",
	}, nil

}

func DockerNamespaceParse(namespace string) (*source, error) {

	var parsingNamespace *source = new(source)
	parsingNamespace.Vendor = "dockerhub"

	splitStr := strings.Split(namespace, "/")
	switch len(splitStr) {
	case 1:
		parsingNamespace.Repo = splitStr[0]
		return parsingNamespace, nil
	case 2:
		parsingNamespace.Owner = splitStr[0]
	case 3:
		parsingNamespace.Hub = splitStr[0]
		parsingNamespace.Owner = splitStr[1]
	default:
		return nil, errors.New("can't parse url")
	}
	repoAndTag := strings.Split(splitStr[len(splitStr)-1], ":")
	parsingNamespace.Repo = repoAndTag[0]
	if len(repoAndTag) == 2 {
		parsingNamespace.Branch = repoAndTag[1]
	}

	return parsingNamespace, nil

}

func EnforcePtr(obj interface{}) (reflect.Value, error) {
	v := reflect.ValueOf(obj)
	if v.Kind() != reflect.Ptr {
		if v.Kind() == reflect.Invalid {
			return reflect.Value{}, fmt.Errorf("expected pointer, but got invalid kind")
		}
		return reflect.Value{}, fmt.Errorf("expected pointer, but got %v type", v.Type())
	}
	if v.IsNil() {
		return reflect.Value{}, fmt.Errorf("expected pointer, but got nil")
	}
	return v.Elem(), nil
}

func StringToFloat(s string) float64 {
	i, _ := strconv.ParseFloat(s, 64)
	return i
}
<|endoftext|>


<commit_before>// Pretty-print objects for easy grepping.

package main

import (
	"fmt"
	"log"
	"sync"

	tor "git.torproject.org/user/phw/zoossh.git"
)

// PrettyPrint prints all objects within the object sets received over the
// given channel.  The output is meant to be human-readable and easy to analyse
// and grep.
func PrettyPrint(channel chan tor.ObjectSet, params *CmdLineParams, group *sync.WaitGroup) {

	defer group.Done()

	counter := 0
	for objects := range channel {
		for object := range objects.Iterate() {
			fmt.Println(object)
			counter += 1
		}
	}
	log.Printf("Printed %d objects.\n", counter)
}
<commit_msg>also print router descriptor if possible.<commit_after>// Pretty-print objects for easy grepping.

package main

import (
	"fmt"
	"log"
	"sync"

	tor "git.torproject.org/user/phw/zoossh.git"
)

var printedBanner bool = false

// PrintInfo prints a router status.  If we also have access to router
// descriptors, we print those too.
func PrintInfo(descriptorDir string, status *tor.RouterStatus) {

	desc, err := tor.LoadDescriptorFromDigest(descriptorDir, status.Digest, status.Publication)
	if err == nil {
		if !printedBanner {
			fmt.Println("fingerprint,nickname,ip_addr,or_port,dir_port,flags,published,version,platform,bandwidthavg,bandwidthburst,uptime,familysize")
			printedBanner = true
		}
		fmt.Printf("%s, %s, %d, %d, %d, %d\n", status, desc.OperatingSystem, desc.BandwidthAvg, desc.BandwidthBurst, desc.Uptime, len(desc.Family))
	} else {
		if !printedBanner {
			fmt.Println("fingerprint,nickname,ip_addr,or_port,dir_port,flags,published,version")
			printedBanner = true
		}
		fmt.Println(status)
	}
}

// PrettyPrint prints all objects within the object sets received over the
// given channel.  The output is meant to be human-readable and easy to analyse
// and grep.
func PrettyPrint(channel chan tor.ObjectSet, params *CmdLineParams, group *sync.WaitGroup) {

	defer group.Done()

	counter := 0
	for objects := range channel {
		for object := range objects.Iterate() {
			counter += 1

			switch obj := object.(type) {
			case *tor.RouterStatus:
				PrintInfo(params.DescriptorDir, obj)
			case *tor.RouterDescriptor:
				fmt.Println(obj)
			}
		}
	}
	log.Printf("Printed %d objects.\n", counter)
}
<|endoftext|>


<commit_before>package transports

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
)

type Proxy struct {
	Port      int
	Transport interface{}
}

func (proxy *Proxy) Listen() {
	fmt.Println("Proxy listening on", proxy.Port)

	err := errors.New("no transport specified")

	if proxy.Transport == nil {
		panic(err)
	}

	// transport := proxy.Transport.(FacebookTransport)
	transport := proxy.Transport.(WhatsappTransport)
	transport.Prepare()

	http.HandleFunc("/", transport.Handler)
	http.ListenAndServe(":8080", nil)

	return
}

func MarshalRequest(request *http.Request) []byte {
	r := Request{
		Method:  request.Method,
		URL:     request.URL.String(),
		Proto:   request.Proto,
		Headers: request.Header,
	}
	output, _ := json.Marshal(r)
	return output
}
<commit_msg>Provide some stats<commit_after>package transports

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"

	"github.com/fukata/golang-stats-api-handler"
)

type Proxy struct {
	Port      int
	Transport interface{}
}

func (proxy *Proxy) Listen() {
	fmt.Println("Proxy listening on", proxy.Port)

	err := errors.New("no transport specified")

	if proxy.Transport == nil {
		panic(err)
	}

	// transport := proxy.Transport.(FacebookTransport)
	transport := proxy.Transport.(WhatsappTransport)
	transport.Prepare()

	http.HandleFunc("/", transport.Handler)
	http.HandleFunc( "/stats", stats_api.Handler )

	http.ListenAndServe(":8080", nil)

	return
}

func MarshalRequest(request *http.Request) []byte {
	r := Request{
		Method:  request.Method,
		URL:     request.URL.String(),
		Proto:   request.Proto,
		Headers: request.Header,
	}
	output, _ := json.Marshal(r)
	return output
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"net/http"
	"net/http/httputil"
	"net/url"
)

func main() {
	port := flag.String("port", "8080", "Specifies which port to proxy in front of")
	cert := flag.String("cert", "cert.pem", "Absolute path to cert.pem")
	key := flag.String("key", "key.pem", "Absolute path to key.pem")
	verbose := flag.Bool("verbose", false, "Log requests to stdout")

	flag.Parse()

	log.Println("Proxying port: " + *port)
	log.Println("Loading cert at: " + *cert)
	log.Println("Loading key at: " + *key)

	remote, err := url.Parse("http://localhost:" + *port)
	if err != nil {
		panic(err)
	}

	proxy := httputil.NewSingleHostReverseProxy(remote)
	http.HandleFunc("/", handler(proxy, verbose))

	go func() {
		err = http.ListenAndServe(":80", nil)
		if err != nil {
			panic(err)
		}
	}()

	err = http.ListenAndServeTLS(":443", *cert, *key, nil)
	if err != nil {
		panic(err)
	}
}

func handler(p *httputil.ReverseProxy, verbose *bool) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		var protocol string
		if r.TLS != nil {
			protocol = "https"
		} else {
			protocol = "http"
		}

    if *verbose {
      log.Println(r.Method, protocol, r.URL)
    }

		r.Header.Add("X-Forwarded-Proto", protocol)
		r.Header.Add("Host", r.Host)
		p.ServeHTTP(w, r)
	}
}
<commit_msg>go fmt<commit_after>package main

import (
	"flag"
	"log"
	"net/http"
	"net/http/httputil"
	"net/url"
)

func main() {
	port := flag.String("port", "8080", "Specifies which port to proxy in front of")
	cert := flag.String("cert", "cert.pem", "Absolute path to cert.pem")
	key := flag.String("key", "key.pem", "Absolute path to key.pem")
	verbose := flag.Bool("verbose", false, "Log requests to stdout")

	flag.Parse()

	log.Println("Proxying port: " + *port)
	log.Println("Loading cert at: " + *cert)
	log.Println("Loading key at: " + *key)

	remote, err := url.Parse("http://localhost:" + *port)
	if err != nil {
		panic(err)
	}

	proxy := httputil.NewSingleHostReverseProxy(remote)
	http.HandleFunc("/", handler(proxy, verbose))

	go func() {
		err = http.ListenAndServe(":80", nil)
		if err != nil {
			panic(err)
		}
	}()

	err = http.ListenAndServeTLS(":443", *cert, *key, nil)
	if err != nil {
		panic(err)
	}
}

func handler(p *httputil.ReverseProxy, verbose *bool) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		var protocol string
		if r.TLS != nil {
			protocol = "https"
		} else {
			protocol = "http"
		}

		if *verbose {
			log.Println(r.Method, protocol, r.URL)
		}

		r.Header.Add("X-Forwarded-Proto", protocol)
		r.Header.Add("Host", r.Host)
		p.ServeHTTP(w, r)
	}
}
<|endoftext|>