<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"fmt"
	"k8s.io/kubernetes/test/e2e/framework/metrics"
	"k8s.io/kubernetes/test/e2e/perftype"
	"math"
	"os"
	"strconv"
	"time"
)

func stripCount(data *perftype.DataItem) {
	delete(data.Labels, "Count")
}

func createRequestCountData(data *perftype.DataItem) error {
	data.Unit = ""
	data.Data = make(map[string]float64)
	requestCountString, ok := data.Labels["Count"]
	if !ok {
		return fmt.Errorf("no 'Count' label")
	}
	requestCount, err := strconv.ParseFloat(requestCountString, 64)
	if err != nil {
		return fmt.Errorf("couldn't parse count: %v", err)
	}
	data.Data["RequestCount"] = requestCount
	return nil
}

func parseResponsivenessData(data []byte, buildNumber int, testResult *BuildData) {
	build := fmt.Sprintf("%d", buildNumber)
	obj := perftype.PerfData{}
	if err := json.Unmarshal(data, &obj); err != nil {
		fmt.Fprintf(os.Stderr, "error parsing JSON in build %d: %v %s\n", buildNumber, err, string(data))
		return
	}
	if testResult.Version == "" {
		testResult.Version = obj.Version
	}
	if testResult.Version == obj.Version {
		for i := range obj.DataItems {
			stripCount(&obj.DataItems[i])
			testResult.Builds[build] = append(testResult.Builds[build], obj.DataItems[i])
		}
	}
}

type resourceUsagePercentiles map[string][]resourceUsages

type resourceUsages struct {
	Name   string  `json:"Name"`
	Cpu    float64 `json:"Cpu"`
	Memory int     `json:"Mem"`
}

type resourceUsage struct {
	Cpu    float64
	Memory float64
}
type usageAtPercentiles map[string]resourceUsage
type podNameToUsage map[string]usageAtPercentiles

func parseResourceUsageData(data []byte, buildNumber int, testResult *BuildData) {
	testResult.Version = "v1"
	build := fmt.Sprintf("%d", buildNumber)
	var obj resourceUsagePercentiles
	if err := json.Unmarshal(data, &obj); err != nil {
		fmt.Fprintf(os.Stderr, "error parsing JSON in build %d: %v %s\n", buildNumber, err, string(data))
		return
	}
	usage := make(podNameToUsage)
	for percentile, items := range obj {
		for _, item := range items {
			name := RemoveDisambiguationInfixes(item.Name)
			if _, ok := usage[name]; !ok {
				usage[name] = make(usageAtPercentiles)
			}
			cpu, memory := float64(item.Cpu), float64(item.Memory)
			if otherUsage, ok := usage[name][percentile]; ok {
				// Note that we take max of each resource separately, potentially manufacturing a
				// "franken-sample" which was never seen in the wild. We do this hoping that such result
				// will be more stable across runs.
				cpu = math.Max(cpu, otherUsage.Cpu)
				memory = math.Max(memory, otherUsage.Memory)
			}
			usage[name][percentile] = resourceUsage{cpu, memory}
		}
	}
	for podName, usageAtPercentiles := range usage {
		cpu := perftype.DataItem{Unit: "cores", Labels: map[string]string{"PodName": podName, "Resource": "CPU"}, Data: make(map[string]float64)}
		memory := perftype.DataItem{Unit: "MiB", Labels: map[string]string{"PodName": podName, "Resource": "memory"}, Data: make(map[string]float64)}
		for percentile, usage := range usageAtPercentiles {
			cpu.Data[percentile] = usage.Cpu
			memory.Data[percentile] = usage.Memory / (1024 * 1024)
		}
		testResult.Builds[build] = append(testResult.Builds[build], cpu)
		testResult.Builds[build] = append(testResult.Builds[build], memory)
	}
}

func parseRequestCountData(data []byte, buildNumber int, testResult *BuildData) {
	build := fmt.Sprintf("%d", buildNumber)
	obj := perftype.PerfData{}
	if err := json.Unmarshal(data, &obj); err != nil {
		fmt.Fprintf(os.Stderr, "error parsing JSON in build %d: %v %s\n", buildNumber, err, string(data))
		return
	}
	if testResult.Version == "" {
		testResult.Version = obj.Version
	}
	if testResult.Version == obj.Version {
		for i := range obj.DataItems {
			if err := createRequestCountData(&obj.DataItems[i]); err != nil {
				fmt.Fprintf(os.Stderr, "error creating request count data in build %d dataItem %d: %v\n", buildNumber, i, err)
				continue
			}
			stripCount(&obj.DataItems[i])
			testResult.Builds[build] = append(testResult.Builds[build], obj.DataItems[i])
		}
	}
}

func parseApiserverRequestCount(data []byte, buildNumber int, testResult *BuildData) {
	testResult.Version = "v1"
	build := fmt.Sprintf("%d", buildNumber)
	var obj metrics.MetricsCollection
	if err := json.Unmarshal(data, &obj); err != nil {
		fmt.Fprintf(os.Stderr, "error parsing JSON in build %d: %v %s\n", buildNumber, err, string(data))
		return
	}
	if obj.ApiServerMetrics == nil {
		fmt.Fprintf(os.Stderr, "no ApiServerMetrics data in build %d", buildNumber)
		return
	}
	metric, ok := obj.ApiServerMetrics["apiserver_request_count"]
	if !ok {
		fmt.Fprintf(os.Stderr, "no apiserver_request_count metric data in build %d", buildNumber)
		return
	}
	for i := range metric {
		perfData := perftype.DataItem{Unit: "", Data: make(map[string]float64), Labels: make(map[string]string)}
		for k, v := range metric[i].Metric {
			perfData.Labels[string(k)] = string(v)
		}
		delete(perfData.Labels, "__name__")
		delete(perfData.Labels, "contentType")
		perfData.Data["RequestCount"] = float64(metric[i].Value)
		testResult.Builds[build] = append(testResult.Builds[build], perfData)
	}
}

// TODO(krzysied): Copy of structures from kuberentes repository.
// At some point latencyMetric and schedulingMetrics should be moved to metric package.
type latencyMetric struct {
	Perc50 time.Duration `json:"Perc50"`
	Perc90 time.Duration `json:"Perc90"`
	Perc99 time.Duration `json:"Perc99"`
}

type schedulingMetrics struct {
	PredicateEvaluationLatency  latencyMetric `json:"predicateEvaluationLatency"`
	PriorityEvaluationLatency   latencyMetric `json:"priorityEvaluationLatency"`
	PreemptionEvaluationLatency latencyMetric `json:"preemptionEvaluationLatency"`
	BindingLatency              latencyMetric `json:"bindingLatency"`
	ThroughputAverage           float64       `json:"throughputAverage"`
	ThroughputPerc50            float64       `json:"throughputPerc50"`
	ThroughputPerc90            float64       `json:"throughputPerc90"`
	ThroughputPerc99            float64       `json:"throughputPerc99"`
}

func parseOperationLatency(latency latencyMetric, operationName string) perftype.DataItem {
	perfData := perftype.DataItem{Unit: "ms", Labels: map[string]string{"Operation": operationName}, Data: make(map[string]float64)}
	perfData.Data["Perc50"] = float64(latency.Perc50) / float64(time.Millisecond)
	perfData.Data["Perc90"] = float64(latency.Perc90) / float64(time.Millisecond)
	perfData.Data["Perc99"] = float64(latency.Perc99) / float64(time.Millisecond)
	return perfData
}

func parseSchedulingLatency(data []byte, buildNumber int, testResult *BuildData) {
	testResult.Version = "v1"
	build := fmt.Sprintf("%d", buildNumber)
	var obj schedulingMetrics
	if err := json.Unmarshal(data, &obj); err != nil {
		fmt.Fprintf(os.Stderr, "error parsing JSON in build %d: %v %s\n", buildNumber, err, string(data))
		return
	}
	predicateEvaluation := parseOperationLatency(obj.PredicateEvaluationLatency, "predicate_evaluation")
	testResult.Builds[build] = append(testResult.Builds[build], predicateEvaluation)
	priorityEvaluation := parseOperationLatency(obj.PriorityEvaluationLatency, "priority_evaluation")
	testResult.Builds[build] = append(testResult.Builds[build], priorityEvaluation)
	preemptionEvaluation := parseOperationLatency(obj.PreemptionEvaluationLatency, "preemption_evaluation")
	testResult.Builds[build] = append(testResult.Builds[build], preemptionEvaluation)
	binding := parseOperationLatency(obj.BindingLatency, "binding")
	testResult.Builds[build] = append(testResult.Builds[build], binding)
}

func pareseSchedulingThroughput(data []byte, buildNumber int, testResult *BuildData) {
	testResult.Version = "v1"
	build := fmt.Sprintf("%d", buildNumber)
	var obj schedulingMetrics
	if err := json.Unmarshal(data, &obj); err != nil {
		fmt.Fprintf(os.Stderr, "error parsing JSON in build %d: %v %s\n", buildNumber, err, string(data))
		return
	}
	perfData := perftype.DataItem{Unit: "1/s", Labels: map[string]string{}, Data: make(map[string]float64)}
	perfData.Data["Perc50"] = obj.ThroughputPerc50
	perfData.Data["Perc90"] = obj.ThroughputPerc90
	perfData.Data["Perc99"] = obj.ThroughputPerc99
	perfData.Data["Average"] = obj.ThroughputAverage
	testResult.Builds[build] = append(testResult.Builds[build], perfData)
}

// TODO(krzysied): This structure also should be moved to metric package.
type histogram struct {
	Labels  map[string]string `json:"labels"`
	Buckets map[string]int    `json:"buckets"`
}

type histogramVec []histogram

type etcdMetrics struct {
	BackendCommitDuration     histogramVec `json:"backendCommitDuration"`
	SnapshotSaveTotalDuration histogramVec `json:"snapshotSaveTotalDuration"`
	PeerRoundTripTime         histogramVec `json:"peerRoundTripTime"`
	WalFsyncDuration          histogramVec `json:"walFsyncDuration"`
	MaxDatabaseSize           float64      `json:"maxDatabaseSize"`
}

func parseHistogramMetric(metricName string) func(data []byte, buildNumber int, testResult *BuildData) {
	return func(data []byte, buildNumber int, testResult *BuildData) {
		fmt.Fprintf(os.Stderr, "parsing metric: %s\n", metricName)
		testResult.Version = "v1"
		build := fmt.Sprintf("%d", buildNumber)
		var obj etcdMetrics
		if err := json.Unmarshal(data, &obj); err != nil {
			fmt.Fprintf(os.Stderr, "error parsing JSON in build %d: %v %s\n", buildNumber, err, string(data))
			return
		}

		var histogramVecMetric histogramVec
		switch metricName {
		case "backendCommitDuration":
			histogramVecMetric = obj.BackendCommitDuration
		case "snapshotSaveTotalDuration":
			histogramVecMetric = obj.SnapshotSaveTotalDuration
		case "peerRoundTripTime":
			histogramVecMetric = obj.PeerRoundTripTime
		case "walFsyncDuration":
			histogramVecMetric = obj.WalFsyncDuration
		default:
			fmt.Fprintf(os.Stderr, "unknown metric name: %s\n", metricName)
		}
		if histogramVecMetric != nil {
			for i := range histogramVecMetric {
				perfData := perftype.DataItem{Unit: "%", Labels: histogramVecMetric[i].Labels, Data: make(map[string]float64)}
				delete(perfData.Labels, "__name__")
				count, exists := histogramVecMetric[i].Buckets["+Inf"]
				if !exists {
					fmt.Fprintf(os.Stderr, "err in build %d: no +Inf bucket: %s\n", buildNumber, string(data))
					continue
				}
				for kBucket, vBucket := range histogramVecMetric[i].Buckets {
					if kBucket != "+Inf" {
						perfData.Data["<= "+kBucket+"s"] = float64(vBucket) / float64(count) * 100
					}
				}
				testResult.Builds[build] = append(testResult.Builds[build], perfData)
			}
		}
	}
}
<commit_msg>removing kubernetes version from client label<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"fmt"
	"k8s.io/kubernetes/test/e2e/framework/metrics"
	"k8s.io/kubernetes/test/e2e/perftype"
	"math"
	"os"
	"regexp"
	"strconv"
	"time"
)

func stripCount(data *perftype.DataItem) {
	delete(data.Labels, "Count")
}

func createRequestCountData(data *perftype.DataItem) error {
	data.Unit = ""
	data.Data = make(map[string]float64)
	requestCountString, ok := data.Labels["Count"]
	if !ok {
		return fmt.Errorf("no 'Count' label")
	}
	requestCount, err := strconv.ParseFloat(requestCountString, 64)
	if err != nil {
		return fmt.Errorf("couldn't parse count: %v", err)
	}
	data.Data["RequestCount"] = requestCount
	return nil
}

func parseResponsivenessData(data []byte, buildNumber int, testResult *BuildData) {
	build := fmt.Sprintf("%d", buildNumber)
	obj := perftype.PerfData{}
	if err := json.Unmarshal(data, &obj); err != nil {
		fmt.Fprintf(os.Stderr, "error parsing JSON in build %d: %v %s\n", buildNumber, err, string(data))
		return
	}
	if testResult.Version == "" {
		testResult.Version = obj.Version
	}
	if testResult.Version == obj.Version {
		for i := range obj.DataItems {
			stripCount(&obj.DataItems[i])
			testResult.Builds[build] = append(testResult.Builds[build], obj.DataItems[i])
		}
	}
}

type resourceUsagePercentiles map[string][]resourceUsages

type resourceUsages struct {
	Name   string  `json:"Name"`
	Cpu    float64 `json:"Cpu"`
	Memory int     `json:"Mem"`
}

type resourceUsage struct {
	Cpu    float64
	Memory float64
}
type usageAtPercentiles map[string]resourceUsage
type podNameToUsage map[string]usageAtPercentiles

func parseResourceUsageData(data []byte, buildNumber int, testResult *BuildData) {
	testResult.Version = "v1"
	build := fmt.Sprintf("%d", buildNumber)
	var obj resourceUsagePercentiles
	if err := json.Unmarshal(data, &obj); err != nil {
		fmt.Fprintf(os.Stderr, "error parsing JSON in build %d: %v %s\n", buildNumber, err, string(data))
		return
	}
	usage := make(podNameToUsage)
	for percentile, items := range obj {
		for _, item := range items {
			name := RemoveDisambiguationInfixes(item.Name)
			if _, ok := usage[name]; !ok {
				usage[name] = make(usageAtPercentiles)
			}
			cpu, memory := float64(item.Cpu), float64(item.Memory)
			if otherUsage, ok := usage[name][percentile]; ok {
				// Note that we take max of each resource separately, potentially manufacturing a
				// "franken-sample" which was never seen in the wild. We do this hoping that such result
				// will be more stable across runs.
				cpu = math.Max(cpu, otherUsage.Cpu)
				memory = math.Max(memory, otherUsage.Memory)
			}
			usage[name][percentile] = resourceUsage{cpu, memory}
		}
	}
	for podName, usageAtPercentiles := range usage {
		cpu := perftype.DataItem{Unit: "cores", Labels: map[string]string{"PodName": podName, "Resource": "CPU"}, Data: make(map[string]float64)}
		memory := perftype.DataItem{Unit: "MiB", Labels: map[string]string{"PodName": podName, "Resource": "memory"}, Data: make(map[string]float64)}
		for percentile, usage := range usageAtPercentiles {
			cpu.Data[percentile] = usage.Cpu
			memory.Data[percentile] = usage.Memory / (1024 * 1024)
		}
		testResult.Builds[build] = append(testResult.Builds[build], cpu)
		testResult.Builds[build] = append(testResult.Builds[build], memory)
	}
}

func parseRequestCountData(data []byte, buildNumber int, testResult *BuildData) {
	build := fmt.Sprintf("%d", buildNumber)
	obj := perftype.PerfData{}
	if err := json.Unmarshal(data, &obj); err != nil {
		fmt.Fprintf(os.Stderr, "error parsing JSON in build %d: %v %s\n", buildNumber, err, string(data))
		return
	}
	if testResult.Version == "" {
		testResult.Version = obj.Version
	}
	if testResult.Version == obj.Version {
		for i := range obj.DataItems {
			if err := createRequestCountData(&obj.DataItems[i]); err != nil {
				fmt.Fprintf(os.Stderr, "error creating request count data in build %d dataItem %d: %v\n", buildNumber, i, err)
				continue
			}
			stripCount(&obj.DataItems[i])
			testResult.Builds[build] = append(testResult.Builds[build], obj.DataItems[i])
		}
	}
}

var versionMatcher = regexp.MustCompile("kubernetes/.{7}")

func parseApiserverRequestCount(data []byte, buildNumber int, testResult *BuildData) {
	testResult.Version = "v1"
	build := fmt.Sprintf("%d", buildNumber)
	var obj metrics.MetricsCollection
	if err := json.Unmarshal(data, &obj); err != nil {
		fmt.Fprintf(os.Stderr, "error parsing JSON in build %d: %v %s\n", buildNumber, err, string(data))
		return
	}
	if obj.ApiServerMetrics == nil {
		fmt.Fprintf(os.Stderr, "no ApiServerMetrics data in build %d", buildNumber)
		return
	}
	metric, ok := obj.ApiServerMetrics["apiserver_request_count"]
	if !ok {
		fmt.Fprintf(os.Stderr, "no apiserver_request_count metric data in build %d", buildNumber)
		return
	}
	for i := range metric {
		perfData := perftype.DataItem{Unit: "", Data: make(map[string]float64), Labels: make(map[string]string)}
		for k, v := range metric[i].Metric {
			perfData.Labels[string(k)] = string(v)
		}
		delete(perfData.Labels, "__name__")
		delete(perfData.Labels, "contentType")
		if client, ok := perfData.Labels["client"]; ok {
			// Client label contains kubernetes version, which is different
			// in every build. This causes unnecessary creation on multiple different label sets
			// for one metric.
			// This fix removes kubernetes version from client label.
			newClient := versionMatcher.ReplaceAllString(client, "kubernetes")
			perfData.Labels["client"] = newClient
		}
		perfData.Data["RequestCount"] = float64(metric[i].Value)
		testResult.Builds[build] = append(testResult.Builds[build], perfData)
	}
}

// TODO(krzysied): Copy of structures from kuberentes repository.
// At some point latencyMetric and schedulingMetrics should be moved to metric package.
type latencyMetric struct {
	Perc50 time.Duration `json:"Perc50"`
	Perc90 time.Duration `json:"Perc90"`
	Perc99 time.Duration `json:"Perc99"`
}

type schedulingMetrics struct {
	PredicateEvaluationLatency  latencyMetric `json:"predicateEvaluationLatency"`
	PriorityEvaluationLatency   latencyMetric `json:"priorityEvaluationLatency"`
	PreemptionEvaluationLatency latencyMetric `json:"preemptionEvaluationLatency"`
	BindingLatency              latencyMetric `json:"bindingLatency"`
	ThroughputAverage           float64       `json:"throughputAverage"`
	ThroughputPerc50            float64       `json:"throughputPerc50"`
	ThroughputPerc90            float64       `json:"throughputPerc90"`
	ThroughputPerc99            float64       `json:"throughputPerc99"`
}

func parseOperationLatency(latency latencyMetric, operationName string) perftype.DataItem {
	perfData := perftype.DataItem{Unit: "ms", Labels: map[string]string{"Operation": operationName}, Data: make(map[string]float64)}
	perfData.Data["Perc50"] = float64(latency.Perc50) / float64(time.Millisecond)
	perfData.Data["Perc90"] = float64(latency.Perc90) / float64(time.Millisecond)
	perfData.Data["Perc99"] = float64(latency.Perc99) / float64(time.Millisecond)
	return perfData
}

func parseSchedulingLatency(data []byte, buildNumber int, testResult *BuildData) {
	testResult.Version = "v1"
	build := fmt.Sprintf("%d", buildNumber)
	var obj schedulingMetrics
	if err := json.Unmarshal(data, &obj); err != nil {
		fmt.Fprintf(os.Stderr, "error parsing JSON in build %d: %v %s\n", buildNumber, err, string(data))
		return
	}
	predicateEvaluation := parseOperationLatency(obj.PredicateEvaluationLatency, "predicate_evaluation")
	testResult.Builds[build] = append(testResult.Builds[build], predicateEvaluation)
	priorityEvaluation := parseOperationLatency(obj.PriorityEvaluationLatency, "priority_evaluation")
	testResult.Builds[build] = append(testResult.Builds[build], priorityEvaluation)
	preemptionEvaluation := parseOperationLatency(obj.PreemptionEvaluationLatency, "preemption_evaluation")
	testResult.Builds[build] = append(testResult.Builds[build], preemptionEvaluation)
	binding := parseOperationLatency(obj.BindingLatency, "binding")
	testResult.Builds[build] = append(testResult.Builds[build], binding)
}

func pareseSchedulingThroughput(data []byte, buildNumber int, testResult *BuildData) {
	testResult.Version = "v1"
	build := fmt.Sprintf("%d", buildNumber)
	var obj schedulingMetrics
	if err := json.Unmarshal(data, &obj); err != nil {
		fmt.Fprintf(os.Stderr, "error parsing JSON in build %d: %v %s\n", buildNumber, err, string(data))
		return
	}
	perfData := perftype.DataItem{Unit: "1/s", Labels: map[string]string{}, Data: make(map[string]float64)}
	perfData.Data["Perc50"] = obj.ThroughputPerc50
	perfData.Data["Perc90"] = obj.ThroughputPerc90
	perfData.Data["Perc99"] = obj.ThroughputPerc99
	perfData.Data["Average"] = obj.ThroughputAverage
	testResult.Builds[build] = append(testResult.Builds[build], perfData)
}

// TODO(krzysied): This structure also should be moved to metric package.
type histogram struct {
	Labels  map[string]string `json:"labels"`
	Buckets map[string]int    `json:"buckets"`
}

type histogramVec []histogram

type etcdMetrics struct {
	BackendCommitDuration     histogramVec `json:"backendCommitDuration"`
	SnapshotSaveTotalDuration histogramVec `json:"snapshotSaveTotalDuration"`
	PeerRoundTripTime         histogramVec `json:"peerRoundTripTime"`
	WalFsyncDuration          histogramVec `json:"walFsyncDuration"`
	MaxDatabaseSize           float64      `json:"maxDatabaseSize"`
}

func parseHistogramMetric(metricName string) func(data []byte, buildNumber int, testResult *BuildData) {
	return func(data []byte, buildNumber int, testResult *BuildData) {
		testResult.Version = "v1"
		build := fmt.Sprintf("%d", buildNumber)
		var obj etcdMetrics
		if err := json.Unmarshal(data, &obj); err != nil {
			fmt.Fprintf(os.Stderr, "error parsing JSON in build %d: %v %s\n", buildNumber, err, string(data))
			return
		}

		var histogramVecMetric histogramVec
		switch metricName {
		case "backendCommitDuration":
			histogramVecMetric = obj.BackendCommitDuration
		case "snapshotSaveTotalDuration":
			histogramVecMetric = obj.SnapshotSaveTotalDuration
		case "peerRoundTripTime":
			histogramVecMetric = obj.PeerRoundTripTime
		case "walFsyncDuration":
			histogramVecMetric = obj.WalFsyncDuration
		default:
			fmt.Fprintf(os.Stderr, "unknown metric name: %s\n", metricName)
		}
		if histogramVecMetric != nil {
			for i := range histogramVecMetric {
				perfData := perftype.DataItem{Unit: "%", Labels: histogramVecMetric[i].Labels, Data: make(map[string]float64)}
				delete(perfData.Labels, "__name__")
				count, exists := histogramVecMetric[i].Buckets["+Inf"]
				if !exists {
					fmt.Fprintf(os.Stderr, "err in build %d: no +Inf bucket: %s\n", buildNumber, string(data))
					continue
				}
				for kBucket, vBucket := range histogramVecMetric[i].Buckets {
					if kBucket != "+Inf" {
						perfData.Data["<= "+kBucket+"s"] = float64(vBucket) / float64(count) * 100
					}
				}
				testResult.Builds[build] = append(testResult.Builds[build], perfData)
			}
		}
	}
}
<|endoftext|>


<commit_before>// This is free and unencumbered software released into the public
// domain.  For more information, see <http://unlicense.org> or the
// accompanying UNLICENSE file.

package pers

import (
	"errors"
	"fmt"
	"reflect"
)

// ConsistentlyReturn will continue adding a given value to the channel
// until the returned done function is called.  You may pass in either
// a channel (in which case you should pass in a single argument) or a
// struct full of channels (in which case you should pass in arguments
// in the order the fields appear in the struct).
//
// After the returned function is called, you will still need to drain
// any remaining calls from the channel(s) before it will start blocking
// again.
func ConsistentlyReturn(mock interface{}, args ...interface{}) (func(), error) {
	cases, err := selectCases(mock, args...)
	if err != nil {
		return nil, err
	}
	done := make(chan struct{})
	go consistentlyReturn(cases, done, args...)
	return func() { close(done) }, nil
}

func selectCases(mock interface{}, args ...interface{}) ([]reflect.SelectCase, error) {
	v := reflect.ValueOf(mock)
	switch v.Kind() {
	case reflect.Chan:
		if len(args) != 1 {
			return nil, fmt.Errorf("expected 1 argument for %#v; got %d", mock, len(args))
		}
		return []reflect.SelectCase{{Dir: reflect.SelectSend, Chan: v, Send: reflect.ValueOf(args[0])}}, nil
	case reflect.Struct:
		if v.NumField() == 0 {
			return nil, errors.New("cannot consistently return on unsupported type struct{}")
		}
		if len(args) != v.NumField() {
			argString := "argument"
			if v.NumField() != 1 {
				argString = "arguments"
			}
			return nil, fmt.Errorf("expected %d %s for %#v; got %d", v.NumField(), argString, mock, len(args))
		}
		var cases []reflect.SelectCase
		for i := 0; i < v.NumField(); i++ {
			c, err := selectCases(v.Field(i).Interface(), args[i])
			if err != nil {
				return nil, err
			}
			cases = append(cases, c...)
		}
		return cases, nil
	default:
		return nil, fmt.Errorf("cannot consistently return on unsupported type %T", mock)
	}
}

func consistentlyReturn(cases []reflect.SelectCase, done chan struct{}, args ...interface{}) {
	doneIdx := len(cases)
	cases = append(cases, reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(done)})
	for {
		chosen, _, _ := reflect.Select(cases)
		if chosen == doneIdx {
			return
		}
	}
}
<commit_msg>pers: make sure that the loop exits before returning from done<commit_after>// This is free and unencumbered software released into the public
// domain.  For more information, see <http://unlicense.org> or the
// accompanying UNLICENSE file.

package pers

import (
	"errors"
	"fmt"
	"reflect"
)

// ConsistentlyReturn will continue adding a given value to the channel
// until the returned done function is called.  You may pass in either
// a channel (in which case you should pass in a single argument) or a
// struct full of channels (in which case you should pass in arguments
// in the order the fields appear in the struct).
//
// After the returned function is called, you will still need to drain
// any remaining calls from the channel(s) before it will start blocking
// again.
func ConsistentlyReturn(mock interface{}, args ...interface{}) (func(), error) {
	cases, err := selectCases(mock, args...)
	if err != nil {
		return nil, err
	}
	done := make(chan struct{})
	exited := make(chan struct{})
	go consistentlyReturn(cases, done, exited, args...)
	return func() {
		close(done)
		<-exited
	}, nil
}

func selectCases(mock interface{}, args ...interface{}) ([]reflect.SelectCase, error) {
	v := reflect.ValueOf(mock)
	switch v.Kind() {
	case reflect.Chan:
		if len(args) != 1 {
			return nil, fmt.Errorf("expected 1 argument for %#v; got %d", mock, len(args))
		}
		return []reflect.SelectCase{{Dir: reflect.SelectSend, Chan: v, Send: reflect.ValueOf(args[0])}}, nil
	case reflect.Struct:
		if v.NumField() == 0 {
			return nil, errors.New("cannot consistently return on unsupported type struct{}")
		}
		if len(args) != v.NumField() {
			argString := "argument"
			if v.NumField() != 1 {
				argString = "arguments"
			}
			return nil, fmt.Errorf("expected %d %s for %#v; got %d", v.NumField(), argString, mock, len(args))
		}
		var cases []reflect.SelectCase
		for i := 0; i < v.NumField(); i++ {
			c, err := selectCases(v.Field(i).Interface(), args[i])
			if err != nil {
				return nil, err
			}
			cases = append(cases, c...)
		}
		return cases, nil
	default:
		return nil, fmt.Errorf("cannot consistently return on unsupported type %T", mock)
	}
}

func consistentlyReturn(cases []reflect.SelectCase, done, exited chan struct{}, args ...interface{}) {
	defer close(exited)
	doneIdx := len(cases)
	cases = append(cases, reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(done)})
	for {
		chosen, _, _ := reflect.Select(cases)
		if chosen == doneIdx {
			return
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 The casbin Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package persist

import (
	"strings"

	"github.com/casbin/casbin/model"
)

// LoadPolicyLine loads a text line as a policy rule to model.
func LoadPolicyLine(line string, model model.Model) {
	if line == "" {
		return
	}

	if strings.HasPrefix(line, "#") {
		return
	}

	tokens := strings.Split(line, ", ")

	key := tokens[0]
	sec := key[:1]
	model[sec][key].Policy = append(model[sec][key].Policy, tokens[1:])
}

// Adapter is the interface for Casbin adapters.
type Adapter interface {
	LoadPolicy(model model.Model) error
	SavePolicy(model model.Model) error

	AddPolicy(sec string, ptype string, rule []string) error
	RemovePolicy(sec string, ptype string, rule []string) error
	RemoveFilteredPolicy(sec string, ptype string, fieldIndex int, fieldValues ...string) error
}
<commit_msg>Add comments to the adapter interface.<commit_after>// Copyright 2017 The casbin Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package persist

import (
	"strings"

	"github.com/casbin/casbin/model"
)

// LoadPolicyLine loads a text line as a policy rule to model.
func LoadPolicyLine(line string, model model.Model) {
	if line == "" {
		return
	}

	if strings.HasPrefix(line, "#") {
		return
	}

	tokens := strings.Split(line, ", ")

	key := tokens[0]
	sec := key[:1]
	model[sec][key].Policy = append(model[sec][key].Policy, tokens[1:])
}

// Adapter is the interface for Casbin adapters.
type Adapter interface {
	// LoadPolicy loads all policy rules from the storage.
	LoadPolicy(model model.Model) error
	// SavePolicy saves all policy rules to the storage.
	SavePolicy(model model.Model) error

	// AddPolicy adds a policy rule to the storage.
	// This is part of the Auto-Save feature.
	AddPolicy(sec string, ptype string, rule []string) error
	// RemovePolicy removes a policy rule from the storage.
	// This is part of the Auto-Save feature.
	RemovePolicy(sec string, ptype string, rule []string) error
	// RemoveFilteredPolicy removes policy rules that match the filter from the storage.
	// This is part of the Auto-Save feature.
	RemoveFilteredPolicy(sec string, ptype string, fieldIndex int, fieldValues ...string) error
}
<|endoftext|>


<commit_before>package fuseshim

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"os/exec"
	"sync"
	"syscall"
)

// Maximum file write size we are prepared to receive from the kernel. Linux
// appears to limit writes to 128 KiB.
const maxWrite = 128 * 1024

func lineLogger(wg *sync.WaitGroup, prefix string, r io.ReadCloser) {
	defer wg.Done()

	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		switch line := scanner.Text(); line {
		case `fusermount: failed to open /etc/fuse.conf: Permission denied`:
			// Silence this particular message, it occurs way too
			// commonly and isn't very relevant to whether the mount
			// succeeds or not.
			continue
		default:
			log.Printf("%s: %s", prefix, line)
		}
	}
	if err := scanner.Err(); err != nil {
		log.Printf("%s, error reading: %v", prefix, err)
	}
}

func mount(dir string, conf *mountConfig, ready chan<- struct{}, errp *error) (fusefd *os.File, err error) {
	// linux mount is never delayed
	close(ready)

	fds, err := syscall.Socketpair(syscall.AF_FILE, syscall.SOCK_STREAM, 0)
	if err != nil {
		return nil, fmt.Errorf("socketpair error: %v", err)
	}

	writeFile := os.NewFile(uintptr(fds[0]), "fusermount-child-writes")
	defer writeFile.Close()

	readFile := os.NewFile(uintptr(fds[1]), "fusermount-parent-reads")
	defer readFile.Close()

	cmd := exec.Command(
		"fusermount",
		"-o", conf.getOptions(),
		"--",
		dir,
	)
	cmd.Env = append(os.Environ(), "_FUSE_COMMFD=3")

	cmd.ExtraFiles = []*os.File{writeFile}

	var wg sync.WaitGroup
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, fmt.Errorf("setting up fusermount stderr: %v", err)
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return nil, fmt.Errorf("setting up fusermount stderr: %v", err)
	}

	if err := cmd.Start(); err != nil {
		return nil, fmt.Errorf("fusermount: %v", err)
	}
	wg.Add(2)
	go lineLogger(&wg, "mount helper output", stdout)
	go lineLogger(&wg, "mount helper error", stderr)
	wg.Wait()
	if err := cmd.Wait(); err != nil {
		return nil, fmt.Errorf("fusermount: %v", err)
	}

	c, err := net.FileConn(readFile)
	if err != nil {
		return nil, fmt.Errorf("FileConn from fusermount socket: %v", err)
	}
	defer c.Close()

	uc, ok := c.(*net.UnixConn)
	if !ok {
		return nil, fmt.Errorf("unexpected FileConn type; expected UnixConn, got %T", c)
	}

	buf := make([]byte, 32) // expect 1 byte
	oob := make([]byte, 32) // expect 24 bytes
	_, oobn, _, _, err := uc.ReadMsgUnix(buf, oob)
	scms, err := syscall.ParseSocketControlMessage(oob[:oobn])
	if err != nil {
		return nil, fmt.Errorf("ParseSocketControlMessage: %v", err)
	}
	if len(scms) != 1 {
		return nil, fmt.Errorf("expected 1 SocketControlMessage; got scms = %#v", scms)
	}
	scm := scms[0]
	gotFds, err := syscall.ParseUnixRights(&scm)
	if err != nil {
		return nil, fmt.Errorf("syscall.ParseUnixRights: %v", err)
	}
	if len(gotFds) != 1 {
		return nil, fmt.Errorf("wanted 1 fd; got %#v", gotFds)
	}
	f := os.NewFile(uintptr(gotFds[0]), "/dev/fuse")
	return f, nil
}
<commit_msg>Fixed the package name.<commit_after>package fuse

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"os/exec"
	"sync"
	"syscall"
)

// Maximum file write size we are prepared to receive from the kernel. Linux
// appears to limit writes to 128 KiB.
const maxWrite = 128 * 1024

func lineLogger(wg *sync.WaitGroup, prefix string, r io.ReadCloser) {
	defer wg.Done()

	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		switch line := scanner.Text(); line {
		case `fusermount: failed to open /etc/fuse.conf: Permission denied`:
			// Silence this particular message, it occurs way too
			// commonly and isn't very relevant to whether the mount
			// succeeds or not.
			continue
		default:
			log.Printf("%s: %s", prefix, line)
		}
	}
	if err := scanner.Err(); err != nil {
		log.Printf("%s, error reading: %v", prefix, err)
	}
}

func mount(dir string, conf *mountConfig, ready chan<- struct{}, errp *error) (fusefd *os.File, err error) {
	// linux mount is never delayed
	close(ready)

	fds, err := syscall.Socketpair(syscall.AF_FILE, syscall.SOCK_STREAM, 0)
	if err != nil {
		return nil, fmt.Errorf("socketpair error: %v", err)
	}

	writeFile := os.NewFile(uintptr(fds[0]), "fusermount-child-writes")
	defer writeFile.Close()

	readFile := os.NewFile(uintptr(fds[1]), "fusermount-parent-reads")
	defer readFile.Close()

	cmd := exec.Command(
		"fusermount",
		"-o", conf.getOptions(),
		"--",
		dir,
	)
	cmd.Env = append(os.Environ(), "_FUSE_COMMFD=3")

	cmd.ExtraFiles = []*os.File{writeFile}

	var wg sync.WaitGroup
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, fmt.Errorf("setting up fusermount stderr: %v", err)
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return nil, fmt.Errorf("setting up fusermount stderr: %v", err)
	}

	if err := cmd.Start(); err != nil {
		return nil, fmt.Errorf("fusermount: %v", err)
	}
	wg.Add(2)
	go lineLogger(&wg, "mount helper output", stdout)
	go lineLogger(&wg, "mount helper error", stderr)
	wg.Wait()
	if err := cmd.Wait(); err != nil {
		return nil, fmt.Errorf("fusermount: %v", err)
	}

	c, err := net.FileConn(readFile)
	if err != nil {
		return nil, fmt.Errorf("FileConn from fusermount socket: %v", err)
	}
	defer c.Close()

	uc, ok := c.(*net.UnixConn)
	if !ok {
		return nil, fmt.Errorf("unexpected FileConn type; expected UnixConn, got %T", c)
	}

	buf := make([]byte, 32) // expect 1 byte
	oob := make([]byte, 32) // expect 24 bytes
	_, oobn, _, _, err := uc.ReadMsgUnix(buf, oob)
	scms, err := syscall.ParseSocketControlMessage(oob[:oobn])
	if err != nil {
		return nil, fmt.Errorf("ParseSocketControlMessage: %v", err)
	}
	if len(scms) != 1 {
		return nil, fmt.Errorf("expected 1 SocketControlMessage; got scms = %#v", scms)
	}
	scm := scms[0]
	gotFds, err := syscall.ParseUnixRights(&scm)
	if err != nil {
		return nil, fmt.Errorf("syscall.ParseUnixRights: %v", err)
	}
	if len(gotFds) != 1 {
		return nil, fmt.Errorf("wanted 1 fd; got %#v", gotFds)
	}
	f := os.NewFile(uintptr(gotFds[0]), "/dev/fuse")
	return f, nil
}
<|endoftext|>


<commit_before>package models

import (
	"github.com/astaxie/beego/orm"
	"github.com/lifei6671/mindoc/conf"
	"time"
)

type MemberRelationshipResult struct {
	MemberId       int             `json:"member_id"`
	Account        string          `json:"account"`
	RealName       string          `json:"real_name"`
	Description    string          `json:"description"`
	Email          string          `json:"email"`
	Phone          string          `json:"phone"`
	Avatar         string          `json:"avatar"`
	Role           conf.SystemRole `json:"role"`   //用户角色：0 管理员/ 1 普通用户
	Status         int             `json:"status"` //用户状态：0 正常/1 禁用
	CreateTime     time.Time       `json:"create_time"`
	CreateAt       int             `json:"create_at"`
	RelationshipId int             `json:"relationship_id"`
	BookId         int             `json:"book_id"`
	// RoleId 角色：0 创始人(创始人不能被移除) / 1 管理员/2 编辑者/3 观察者
	RoleId   conf.BookRole `json:"role_id"`
	RoleName string        `json:"role_name"`
}

type SelectMemberResult struct {
	Result []KeyValueItem `json:"results"`
}
type KeyValueItem struct {
	Id   int    `json:"id"`
	Text string `json:"text"`
}

func NewMemberRelationshipResult() *MemberRelationshipResult {
	return &MemberRelationshipResult{}
}

func (m *MemberRelationshipResult) FromMember(member *Member) *MemberRelationshipResult {
	m.MemberId = member.MemberId
	m.Account = member.Account
	m.Description = member.Description
	m.Email = member.Email
	m.Phone = member.Phone
	m.Avatar = member.Avatar
	m.Role = member.Role
	m.Status = member.Status
	m.CreateTime = member.CreateTime
	m.CreateAt = member.CreateAt
	m.RealName = member.RealName

	return m
}

func (m *MemberRelationshipResult) ResolveRoleName() *MemberRelationshipResult {
	if m.RoleId == conf.BookAdmin {
		m.RoleName = "管理者"
	} else if m.RoleId == conf.BookEditor {
		m.RoleName = "编辑者"
	} else if m.RoleId == conf.BookObserver {
		m.RoleName = "观察者"
	}
	return m
}

// 根据项目ID查询用户
func (m *MemberRelationshipResult) FindForUsersByBookId(bookId, pageIndex, pageSize int) ([]*MemberRelationshipResult, int, error) {
	o := orm.NewOrm()

	var members []*MemberRelationshipResult

	sql1 := "SELECT * FROM md_relationship AS rel LEFT JOIN md_members as member ON rel.member_id = member.member_id WHERE rel.book_id = ? ORDER BY rel.relationship_id DESC  LIMIT ?,?"

	sql2 := "SELECT count(*) AS total_count FROM md_relationship AS rel LEFT JOIN md_members as member ON rel.member_id = member.member_id WHERE rel.book_id = ?"

	var total_count int

	err := o.Raw(sql2, bookId).QueryRow(&total_count)

	if err != nil {
		return members, 0, err
	}

	offset := (pageIndex - 1) * pageSize

	_, err = o.Raw(sql1, bookId, offset, pageSize).QueryRows(&members)

	if err != nil {
		return members, 0, err
	}

	for _, item := range members {
		item.ResolveRoleName()
	}
	return members, total_count, nil
}

// 查询指定文档中不存在的用户列表
func (m *MemberRelationshipResult) FindNotJoinUsersByAccount(bookId, limit int, account string) ([]*Member, error) {
	o := orm.NewOrm()

	sql := "SELECT m.* FROM md_members as m LEFT JOIN md_relationship as rel ON m.member_id=rel.member_id AND rel.book_id = ? WHERE rel.relationship_id IS NULL AND m.account LIKE ? LIMIT 0,?;"

	var members []*Member

	_, err := o.Raw(sql, bookId, account, limit).QueryRows(&members)

	return members, err
}
<commit_msg>feat:根据姓名以及用户名模糊查询指定文档中不存在的用户列表<commit_after>package models

import (
	"github.com/astaxie/beego/orm"
	"github.com/lifei6671/mindoc/conf"
	"time"
)

type MemberRelationshipResult struct {
	MemberId       int             `json:"member_id"`
	Account        string          `json:"account"`
	RealName       string          `json:"real_name"`
	Description    string          `json:"description"`
	Email          string          `json:"email"`
	Phone          string          `json:"phone"`
	Avatar         string          `json:"avatar"`
	Role           conf.SystemRole `json:"role"`   //用户角色：0 管理员/ 1 普通用户
	Status         int             `json:"status"` //用户状态：0 正常/1 禁用
	CreateTime     time.Time       `json:"create_time"`
	CreateAt       int             `json:"create_at"`
	RelationshipId int             `json:"relationship_id"`
	BookId         int             `json:"book_id"`
	// RoleId 角色：0 创始人(创始人不能被移除) / 1 管理员/2 编辑者/3 观察者
	RoleId   conf.BookRole `json:"role_id"`
	RoleName string        `json:"role_name"`
}

type SelectMemberResult struct {
	Result []KeyValueItem `json:"results"`
}
type KeyValueItem struct {
	Id   int    `json:"id"`
	Text string `json:"text"`
}

func NewMemberRelationshipResult() *MemberRelationshipResult {
	return &MemberRelationshipResult{}
}

func (m *MemberRelationshipResult) FromMember(member *Member) *MemberRelationshipResult {
	m.MemberId = member.MemberId
	m.Account = member.Account
	m.Description = member.Description
	m.Email = member.Email
	m.Phone = member.Phone
	m.Avatar = member.Avatar
	m.Role = member.Role
	m.Status = member.Status
	m.CreateTime = member.CreateTime
	m.CreateAt = member.CreateAt
	m.RealName = member.RealName

	return m
}

func (m *MemberRelationshipResult) ResolveRoleName() *MemberRelationshipResult {
	if m.RoleId == conf.BookAdmin {
		m.RoleName = "管理者"
	} else if m.RoleId == conf.BookEditor {
		m.RoleName = "编辑者"
	} else if m.RoleId == conf.BookObserver {
		m.RoleName = "观察者"
	}
	return m
}

// 根据项目ID查询用户
func (m *MemberRelationshipResult) FindForUsersByBookId(bookId, pageIndex, pageSize int) ([]*MemberRelationshipResult, int, error) {
	o := orm.NewOrm()

	var members []*MemberRelationshipResult

	sql1 := "SELECT * FROM md_relationship AS rel LEFT JOIN md_members as member ON rel.member_id = member.member_id WHERE rel.book_id = ? ORDER BY rel.relationship_id DESC  LIMIT ?,?"

	sql2 := "SELECT count(*) AS total_count FROM md_relationship AS rel LEFT JOIN md_members as member ON rel.member_id = member.member_id WHERE rel.book_id = ?"

	var total_count int

	err := o.Raw(sql2, bookId).QueryRow(&total_count)

	if err != nil {
		return members, 0, err
	}

	offset := (pageIndex - 1) * pageSize

	_, err = o.Raw(sql1, bookId, offset, pageSize).QueryRows(&members)

	if err != nil {
		return members, 0, err
	}

	for _, item := range members {
		item.ResolveRoleName()
	}
	return members, total_count, nil
}

// 查询指定文档中不存在的用户列表
func (m *MemberRelationshipResult) FindNotJoinUsersByAccount(bookId, limit int, account string) ([]*Member, error) {
	o := orm.NewOrm()

	sql := "SELECT m.* FROM md_members as m LEFT JOIN md_relationship as rel ON m.member_id=rel.member_id AND rel.book_id = ? WHERE rel.relationship_id IS NULL AND m.account LIKE ? LIMIT 0,?;"

	var members []*Member

	_, err := o.Raw(sql, bookId, account, limit).QueryRows(&members)

	return members, err
}

// 根据姓名以及用户名模糊查询指定文档中不存在的用户列表
func (m *MemberRelationshipResult) FindNotJoinUsersByAccountOrRealName(bookId, limit int, keyWord string) ([]*Member, error) {
	o := orm.NewOrm()

	sql := "SELECT m.* FROM md_members as m LEFT JOIN md_relationship as rel ON rel.member_id = m.member_id AND rel.book_id = ? WHERE rel.relationship_id IS NULL AND (m.real_name LIKE ?) OR (m.account LIKE ?) LIMIT 0,?;"

	var members []*Member

	_, err := o.Raw(sql, bookId, keyWord,keyWord, limit).QueryRows(&members)

	return members, err
}

<|endoftext|>


<commit_before>package cloudflare

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"time"

	"github.com/google/go-querystring/query"
	"github.com/pkg/errors"
)

// Tunnel is the struct definition of a tunnel.
type Tunnel struct {
	ID             string             `json:"id,omitempty"`
	Name           string             `json:"name,omitempty"`
	Secret         string             `json:"tunnel_secret,omitempty"`
	CreatedAt      *time.Time         `json:"created_at,omitempty"`
	DeletedAt      *time.Time         `json:"deleted_at,omitempty"`
	Connections    []TunnelConnection `json:"connections,omitempty"`
	ConnsActiveAt  *time.Time         `json:"conns_active_at,omitempty"`
	ConnInactiveAt *time.Time         `json:"conns_inactive_at,omitempty"`
}

// TunnelConnection represents the connections associated with a tunnel.
type TunnelConnection struct {
	ColoName           string `json:"colo_name"`
	ID                 string `json:"id"`
	IsPendingReconnect bool   `json:"is_pending_reconnect"`
	ClientID           string `json:"client_id"`
	ClientVersion      string `json:"client_version"`
	OpenedAt           string `json:"opened_at"`
	OriginIP           string `json:"origin_ip"`
}

// TunnelsDetailResponse is used for representing the API response payload for
// multiple tunnels.
type TunnelsDetailResponse struct {
	Result []Tunnel `json:"result"`
	Response
}

// TunnelDetailResponse is used for representing the API response payload for
// a single tunnel.
type TunnelDetailResponse struct {
	Result Tunnel `json:"result"`
	Response
}

type TunnelParams struct {
	AccountID string
	UUID      string
}

type TunnelCreateParams struct {
	AccountID string `json:"-"`
	Name      string `json:"name,omitempty"`
	Secret    string `json:"tunnel_secret,omitempty"`
}

type TunnelUpdateParams struct {
	AccountID string `json:"-"`
	Name      string `json:"name,omitempty"`
	Secret    string `json:"tunnel_secret,omitempty"`
}

type TunnelDeleteParams struct {
	AccountID string
	UUID      string
}

type TunnelCleanupParams struct {
	AccountID string
	UUID      string
}

type TunnelTokenParams struct {
	AccountID string
	UUID      string
}

type TunnelListParams struct {
	AccountID string
	Name      string     `url:"name,omitempty"`
	UUID      string     `url:"uuid,omitempty"`
	IsDeleted bool       `url:"is_deleted,omitempty"`
	ExistedAt *time.Time `url:"existed_at,omitempty"`
}

// Tunnels lists all tunnels.
//
// API reference: https://api.cloudflare.com/#cloudflare-tunnel-list-cloudflare-tunnels
func (api *API) Tunnels(ctx context.Context, params TunnelListParams) ([]Tunnel, error) {
	if params.AccountID == "" {
		return []Tunnel{}, ErrMissingAccountID
	}

	v, _ := query.Values(params)
	queryParams := v.Encode()
	if queryParams != "" {
		queryParams = "?" + queryParams
	}

	uri := fmt.Sprintf("/accounts/%s/cfd_tunnel", params.AccountID)

	res, err := api.makeRequestContextWithHeaders(ctx, http.MethodGet, uri+queryParams, nil, nil)
	if err != nil {
		return []Tunnel{}, err
	}

	var argoDetailsResponse TunnelsDetailResponse
	err = json.Unmarshal(res, &argoDetailsResponse)
	if err != nil {
		return []Tunnel{}, errors.Wrap(err, errUnmarshalError)
	}
	return argoDetailsResponse.Result, nil
}

// Tunnel returns a single Argo tunnel.
//
// API reference: https://api.cloudflare.com/#cloudflare-tunnel-get-cloudflare-tunnel
func (api *API) Tunnel(ctx context.Context, params TunnelParams) (Tunnel, error) {
	if params.AccountID == "" {
		return Tunnel{}, ErrMissingAccountID
	}

	if params.UUID == "" {
		return Tunnel{}, errors.New("missing tunnel UUID")
	}

	uri := fmt.Sprintf("/accounts/%s/cfd_tunnel/%s", params.AccountID, params.UUID)

	res, err := api.makeRequestContextWithHeaders(ctx, http.MethodGet, uri, nil, nil)
	if err != nil {
		return Tunnel{}, err
	}

	var argoDetailsResponse TunnelDetailResponse
	err = json.Unmarshal(res, &argoDetailsResponse)
	if err != nil {
		return Tunnel{}, errors.Wrap(err, errUnmarshalError)
	}
	return argoDetailsResponse.Result, nil
}

// CreateTunnel creates a new tunnel for the account.
//
// API reference: https://api.cloudflare.com/#cloudflare-tunnel-create-cloudflare-tunnel
func (api *API) CreateTunnel(ctx context.Context, params TunnelCreateParams) (Tunnel, error) {
	if params.AccountID == "" {
		return Tunnel{}, ErrMissingAccountID
	}

	if params.Name == "" {
		return Tunnel{}, errors.New("missing tunnel name")
	}

	if params.Secret == "" {
		return Tunnel{}, errors.New("missing tunnel secret")
	}

	uri := fmt.Sprintf("/accounts/%s/cfd_tunnel", params.AccountID)

	tunnel := Tunnel{Name: params.Name, Secret: params.Secret}

	res, err := api.makeRequestContextWithHeaders(ctx, http.MethodPost, uri, tunnel, nil)
	if err != nil {
		return Tunnel{}, err
	}

	var argoDetailsResponse TunnelDetailResponse
	err = json.Unmarshal(res, &argoDetailsResponse)
	if err != nil {
		return Tunnel{}, errors.Wrap(err, errUnmarshalError)
	}

	return argoDetailsResponse.Result, nil
}

// UpdateTunnel updates an existing tunnel for the account.
//
// API reference: https://api.cloudflare.com/#cloudflare-tunnel-update-cloudflare-tunnel
func (api *API) UpdateTunnel(ctx context.Context, params TunnelUpdateParams) (Tunnel, error) {
	if params.AccountID == "" {
		return Tunnel{}, ErrMissingAccountID
	}

	uri := fmt.Sprintf("/accounts/%s/cfd_tunnel", params.AccountID)

	var tunnel Tunnel

	if params.Name != "" {
		tunnel.Name = params.Name
	}

	if params.Secret != "" {
		tunnel.Secret = params.Secret
	}

	res, err := api.makeRequestContextWithHeaders(ctx, http.MethodPatch, uri, tunnel, nil)
	if err != nil {
		return Tunnel{}, err
	}

	var argoDetailsResponse TunnelDetailResponse
	err = json.Unmarshal(res, &argoDetailsResponse)
	if err != nil {
		return Tunnel{}, errors.Wrap(err, errUnmarshalError)
	}

	return argoDetailsResponse.Result, nil
}

// DeleteTunnel removes a single Argo tunnel.
//
// API reference: https://api.cloudflare.com/#cloudflare-tunnel-delete-cloudflare-tunnel
func (api *API) DeleteTunnel(ctx context.Context, params TunnelDeleteParams) error {
	uri := fmt.Sprintf("/accounts/%s/cfd_tunnel/%s", params.AccountID, params.UUID)

	res, err := api.makeRequestContextWithHeaders(ctx, http.MethodDelete, uri, nil, nil)
	if err != nil {
		return err
	}

	var argoDetailsResponse TunnelDetailResponse
	err = json.Unmarshal(res, &argoDetailsResponse)
	if err != nil {
		return errors.Wrap(err, errUnmarshalError)
	}

	return nil
}

// CleanupTunnelConnections deletes any inactive connections on a tunnel.
//
// API reference: https://api.cloudflare.com/#cloudflare-tunnel-clean-up-cloudflare-tunnel-connections
func (api *API) CleanupTunnelConnections(ctx context.Context, params TunnelCleanupParams) error {
	if params.AccountID == "" {
		return ErrMissingAccountID
	}

	if params.UUID == "" {
		return errors.New("missing tunnel UUID")
	}

	uri := fmt.Sprintf("/accounts/%s/cfd_tunnel/%s/connections", params.AccountID, params.UUID)

	res, err := api.makeRequestContextWithHeaders(ctx, http.MethodDelete, uri, nil, nil)
	if err != nil {
		return err
	}

	var argoDetailsResponse TunnelDetailResponse
	err = json.Unmarshal(res, &argoDetailsResponse)
	if err != nil {
		return errors.Wrap(err, errUnmarshalError)
	}

	return nil
}

// TunnelToken gets the cloudflared token for a tunnel.
//
// API reference: https://api.cloudflare.com/#cloudflare-tunnel-get-cloudflare-tunnel-token
func (api *API) TunnelToken(ctx context.Context, params TunnelTokenParams) error {
	if params.AccountID == "" {
		return ErrMissingAccountID
	}

	if params.UUID == "" {
		return errors.New("missing tunnel UUID")
	}

	uri := fmt.Sprintf("accounts/%s/cfd_tunnel/%s/token", params.AccountID, params.UUID)

	res, err := api.makeRequestContextWithHeaders(ctx, http.MethodDelete, uri, nil, nil)
	if err != nil {
		return err
	}

	var argoDetailsResponse TunnelDetailResponse
	err = json.Unmarshal(res, &argoDetailsResponse)
	if err != nil {
		return errors.Wrap(err, errUnmarshalError)
	}

	return nil
}
<commit_msg>swap get token to use GET<commit_after>package cloudflare

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"time"

	"github.com/google/go-querystring/query"
	"github.com/pkg/errors"
)

// Tunnel is the struct definition of a tunnel.
type Tunnel struct {
	ID             string             `json:"id,omitempty"`
	Name           string             `json:"name,omitempty"`
	Secret         string             `json:"tunnel_secret,omitempty"`
	CreatedAt      *time.Time         `json:"created_at,omitempty"`
	DeletedAt      *time.Time         `json:"deleted_at,omitempty"`
	Connections    []TunnelConnection `json:"connections,omitempty"`
	ConnsActiveAt  *time.Time         `json:"conns_active_at,omitempty"`
	ConnInactiveAt *time.Time         `json:"conns_inactive_at,omitempty"`
}

// TunnelConnection represents the connections associated with a tunnel.
type TunnelConnection struct {
	ColoName           string `json:"colo_name"`
	ID                 string `json:"id"`
	IsPendingReconnect bool   `json:"is_pending_reconnect"`
	ClientID           string `json:"client_id"`
	ClientVersion      string `json:"client_version"`
	OpenedAt           string `json:"opened_at"`
	OriginIP           string `json:"origin_ip"`
}

// TunnelsDetailResponse is used for representing the API response payload for
// multiple tunnels.
type TunnelsDetailResponse struct {
	Result []Tunnel `json:"result"`
	Response
}

// TunnelDetailResponse is used for representing the API response payload for
// a single tunnel.
type TunnelDetailResponse struct {
	Result Tunnel `json:"result"`
	Response
}

type TunnelParams struct {
	AccountID string
	UUID      string
}

type TunnelCreateParams struct {
	AccountID string `json:"-"`
	Name      string `json:"name,omitempty"`
	Secret    string `json:"tunnel_secret,omitempty"`
}

type TunnelUpdateParams struct {
	AccountID string `json:"-"`
	Name      string `json:"name,omitempty"`
	Secret    string `json:"tunnel_secret,omitempty"`
}

type TunnelDeleteParams struct {
	AccountID string
	UUID      string
}

type TunnelCleanupParams struct {
	AccountID string
	UUID      string
}

type TunnelTokenParams struct {
	AccountID string
	UUID      string
}

type TunnelListParams struct {
	AccountID string
	Name      string     `url:"name,omitempty"`
	UUID      string     `url:"uuid,omitempty"`
	IsDeleted bool       `url:"is_deleted,omitempty"`
	ExistedAt *time.Time `url:"existed_at,omitempty"`
}

// Tunnels lists all tunnels.
//
// API reference: https://api.cloudflare.com/#cloudflare-tunnel-list-cloudflare-tunnels
func (api *API) Tunnels(ctx context.Context, params TunnelListParams) ([]Tunnel, error) {
	if params.AccountID == "" {
		return []Tunnel{}, ErrMissingAccountID
	}

	v, _ := query.Values(params)
	queryParams := v.Encode()
	if queryParams != "" {
		queryParams = "?" + queryParams
	}

	uri := fmt.Sprintf("/accounts/%s/cfd_tunnel", params.AccountID)

	res, err := api.makeRequestContextWithHeaders(ctx, http.MethodGet, uri+queryParams, nil, nil)
	if err != nil {
		return []Tunnel{}, err
	}

	var argoDetailsResponse TunnelsDetailResponse
	err = json.Unmarshal(res, &argoDetailsResponse)
	if err != nil {
		return []Tunnel{}, errors.Wrap(err, errUnmarshalError)
	}
	return argoDetailsResponse.Result, nil
}

// Tunnel returns a single Argo tunnel.
//
// API reference: https://api.cloudflare.com/#cloudflare-tunnel-get-cloudflare-tunnel
func (api *API) Tunnel(ctx context.Context, params TunnelParams) (Tunnel, error) {
	if params.AccountID == "" {
		return Tunnel{}, ErrMissingAccountID
	}

	if params.UUID == "" {
		return Tunnel{}, errors.New("missing tunnel UUID")
	}

	uri := fmt.Sprintf("/accounts/%s/cfd_tunnel/%s", params.AccountID, params.UUID)

	res, err := api.makeRequestContextWithHeaders(ctx, http.MethodGet, uri, nil, nil)
	if err != nil {
		return Tunnel{}, err
	}

	var argoDetailsResponse TunnelDetailResponse
	err = json.Unmarshal(res, &argoDetailsResponse)
	if err != nil {
		return Tunnel{}, errors.Wrap(err, errUnmarshalError)
	}
	return argoDetailsResponse.Result, nil
}

// CreateTunnel creates a new tunnel for the account.
//
// API reference: https://api.cloudflare.com/#cloudflare-tunnel-create-cloudflare-tunnel
func (api *API) CreateTunnel(ctx context.Context, params TunnelCreateParams) (Tunnel, error) {
	if params.AccountID == "" {
		return Tunnel{}, ErrMissingAccountID
	}

	if params.Name == "" {
		return Tunnel{}, errors.New("missing tunnel name")
	}

	if params.Secret == "" {
		return Tunnel{}, errors.New("missing tunnel secret")
	}

	uri := fmt.Sprintf("/accounts/%s/cfd_tunnel", params.AccountID)

	tunnel := Tunnel{Name: params.Name, Secret: params.Secret}

	res, err := api.makeRequestContextWithHeaders(ctx, http.MethodPost, uri, tunnel, nil)
	if err != nil {
		return Tunnel{}, err
	}

	var argoDetailsResponse TunnelDetailResponse
	err = json.Unmarshal(res, &argoDetailsResponse)
	if err != nil {
		return Tunnel{}, errors.Wrap(err, errUnmarshalError)
	}

	return argoDetailsResponse.Result, nil
}

// UpdateTunnel updates an existing tunnel for the account.
//
// API reference: https://api.cloudflare.com/#cloudflare-tunnel-update-cloudflare-tunnel
func (api *API) UpdateTunnel(ctx context.Context, params TunnelUpdateParams) (Tunnel, error) {
	if params.AccountID == "" {
		return Tunnel{}, ErrMissingAccountID
	}

	uri := fmt.Sprintf("/accounts/%s/cfd_tunnel", params.AccountID)

	var tunnel Tunnel

	if params.Name != "" {
		tunnel.Name = params.Name
	}

	if params.Secret != "" {
		tunnel.Secret = params.Secret
	}

	res, err := api.makeRequestContextWithHeaders(ctx, http.MethodPatch, uri, tunnel, nil)
	if err != nil {
		return Tunnel{}, err
	}

	var argoDetailsResponse TunnelDetailResponse
	err = json.Unmarshal(res, &argoDetailsResponse)
	if err != nil {
		return Tunnel{}, errors.Wrap(err, errUnmarshalError)
	}

	return argoDetailsResponse.Result, nil
}

// DeleteTunnel removes a single Argo tunnel.
//
// API reference: https://api.cloudflare.com/#cloudflare-tunnel-delete-cloudflare-tunnel
func (api *API) DeleteTunnel(ctx context.Context, params TunnelDeleteParams) error {
	uri := fmt.Sprintf("/accounts/%s/cfd_tunnel/%s", params.AccountID, params.UUID)

	res, err := api.makeRequestContextWithHeaders(ctx, http.MethodDelete, uri, nil, nil)
	if err != nil {
		return err
	}

	var argoDetailsResponse TunnelDetailResponse
	err = json.Unmarshal(res, &argoDetailsResponse)
	if err != nil {
		return errors.Wrap(err, errUnmarshalError)
	}

	return nil
}

// CleanupTunnelConnections deletes any inactive connections on a tunnel.
//
// API reference: https://api.cloudflare.com/#cloudflare-tunnel-clean-up-cloudflare-tunnel-connections
func (api *API) CleanupTunnelConnections(ctx context.Context, params TunnelCleanupParams) error {
	if params.AccountID == "" {
		return ErrMissingAccountID
	}

	if params.UUID == "" {
		return errors.New("missing tunnel UUID")
	}

	uri := fmt.Sprintf("/accounts/%s/cfd_tunnel/%s/connections", params.AccountID, params.UUID)

	res, err := api.makeRequestContextWithHeaders(ctx, http.MethodDelete, uri, nil, nil)
	if err != nil {
		return err
	}

	var argoDetailsResponse TunnelDetailResponse
	err = json.Unmarshal(res, &argoDetailsResponse)
	if err != nil {
		return errors.Wrap(err, errUnmarshalError)
	}

	return nil
}

// TunnelToken gets the cloudflared token for a tunnel.
//
// API reference: https://api.cloudflare.com/#cloudflare-tunnel-get-cloudflare-tunnel-token
func (api *API) TunnelToken(ctx context.Context, params TunnelTokenParams) error {
	if params.AccountID == "" {
		return ErrMissingAccountID
	}

	if params.UUID == "" {
		return errors.New("missing tunnel UUID")
	}

	uri := fmt.Sprintf("accounts/%s/cfd_tunnel/%s/token", params.AccountID, params.UUID)

	res, err := api.makeRequestContextWithHeaders(ctx, http.MethodGet, uri, nil, nil)
	if err != nil {
		return err
	}

	var argoDetailsResponse TunnelDetailResponse
	err = json.Unmarshal(res, &argoDetailsResponse)
	if err != nil {
		return errors.Wrap(err, errUnmarshalError)
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package asset

import (
	"bytes"
	"encoding/json"
	"regexp"
	"strings"

	"github.com/forseti-security/config-validator/pkg/api/validator"
	"github.com/forseti-security/config-validator/pkg/gcv/configs"
	"github.com/golang/glog"
	"github.com/golang/protobuf/jsonpb"
	"github.com/hashicorp/go-multierror"
	"github.com/pkg/errors"
	admissionv1beta1 "k8s.io/api/admission/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
)

const logRequestsVerboseLevel = 2

func ValidateAsset(asset *validator.Asset) error {
	var result *multierror.Error
	if asset.GetName() == "" {
		result = multierror.Append(result, errors.New("missing asset name"))
	}
	if asset.GetAncestryPath() == "" {
		result = multierror.Append(result, errors.Errorf("asset %q missing ancestry path", asset.GetName()))
	}
	if asset.GetAssetType() == "" {
		result = multierror.Append(result, errors.Errorf("asset %q missing type", asset.GetName()))
	}
	if asset.GetResource() == nil && asset.GetIamPolicy() == nil {
		result = multierror.Append(result, errors.Errorf("asset %q missing both resource and IAM policy", asset.GetName()))
	}
	return result.ErrorOrNil()
}

func ConvertResourceViaJSONToInterface(asset *validator.Asset) (interface{}, error) {
	if asset == nil {
		return nil, nil
	}
	m := &jsonpb.Marshaler{
		OrigName: true,
	}
	if asset.Resource != nil {
		CleanStructValue(asset.Resource.Data)
	}
	glog.V(logRequestsVerboseLevel).Infof("converting asset to golang interface: %v", asset)
	var buf bytes.Buffer
	if err := m.Marshal(&buf, asset); err != nil {
		return nil, errors.Wrapf(err, "marshalling to json with asset %s: %v", asset.Name, asset)
	}
	var f interface{}
	err := json.Unmarshal(buf.Bytes(), &f)
	if err != nil {
		return nil, errors.Wrapf(err, "marshalling from json with asset %s: %v", asset.Name, asset)
	}
	return f, nil
}

// SanitizeAncestryPath will populate the AncestryPath field from the ancestors list, or fix the pre-populated one
// if no ancestry list is provided.
func SanitizeAncestryPath(asset *validator.Asset) error {
	if len(asset.Ancestors) != 0 {
		asset.AncestryPath = AncestryPath(asset.Ancestors)
		return nil
	}

	if asset.AncestryPath != "" {
		asset.AncestryPath = configs.NormalizeAncestry(asset.AncestryPath)
		return nil
	}

	return errors.Errorf("no ancestry information for asset %s", asset.String())
}

// AncestryPath returns the ancestry path from a given ancestors list
func AncestryPath(ancestors []string) string {
	cnt := len(ancestors)
	revAncestors := make([]string, len(ancestors))
	for idx := 0; idx < cnt; idx++ {
		revAncestors[cnt-idx-1] = ancestors[idx]
	}
	return strings.Join(revAncestors, "/")
}

// UnwrapCAIResource will unwrap a K8S resource from the CAI payload and populate any omitted fields.
func UnwrapCAIResource(asset map[string]interface{}) (*unstructured.Unstructured, error) {
	groupKind, found, err := unstructured.NestedString(asset, "asset_type")
	if err != nil {
		return nil, errors.Wrapf(err, "failed to access asset_type field")
	}
	if !found {
		return nil, errors.Errorf("asset_type field not found")
	}

	parts := strings.Split(groupKind, "/")
	if len(parts) != 2 {
		return nil, errors.Errorf("expected asset_type to be of form \"<group>/<kind>\", got %s", groupKind)
	}

	group := parts[0]
	kind := parts[1]
	// CAI pretends that the core resources are part of the "k8s.io" apiGroup.  For compatibility with what one would
	// see in kubernetes, we set the group to empty string ("").
	if group == "k8s.io" {
		group = ""
	}

	version, found, err := unstructured.NestedString(asset, "resource", "version")
	if err != nil {
		return nil, errors.Wrapf(err, "failed to access resource.version field")
	}
	if !found {
		return nil, errors.Errorf("resource.version field not found")
	}

	resource, found, err := unstructured.NestedMap(asset, "resource", "data")
	if err != nil {
		return nil, errors.Wrapf(err, "failed to access resource.data field")
	}
	if !found {
		return nil, errors.Errorf("resource.data field not found")
	}

	u := &unstructured.Unstructured{Object: resource}
	u.SetGroupVersionKind(schema.GroupVersionKind{
		Group:   group,
		Version: version,
		Kind:    kind,
	})

	ancestors, found, err := unstructured.NestedStringSlice(asset, "ancestors")
	if err != nil {
		return nil, errors.Wrapf(err, "failed to access resource.data field")
	}
	if !found {
		return nil, errors.Errorf("resource.data field not found")
	}

	annotations := u.GetAnnotations()
	if annotations == nil {
		annotations = map[string]string{}
	}
	annotations["validator.forsetisecurity.org/ancestorPath"] = AncestryPath(ancestors)
	u.SetAnnotations(annotations)

	return u, nil
}

// ConvertToAdmissionRequest converts a CAI asset containing a K8S type to an AdmissionRequest which is the format that
// the Gatekeeper Constraint Framework target expects.
func ConvertToAdmissionRequest(asset map[string]interface{}) (*admissionv1beta1.AdmissionRequest, error) {
	resource, err := UnwrapCAIResource(asset)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to unwrap k8s resource from CAI asset")
	}

	resourceJSON, err := json.Marshal(resource.Object)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to convert unwrapped resource to JSON")
	}

	gvk := resource.GroupVersionKind()
	req := &admissionv1beta1.AdmissionRequest{
		Kind: metav1.GroupVersionKind{
			Group:   gvk.Group,
			Version: gvk.Version,
			Kind:    gvk.Kind,
		},
		Object: runtime.RawExtension{
			Raw: resourceJSON,
		},
		Name: resource.GetName(),
	}
	return req, nil
}

// k8s assset names will follow pattern:
// //container.googleapis.com/projects/*/(locations|zones)/*/clusters/*/k8s
var assetPath = regexp.MustCompile(`^//container.googleapis.com/projects/[^/]*/(locations|zones)/[^/]*/clusters/[^/]*/k8s`)

// IsK8S returns true if the CAI asset is an asset from a kubernetes cluster.
func IsK8S(asset map[string]interface{}) bool {
	assetName, found, err := unstructured.NestedString(asset, "name")
	if !found || err != nil {
		return false
	}
	return assetPath.MatchString(assetName)
}
<commit_msg>accept org policy & access policy<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package asset

import (
	"bytes"
	"encoding/json"
	"regexp"
	"strings"

	"github.com/forseti-security/config-validator/pkg/api/validator"
	"github.com/forseti-security/config-validator/pkg/gcv/configs"
	"github.com/golang/glog"
	"github.com/golang/protobuf/jsonpb"
	"github.com/hashicorp/go-multierror"
	"github.com/pkg/errors"
	admissionv1beta1 "k8s.io/api/admission/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
)

const logRequestsVerboseLevel = 2

func ValidateAsset(asset *validator.Asset) error {
	var result *multierror.Error
	if asset.GetName() == "" {
		result = multierror.Append(result, errors.New("missing asset name"))
	}
	if asset.GetAncestryPath() == "" {
		result = multierror.Append(result, errors.Errorf("asset %q missing ancestry path", asset.GetName()))
	}
	if asset.GetAssetType() == "" {
		result = multierror.Append(result, errors.Errorf("asset %q missing type", asset.GetName()))
	}
	if asset.GetResource() == nil && asset.GetIamPolicy() == nil && asset.GetOrgPolicy() == nil && asset.GetAccessContextPolicy() == nil {
		result = multierror.Append(result, errors.Errorf("asset %q missing all of these: resource, IAM policy, Org Policy, Access Policy", asset.GetName()))
	}
	return result.ErrorOrNil()
}

func ConvertResourceViaJSONToInterface(asset *validator.Asset) (interface{}, error) {
	if asset == nil {
		return nil, nil
	}
	m := &jsonpb.Marshaler{
		OrigName: true,
	}
	if asset.Resource != nil {
		CleanStructValue(asset.Resource.Data)
	}
	glog.V(logRequestsVerboseLevel).Infof("converting asset to golang interface: %v", asset)
	var buf bytes.Buffer
	if err := m.Marshal(&buf, asset); err != nil {
		return nil, errors.Wrapf(err, "marshalling to json with asset %s: %v", asset.Name, asset)
	}
	var f interface{}
	err := json.Unmarshal(buf.Bytes(), &f)
	if err != nil {
		return nil, errors.Wrapf(err, "marshalling from json with asset %s: %v", asset.Name, asset)
	}
	return f, nil
}

// SanitizeAncestryPath will populate the AncestryPath field from the ancestors list, or fix the pre-populated one
// if no ancestry list is provided.
func SanitizeAncestryPath(asset *validator.Asset) error {
	if len(asset.Ancestors) != 0 {
		asset.AncestryPath = AncestryPath(asset.Ancestors)
		return nil
	}

	if asset.AncestryPath != "" {
		asset.AncestryPath = configs.NormalizeAncestry(asset.AncestryPath)
		return nil
	}

	return errors.Errorf("no ancestry information for asset %s", asset.String())
}

// AncestryPath returns the ancestry path from a given ancestors list
func AncestryPath(ancestors []string) string {
	cnt := len(ancestors)
	revAncestors := make([]string, len(ancestors))
	for idx := 0; idx < cnt; idx++ {
		revAncestors[cnt-idx-1] = ancestors[idx]
	}
	return strings.Join(revAncestors, "/")
}

// UnwrapCAIResource will unwrap a K8S resource from the CAI payload and populate any omitted fields.
func UnwrapCAIResource(asset map[string]interface{}) (*unstructured.Unstructured, error) {
	groupKind, found, err := unstructured.NestedString(asset, "asset_type")
	if err != nil {
		return nil, errors.Wrapf(err, "failed to access asset_type field")
	}
	if !found {
		return nil, errors.Errorf("asset_type field not found")
	}

	parts := strings.Split(groupKind, "/")
	if len(parts) != 2 {
		return nil, errors.Errorf("expected asset_type to be of form \"<group>/<kind>\", got %s", groupKind)
	}

	group := parts[0]
	kind := parts[1]
	// CAI pretends that the core resources are part of the "k8s.io" apiGroup.  For compatibility with what one would
	// see in kubernetes, we set the group to empty string ("").
	if group == "k8s.io" {
		group = ""
	}

	version, found, err := unstructured.NestedString(asset, "resource", "version")
	if err != nil {
		return nil, errors.Wrapf(err, "failed to access resource.version field")
	}
	if !found {
		return nil, errors.Errorf("resource.version field not found")
	}

	resource, found, err := unstructured.NestedMap(asset, "resource", "data")
	if err != nil {
		return nil, errors.Wrapf(err, "failed to access resource.data field")
	}
	if !found {
		return nil, errors.Errorf("resource.data field not found")
	}

	u := &unstructured.Unstructured{Object: resource}
	u.SetGroupVersionKind(schema.GroupVersionKind{
		Group:   group,
		Version: version,
		Kind:    kind,
	})

	ancestors, found, err := unstructured.NestedStringSlice(asset, "ancestors")
	if err != nil {
		return nil, errors.Wrapf(err, "failed to access resource.data field")
	}
	if !found {
		return nil, errors.Errorf("resource.data field not found")
	}

	annotations := u.GetAnnotations()
	if annotations == nil {
		annotations = map[string]string{}
	}
	annotations["validator.forsetisecurity.org/ancestorPath"] = AncestryPath(ancestors)
	u.SetAnnotations(annotations)

	return u, nil
}

// ConvertToAdmissionRequest converts a CAI asset containing a K8S type to an AdmissionRequest which is the format that
// the Gatekeeper Constraint Framework target expects.
func ConvertToAdmissionRequest(asset map[string]interface{}) (*admissionv1beta1.AdmissionRequest, error) {
	resource, err := UnwrapCAIResource(asset)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to unwrap k8s resource from CAI asset")
	}

	resourceJSON, err := json.Marshal(resource.Object)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to convert unwrapped resource to JSON")
	}

	gvk := resource.GroupVersionKind()
	req := &admissionv1beta1.AdmissionRequest{
		Kind: metav1.GroupVersionKind{
			Group:   gvk.Group,
			Version: gvk.Version,
			Kind:    gvk.Kind,
		},
		Object: runtime.RawExtension{
			Raw: resourceJSON,
		},
		Name: resource.GetName(),
	}
	return req, nil
}

// k8s assset names will follow pattern:
// //container.googleapis.com/projects/*/(locations|zones)/*/clusters/*/k8s
var assetPath = regexp.MustCompile(`^//container.googleapis.com/projects/[^/]*/(locations|zones)/[^/]*/clusters/[^/]*/k8s`)

// IsK8S returns true if the CAI asset is an asset from a kubernetes cluster.
func IsK8S(asset map[string]interface{}) bool {
	assetName, found, err := unstructured.NestedString(asset, "name")
	if !found || err != nil {
		return false
	}
	return assetPath.MatchString(assetName)
}
<|endoftext|>


<commit_before>package files

import (
	"errors"
	"io"
	"time"

	"honnef.co/go/js/dom"

	"github.com/gopherjs/gopherjs/js"
)

type Blob struct {
	b *js.Object
}

func NewBlob(b *js.Object) Blob {
	return Blob{b}
}

func (b Blob) Len() int {
	return b.b.Get("size").Int()
}

func (b Blob) Type() string {
	return b.b.Get("type").String()
}

func (b Blob) Slice(i, j int) Blob {
	return NewBlob(b.b.Call("slice", i, j))
}

type File struct {
	Blob
}

func NewFile(f *dom.File) File {
	return File{NewBlob(f.Object)}
}

func (f File) LastModifiedDate() time.Time {
	return f.b.Get("lastModifiedDate").Interface().(time.Time)
}

func (f File) Name() string {
	return f.b.Get("name").String()
}

type FileReader struct {
	File
	fr           *js.Object
	offset, size int64
}

func NewFileReader(f File) *FileReader {
	return &FileReader{
		f,
		js.Global.Get("FileReader").New(),
		0,
		int64(f.Len()),
	}
}

func (fr *FileReader) Close() error {
	fr.fr = nil
	return nil
}

func (fr *FileReader) Read(b []byte) (int, error) {
	n, err := fr.ReadAt(b, int64(fr.offset))
	fr.offset += int64(n)
	return n, err
}

func (fr *FileReader) ReadAt(b []byte, off int64) (int, error) {
	if fr.fr == nil {
		return 0, ErrClosed
	}
	if off >= fr.size {
		return 0, io.EOF
	}

	type readResult struct {
		size int
		err  error
	}

	c := make(chan readResult)
	fr.fr.Set("onloadend", func(*js.Object) {
		arr := js.Global.Get("Uint8Array").New(fr.fr.Get("result"))
		buf := arr.Interface().([]byte)
		go func() {
			if len(buf) == 0 {
				c <- readResult{0, io.EOF}
			} else {
				copy(b, buf)
				c <- readResult{len(buf), nil}
			}
		}()
	})
	e := off + int64(len(b))
	if e > fr.size {
		e = fr.size
	}
	blob := fr.f.b.Call("slice", fr.offset, e)
	fr.fr.Call("readAsArrayBuffer", blob)
	r := <-c
	return r.size, r.err
}

func (fr *FileReader) Seek(offset int64, whence int) (int64, error) {
	if fr.fr == nil {
		return 0, ErrClosed
	}
	switch whence {
	case 0:
		fr.offset = offset
	case 1:
		fr.offset += offset
	case 2:
		fr.offset = fr.size + offset
	}
	if fr.offset < 0 {
		fr.offset = 0
	}
	return fr.offset, nil
}

// Errors
var (
	ErrClosed = errors.New("file closed")
)
<commit_msg>Propegated struct change<commit_after>package files

import (
	"errors"
	"io"
	"time"

	"honnef.co/go/js/dom"

	"github.com/gopherjs/gopherjs/js"
)

type Blob struct {
	b *js.Object
}

func NewBlob(b *js.Object) Blob {
	return Blob{b}
}

func (b Blob) Len() int {
	return b.b.Get("size").Int()
}

func (b Blob) Type() string {
	return b.b.Get("type").String()
}

func (b Blob) Slice(i, j int) Blob {
	return NewBlob(b.b.Call("slice", i, j))
}

type File struct {
	Blob
}

func NewFile(f *dom.File) File {
	return File{NewBlob(f.Object)}
}

func (f File) LastModifiedDate() time.Time {
	return f.b.Get("lastModifiedDate").Interface().(time.Time)
}

func (f File) Name() string {
	return f.b.Get("name").String()
}

type FileReader struct {
	File
	fr           *js.Object
	offset, size int64
}

func NewFileReader(f File) *FileReader {
	return &FileReader{
		f,
		js.Global.Get("FileReader").New(),
		0,
		int64(f.Len()),
	}
}

func (fr *FileReader) Close() error {
	fr.fr = nil
	return nil
}

func (fr *FileReader) Read(b []byte) (int, error) {
	n, err := fr.ReadAt(b, int64(fr.offset))
	fr.offset += int64(n)
	return n, err
}

func (fr *FileReader) ReadAt(b []byte, off int64) (int, error) {
	if fr.fr == nil {
		return 0, ErrClosed
	}
	if off >= fr.size {
		return 0, io.EOF
	}

	type readResult struct {
		size int
		err  error
	}

	c := make(chan readResult)
	fr.fr.Set("onloadend", func(*js.Object) {
		arr := js.Global.Get("Uint8Array").New(fr.fr.Get("result"))
		buf := arr.Interface().([]byte)
		go func() {
			if len(buf) == 0 {
				c <- readResult{0, io.EOF}
			} else {
				copy(b, buf)
				c <- readResult{len(buf), nil}
			}
		}()
	})
	e := off + int64(len(b))
	if e > fr.size {
		e = fr.size
	}
	blob := fr.b.Call("slice", fr.offset, e)
	fr.fr.Call("readAsArrayBuffer", blob)
	r := <-c
	return r.size, r.err
}

func (fr *FileReader) Seek(offset int64, whence int) (int64, error) {
	if fr.fr == nil {
		return 0, ErrClosed
	}
	switch whence {
	case 0:
		fr.offset = offset
	case 1:
		fr.offset += offset
	case 2:
		fr.offset = fr.size + offset
	}
	if fr.offset < 0 {
		fr.offset = 0
	}
	return fr.offset, nil
}

// Errors
var (
	ErrClosed = errors.New("file closed")
)
<|endoftext|>


<commit_before>package commands

import (
	"context"
	"fmt"
	"log"
	"net/url"
	"os"
	"text/tabwriter"

	"github.com/argoproj/argo-cd/errors"
	argocdclient "github.com/argoproj/argo-cd/pkg/apiclient"
	argoappv1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
	"github.com/argoproj/argo-cd/server/application"
	"github.com/argoproj/argo-cd/util"
	"github.com/argoproj/argo-cd/util/cli"
	"github.com/argoproj/argo-cd/util/diff"
	"github.com/spf13/cobra"
	"github.com/yudai/gojsondiff/formatter"
	"golang.org/x/crypto/ssh/terminal"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// NewApplicationCommand returns a new instance of an `argocd app` command
func NewApplicationCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command {
	var command = &cobra.Command{
		Use:   "app",
		Short: fmt.Sprintf("%s app COMMAND", cliName),
		Run: func(c *cobra.Command, args []string) {
			c.HelpFunc()(c, args)
			os.Exit(1)
		},
	}

	command.AddCommand(NewApplicationCreateCommand(clientOpts))
	command.AddCommand(NewApplicationGetCommand(clientOpts))
	command.AddCommand(NewApplicationDiffCommand(clientOpts))
	command.AddCommand(NewApplicationSyncCommand(clientOpts))
	command.AddCommand(NewApplicationListCommand(clientOpts))
	command.AddCommand(NewApplicationDeleteCommand(clientOpts))
	return command
}

// NewApplicationCreateCommand returns a new instance of an `argocd app create` command
func NewApplicationCreateCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command {
	var (
		fileURL       string
		repoURL       string
		appPath       string
		appName       string
		env           string
		destServer    string
		destNamespace string
		syncPolicy    string
	)
	var command = &cobra.Command{
		Use:   "create",
		Short: fmt.Sprintf("%s app create", cliName),
		Run: func(c *cobra.Command, args []string) {
			if len(args) != 0 {
				c.HelpFunc()(c, args)
				os.Exit(1)
			}
			var app argoappv1.Application
			if fileURL != "" {
				_, err := url.ParseRequestURI(fileURL)
				if err != nil {
					err = cli.UnmarshalLocalFile(fileURL, &app)
				} else {
					err = cli.UnmarshalRemoteFile(fileURL, &app)
				}
				if err != nil {
					log.Fatal(err)
				}

			} else {
				if syncPolicy != "" && syncPolicy != "Always" {
					c.HelpFunc()(c, args)
					os.Exit(1)
				}
				// all these params are required if we're here
				if repoURL == "" || appPath == "" || appName == "" {
					c.HelpFunc()(c, args)
					os.Exit(1)
				}
				app = argoappv1.Application{
					ObjectMeta: metav1.ObjectMeta{
						Name: appName,
					},
					Spec: argoappv1.ApplicationSpec{
						Source: argoappv1.ApplicationSource{
							RepoURL:     repoURL,
							Path:        appPath,
							Environment: env,
						},
						SyncPolicy: syncPolicy,
					},
				}
			}
			if destServer != "" || destNamespace != "" {
				app.Spec.Destination = &argoappv1.ApplicationDestination{
					Server:    destServer,
					Namespace: destNamespace,
				}
			}
			conn, appIf := argocdclient.NewClientOrDie(clientOpts).NewApplicationClientOrDie()
			defer util.Close(conn)
			_, err := appIf.Create(context.Background(), &app)
			errors.CheckError(err)
		},
	}
	command.Flags().StringVarP(&fileURL, "file", "f", "", "Filename or URL to Kubernetes manifests for the app")
	command.Flags().StringVar(&appName, "name", "", "A name for the app, ignored if a file is set")
	command.Flags().StringVar(&repoURL, "repo", "", "Repository URL, ignored if a file is set")
	command.Flags().StringVar(&appPath, "path", "", "Path in repository to the ksonnet app directory, ignored if a file is set")
	command.Flags().StringVar(&env, "env", "", "Application environment to monitor")
	command.Flags().StringVar(&destServer, "dest-server", "", "K8s cluster URL (overrides the server URL specified in the ksonnet app.yaml)")
	command.Flags().StringVar(&destNamespace, "dest-namespace", "", "K8s target namespace (overrides the namespace specified in the ksonnet app.yaml)")
	command.Flags().StringVar(&syncPolicy, "sync-policy", "", "Synchronization policy for application (e.g., Always)")
	return command
}

// NewApplicationGetCommand returns a new instance of an `argocd app get` command
func NewApplicationGetCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command {
	var command = &cobra.Command{
		Use:   "get",
		Short: fmt.Sprintf("%s app get APPNAME", cliName),
		Run: func(c *cobra.Command, args []string) {
			if len(args) == 0 {
				c.HelpFunc()(c, args)
				os.Exit(1)
			}
			conn, appIf := argocdclient.NewClientOrDie(clientOpts).NewApplicationClientOrDie()
			defer util.Close(conn)
			appName := args[0]
			app, err := appIf.Get(context.Background(), &application.ApplicationQuery{Name: appName})
			errors.CheckError(err)
			format := "%-15s%s\n"
			fmt.Printf(format, "Name:", app.Name)
			fmt.Printf(format, "Environment:", app.Spec.Source.Environment)
			fmt.Printf(format, "Repo:", app.Spec.Source.RepoURL)
			fmt.Printf(format, "Path:", app.Spec.Source.Path)
			if app.Spec.Source.TargetRevision == "" {
				fmt.Printf(format, "Target:", "HEAD")
			} else {
				fmt.Printf(format, "Target:", app.Spec.Source.TargetRevision)
			}
			if app.Status.ComparisonResult.Error != "" {
				fmt.Printf(format, "Error:", app.Status.ComparisonResult.Error)
			}
			if app.Status.ComparisonResult.Server != "" {
				fmt.Printf(format, "Server:", app.Status.ComparisonResult.Server)
			}
			if app.Status.ComparisonResult.Namespace != "" {
				fmt.Printf(format, "Namespace:", app.Status.ComparisonResult.Namespace)
			}
			if len(app.Status.ComparisonResult.Resources) > 0 {
				fmt.Println()
				w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
				fmt.Fprintf(w, "KIND\tNAME\tSTATUS\n")
				for _, res := range app.Status.ComparisonResult.Resources {
					targetObj, err := argoappv1.UnmarshalToUnstructured(res.TargetState)
					errors.CheckError(err)
					fmt.Fprintf(w, "%s\t%s\t%s\n", targetObj.GetKind(), targetObj.GetName(), res.Status)
				}
				_ = w.Flush()
			}
		},
	}
	return command
}

// NewApplicationDiffCommand returns a new instance of an `argocd app diff` command
func NewApplicationDiffCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command {
	var command = &cobra.Command{
		Use:   "diff",
		Short: fmt.Sprintf("%s app diff APPNAME", cliName),
		Run: func(c *cobra.Command, args []string) {
			if len(args) == 0 {
				c.HelpFunc()(c, args)
				os.Exit(1)
			}
			conn, appIf := argocdclient.NewClientOrDie(clientOpts).NewApplicationClientOrDie()
			defer util.Close(conn)
			appName := args[0]
			app, err := appIf.Get(context.Background(), &application.ApplicationQuery{Name: appName})
			errors.CheckError(err)
			targetObjs, err := app.Status.ComparisonResult.TargetObjects()
			errors.CheckError(err)
			liveObjs, err := app.Status.ComparisonResult.LiveObjects()
			errors.CheckError(err)
			diffResults, err := diff.DiffArray(targetObjs, liveObjs)
			errors.CheckError(err)
			for i := 0; i < len(targetObjs); i++ {
				targetObj := targetObjs[i]
				diffRes := diffResults.Diffs[i]
				fmt.Printf("===== %s %s ======\n", targetObj.GetKind(), targetObj.GetName())
				if diffRes.Modified {
					formatOpts := formatter.AsciiFormatterConfig{
						Coloring: terminal.IsTerminal(int(os.Stdout.Fd())),
					}
					out, err := diffResults.Diffs[i].ASCIIFormat(targetObj, formatOpts)
					errors.CheckError(err)
					fmt.Println(out)
				}
			}
		},
	}
	return command
}

// NewApplicationDeleteCommand returns a new instance of an `argocd app delete` command
func NewApplicationDeleteCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command {
	var (
		force bool
	)
	var command = &cobra.Command{
		Use:   "delete",
		Short: fmt.Sprintf("%s app delete APPNAME", cliName),
		Run: func(c *cobra.Command, args []string) {
			if len(args) == 0 {
				c.HelpFunc()(c, args)
				os.Exit(1)
			}
			conn, appIf := argocdclient.NewClientOrDie(clientOpts).NewApplicationClientOrDie()
			defer util.Close(conn)
			for _, appName := range args {
				appDeleteReq := application.DeleteApplicationRequest{
					Name:  appName,
					Force: force,
				}
				_, err := appIf.Delete(context.Background(), &appDeleteReq)
				errors.CheckError(err)
			}
		},
	}
	command.Flags().BoolVar(&force, "force", false, "Force delete application even if cascaded deletion unsuccessful")
	return command
}

// NewApplicationListCommand returns a new instance of an `argocd app list` command
func NewApplicationListCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command {
	var command = &cobra.Command{
		Use:   "list",
		Short: fmt.Sprintf("%s app list", cliName),
		Run: func(c *cobra.Command, args []string) {
			conn, appIf := argocdclient.NewClientOrDie(clientOpts).NewApplicationClientOrDie()
			defer util.Close(conn)
			apps, err := appIf.List(context.Background(), &application.ApplicationQuery{})
			errors.CheckError(err)
			w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
			fmt.Fprintf(w, "NAME\tENVIRONMENT\tTARGET\tCLUSTER\tNAMESPACE\tSTATUS\n")
			for _, app := range apps.Items {
				targetRev := app.Spec.Source.TargetRevision
				if targetRev == "" {
					targetRev = "HEAD"
				}
				fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n",
					app.Name,
					app.Spec.Source.Environment,
					targetRev,
					app.Status.ComparisonResult.Server,
					app.Status.ComparisonResult.Namespace,
					app.Status.ComparisonResult.Status,
				)
			}
			_ = w.Flush()
		},
	}
	return command
}

// NewApplicationSyncCommand returns a new instance of an `argocd app sync` command
func NewApplicationSyncCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command {
	var (
		dryRun bool
	)
	var command = &cobra.Command{
		Use:   "sync",
		Short: fmt.Sprintf("%s app sync APPNAME", cliName),
		Run: func(c *cobra.Command, args []string) {
			if len(args) == 0 {
				c.HelpFunc()(c, args)
				os.Exit(1)
			}
			conn, appIf := argocdclient.NewClientOrDie(clientOpts).NewApplicationClientOrDie()
			defer util.Close(conn)
			appName := args[0]
			syncReq := application.ApplicationSyncRequest{
				Name:   appName,
				DryRun: dryRun,
			}
			syncRes, err := appIf.Sync(context.Background(), &syncReq)
			errors.CheckError(err)
			fmt.Printf("%s %s\n", appName, syncRes.Message)
			w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
			fmt.Fprintf(w, "NAME\tKIND\tMESSAGE\n")
			for _, resDetails := range syncRes.Resources {
				fmt.Fprintf(w, "%s\t%s\t%s\n", resDetails.Name, resDetails.Kind, resDetails.Message)
			}
			_ = w.Flush()
		},
	}
	command.Flags().BoolVar(&dryRun, "dry-run", false, "Preview apply without affecting cluster")
	return command
}
<commit_msg>Add ability to set the tracking revision during app creation<commit_after>package commands

import (
	"context"
	"fmt"
	"log"
	"net/url"
	"os"
	"text/tabwriter"

	"github.com/argoproj/argo-cd/errors"
	argocdclient "github.com/argoproj/argo-cd/pkg/apiclient"
	argoappv1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
	"github.com/argoproj/argo-cd/server/application"
	"github.com/argoproj/argo-cd/util"
	"github.com/argoproj/argo-cd/util/cli"
	"github.com/argoproj/argo-cd/util/diff"
	"github.com/spf13/cobra"
	"github.com/yudai/gojsondiff/formatter"
	"golang.org/x/crypto/ssh/terminal"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// NewApplicationCommand returns a new instance of an `argocd app` command
func NewApplicationCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command {
	var command = &cobra.Command{
		Use:   "app",
		Short: fmt.Sprintf("%s app COMMAND", cliName),
		Run: func(c *cobra.Command, args []string) {
			c.HelpFunc()(c, args)
			os.Exit(1)
		},
	}

	command.AddCommand(NewApplicationCreateCommand(clientOpts))
	command.AddCommand(NewApplicationGetCommand(clientOpts))
	command.AddCommand(NewApplicationDiffCommand(clientOpts))
	command.AddCommand(NewApplicationSyncCommand(clientOpts))
	command.AddCommand(NewApplicationListCommand(clientOpts))
	command.AddCommand(NewApplicationDeleteCommand(clientOpts))
	return command
}

// NewApplicationCreateCommand returns a new instance of an `argocd app create` command
func NewApplicationCreateCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command {
	var (
		fileURL       string
		repoURL       string
		appPath       string
		appName       string
		env           string
		revision      string
		destServer    string
		destNamespace string
		syncPolicy    string
	)
	var command = &cobra.Command{
		Use:   "create",
		Short: fmt.Sprintf("%s app create", cliName),
		Run: func(c *cobra.Command, args []string) {
			if len(args) != 0 {
				c.HelpFunc()(c, args)
				os.Exit(1)
			}
			var app argoappv1.Application
			if fileURL != "" {
				_, err := url.ParseRequestURI(fileURL)
				if err != nil {
					err = cli.UnmarshalLocalFile(fileURL, &app)
				} else {
					err = cli.UnmarshalRemoteFile(fileURL, &app)
				}
				if err != nil {
					log.Fatal(err)
				}

			} else {
				if syncPolicy != "" && syncPolicy != "Always" {
					c.HelpFunc()(c, args)
					os.Exit(1)
				}
				// all these params are required if we're here
				if repoURL == "" || appPath == "" || appName == "" {
					c.HelpFunc()(c, args)
					os.Exit(1)
				}
				app = argoappv1.Application{
					ObjectMeta: metav1.ObjectMeta{
						Name: appName,
					},
					Spec: argoappv1.ApplicationSpec{
						Source: argoappv1.ApplicationSource{
							RepoURL:        repoURL,
							Path:           appPath,
							Environment:    env,
							TargetRevision: revision,
						},
						SyncPolicy: syncPolicy,
					},
				}
			}
			if destServer != "" || destNamespace != "" {
				app.Spec.Destination = &argoappv1.ApplicationDestination{
					Server:    destServer,
					Namespace: destNamespace,
				}
			}
			conn, appIf := argocdclient.NewClientOrDie(clientOpts).NewApplicationClientOrDie()
			defer util.Close(conn)
			_, err := appIf.Create(context.Background(), &app)
			errors.CheckError(err)
		},
	}
	command.Flags().StringVarP(&fileURL, "file", "f", "", "Filename or URL to Kubernetes manifests for the app")
	command.Flags().StringVar(&appName, "name", "", "A name for the app, ignored if a file is set")
	command.Flags().StringVar(&repoURL, "repo", "", "Repository URL, ignored if a file is set")
	command.Flags().StringVar(&appPath, "path", "", "Path in repository to the ksonnet app directory, ignored if a file is set")
	command.Flags().StringVar(&env, "env", "", "Application environment to monitor")
	command.Flags().StringVar(&revision, "revision", "HEAD", "The tracking source branch, tag, or commit the application will sync to")
	command.Flags().StringVar(&destServer, "dest-server", "", "K8s cluster URL (overrides the server URL specified in the ksonnet app.yaml)")
	command.Flags().StringVar(&destNamespace, "dest-namespace", "", "K8s target namespace (overrides the namespace specified in the ksonnet app.yaml)")
	//command.Flags().StringVar(&syncPolicy, "sync-policy", "", "Synchronization policy for application (e.g., Always)")
	return command
}

// NewApplicationGetCommand returns a new instance of an `argocd app get` command
func NewApplicationGetCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command {
	var command = &cobra.Command{
		Use:   "get",
		Short: fmt.Sprintf("%s app get APPNAME", cliName),
		Run: func(c *cobra.Command, args []string) {
			if len(args) == 0 {
				c.HelpFunc()(c, args)
				os.Exit(1)
			}
			conn, appIf := argocdclient.NewClientOrDie(clientOpts).NewApplicationClientOrDie()
			defer util.Close(conn)
			appName := args[0]
			app, err := appIf.Get(context.Background(), &application.ApplicationQuery{Name: appName})
			errors.CheckError(err)
			format := "%-15s%s\n"
			fmt.Printf(format, "Name:", app.Name)
			fmt.Printf(format, "Environment:", app.Spec.Source.Environment)
			fmt.Printf(format, "Repo:", app.Spec.Source.RepoURL)
			fmt.Printf(format, "Path:", app.Spec.Source.Path)
			if app.Spec.Source.TargetRevision == "" {
				fmt.Printf(format, "Target:", "HEAD")
			} else {
				fmt.Printf(format, "Target:", app.Spec.Source.TargetRevision)
			}
			if app.Status.ComparisonResult.Error != "" {
				fmt.Printf(format, "Error:", app.Status.ComparisonResult.Error)
			}
			if app.Status.ComparisonResult.Server != "" {
				fmt.Printf(format, "Server:", app.Status.ComparisonResult.Server)
			}
			if app.Status.ComparisonResult.Namespace != "" {
				fmt.Printf(format, "Namespace:", app.Status.ComparisonResult.Namespace)
			}
			if len(app.Status.ComparisonResult.Resources) > 0 {
				fmt.Println()
				w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
				fmt.Fprintf(w, "KIND\tNAME\tSTATUS\n")
				for _, res := range app.Status.ComparisonResult.Resources {
					targetObj, err := argoappv1.UnmarshalToUnstructured(res.TargetState)
					errors.CheckError(err)
					fmt.Fprintf(w, "%s\t%s\t%s\n", targetObj.GetKind(), targetObj.GetName(), res.Status)
				}
				_ = w.Flush()
			}
		},
	}
	return command
}

// NewApplicationDiffCommand returns a new instance of an `argocd app diff` command
func NewApplicationDiffCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command {
	var command = &cobra.Command{
		Use:   "diff",
		Short: fmt.Sprintf("%s app diff APPNAME", cliName),
		Run: func(c *cobra.Command, args []string) {
			if len(args) == 0 {
				c.HelpFunc()(c, args)
				os.Exit(1)
			}
			conn, appIf := argocdclient.NewClientOrDie(clientOpts).NewApplicationClientOrDie()
			defer util.Close(conn)
			appName := args[0]
			app, err := appIf.Get(context.Background(), &application.ApplicationQuery{Name: appName})
			errors.CheckError(err)
			targetObjs, err := app.Status.ComparisonResult.TargetObjects()
			errors.CheckError(err)
			liveObjs, err := app.Status.ComparisonResult.LiveObjects()
			errors.CheckError(err)
			diffResults, err := diff.DiffArray(targetObjs, liveObjs)
			errors.CheckError(err)
			for i := 0; i < len(targetObjs); i++ {
				targetObj := targetObjs[i]
				diffRes := diffResults.Diffs[i]
				fmt.Printf("===== %s %s ======\n", targetObj.GetKind(), targetObj.GetName())
				if diffRes.Modified {
					formatOpts := formatter.AsciiFormatterConfig{
						Coloring: terminal.IsTerminal(int(os.Stdout.Fd())),
					}
					out, err := diffResults.Diffs[i].ASCIIFormat(targetObj, formatOpts)
					errors.CheckError(err)
					fmt.Println(out)
				}
			}
		},
	}
	return command
}

// NewApplicationDeleteCommand returns a new instance of an `argocd app delete` command
func NewApplicationDeleteCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command {
	var (
		force bool
	)
	var command = &cobra.Command{
		Use:   "delete",
		Short: fmt.Sprintf("%s app delete APPNAME", cliName),
		Run: func(c *cobra.Command, args []string) {
			if len(args) == 0 {
				c.HelpFunc()(c, args)
				os.Exit(1)
			}
			conn, appIf := argocdclient.NewClientOrDie(clientOpts).NewApplicationClientOrDie()
			defer util.Close(conn)
			for _, appName := range args {
				appDeleteReq := application.DeleteApplicationRequest{
					Name:  appName,
					Force: force,
				}
				_, err := appIf.Delete(context.Background(), &appDeleteReq)
				errors.CheckError(err)
			}
		},
	}
	command.Flags().BoolVar(&force, "force", false, "Force delete application even if cascaded deletion unsuccessful")
	return command
}

// NewApplicationListCommand returns a new instance of an `argocd app list` command
func NewApplicationListCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command {
	var command = &cobra.Command{
		Use:   "list",
		Short: fmt.Sprintf("%s app list", cliName),
		Run: func(c *cobra.Command, args []string) {
			conn, appIf := argocdclient.NewClientOrDie(clientOpts).NewApplicationClientOrDie()
			defer util.Close(conn)
			apps, err := appIf.List(context.Background(), &application.ApplicationQuery{})
			errors.CheckError(err)
			w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
			fmt.Fprintf(w, "NAME\tENVIRONMENT\tTARGET\tCLUSTER\tNAMESPACE\tSTATUS\n")
			for _, app := range apps.Items {
				targetRev := app.Spec.Source.TargetRevision
				if targetRev == "" {
					targetRev = "HEAD"
				}
				fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n",
					app.Name,
					app.Spec.Source.Environment,
					targetRev,
					app.Status.ComparisonResult.Server,
					app.Status.ComparisonResult.Namespace,
					app.Status.ComparisonResult.Status,
				)
			}
			_ = w.Flush()
		},
	}
	return command
}

// NewApplicationSyncCommand returns a new instance of an `argocd app sync` command
func NewApplicationSyncCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command {
	var (
		dryRun bool
	)
	var command = &cobra.Command{
		Use:   "sync",
		Short: fmt.Sprintf("%s app sync APPNAME", cliName),
		Run: func(c *cobra.Command, args []string) {
			if len(args) == 0 {
				c.HelpFunc()(c, args)
				os.Exit(1)
			}
			conn, appIf := argocdclient.NewClientOrDie(clientOpts).NewApplicationClientOrDie()
			defer util.Close(conn)
			appName := args[0]
			syncReq := application.ApplicationSyncRequest{
				Name:   appName,
				DryRun: dryRun,
			}
			syncRes, err := appIf.Sync(context.Background(), &syncReq)
			errors.CheckError(err)
			fmt.Printf("%s %s\n", appName, syncRes.Message)
			w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
			fmt.Fprintf(w, "NAME\tKIND\tMESSAGE\n")
			for _, resDetails := range syncRes.Resources {
				fmt.Fprintf(w, "%s\t%s\t%s\n", resDetails.Name, resDetails.Kind, resDetails.Message)
			}
			_ = w.Flush()
		},
	}
	command.Flags().BoolVar(&dryRun, "dry-run", false, "Preview apply without affecting cluster")
	return command
}
<|endoftext|>