<commit_before>// Application that does metrics analysis as described in the design doc:
// go/ct_metrics_analysis
//
// Can be tested locally with:
// $ go run go/worker_scripts/metrics_analysis/main.go --start_range=1 --num=3 --run_id=rmistry-test1 --benchmark_extra_args="--output-format=csv" --metric_name="loadingMetric" --logtostderr=true --local --chromium_hash=cab48032c45695c73b202af7d472a52808d551ca
//
package main

import (
	"bytes"
	"context"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"go.skia.org/infra/go/sklog"

	"go.skia.org/infra/ct/go/util"
	"go.skia.org/infra/ct/go/worker_scripts/worker_common"
	"go.skia.org/infra/go/common"
	skutil "go.skia.org/infra/go/util"
)

const (
	// The number of goroutines that will run in parallel to download traces and
	// run metrics analysis.
	WORKER_POOL_SIZE = 10

	METRICS_BENCHMARK_TIMEOUT_SECS = 300

	TRACE_OUTPUT_BUCKET = "chrome-telemetry-output"
)

var (
	startRange         = flag.Int("start_range", 1, "The number this worker will run metrics analysis from.")
	num                = flag.Int("num", 100, "The total number of traces to run metrics analysis from the start_range.")
	runID              = flag.String("run_id", "", "The unique run id (typically requester + timestamp).")
	benchmarkExtraArgs = flag.String("benchmark_extra_args", "", "The extra arguments that are passed to the specified benchmark.")
	metricName         = flag.String("metric_name", "", "The metric to parse the traces with. Eg: loadingMetric")
	chromiumHash       = flag.String("chromium_hash", "", "The chromium commit hash to sync the local checkout to. Done so that all slaves operate on the same revision.")
)

func metricsAnalysis() error {
	defer common.LogPanic()
	worker_common.Init()
	if !*worker_common.Local {
		defer util.CleanTmpDir()
	}
	defer util.TimeTrack(time.Now(), "Metrics Analysis")
	defer sklog.Flush()

	// Validate required arguments.
	if *runID == "" {
		return errors.New("Must specify --run_id")
	}
	if *metricName == "" {
		return errors.New("Must specify --metric_name")
	}
	if *chromiumHash == "" {
		return errors.New("Must specify --chromium_hash")
	}

	ctx := context.Background()

	// Reset the local chromium checkout.
	if err := util.ResetChromiumCheckout(ctx, util.ChromiumSrcDir); err != nil {
		return fmt.Errorf("Could not reset %s: %s", util.ChromiumSrcDir, err)
	}
	// Sync the local chromium checkout.
	if err := util.SyncDir(ctx, util.ChromiumSrcDir, map[string]string{"src": *chromiumHash}, []string{}); err != nil {
		return fmt.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err)
	}

	// Instantiate GcsUtil object.
	gs, err := util.NewGcsUtil(nil)
	if err != nil {
		return fmt.Errorf("GcsUtil instantiation failed: %s", err)
	}

	// Download the trace URLs for this run from Google storage.
	tracesFilename := *runID + ".traces.csv"
	tmpDir, err := ioutil.TempDir(util.PagesetsDir, "traces")
	if err != nil {
		return fmt.Errorf("Could not create tmpdir: %s", err)
	}
	defer skutil.RemoveAll(tmpDir)
	remotePatchesDir := filepath.Join(util.BenchmarkRunsDir, *runID)

	// Download the catapult patch for this run from Google storage.
	catapultPatchName := *runID + ".catapult.patch"
	if err := util.DownloadAndApplyPatch(ctx, catapultPatchName, tmpDir, remotePatchesDir, util.CatapultSrcDir, gs); err != nil {
		return fmt.Errorf("Could not apply %s: %s", catapultPatchName, err)
	}

	// Download the chromium patch for this run from Google storage.
	chromiumPatchName := *runID + ".chromium.patch"
	if err := util.DownloadAndApplyPatch(ctx, chromiumPatchName, tmpDir, remotePatchesDir, util.ChromiumSrcDir, gs); err != nil {
		return fmt.Errorf("Could not apply %s: %s", chromiumPatchName, err)
	}

	// Download traces.
	if _, err := util.DownloadPatch(filepath.Join(tmpDir, tracesFilename), filepath.Join(remotePatchesDir, tracesFilename), gs); err != nil {
		return fmt.Errorf("Could not download %s: %s", tracesFilename, err)
	}
	traces, err := util.GetCustomPages(filepath.Join(tmpDir, tracesFilename))
	if err != nil {
		return fmt.Errorf("Could not read custom traces file %s: %s", tracesFilename, err)
	}
	if len(traces) == 0 {
		return errors.New(fmt.Sprintf("No traces found in %s", tracesFilename))
	}
	traces = util.GetCustomPagesWithinRange(*startRange, *num, traces)
	sklog.Infof("Using %d traces", len(traces))

	// Establish output paths for pdf downloads and metrics.
	traceDownloadDir := filepath.Join(util.StorageDir, util.TraceDownloadsDir, *runID)
	skutil.RemoveAll(traceDownloadDir)
	skutil.MkdirAll(traceDownloadDir, 0700)
	defer skutil.RemoveAll(traceDownloadDir)

	localOutputDir := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, *runID)
	skutil.RemoveAll(localOutputDir)
	skutil.MkdirAll(localOutputDir, 0700)
	defer skutil.RemoveAll(localOutputDir)
	remoteDir := filepath.Join(util.BenchmarkRunsDir, *runID)

	sklog.Infof("===== Going to run the task with %d parallel goroutines =====", WORKER_POOL_SIZE)
	// Create channel that contains all trace ULs. This channel will
	// be consumed by the worker pool.
	traceRequests := getClosedChannelOfTraces(traces)

	var wg sync.WaitGroup

	// If not a single benchmark run succeeds then throw at error at the end.
	atleastOneBenchmarkSucceeded := false
	// Gather traceURLs that could not be downloaded.
	erroredTraces := []string{}
	// Mutex to control access to the above slice.
	var erroredTracesMutex sync.Mutex

	// Loop through workers in the worker pool.
	for i := 0; i < WORKER_POOL_SIZE; i++ {
		// Increment the WaitGroup counter.
		wg.Add(1)

		// Create and run a goroutine closure that runs the analysis benchmark.
		go func() {
			// Decrement the WaitGroup counter when the goroutine completes.
			defer wg.Done()

			for t := range traceRequests {
				sklog.Infof("========== Downloading trace %s ==========", t)
				downloadedTrace, err := downloadTrace(t, traceDownloadDir, gs)
				if err != nil {
					sklog.Errorf("Could not download %s: %s", t, err)
					erroredTracesMutex.Lock()
					erroredTraces = append(erroredTraces, t)
					erroredTracesMutex.Unlock()
					continue
				}

				sklog.Infof("========== Processing %s ==========", t)
				if err := runMetricsAnalysisBenchmark(ctx, localOutputDir, downloadedTrace, t); err != nil {
					sklog.Errorf("Error during run_benchmark: %s", err)
				} else {
					atleastOneBenchmarkSucceeded = true
				}
			}
		}()
	}

	// Wait for all spawned goroutines to complete.
	wg.Wait()

	// Summarize errors.
	if len(erroredTraces) > 0 {
		sklog.Error("The following traces could not be downloaded:")
		for _, erroredTrace := range erroredTraces {
			sklog.Errorf("\t%s", erroredTrace)
		}
	}
	if !atleastOneBenchmarkSucceeded {
		return errors.New("Not a single benchmark run was successful. Something is wrong.")
	}

	// If "--output-format=csv" was specified then merge all CSV files and upload.
	if strings.Contains(*benchmarkExtraArgs, "--output-format=csv") {
		// Construct path to CT's python scripts.
		pathToPyFiles := util.GetPathToPyFiles(!*worker_common.Local)
		if err := util.MergeUploadCSVFilesOnWorkers(ctx, localOutputDir, pathToPyFiles, *runID, remoteDir, gs, *startRange, true /* handleStrings */, false /* addRanks */, map[string]map[string]string{} /* pageRankToAdditionalFields */); err != nil {
			return fmt.Errorf("Error while processing withpatch CSV files: %s", err)
		}
	}

	return nil
}

// runMetricsAnalysisBenchmark runs the analysis_metrics_ct benchmark on the provided trace.
func runMetricsAnalysisBenchmark(ctx context.Context, outputPath, downloadedTrace, cloudTraceLink string) error {
	args := []string{
		filepath.Join(util.TelemetryBinariesDir, util.BINARY_RUN_BENCHMARK),
		util.BENCHMARK_METRICS_ANALYSIS,
		"--local-trace-path", fmt.Sprintf("file://%s", downloadedTrace),
		"--cloud-trace-link", cloudTraceLink,
		"--metric-name", *metricName,
		"--browser", "system", // No browser is brought up but unfortunately this is needed by the framework.
		"--output-dir", filepath.Join(outputPath, getTraceName(downloadedTrace)),
	}
	// Split benchmark args if not empty and append to args.
	if *benchmarkExtraArgs != "" {
		args = append(args, strings.Fields(*benchmarkExtraArgs)...)
	}
	// Set the PYTHONPATH to the telemetry dirs.
	env := []string{
		fmt.Sprintf("PYTHONPATH=%s:%s:%s:$PYTHONPATH", util.TelemetryBinariesDir, util.TelemetrySrcDir, util.CatapultSrcDir),
		"DISPLAY=:0",
	}
	// Append the original environment as well.
	for _, e := range os.Environ() {
		env = append(env, e)
	}

	// Create buffer for capturing the stdout and stderr of the benchmark run.
	var b bytes.Buffer
	if _, err := b.WriteString(fmt.Sprintf("========== Stdout and stderr for %s ==========\n", downloadedTrace)); err != nil {
		return fmt.Errorf("Error writing to output buffer: %s", err)
	}
	if err := util.ExecuteCmdWithConfigurableLogging(ctx, "python", args, env, METRICS_BENCHMARK_TIMEOUT_SECS*time.Second, &b, &b, false, false); err != nil {
		output, getErr := util.GetRunBenchmarkOutput(b)
		skutil.LogErr(getErr)
		fmt.Println(output)
		return fmt.Errorf("Run benchmark command failed with: %s", err)
	}
	output, err := util.GetRunBenchmarkOutput(b)
	if err != nil {
		return fmt.Errorf("Could not get run benchmark output: %s", err)
	}
	// Print the output and return.
	fmt.Println(output)
	return nil
}

// downloadTrace downloads the traceURL from google storage into the specified
// destDir.
func downloadTrace(traceURL, destDir string, gs *util.GcsUtil) (string, error) {
	traceName := getTraceName(traceURL)
	traceDest := filepath.Join(destDir, traceName)
	if err := gs.DownloadRemoteFileFromBucket(TRACE_OUTPUT_BUCKET, traceName, traceDest); err != nil {
		return "", fmt.Errorf("Error downloading %s from %s to %s: %s", traceName, TRACE_OUTPUT_BUCKET, traceDest, err)
	}
	return traceDest, nil
}

// getTraceName parses the provided traceURI and returns the name of the trace.
// traceURI could be a file path or a URL.
func getTraceName(traceURI string) string {
	traceTokens := strings.Split(traceURI, "/")
	return traceTokens[len(traceTokens)-1]
}

// getClosedChannelOfTraces returns a channel that contains all trace URLs.
func getClosedChannelOfTraces(traces []string) chan string {
	tracesChannel := make(chan string, len(traces))
	for _, t := range traces {
		tracesChannel <- t
	}
	close(tracesChannel)
	return tracesChannel
}

func main() {
	retCode := 0
	if err := metricsAnalysis(); err != nil {
		sklog.Errorf("Error while running metrics analysis: %s", err)
		retCode = 255
	}
	os.Exit(retCode)
}
<commit_msg>[CT] Experiment by increasing to 20 parallel goroutines in metrics analysis<commit_after>// Application that does metrics analysis as described in the design doc:
// go/ct_metrics_analysis
//
// Can be tested locally with:
// $ go run go/worker_scripts/metrics_analysis/main.go --start_range=1 --num=3 --run_id=rmistry-test1 --benchmark_extra_args="--output-format=csv" --metric_name="loadingMetric" --logtostderr=true --local --chromium_hash=cab48032c45695c73b202af7d472a52808d551ca
//
package main

import (
	"bytes"
	"context"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"go.skia.org/infra/go/sklog"

	"go.skia.org/infra/ct/go/util"
	"go.skia.org/infra/ct/go/worker_scripts/worker_common"
	"go.skia.org/infra/go/common"
	skutil "go.skia.org/infra/go/util"
)

const (
	// The number of goroutines that will run in parallel to download traces and
	// run metrics analysis.
	WORKER_POOL_SIZE = 20

	METRICS_BENCHMARK_TIMEOUT_SECS = 300

	TRACE_OUTPUT_BUCKET = "chrome-telemetry-output"
)

var (
	startRange         = flag.Int("start_range", 1, "The number this worker will run metrics analysis from.")
	num                = flag.Int("num", 100, "The total number of traces to run metrics analysis from the start_range.")
	runID              = flag.String("run_id", "", "The unique run id (typically requester + timestamp).")
	benchmarkExtraArgs = flag.String("benchmark_extra_args", "", "The extra arguments that are passed to the specified benchmark.")
	metricName         = flag.String("metric_name", "", "The metric to parse the traces with. Eg: loadingMetric")
	chromiumHash       = flag.String("chromium_hash", "", "The chromium commit hash to sync the local checkout to. Done so that all slaves operate on the same revision.")
)

func metricsAnalysis() error {
	defer common.LogPanic()
	worker_common.Init()
	if !*worker_common.Local {
		defer util.CleanTmpDir()
	}
	defer util.TimeTrack(time.Now(), "Metrics Analysis")
	defer sklog.Flush()

	// Validate required arguments.
	if *runID == "" {
		return errors.New("Must specify --run_id")
	}
	if *metricName == "" {
		return errors.New("Must specify --metric_name")
	}
	if *chromiumHash == "" {
		return errors.New("Must specify --chromium_hash")
	}

	ctx := context.Background()

	// Reset the local chromium checkout.
	if err := util.ResetChromiumCheckout(ctx, util.ChromiumSrcDir); err != nil {
		return fmt.Errorf("Could not reset %s: %s", util.ChromiumSrcDir, err)
	}
	// Sync the local chromium checkout.
	if err := util.SyncDir(ctx, util.ChromiumSrcDir, map[string]string{"src": *chromiumHash}, []string{}); err != nil {
		return fmt.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err)
	}

	// Instantiate GcsUtil object.
	gs, err := util.NewGcsUtil(nil)
	if err != nil {
		return fmt.Errorf("GcsUtil instantiation failed: %s", err)
	}

	// Download the trace URLs for this run from Google storage.
	tracesFilename := *runID + ".traces.csv"
	tmpDir, err := ioutil.TempDir(util.PagesetsDir, "traces")
	if err != nil {
		return fmt.Errorf("Could not create tmpdir: %s", err)
	}
	defer skutil.RemoveAll(tmpDir)
	remotePatchesDir := filepath.Join(util.BenchmarkRunsDir, *runID)

	// Download the catapult patch for this run from Google storage.
	catapultPatchName := *runID + ".catapult.patch"
	if err := util.DownloadAndApplyPatch(ctx, catapultPatchName, tmpDir, remotePatchesDir, util.CatapultSrcDir, gs); err != nil {
		return fmt.Errorf("Could not apply %s: %s", catapultPatchName, err)
	}

	// Download the chromium patch for this run from Google storage.
	chromiumPatchName := *runID + ".chromium.patch"
	if err := util.DownloadAndApplyPatch(ctx, chromiumPatchName, tmpDir, remotePatchesDir, util.ChromiumSrcDir, gs); err != nil {
		return fmt.Errorf("Could not apply %s: %s", chromiumPatchName, err)
	}

	// Download traces.
	if _, err := util.DownloadPatch(filepath.Join(tmpDir, tracesFilename), filepath.Join(remotePatchesDir, tracesFilename), gs); err != nil {
		return fmt.Errorf("Could not download %s: %s", tracesFilename, err)
	}
	traces, err := util.GetCustomPages(filepath.Join(tmpDir, tracesFilename))
	if err != nil {
		return fmt.Errorf("Could not read custom traces file %s: %s", tracesFilename, err)
	}
	if len(traces) == 0 {
		return errors.New(fmt.Sprintf("No traces found in %s", tracesFilename))
	}
	traces = util.GetCustomPagesWithinRange(*startRange, *num, traces)
	sklog.Infof("Using %d traces", len(traces))

	// Establish output paths for pdf downloads and metrics.
	traceDownloadDir := filepath.Join(util.StorageDir, util.TraceDownloadsDir, *runID)
	skutil.RemoveAll(traceDownloadDir)
	skutil.MkdirAll(traceDownloadDir, 0700)
	defer skutil.RemoveAll(traceDownloadDir)

	localOutputDir := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, *runID)
	skutil.RemoveAll(localOutputDir)
	skutil.MkdirAll(localOutputDir, 0700)
	defer skutil.RemoveAll(localOutputDir)
	remoteDir := filepath.Join(util.BenchmarkRunsDir, *runID)

	sklog.Infof("===== Going to run the task with %d parallel goroutines =====", WORKER_POOL_SIZE)
	// Create channel that contains all trace ULs. This channel will
	// be consumed by the worker pool.
	traceRequests := getClosedChannelOfTraces(traces)

	var wg sync.WaitGroup

	// If not a single benchmark run succeeds then throw at error at the end.
	atleastOneBenchmarkSucceeded := false
	// Gather traceURLs that could not be downloaded.
	erroredTraces := []string{}
	// Mutex to control access to the above slice.
	var erroredTracesMutex sync.Mutex

	// Loop through workers in the worker pool.
	for i := 0; i < WORKER_POOL_SIZE; i++ {
		// Increment the WaitGroup counter.
		wg.Add(1)

		// Create and run a goroutine closure that runs the analysis benchmark.
		go func() {
			// Decrement the WaitGroup counter when the goroutine completes.
			defer wg.Done()

			for t := range traceRequests {
				sklog.Infof("========== Downloading trace %s ==========", t)
				downloadedTrace, err := downloadTrace(t, traceDownloadDir, gs)
				if err != nil {
					sklog.Errorf("Could not download %s: %s", t, err)
					erroredTracesMutex.Lock()
					erroredTraces = append(erroredTraces, t)
					erroredTracesMutex.Unlock()
					continue
				}

				sklog.Infof("========== Processing %s ==========", t)
				if err := runMetricsAnalysisBenchmark(ctx, localOutputDir, downloadedTrace, t); err != nil {
					sklog.Errorf("Error during run_benchmark: %s", err)
				} else {
					atleastOneBenchmarkSucceeded = true
				}
			}
		}()
	}

	// Wait for all spawned goroutines to complete.
	wg.Wait()

	// Summarize errors.
	if len(erroredTraces) > 0 {
		sklog.Error("The following traces could not be downloaded:")
		for _, erroredTrace := range erroredTraces {
			sklog.Errorf("\t%s", erroredTrace)
		}
	}
	if !atleastOneBenchmarkSucceeded {
		return errors.New("Not a single benchmark run was successful. Something is wrong.")
	}

	// If "--output-format=csv" was specified then merge all CSV files and upload.
	if strings.Contains(*benchmarkExtraArgs, "--output-format=csv") {
		// Construct path to CT's python scripts.
		pathToPyFiles := util.GetPathToPyFiles(!*worker_common.Local)
		if err := util.MergeUploadCSVFilesOnWorkers(ctx, localOutputDir, pathToPyFiles, *runID, remoteDir, gs, *startRange, true /* handleStrings */, false /* addRanks */, map[string]map[string]string{} /* pageRankToAdditionalFields */); err != nil {
			return fmt.Errorf("Error while processing withpatch CSV files: %s", err)
		}
	}

	return nil
}

// runMetricsAnalysisBenchmark runs the analysis_metrics_ct benchmark on the provided trace.
func runMetricsAnalysisBenchmark(ctx context.Context, outputPath, downloadedTrace, cloudTraceLink string) error {
	args := []string{
		filepath.Join(util.TelemetryBinariesDir, util.BINARY_RUN_BENCHMARK),
		util.BENCHMARK_METRICS_ANALYSIS,
		"--local-trace-path", fmt.Sprintf("file://%s", downloadedTrace),
		"--cloud-trace-link", cloudTraceLink,
		"--metric-name", *metricName,
		"--browser", "system", // No browser is brought up but unfortunately this is needed by the framework.
		"--output-dir", filepath.Join(outputPath, getTraceName(downloadedTrace)),
	}
	// Split benchmark args if not empty and append to args.
	if *benchmarkExtraArgs != "" {
		args = append(args, strings.Fields(*benchmarkExtraArgs)...)
	}
	// Set the PYTHONPATH to the telemetry dirs.
	env := []string{
		fmt.Sprintf("PYTHONPATH=%s:%s:%s:$PYTHONPATH", util.TelemetryBinariesDir, util.TelemetrySrcDir, util.CatapultSrcDir),
		"DISPLAY=:0",
	}
	// Append the original environment as well.
	for _, e := range os.Environ() {
		env = append(env, e)
	}

	// Create buffer for capturing the stdout and stderr of the benchmark run.
	var b bytes.Buffer
	if _, err := b.WriteString(fmt.Sprintf("========== Stdout and stderr for %s ==========\n", downloadedTrace)); err != nil {
		return fmt.Errorf("Error writing to output buffer: %s", err)
	}
	if err := util.ExecuteCmdWithConfigurableLogging(ctx, "python", args, env, METRICS_BENCHMARK_TIMEOUT_SECS*time.Second, &b, &b, false, false); err != nil {
		output, getErr := util.GetRunBenchmarkOutput(b)
		skutil.LogErr(getErr)
		fmt.Println(output)
		return fmt.Errorf("Run benchmark command failed with: %s", err)
	}
	output, err := util.GetRunBenchmarkOutput(b)
	if err != nil {
		return fmt.Errorf("Could not get run benchmark output: %s", err)
	}
	// Print the output and return.
	fmt.Println(output)
	return nil
}

// downloadTrace downloads the traceURL from google storage into the specified
// destDir.
func downloadTrace(traceURL, destDir string, gs *util.GcsUtil) (string, error) {
	traceName := getTraceName(traceURL)
	traceDest := filepath.Join(destDir, traceName)
	if err := gs.DownloadRemoteFileFromBucket(TRACE_OUTPUT_BUCKET, traceName, traceDest); err != nil {
		return "", fmt.Errorf("Error downloading %s from %s to %s: %s", traceName, TRACE_OUTPUT_BUCKET, traceDest, err)
	}
	return traceDest, nil
}

// getTraceName parses the provided traceURI and returns the name of the trace.
// traceURI could be a file path or a URL.
func getTraceName(traceURI string) string {
	traceTokens := strings.Split(traceURI, "/")
	return traceTokens[len(traceTokens)-1]
}

// getClosedChannelOfTraces returns a channel that contains all trace URLs.
func getClosedChannelOfTraces(traces []string) chan string {
	tracesChannel := make(chan string, len(traces))
	for _, t := range traces {
		tracesChannel <- t
	}
	close(tracesChannel)
	return tracesChannel
}

func main() {
	retCode := 0
	if err := metricsAnalysis(); err != nil {
		sklog.Errorf("Error while running metrics analysis: %s", err)
		retCode = 255
	}
	os.Exit(retCode)
}
<|endoftext|>


<commit_before>package slack

import (
	"context"
	"errors"
	"net/url"
	"time"
)

type Reminder struct {
	ID         string    `json:"id"`
	Creator    string    `json:"creator"`
	User       string    `json:"user"`
	Text       string    `json:"text"`
	Recurring  bool      `json:"recurring"`
	Time       time.Time `json:"time"`
	CompleteTS int       `json:"complete_ts"`
}

type reminderResp struct {
	SlackResponse
	Reminder Reminder `json:"reminder"`
}

func (api *Client) doReminder(ctx context.Context, path string, values url.Values) (*Reminder, error) {
	response := &reminderResp{}
	if err := postSlackMethod(ctx, api.httpclient, path, values, response, api); err != nil {
		return nil, err
	}
	return &response.Reminder, response.Err()
}

// AddChannelReminder adds a reminder for a channel.
//
// See https://api.slack.com/methods/reminders.add (NOTE: the ability to set
// reminders on a channel is currently undocumented but has been tested to
// work)
func (api *Client) AddChannelReminder(channelID, text, time string) (*Reminder, error) {
	values := url.Values{
		"token":   {api.token},
		"text":    {text},
		"time":    {time},
		"channel": {channelID},
	}
	return api.doReminder(context.Background(), "reminders.add", values)
}

// AddUserReminder adds a reminder for a user.
//
// See https://api.slack.com/methods/reminders.add (NOTE: the ability to set
// reminders on a channel is currently undocumented but has been tested to
// work)
func (api *Client) AddUserReminder(userID, text, time string) (*Reminder, error) {
	values := url.Values{
		"token": {api.token},
		"text":  {text},
		"time":  {time},
		"user":  {userID},
	}
	return api.doReminder(context.Background(), "reminders.add", values)
}

// DeleteReminder deletes an existing reminder.
//
// See https://api.slack.com/methods/reminders.delete
func (api *Client) DeleteReminder(id string) error {
	values := url.Values{
		"token":    {api.token},
		"reminder": {id},
	}
	response := &SlackResponse{}
	if err := postSlackMethod(context.Background(), api.httpclient, "reminders.delete", values, response, api); err != nil {
		return err
	}
	return response.Err()
}
<commit_msg>remove unused import<commit_after>package slack

import (
	"context"
	"net/url"
	"time"
)

type Reminder struct {
	ID         string    `json:"id"`
	Creator    string    `json:"creator"`
	User       string    `json:"user"`
	Text       string    `json:"text"`
	Recurring  bool      `json:"recurring"`
	Time       time.Time `json:"time"`
	CompleteTS int       `json:"complete_ts"`
}

type reminderResp struct {
	SlackResponse
	Reminder Reminder `json:"reminder"`
}

func (api *Client) doReminder(ctx context.Context, path string, values url.Values) (*Reminder, error) {
	response := &reminderResp{}
	if err := postSlackMethod(ctx, api.httpclient, path, values, response, api); err != nil {
		return nil, err
	}
	return &response.Reminder, response.Err()
}

// AddChannelReminder adds a reminder for a channel.
//
// See https://api.slack.com/methods/reminders.add (NOTE: the ability to set
// reminders on a channel is currently undocumented but has been tested to
// work)
func (api *Client) AddChannelReminder(channelID, text, time string) (*Reminder, error) {
	values := url.Values{
		"token":   {api.token},
		"text":    {text},
		"time":    {time},
		"channel": {channelID},
	}
	return api.doReminder(context.Background(), "reminders.add", values)
}

// AddUserReminder adds a reminder for a user.
//
// See https://api.slack.com/methods/reminders.add (NOTE: the ability to set
// reminders on a channel is currently undocumented but has been tested to
// work)
func (api *Client) AddUserReminder(userID, text, time string) (*Reminder, error) {
	values := url.Values{
		"token": {api.token},
		"text":  {text},
		"time":  {time},
		"user":  {userID},
	}
	return api.doReminder(context.Background(), "reminders.add", values)
}

// DeleteReminder deletes an existing reminder.
//
// See https://api.slack.com/methods/reminders.delete
func (api *Client) DeleteReminder(id string) error {
	values := url.Values{
		"token":    {api.token},
		"reminder": {id},
	}
	response := &SlackResponse{}
	if err := postSlackMethod(context.Background(), api.httpclient, "reminders.delete", values, response, api); err != nil {
		return err
	}
	return response.Err()
}
<|endoftext|>


<commit_before>package walgo

import (
	"bytes"
	"errors"
	"io/ioutil"
	"net/http"
	"time"
)

var (
	defaultRequester Requester
)

func init() {
	defaultRequester = NewRequester(http.DefaultClient, "walgo", "")
}

type Requester interface {
	Get(url string, p ParameterMap) (res Response, err error)
	Post(url string, p ParameterMap, l Payload) (r Response, err error)
	Put(url string, p ParameterMap, l Payload) (r Response, err error)
	Delete(url string, p ParameterMap) (r Response, err error)

	makeRequest(url string, p ParameterMap, method string, l Payload) (r Response, err error)
}

type requesterImpl struct {
	client    *http.Client
	userAgent string
	authToken string
}

func NewRequester(c *http.Client, userAgent, authToken string) (r Requester) {
	return &requesterImpl{
		client:    c,
		userAgent: userAgent,
		authToken: authToken,
	}
}

func (f *requesterImpl) makeRequest(url string, p ParameterMap, method string, l Payload) (r Response, err error) {
	u, err := createParameterUrl(url, p)
	if err != nil {
		return nil, err
	}

	startTime := time.Now()
	code := -1
	var output []byte

	buffer := &bytes.Buffer{}

	if l != nil {
		data := l.getData()
		c, err := buffer.Write(data)
		if c != len(data) || err != nil {
			return nil, errors.New("Error creating data buffer.")
		}
	}

	req, err := http.NewRequest(method, u.String(), buffer)
	if err != nil {
		return nil, err
	}

	req.Header.Add("User-Agent", f.userAgent)
	if "" != f.authToken {
		req.Header.Add("Authorizastion", "Bearer "+f.authToken)
	}

	resp, err := f.client.Do(req)
	if err != nil {
		return nil, err
	}
	if resp != nil && resp.Body != nil {
		code = resp.StatusCode
		output, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, err
		}
	}

	duration := time.Now().Sub(startTime)

	r = responseImpl{
		data:     output,
		code:     code,
		duration: duration,
	}

	return r, err
}
<commit_msg>Added content type header.<commit_after>package walgo

import (
	"bytes"
	"errors"
	"io/ioutil"
	"net/http"
	"time"
)

var (
	defaultRequester Requester
)

func init() {
	defaultRequester = NewRequester(http.DefaultClient, "walgo", "")
}

type Requester interface {
	Get(url string, p ParameterMap) (res Response, err error)
	Post(url string, p ParameterMap, l Payload) (r Response, err error)
	Put(url string, p ParameterMap, l Payload) (r Response, err error)
	Delete(url string, p ParameterMap) (r Response, err error)

	makeRequest(url string, p ParameterMap, method string, l Payload) (r Response, err error)
}

type requesterImpl struct {
	client    *http.Client
	userAgent string
	authToken string
}

func NewRequester(c *http.Client, userAgent, authToken string) (r Requester) {
	return &requesterImpl{
		client:    c,
		userAgent: userAgent,
		authToken: authToken,
	}
}

func (f *requesterImpl) makeRequest(url string, p ParameterMap, method string, l Payload) (r Response, err error) {
	u, err := createParameterUrl(url, p)
	if err != nil {
		return nil, err
	}

	startTime := time.Now()
	code := -1
	var output []byte

	buffer := &bytes.Buffer{}

	if l != nil {
		data := l.getData()
		c, err := buffer.Write(data)
		if c != len(data) || err != nil {
			return nil, errors.New("Error creating data buffer.")
		}
	}

	req, err := http.NewRequest(method, u.String(), buffer)
	if err != nil {
		return nil, err
	}

	if l != nil {
		req.Header.Add("Content-Type", l.getContentType())
	}

	req.Header.Add("User-Agent", f.userAgent)
	if "" != f.authToken {
		req.Header.Add("Authorizastion", "Bearer "+f.authToken)
	}

	resp, err := f.client.Do(req)
	if err != nil {
		return nil, err
	}
	if resp != nil && resp.Body != nil {
		code = resp.StatusCode
		output, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, err
		}
	}

	duration := time.Now().Sub(startTime)

	r = responseImpl{
		data:     output,
		code:     code,
		duration: duration,
	}

	return r, err
}
<|endoftext|>


<commit_before>package probstruct

import (
	"math/rand"
)

// Reservoir implements simple reservoir filter
type Reservoir struct {
	k      int
	total  uint64
	sample []interface{}
}

// InitReservoir instantiates new Reservoir struct
func InitReservoir(k int) (r *Reservoir, err error) {
	r = &Reservoir{
		k:      k,
		total:  0,
		sample: make([]interface{}, k),
	}
	return r, nil
}

// Add new item to reservoir
func (r *Reservoir) Add(item interface{}) *Reservoir {
	r.total++
	if len(r.sample) < r.k {
		r.sample = append(r.sample, item)
	} else {
		if rand.Float64() < (float64(r.k) / float64(r.total)) {
			r.sample[rand.Intn(r.k)] = item
		}
	}
	return r
}

// GetSample is a helper to return size of sampled data
func (r *Reservoir) GetSample() []interface{} {
	return r.sample
}

// GetK is a helper to return all sampled values
func (r *Reservoir) GetK() int {
	return r.k
}

func (r *Reservoir) randomRange() int {
	return rand.Intn(r.k)
}
<commit_msg>cleanup<commit_after>package probstruct

import (
	"math/rand"
)

// Reservoir implements simple reservoir filter
type Reservoir struct {
	k      int
	total  uint64
	sample []interface{}
}

// InitReservoir instantiates new Reservoir struct
func InitReservoir(k int) (r *Reservoir, err error) {
	r = &Reservoir{
		k:      k,
		total:  0,
		sample: make([]interface{}, k),
	}
	return r, nil
}

// Add new item to reservoir
func (r *Reservoir) Add(item interface{}) *Reservoir {
	r.total++
	if len(r.sample) < r.k {
		r.sample = append(r.sample, item)
	} else {
		if rand.Float64() < (float64(r.k) / float64(r.total)) {
			r.sample[rand.Intn(r.k)] = item
		}
	}
	return r
}

// GetSample is a helper to return size of sampled data
func (r *Reservoir) GetSample() []interface{} {
	return r.sample
}

// GetK is a helper to return all sampled values
func (r *Reservoir) GetK() int {
	return r.k
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package kubernetes

import (
	"encoding/json"
	"fmt"
	"github.com/romana/core/common"
	"github.com/romana/core/tenant"
	"log"
	"net/http"
)

const (
	selector = "podSelector"
)

// Done is an alias for empty struct, used to make broadcast channels
// for terminating goroutines.
type Done struct{}

// Event is a representation of a structure that we receive from kubernetes API.
type Event struct {
	Type   string     `json:"Type"`
	Object KubeObject `json:"object"`
}

const (
	KubeEventAdded         = "ADDED"
	KubeEventDeleted       = "DELETED"
	KubeEventModified      = "MODIFIED"
	InternalEventDeleteAll = "_DELETE_ALL"
)

// KubeObject is a representation of object in kubernetes.
type KubeObject struct {
	Kind       string            `json:"kind"`
	Spec       Spec              `json:"spec"`
	ApiVersion string            `json:"apiVersion"`
	Metadata   Metadata          `json:"metadata"`
	Status     map[string]string `json:"status,omitempty"`
}

// makeId makes id to identify kube object.
func (o KubeObject) makeId() string {
	id := o.Metadata.Name + "/" + o.Metadata.Namespace
	return id
}

type PodSelector struct {
	MatchLabels map[string]string `json:"matchLabels"`
}

type FromEntry struct {
	Pods PodSelector `json:"podSelector"`
}

type Ingress struct {
	From    []FromEntry `json:"from"`
	ToPorts []ToPort    `json:"ports"`
}

type ToPort struct {
	Port     uint   `json:"port"`
	Protocol string `json:"protocol"`
}

// TODO need to find a way to use different specs for different resources.
type Spec struct {
	Ingress     []Ingress   `json:"ingress"`
	PodSelector PodSelector `json:"podSelector"`
}

// Metadata is a representation of metadata in kubernetes object
type Metadata struct {
	Name              string            `json:"name"`
	Namespace         string            `json:"namespace"`
	SelfLink          string            `json:"selfLink"`
	Uid               string            `json:"uid"`
	ResourceVersion   string            `json:"resourceVersion"`
	CreationTimestamp string            `json:"creationTimestamp"`
	Labels            map[string]string `json:"labels"`
	Annotations       map[string]string `json:"annotations,omitempty"`
}

// handle kubernetes events according to their type.
func (e Event) handle(l *kubeListener) {
	log.Printf("Processing %s request for %s", e.Type, e.Object.Metadata.Name)

	if e.Object.Kind == "NetworkPolicy" && e.Type != KubeEventModified {
		e.handleNetworkPolicyEvent(l)
	} else if e.Object.Kind == "Namespace" {
		e.handleNamespaceEvent(l)
	} else {
		log.Printf("Received unindentified request %s for %s", e.Type, e.Object.Metadata.Name)
	}
}

// handleNetworkPolicyEvent by creating or deleting romana policies.
func (e Event) handleNetworkPolicyEvent(l *kubeListener) {
	var action networkPolicyAction
	if e.Type == KubeEventAdded {
		action = networkPolicyActionAdd
	} else {
		action = networkPolicyActionDelete
	}
	policy, err := l.translateNetworkPolicy(&e.Object)
	if err == nil {
		j1, _ := json.Marshal(e)
		j2, _ := json.Marshal(policy)
		log.Printf("handleNetworkPolicyEvent(): translated\n\t%s\n\tto\n\t%s", j1, j2)
		l.applyNetworkPolicy(action, policy)
	} else {
		log.Println(err)
	}
}

// handleNamespaceEvent by creating or deleting romana tenants.
func (e Event) handleNamespaceEvent(l *kubeListener) {
	log.Printf("KubeEvent: Processing namespace event == %v and phase %v", e.Type, e.Object.Status)

	if e.Type == KubeEventAdded {
		tenantReq := tenant.Tenant{Name: e.Object.Metadata.Name, ExternalID: e.Object.Metadata.Uid}
		tenantResp := tenant.Tenant{}
		log.Printf("KubeEventAdded: Posting to /tenants: %+v", tenantReq)
		tenantUrl, err := l.restClient.GetServiceUrl("tenant")
		if err != nil {
			log.Printf("KubeEventAdded:Error adding tenant %s: %+v", tenantReq.Name, err)
		} else {
			err := l.restClient.Post(fmt.Sprintf("%s/tenants", tenantUrl), tenantReq, &tenantResp)
			if err != nil {
				log.Printf("KubeEventAdded: Error adding tenant %s: %+v", tenantReq.Name, err)
			} else {
				log.Printf("KubeEventAdded: Added tenant: %+v", tenantResp)
			}
		}
	} else if e.Type == KubeEventDeleted {
		// TODO
	}

	// Ignore repeated events during namespace termination
	if e.Object.Status["phase"] == "Terminating" {
		if e.Type != KubeEventModified {
			e.Object.handleAnnotations(l)
		}
	} else {
		e.Object.handleAnnotations(l)
	}

}

// handleAnnotations on a namespace by implementing extra features requested through the annotation
func (o KubeObject) handleAnnotations(l *kubeListener) {
	log.Printf("In handleAnnotations")

	if o.Kind != "Namespace" {
		log.Printf("Error handling annotations on a namespace - object is not a namespace %s \n", o.Kind)
		return
	}

	CreateDefaultPolicy(o, l)
}

func CreateDefaultPolicy(o KubeObject, l *kubeListener) {
	log.Printf("In CreateDefaultPolicy for %v\n", o)
	tenant, err := l.resolveTenantByName(o.Metadata.Name)
	if err != nil {
		log.Printf("In CreateDefaultPolicy :: Error :: failed to resolve tenant %s \n", err)
		return
	}

	policyName := fmt.Sprintf("ns%d", tenant.Seq)

	romanaPolicy := &common.Policy{
		Direction: common.PolicyDirectionIngress,
		Name:      policyName,
		AppliedTo: []common.Endpoint{{TenantNetworkID: &tenant.Seq}},
		Peers:     []common.Endpoint{{Peer: common.Wildcard}},
		Rules:     []common.Rule{{Protocol: common.Wildcard}},
	}

	log.Printf("In CreateDefaultPolicy with policy %v\n", romanaPolicy)

	var desiredAction networkPolicyAction

	if isolation, ok := o.Metadata.Annotations["net.alpha.kubernetes.io/network-isolation"]; ok {
		log.Printf("Handling default policy on a namespace %s, isolation is now %s \n", o.Metadata.Name, isolation)
		switch isolation {
		case "on":
			desiredAction = networkPolicyActionDelete
		case "off":
			desiredAction = networkPolicyActionAdd
		default:
			log.Printf("In CreateDefaultPolicy :: Error :: unrecognised annotation on a namespace %s is %s (expected on|off) \n",
				o.Metadata.Name, isolation)
			return
		}

	} else {
		log.Printf("Handling default policy on a namespace, no annotation detected assuming non isolated namespace\n")
		desiredAction = networkPolicyActionAdd
	}

	if err2 := l.applyNetworkPolicy(desiredAction, *romanaPolicy); err2 != nil {
		log.Printf("In CreateDefaultPolicy :: Error :: failed to apply %v to the policy %s \n", desiredAction, err2)
	}
}

// watchEvents maintains goroutine fired by NsWatch, restarts it in case HTTP GET times out.
func (l *kubeListener) watchEvents(done <-chan Done, url string, resp *http.Response, out chan Event) {
	log.Println("Received namespace related event from kubernetes", resp.Body)

	dec := json.NewDecoder(resp.Body)
	var e Event

	for {
		select {
		case <-done:
			return
		default:
			// Flush e to ensure nothing gets carried over
			e = Event{}

			// Attempting to read event from HTTP connection
			err := dec.Decode(&e)
			if err != nil {
				// If fail
				log.Printf("Failed to decode message from connection %s due to %s\n. Attempting to re-establish", url, err)
				// Then stop all goroutines
				out <- Event{Type: InternalEventDeleteAll}

				// And try to re-establish HTTP connection
				resp, err2 := http.Get(url)
				if err2 != nil {
					log.Printf("Failed establish connection %s due to %s\n.", url, err)
				} else if err2 == nil {
					dec = json.NewDecoder(resp.Body)
				}
			} else {
				// Else submit event
				out <- e
			}
		}

	}
}

// NsWatch is a generator that watches namespace related events in
// kubernetes API and publishes this events to a channel.
func (l *kubeListener) nsWatch(done <-chan Done, url string) (<-chan Event, error) {
	out := make(chan Event)

	resp, err := http.Get(url)
	if err != nil {
		return out, err
	}

	go l.watchEvents(done, url, resp, out)

	return out, nil
}

// Produce method listens for resource updates happening within given namespace
// and publishes these updates in a channel.
func (ns KubeObject) produce(out chan Event, done <-chan Done, kubeListener *kubeListener) error {
	url, err := common.CleanURL(fmt.Sprintf("%s/%s/%s%s", kubeListener.kubeURL, kubeListener.policyNotificationPathPrefix, ns.Metadata.Name, kubeListener.policyNotificationPathPostfix))
	if err != nil {
		return err
	}
	log.Printf("Launching producer to listen for policy notifications on namespace %s at URL %s ", ns.Metadata.Name, url)
	resp, err := http.Get(url)
	if err != nil {
		return err
	}

	go kubeListener.watchEvents(done, url, resp, out)

	return nil
}
<commit_msg>Extract policy and check for DefaultDeny.<commit_after>// Copyright (c) 2016 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package kubernetes

import (
	"encoding/json"
	"fmt"
	"github.com/romana/core/common"
	"github.com/romana/core/tenant"
	"log"
	"net/http"
	"strings"
)

const (
	selector = "podSelector"
)

// Done is an alias for empty struct, used to make broadcast channels
// for terminating goroutines.
type Done struct{}

// Event is a representation of a structure that we receive from kubernetes API.
type Event struct {
	Type   string     `json:"Type"`
	Object KubeObject `json:"object"`
}

const (
	KubeEventAdded         = "ADDED"
	KubeEventDeleted       = "DELETED"
	KubeEventModified      = "MODIFIED"
	InternalEventDeleteAll = "_DELETE_ALL"
)

// KubeObject is a representation of object in kubernetes.
type KubeObject struct {
	Kind       string            `json:"kind"`
	Spec       Spec              `json:"spec"`
	ApiVersion string            `json:"apiVersion"`
	Metadata   Metadata          `json:"metadata"`
	Status     map[string]string `json:"status,omitempty"`
}

// makeId makes id to identify kube object.
func (o KubeObject) makeId() string {
	id := o.Metadata.Name + "/" + o.Metadata.Namespace
	return id
}

type PodSelector struct {
	MatchLabels map[string]string `json:"matchLabels"`
}

type FromEntry struct {
	Pods PodSelector `json:"podSelector"`
}

type Ingress struct {
	From    []FromEntry `json:"from"`
	ToPorts []ToPort    `json:"ports"`
}

type ToPort struct {
	Port     uint   `json:"port"`
	Protocol string `json:"protocol"`
}

// TODO need to find a way to use different specs for different resources.
type Spec struct {
	Ingress     []Ingress   `json:"ingress"`
	PodSelector PodSelector `json:"podSelector"`
}

// Metadata is a representation of metadata in kubernetes object
type Metadata struct {
	Name              string            `json:"name"`
	Namespace         string            `json:"namespace"`
	SelfLink          string            `json:"selfLink"`
	Uid               string            `json:"uid"`
	ResourceVersion   string            `json:"resourceVersion"`
	CreationTimestamp string            `json:"creationTimestamp"`
	Labels            map[string]string `json:"labels"`
	Annotations       map[string]string `json:"annotations,omitempty"`
}

// handle kubernetes events according to their type.
func (e Event) handle(l *kubeListener) {
	log.Printf("Processing %s request for %s", e.Type, e.Object.Metadata.Name)

	if e.Object.Kind == "NetworkPolicy" && e.Type != KubeEventModified {
		e.handleNetworkPolicyEvent(l)
	} else if e.Object.Kind == "Namespace" {
		e.handleNamespaceEvent(l)
	} else {
		log.Printf("Received unindentified request %s for %s", e.Type, e.Object.Metadata.Name)
	}
}

// handleNetworkPolicyEvent by creating or deleting romana policies.
func (e Event) handleNetworkPolicyEvent(l *kubeListener) {
	var action networkPolicyAction
	if e.Type == KubeEventAdded {
		action = networkPolicyActionAdd
	} else {
		action = networkPolicyActionDelete
	}
	policy, err := l.translateNetworkPolicy(&e.Object)
	if err == nil {
		j1, _ := json.Marshal(e)
		j2, _ := json.Marshal(policy)
		log.Printf("handleNetworkPolicyEvent(): translated\n\t%s\n\tto\n\t%s", j1, j2)
		l.applyNetworkPolicy(action, policy)
	} else {
		log.Println(err)
	}
}

// handleNamespaceEvent by creating or deleting romana tenants.
func (e Event) handleNamespaceEvent(l *kubeListener) {
	log.Printf("KubeEvent: Processing namespace event == %v and phase %v", e.Type, e.Object.Status)

	if e.Type == KubeEventAdded {
		tenantReq := tenant.Tenant{Name: e.Object.Metadata.Name, ExternalID: e.Object.Metadata.Uid}
		tenantResp := tenant.Tenant{}
		log.Printf("KubeEventAdded: Posting to /tenants: %+v", tenantReq)
		tenantUrl, err := l.restClient.GetServiceUrl("tenant")
		if err != nil {
			log.Printf("KubeEventAdded:Error adding tenant %s: %+v", tenantReq.Name, err)
		} else {
			err := l.restClient.Post(fmt.Sprintf("%s/tenants", tenantUrl), tenantReq, &tenantResp)
			if err != nil {
				log.Printf("KubeEventAdded: Error adding tenant %s: %+v", tenantReq.Name, err)
			} else {
				log.Printf("KubeEventAdded: Added tenant: %+v", tenantResp)
			}
		}
	} else if e.Type == KubeEventDeleted {
		// TODO
	}

	// Ignore repeated events during namespace termination
	if e.Object.Status["phase"] == "Terminating" {
		if e.Type != KubeEventModified {
			e.Object.handleAnnotations(l)
		}
	} else {
		e.Object.handleAnnotations(l)
	}

}

// handleAnnotations on a namespace by implementing extra features requested through the annotation
func (o KubeObject) handleAnnotations(l *kubeListener) {
	log.Printf("In handleAnnotations")

	if o.Kind != "Namespace" {
		log.Printf("Error handling annotations on a namespace - object is not a namespace %s \n", o.Kind)
		return
	}

	CreateDefaultPolicy(o, l)
}

func CreateDefaultPolicy(o KubeObject, l *kubeListener) {
	log.Printf("In CreateDefaultPolicy for %v\n", o)
	tenant, err := l.resolveTenantByName(o.Metadata.Name)
	if err != nil {
		log.Printf("In CreateDefaultPolicy :: Error :: failed to resolve tenant %s \n", err)
		return
	}

	policyName := fmt.Sprintf("ns%d", tenant.Seq)

	romanaPolicy := &common.Policy{
		Direction: common.PolicyDirectionIngress,
		Name:      policyName,
		AppliedTo: []common.Endpoint{{TenantNetworkID: &tenant.Seq}},
		Peers:     []common.Endpoint{{Peer: common.Wildcard}},
		Rules:     []common.Rule{{Protocol: common.Wildcard}},
	}

	log.Printf("In CreateDefaultPolicy with policy %v\n", romanaPolicy)

	var desiredAction networkPolicyAction

	if np, ok := o.Metadata.Annotations["net.beta.kubernetes.io/networkpolicy"]; ok {
		log.Printf("Handling default policy on a namespace %s, policy is now %s \n", o.Metadata.Name, np)
		policy := struct {
			Ingress struct {
				Isolation string `json:"isolation"`
			} `json:"ingress"`
		}{}
		err := json.NewDecoder(strings.NewReader(np)).Decode(&policy)
		if err != nil {
			log.Printf("In CreateDefaultPolicy :: Error decoding network policy: %s", err)
			return
		}

		log.Println("Decoded to policy:", policy)
		if policy.Ingress.Isolation == "DefaultDeny" {
			log.Println("Isolation enabled")
			desiredAction = networkPolicyActionDelete
		} else {
			desiredAction = networkPolicyActionAdd
		}
	} else {
		log.Printf("Handling default policy on a namespace, no annotation detected assuming non isolated namespace\n")
		desiredAction = networkPolicyActionAdd
	}

	if err2 := l.applyNetworkPolicy(desiredAction, *romanaPolicy); err2 != nil {
		log.Printf("In CreateDefaultPolicy :: Error :: failed to apply %v to the policy %s \n", desiredAction, err2)
	}
}

// watchEvents maintains goroutine fired by NsWatch, restarts it in case HTTP GET times out.
func (l *kubeListener) watchEvents(done <-chan Done, url string, resp *http.Response, out chan Event) {
	log.Println("Received namespace related event from kubernetes", resp.Body)

	dec := json.NewDecoder(resp.Body)
	var e Event

	for {
		select {
		case <-done:
			return
		default:
			// Flush e to ensure nothing gets carried over
			e = Event{}

			// Attempting to read event from HTTP connection
			err := dec.Decode(&e)
			if err != nil {
				// If fail
				log.Printf("Failed to decode message from connection %s due to %s\n. Attempting to re-establish", url, err)
				// Then stop all goroutines
				out <- Event{Type: InternalEventDeleteAll}

				// And try to re-establish HTTP connection
				resp, err2 := http.Get(url)
				if err2 != nil {
					log.Printf("Failed establish connection %s due to %s\n.", url, err)
				} else if err2 == nil {
					dec = json.NewDecoder(resp.Body)
				}
			} else {
				// Else submit event
				out <- e
			}
		}

	}
}

// NsWatch is a generator that watches namespace related events in
// kubernetes API and publishes this events to a channel.
func (l *kubeListener) nsWatch(done <-chan Done, url string) (<-chan Event, error) {
	out := make(chan Event)

	resp, err := http.Get(url)
	if err != nil {
		return out, err
	}

	go l.watchEvents(done, url, resp, out)

	return out, nil
}

// Produce method listens for resource updates happening within given namespace
// and publishes these updates in a channel.
func (ns KubeObject) produce(out chan Event, done <-chan Done, kubeListener *kubeListener) error {
	url, err := common.CleanURL(fmt.Sprintf("%s/%s/%s%s", kubeListener.kubeURL, kubeListener.policyNotificationPathPrefix, ns.Metadata.Name, kubeListener.policyNotificationPathPostfix))
	if err != nil {
		return err
	}
	log.Printf("Launching producer to listen for policy notifications on namespace %s at URL %s ", ns.Metadata.Name, url)
	resp, err := http.Get(url)
	if err != nil {
		return err
	}

	go kubeListener.watchEvents(done, url, resp, out)

	return nil
}
<|endoftext|>


<commit_before>// Package resources provides unfancy resources embedding with Go.
package resources

import (
	"bytes"
	"fmt"
	"go/format"
	"io"
	"log"
	"os"
	"strings"
	"text/template"
)

// File mimicks the os.File and http.File interface.
type File interface {
	io.Reader
	Stat() (os.FileInfo, error)
}

// New creates a new Package.
func New() *Package {
	return &Package{
		Config: Config{
			Pkg:     "resources",
			Var:     "FS",
			Declare: true,
		},
		Files: make(map[string]File),
	}
}

// Config defines some details about the output file
type Config struct {
	Pkg     string // Pkg holds the package name
	Var     string // Var holds the variable name for the virtual filesystem
	Tag     string // Tag may hold an optional build tag, unless empty
	Declare bool   // Declare controls if the Var should be declared as well
	Format  bool   // Format controls, whether gofmt should be applied to the output
}

// A Package describes a collection of files and how they should be tranformed
// to an output.
type Package struct {
	Config
	Files map[string]File
}

// Add a file to the package at the give path.
func (p *Package) Add(path string, file File) error {
	p.Files[path] = file
	return nil
}

// AddFile is a helper function that adds the files from the path into the
// package under the path file.
func (p *Package) AddFile(path string, file string) error {
	f, err := os.Open(file)
	if err != nil {
		return err
	}
	return p.Add(path, f)
}

// Build compiles the package and writes it into an io.Writer.
func (p *Package) Build(out io.Writer) error {
	return pkg.Execute(out, p)
}

// Write builds the package (via Build) and writes the output the the file
// given by the path argument.
func (p *Package) Write(path string) error {
	f, err := os.Create(path)
	if err != nil {
		return err
	}
	defer func() {
		err := f.Close()
		if err != nil {
			log.Panicf("Failed to close file: %s", err)
		}
	}()

	if !p.Format {
		return p.Build(f)
	}

	buf := &bytes.Buffer{}
	if e := p.Build(buf); e != nil {
		return e
	}

	fmted, e := format.Source(buf.Bytes())
	if e != nil {
		return e
	}
	_, e = f.Write(fmted)
	return e
}

var (
	// Template
	pkg *template.Template

	// BlockWidth allows to adjust the number of bytes per line in the generated file
	BlockWidth = 12
)

func reader(input io.Reader, indent int) (string, error) {
	var (
		buff      bytes.Buffer
		err       error
		curblock  = 0
		linebreak = "\n" + strings.Repeat("\t", indent)
	)

	b := make([]byte, BlockWidth)

	for n, e := input.Read(b); e == nil; n, e = input.Read(b) {
		for i := 0; i < n; i++ {
			_, e = fmt.Fprintf(&buff, "0x%02x,", b[i])
			if e != nil {
				err = e
				break
			}
			curblock++
			if curblock < BlockWidth {
				buff.WriteRune(' ')
				continue
			}
			buff.WriteString(linebreak)
			curblock = 0
		}
	}

	return buff.String(), err
}

func init() {
	pkg = template.Must(template.New("file").Funcs(template.FuncMap{"reader": reader}).Parse(`File{
				data: []byte{
					{{ reader . 5 }}
				},
				fi: FileInfo{
					name:    "{{ .Stat.Name }}",
					size:    {{ .Stat.Size }},
					modTime: time.Unix(0, {{ .Stat.ModTime.UnixNano }}),
					isDir:   {{ .Stat.IsDir }},
				},
			}`))

	pkg = template.Must(pkg.New("pkg").Parse(`{{ if .Tag }}// +build {{ .Tag }}

{{ end }}// Package {{.Pkg }} is generated by github.com/omeid/go-resources
package {{ .Pkg }}

import (
	"bytes"
	"errors"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"
)

// FileSystem is an http.FileSystem implementation.
type FileSystem struct {
	files map[string]File
}

// String returns the content of the file as string.
func (fs *FileSystem) String(name string) (string, bool) {
	if filepath.Separator != '/' && strings.IndexRune(name, filepath.Separator) >= 0 ||
		strings.Contains(name, "\x00") {
		return "", false
	}

	file, ok := fs.files[name]

	if !ok {
		return "", false
	}

	return string(file.data), true
}

// Open implements http.FileSystem.Open
func (fs *FileSystem) Open(name string) (http.File, error) {
	if filepath.Separator != '/' && strings.IndexRune(name, filepath.Separator) >= 0 ||
		strings.Contains(name, "\x00") {
		return nil, errors.New("http: invalid character in file path")
	}
	file, ok := fs.files[name]
	if !ok {
		files := []os.FileInfo{}
		for path, file := range fs.files {
			if strings.HasPrefix(path, name) {
				fi := file.fi
				files = append(files, &fi)
			}
		}

		if len(files) == 0 {
			return nil, os.ErrNotExist
		}

		//We have a directory.
		return &File{
			fi: FileInfo{
				isDir: true,
				files: files,
			}}, nil
	}
	file.Reader = bytes.NewReader(file.data)
	return &file, nil
}

// File implements http.File
type File struct {
	*bytes.Reader
	data []byte
	fi   FileInfo
}

// Close is a noop-closer.
func (f *File) Close() error {
	return nil
}

// Readdir implements http.File.Readdir
func (f *File) Readdir(count int) ([]os.FileInfo, error) {
	return nil, os.ErrNotExist
}

// Stat implements http.Stat.Readdir
func (f *File) Stat() (os.FileInfo, error) {
	return &f.fi, nil
}

// FileInfo implements the os.FileInfo interface.
type FileInfo struct {
	name    string
	size    int64
	mode    os.FileMode
	modTime time.Time
	isDir   bool
	sys     interface{}

	files []os.FileInfo
}

// Name implements os.FileInfo.Name
func (f *FileInfo) Name() string {
	return f.name
}

// Size implements os.FileInfo.Size
func (f *FileInfo) Size() int64 {
	return f.size
}

// Mode implements os.FileInfo.Mode
func (f *FileInfo) Mode() os.FileMode {
	return f.mode
}

// ModTime implements os.FileInfo.ModTime
func (f *FileInfo) ModTime() time.Time {
	return f.modTime
}

// IsDir implements os.FileInfo.IsDir
func (f *FileInfo) IsDir() bool {
	return f.isDir
}

// Readdir implements os.FileInfo.Readdir
func (f *FileInfo) Readdir(count int) ([]os.FileInfo, error) {
	return f.files, nil
}

// Sys returns the underlying value.
func (f *FileInfo) Sys() interface{} {
	return f.sys
}

{{ if .Declare }}var {{ .Var }} *FileSystem{{ end }}

func init() {
	{{ .Var }} = &FileSystem{
		files: map[string]File{
			{{range $path, $file := .Files }}"/{{ $path }}": {{ template "file" $file }},{{ end }}
		},
	}
}
`))
}
<commit_msg>seperate init code and inline templates<commit_after>// Package resources provides unfancy resources embedding with Go.
package resources

import (
	"bytes"
	"fmt"
	"go/format"
	"io"
	"log"
	"os"
	"strings"
	"text/template"
)

// File mimicks the os.File and http.File interface.
type File interface {
	io.Reader
	Stat() (os.FileInfo, error)
}

// New creates a new Package.
func New() *Package {
	return &Package{
		Config: Config{
			Pkg:     "resources",
			Var:     "FS",
			Declare: true,
		},
		Files: make(map[string]File),
	}
}

// Config defines some details about the output file
type Config struct {
	Pkg     string // Pkg holds the package name
	Var     string // Var holds the variable name for the virtual filesystem
	Tag     string // Tag may hold an optional build tag, unless empty
	Declare bool   // Declare controls if the Var should be declared as well
	Format  bool   // Format controls, whether gofmt should be applied to the output
}

// A Package describes a collection of files and how they should be tranformed
// to an output.
type Package struct {
	Config
	Files map[string]File
}

// Add a file to the package at the give path.
func (p *Package) Add(path string, file File) error {
	p.Files[path] = file
	return nil
}

// AddFile is a helper function that adds the files from the path into the
// package under the path file.
func (p *Package) AddFile(path string, file string) error {
	f, err := os.Open(file)
	if err != nil {
		return err
	}
	return p.Add(path, f)
}

// Build compiles the package and writes it into an io.Writer.
func (p *Package) Build(out io.Writer) error {
	return pkg.Execute(out, p)
}

// Write builds the package (via Build) and writes the output the the file
// given by the path argument.
func (p *Package) Write(path string) error {
	f, err := os.Create(path)
	if err != nil {
		return err
	}
	defer func() {
		err := f.Close()
		if err != nil {
			log.Panicf("Failed to close file: %s", err)
		}
	}()

	if !p.Format {
		return p.Build(f)
	}

	buf := &bytes.Buffer{}
	if e := p.Build(buf); e != nil {
		return e
	}

	fmted, e := format.Source(buf.Bytes())
	if e != nil {
		return e
	}
	_, e = f.Write(fmted)
	return e
}

var (
	// Template
	pkg *template.Template

	// BlockWidth allows to adjust the number of bytes per line in the generated file
	BlockWidth = 12
)

func reader(input io.Reader, indent int) (string, error) {
	var (
		buff      bytes.Buffer
		err       error
		curblock  = 0
		linebreak = "\n" + strings.Repeat("\t", indent)
	)

	b := make([]byte, BlockWidth)

	for n, e := input.Read(b); e == nil; n, e = input.Read(b) {
		for i := 0; i < n; i++ {
			_, e = fmt.Fprintf(&buff, "0x%02x,", b[i])
			if e != nil {
				err = e
				break
			}
			curblock++
			if curblock < BlockWidth {
				buff.WriteRune(' ')
				continue
			}
			buff.WriteString(linebreak)
			curblock = 0
		}
	}

	return buff.String(), err
}

func init() {
	pkg = template.Must(template.New("file").Funcs(template.FuncMap{"reader": reader}).Parse(fileTemplate))
	pkg = template.Must(pkg.New("pkg").Parse(pkgTemplate))
}

const fileTemplate = `File{
				data: []byte{
					{{ reader . 5 }}
				},
				fi: FileInfo{
					name:    "{{ .Stat.Name }}",
					size:    {{ .Stat.Size }},
					modTime: time.Unix(0, {{ .Stat.ModTime.UnixNano }}),
					isDir:   {{ .Stat.IsDir }},
				},
			}`

const pkgTemplate = `{{ if .Tag }}// +build {{ .Tag }}

{{ end }}// Package {{ .Pkg }} is generated by github.com/omeid/go-resources
package {{ .Pkg }}

import (
	"bytes"
	"errors"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"
)

// FileSystem is an http.FileSystem implementation.
type FileSystem struct {
	files map[string]File
}

// String returns the content of the file as string.
func (fs *FileSystem) String(name string) (string, bool) {
	if filepath.Separator != '/' && strings.IndexRune(name, filepath.Separator) >= 0 ||
		strings.Contains(name, "\x00") {
		return "", false
	}

	file, ok := fs.files[name]

	if !ok {
		return "", false
	}

	return string(file.data), true
}

// Open implements http.FileSystem.Open
func (fs *FileSystem) Open(name string) (http.File, error) {
	if filepath.Separator != '/' && strings.IndexRune(name, filepath.Separator) >= 0 ||
		strings.Contains(name, "\x00") {
		return nil, errors.New("http: invalid character in file path")
	}
	file, ok := fs.files[name]
	if !ok {
		files := []os.FileInfo{}
		for path, file := range fs.files {
			if strings.HasPrefix(path, name) {
				fi := file.fi
				files = append(files, &fi)
			}
		}

		if len(files) == 0 {
			return nil, os.ErrNotExist
		}

		//We have a directory.
		return &File{
			fi: FileInfo{
				isDir: true,
				files: files,
			}}, nil
	}
	file.Reader = bytes.NewReader(file.data)
	return &file, nil
}

// File implements http.File
type File struct {
	*bytes.Reader
	data []byte
	fi   FileInfo
}

// Close is a noop-closer.
func (f *File) Close() error {
	return nil
}

// Readdir implements http.File.Readdir
func (f *File) Readdir(count int) ([]os.FileInfo, error) {
	return nil, os.ErrNotExist
}

// Stat implements http.Stat.Readdir
func (f *File) Stat() (os.FileInfo, error) {
	return &f.fi, nil
}

// FileInfo implements the os.FileInfo interface.
type FileInfo struct {
	name    string
	size    int64
	mode    os.FileMode
	modTime time.Time
	isDir   bool
	sys     interface{}

	files []os.FileInfo
}

// Name implements os.FileInfo.Name
func (f *FileInfo) Name() string {
	return f.name
}

// Size implements os.FileInfo.Size
func (f *FileInfo) Size() int64 {
	return f.size
}

// Mode implements os.FileInfo.Mode
func (f *FileInfo) Mode() os.FileMode {
	return f.mode
}

// ModTime implements os.FileInfo.ModTime
func (f *FileInfo) ModTime() time.Time {
	return f.modTime
}

// IsDir implements os.FileInfo.IsDir
func (f *FileInfo) IsDir() bool {
	return f.isDir
}

// Readdir implements os.FileInfo.Readdir
func (f *FileInfo) Readdir(count int) ([]os.FileInfo, error) {
	return f.files, nil
}

// Sys returns the underlying value.
func (f *FileInfo) Sys() interface{} {
	return f.sys
}

{{ if .Declare }}var {{ .Var }} *FileSystem{{ end }}

func init() {
	{{ .Var }} = &FileSystem{
		files: map[string]File{
			{{range $path, $file := .Files }}"/{{ $path }}": {{ template "file" $file }},{{ end }}
		},
	}
}
`
<|endoftext|>


<commit_before>/*
Resource abstractions.

Author:		Alastair Hughes
Contact:	<hobbitalastair at yandex dot com>
*/

package main

import (
	"database/sql"
	"fmt"
	"regexp"
	"strconv"

	_ "github.com/mattn/go-sqlite3"
)

var InvalidResource error = fmt.Errorf("Invalid resource\n")
var InvalidBody error = fmt.Errorf("Invalid body\n")

// Get/Set permission types.
const (
	Get = 1 << iota
	Set
	Create
)

// Interface abstracting encoders.
type Encoder interface {
	Encode(interface{}) error
}
type Decoder interface {
	Decode(interface{}) error
	More() bool
}

// Interface for the various resource types.
type Resource interface {
	Permissions() int    // Return a combination of Read and Write.
	Get(Encoder) error // Writes the resource to the given encoder.
	Set(Decoder) error // Handles an attempt to set the resource.
	Create(Decoder) error
}

// Regular expressions for the various resources.
var (
	projectListRe = regexp.MustCompile(`\A/projects/?\z`)
	projectRe     = regexp.MustCompile(`\A/projects/(\d+)/?\z`)
	flagRe        = regexp.MustCompile(`\A/projects/(\d+)/flag\z`)
	deliverableRe = regexp.MustCompile(`\A/projects/(\d+)/deliverables\z`)
)

type projectList struct {
	user string
	db   *sql.DB
}

func (_ *projectList) Permissions() int {
	return Get | Set
}

func (l *projectList) Get(enc Encoder) error {
	for _, table := range []string{"views", "owns"} {
		rows, err := l.db.Query(fmt.Sprintf("SELECT pid FROM %s WHERE name=?", table), l.user)
		if err != nil {
			return err
		}
		defer rows.Close()

		for rows.Next() {
			id := -1
			err = rows.Scan(&id)
			if err != nil {
				return err
			}
			err = enc.Encode(id)
			if err != nil {
				return err
			}
		}
		if rows.Err() != nil {
			return rows.Err()
		}
	}
	return nil
}

// Set for a projectList allows a login to unsubscribe themselves from projects.
func (l *projectList) Set(dec Decoder) error {
	// TODO: We don't implement this as it is nontrivial...
	return nil
}

func (l *projectList) Create(dec Decoder) error {
	return nil
}

// removeUser removes the given user from the given project.
// It also garbage-collects the project by decrementing and checking the
// viewing counter.
func removeUser(user string, pid uint, db *sql.DB) error {
	_, err := db.Exec("DELETE FROM owns WHERE name=? and pid=?", user, pid)
	if err != nil { return err }
	_, err = db.Exec("DELETE FROM views WHERE name=? and pid=?", user, pid)
	return err
}

type project struct {
	pid         uint
	permissions int
	db          *sql.DB
}

func (p *project) Permissions() int {
	return p.permissions
}

func (p *project) Get(enc Encoder) error {
	name, percentage, description := "", "", ""
	err := p.db.QueryRow("SELECT name, percentage, description FROM projects WHERE id=?", p.pid).Scan(&name, &percentage, &description)
	if err != nil {
		return err
	}
	err = enc.Encode(name)
	if err != nil {
		return err
	}
	err = enc.Encode(percentage)
	if err != nil {
		return err
	}
	return enc.Encode(description)
}

// Set the project state on the server.
// We override any existing state as I have not implemented any kind
// of synchronisation.
// FIXME: Add synchronisation.
func (p *project) Set(dec Decoder) error {
	name, percentage, description := "", "", ""
	err := dec.Decode(&name)
	if err != nil {
		return err
	}
	err = dec.Decode(&percentage)
	if err != nil {
		return err
	}
	err = dec.Decode(&description)
	if err != nil {
		return err
	}
	_, err = p.db.Exec("UPDATE projects SET name=?, percentage=?, description=? WHERE pid=?", name, percentage, description, p.pid)
	return err
}

func (p *project) Create(dec Decoder) error {
	return nil
}

func NewProject(user string, pid uint, db *sql.DB) (Resource, error) {
	p := project{pid, 0, db}
	dbpid := 0
	err := db.QueryRow("SELECT pid FROM views WHERE name=? and pid=?", user, pid).Scan(&dbpid)
	if err == nil {
		p.permissions |= Get
	} else if err != sql.ErrNoRows {
		return nil, err
	}
	return &p, nil
}

type flag struct {
	pid     uint
	project Resource
	db      *sql.DB
}

func (f *flag) Permissions() int {
	// Everyone can read and write to the flag.
	if Get&f.project.Permissions() != 0 {
		return Get | Set
	}
	return 0
}

func (f *flag) Get(enc Encoder) error {
	flag := false
	err := f.db.QueryRow("SELECT flag FROM projects WHERE id=?", f.pid).Scan(&flag)
	if err != nil {
		return err
	}
	return enc.Encode(flag)
}

func (f *flag) Set(dec Decoder) error {
	return nil
}

func (f *flag) Create(dec Decoder) error {
	return nil
}

func NewFlag(user string, pid uint, db *sql.DB) (Resource, error) {
	proj, err := NewProject(user, pid, db)
	return &flag{pid, proj, db}, err
}

// FromURI returns the resource corresponding to the given URI.
func FromURI(user, uri string, db *sql.DB) (Resource, error) {
	// Match the path to the regular expressions.
	if projectListRe.MatchString(uri) {
		return &projectList{user, db}, nil
	} else if projectRe.MatchString(uri) {
		pid, err := strconv.Atoi(projectRe.FindStringSubmatch(uri)[1])
		if err != nil {
			return nil, err
		}
		return NewProject(user, uint(pid), db)
	} else if flagRe.MatchString(uri) {
		pid, err := strconv.Atoi(flagRe.FindStringSubmatch(uri)[1])
		if err != nil {
			return nil, err
		}
		return NewFlag(user, uint(pid), db)
	} else {
		return nil, InvalidResource
	}
}

// vim: sw=4 ts=4 noexpandtab
<commit_msg>Remove excess comments, add FIXME for gc<commit_after>/*
Resource abstractions.

Author:		Alastair Hughes
Contact:	<hobbitalastair at yandex dot com>
*/

package main

import (
	"database/sql"
	"fmt"
	"regexp"
	"strconv"

	_ "github.com/mattn/go-sqlite3"
)

var InvalidResource error = fmt.Errorf("Invalid resource\n")
var InvalidBody error = fmt.Errorf("Invalid body\n")

// Get/Set permission types.
const (
	Get = 1 << iota
	Set
	Create
)

// Interface abstracting encoders.
type Encoder interface {
	Encode(interface{}) error
}
type Decoder interface {
	Decode(interface{}) error
	More() bool
}

// Interface for the various resource types.
type Resource interface {
	Permissions() int
	Get(Encoder) error
	Set(Decoder) error
	Create(Decoder) error
}

// Regular expressions for the various resources.
var (
	projectListRe = regexp.MustCompile(`\A/projects/?\z`)
	projectRe     = regexp.MustCompile(`\A/projects/(\d+)/?\z`)
	flagRe        = regexp.MustCompile(`\A/projects/(\d+)/flag\z`)
	deliverableRe = regexp.MustCompile(`\A/projects/(\d+)/deliverables\z`)
)

type projectList struct {
	user string
	db   *sql.DB
}

func (_ *projectList) Permissions() int {
	return Get | Set
}

func (l *projectList) Get(enc Encoder) error {
	for _, table := range []string{"views", "owns"} {
		rows, err := l.db.Query(fmt.Sprintf("SELECT pid FROM %s WHERE name=?", table), l.user)
		if err != nil {
			return err
		}
		defer rows.Close()

		for rows.Next() {
			id := -1
			err = rows.Scan(&id)
			if err != nil {
				return err
			}
			err = enc.Encode(id)
			if err != nil {
				return err
			}
		}
		if rows.Err() != nil {
			return rows.Err()
		}
	}
	return nil
}

// Set for a projectList allows a login to unsubscribe themselves from projects.
func (l *projectList) Set(dec Decoder) error {
	// TODO: We don't implement this as it is nontrivial...
	return nil
}

func (l *projectList) Create(dec Decoder) error {
	return nil
}

// removeUser removes the given user from the given project.
// It also garbage-collects the project by decrementing and checking the
// viewing counter.
// FIXME: Implement the garbage collection...
func removeUser(user string, pid uint, db *sql.DB) error {
	_, err := db.Exec("DELETE FROM owns WHERE name=? and pid=?", user, pid)
	if err != nil { return err }
	_, err = db.Exec("DELETE FROM views WHERE name=? and pid=?", user, pid)
	return err
}

type project struct {
	pid         uint
	permissions int
	db          *sql.DB
}

func (p *project) Permissions() int {
	return p.permissions
}

func (p *project) Get(enc Encoder) error {
	name, percentage, description := "", "", ""
	err := p.db.QueryRow("SELECT name, percentage, description FROM projects WHERE id=?", p.pid).Scan(&name, &percentage, &description)
	if err != nil {
		return err
	}
	err = enc.Encode(name)
	if err != nil {
		return err
	}
	err = enc.Encode(percentage)
	if err != nil {
		return err
	}
	return enc.Encode(description)
}

// Set the project state on the server.
// We override any existing state as I have not implemented any kind
// of synchronisation.
// FIXME: Add synchronisation.
func (p *project) Set(dec Decoder) error {
	name, percentage, description := "", "", ""
	err := dec.Decode(&name)
	if err != nil {
		return err
	}
	err = dec.Decode(&percentage)
	if err != nil {
		return err
	}
	err = dec.Decode(&description)
	if err != nil {
		return err
	}
	_, err = p.db.Exec("UPDATE projects SET name=?, percentage=?, description=? WHERE pid=?", name, percentage, description, p.pid)
	return err
}

func (p *project) Create(dec Decoder) error {
	return nil
}

func NewProject(user string, pid uint, db *sql.DB) (Resource, error) {
	p := project{pid, 0, db}
	dbpid := 0
	err := db.QueryRow("SELECT pid FROM views WHERE name=? and pid=?", user, pid).Scan(&dbpid)
	if err == nil {
		p.permissions |= Get
	} else if err != sql.ErrNoRows {
		return nil, err
	}
	return &p, nil
}

type flag struct {
	pid     uint
	project Resource
	db      *sql.DB
}

func (f *flag) Permissions() int {
	// Everyone can read and write to the flag.
	if Get&f.project.Permissions() != 0 {
		return Get | Set
	}
	return 0
}

func (f *flag) Get(enc Encoder) error {
	flag := false
	err := f.db.QueryRow("SELECT flag FROM projects WHERE id=?", f.pid).Scan(&flag)
	if err != nil {
		return err
	}
	return enc.Encode(flag)
}

func (f *flag) Set(dec Decoder) error {
	return nil
}

func (f *flag) Create(dec Decoder) error {
	return nil
}

func NewFlag(user string, pid uint, db *sql.DB) (Resource, error) {
	proj, err := NewProject(user, pid, db)
	return &flag{pid, proj, db}, err
}

// FromURI returns the resource corresponding to the given URI.
func FromURI(user, uri string, db *sql.DB) (Resource, error) {
	// Match the path to the regular expressions.
	if projectListRe.MatchString(uri) {
		return &projectList{user, db}, nil
	} else if projectRe.MatchString(uri) {
		pid, err := strconv.Atoi(projectRe.FindStringSubmatch(uri)[1])
		if err != nil {
			return nil, err
		}
		return NewProject(user, uint(pid), db)
	} else if flagRe.MatchString(uri) {
		pid, err := strconv.Atoi(flagRe.FindStringSubmatch(uri)[1])
		if err != nil {
			return nil, err
		}
		return NewFlag(user, uint(pid), db)
	} else {
		return nil, InvalidResource
	}
}

// vim: sw=4 ts=4 noexpandtab
<|endoftext|>


<commit_before>package main

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"

	"golang.org/x/crypto/ssh"
)

var version = "head"

type butlerError struct {
	Error string
}

type butlerDownloadStatus struct {
	Percent int
}

const bufferSize = 128 * 1024

func main() {
	if len(os.Args) < 2 {
		err("Missing command")
	}
	cmd := os.Args[1]

	switch cmd {
	case "version":
		fmt.Println(fmt.Sprintf("butler version %s", version))
	case "dl":
		dl()
	case "test-ssh":
		testSSH()
	default:
		err("Invalid command")
	}
}

func send(v interface{}) {
	j, _ := json.Marshal(v)
	fmt.Println(string(j))
}

func err(msg string) {
	e := &butlerError{Error: msg}
	send(e)
	os.Exit(1)
}

func dl() {
	if len(os.Args) < 4 {
		err("Missing url or dest for dl command")
	}
	url := os.Args[2]
	dest := os.Args[3]

	initialBytes := int64(0)
	stats, err := os.Lstat(dest)
	if err == nil {
		initialBytes = stats.Size()
	}

	bytesWritten := initialBytes

	out, _ := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
	defer out.Close()

	client := &http.Client{}
	req, _ := http.NewRequest("GET", url, nil)
	req.Header.Set("Range", fmt.Sprintf("bytes=%d-", bytesWritten))
	resp, _ := client.Do(req)
	defer resp.Body.Close()

	for {
		n, _ := io.CopyN(out, resp.Body, bufferSize)
		bytesWritten += n

		totalBytes := (initialBytes + resp.ContentLength)
		status := &butlerDownloadStatus{
			Percent: int(bytesWritten * 100 / totalBytes)}
		send(status)

		if n == 0 {
			break
		}
	}
}

func publicKeyFile(file string) ssh.AuthMethod {
	buffer, err := ioutil.ReadFile(file)
	if err != nil {
		return nil
	}

	key, err := ssh.ParsePrivateKey(buffer)
	if err != nil {
		return nil
	}

	log.Println("Our public key is", base64.StdEncoding.EncodeToString(key.PublicKey().Marshal()))
	return ssh.PublicKeys(key)
}

func testSSH() {
	host := "butler.itch.zone"
	port := 2222
	serverString := fmt.Sprintf("%s:%d", host, port)
	fmt.Printf("Trying to connect to %s\n", serverString)

	keyPath := fmt.Sprintf("%s/%s", os.Getenv("HOME"), ".ssh/id_rsa")
	key := publicKeyFile(keyPath)
	auth := []ssh.AuthMethod{key}

	sshConfig := &ssh.ClientConfig{
		User: "butler",
		Auth: auth,
	}

	serverConn, err := ssh.Dial("tcp", serverString, sshConfig)
	if err != nil {
		fmt.Printf("Server dial error: %s\n", err)
		return
	}
	fmt.Printf("Connected!\n")

	ch, _, err := serverConn.OpenChannel("butler", []byte{})
	if err != nil {
		panic(err)
	}

	ch.Write([]byte("Hi"))
	ch.Close()

	serverConn.Close()
}
<commit_msg>Test the brotli compression algorithm<commit_after>package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/kothar/brotli-go/dec"
	"github.com/kothar/brotli-go/enc"
	"golang.org/x/crypto/ssh"
)

var version = "head"

type butlerError struct {
	Error string
}

type butlerDownloadStatus struct {
	Percent int
}

const bufferSize = 128 * 1024

func main() {
	if len(os.Args) < 2 {
		err("Missing command")
	}
	cmd := os.Args[1]

	switch cmd {
	case "version":
		fmt.Println(fmt.Sprintf("butler version %s", version))
	case "dl":
		dl()
	case "test-ssh":
		testSSH()
	case "test-brotli":
		testBrotli()
	default:
		err("Invalid command")
	}
}

func send(v interface{}) {
	j, _ := json.Marshal(v)
	fmt.Println(string(j))
}

func err(msg string) {
	e := &butlerError{Error: msg}
	send(e)
	os.Exit(1)
}

func dl() {
	if len(os.Args) < 4 {
		err("Missing url or dest for dl command")
	}
	url := os.Args[2]
	dest := os.Args[3]

	initialBytes := int64(0)
	stats, err := os.Lstat(dest)
	if err == nil {
		initialBytes = stats.Size()
	}

	bytesWritten := initialBytes

	out, _ := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
	defer out.Close()

	client := &http.Client{}
	req, _ := http.NewRequest("GET", url, nil)
	req.Header.Set("Range", fmt.Sprintf("bytes=%d-", bytesWritten))
	resp, _ := client.Do(req)
	defer resp.Body.Close()

	for {
		n, _ := io.CopyN(out, resp.Body, bufferSize)
		bytesWritten += n

		totalBytes := (initialBytes + resp.ContentLength)
		status := &butlerDownloadStatus{
			Percent: int(bytesWritten * 100 / totalBytes)}
		send(status)

		if n == 0 {
			break
		}
	}
}

func publicKeyFile(file string) ssh.AuthMethod {
	buffer, err := ioutil.ReadFile(file)
	if err != nil {
		return nil
	}

	key, err := ssh.ParsePrivateKey(buffer)
	if err != nil {
		return nil
	}

	log.Println("Our public key is", base64.StdEncoding.EncodeToString(key.PublicKey().Marshal()))
	return ssh.PublicKeys(key)
}

func testSSH() {
	host := "butler.itch.zone"
	port := 2222
	serverString := fmt.Sprintf("%s:%d", host, port)
	fmt.Printf("Trying to connect to %s\n", serverString)

	keyPath := fmt.Sprintf("%s/%s", os.Getenv("HOME"), ".ssh/id_rsa")
	key := publicKeyFile(keyPath)
	auth := []ssh.AuthMethod{key}

	sshConfig := &ssh.ClientConfig{
		User: "butler",
		Auth: auth,
	}

	serverConn, err := ssh.Dial("tcp", serverString, sshConfig)
	if err != nil {
		fmt.Printf("Server dial error: %s\n", err)
		return
	}
	fmt.Printf("Connected!\n")

	ch, _, err := serverConn.OpenChannel("butler", []byte{})
	if err != nil {
		panic(err)
	}

	ch.Write([]byte("Hi"))
	ch.Close()

	serverConn.Close()
}

func testBrotli() {
	start := time.Now()

	src := os.Args[2]
	inputBuffer, err := ioutil.ReadFile(src)
	if err != nil {
		panic(err)
	}

	log.Println("Read file in", time.Since(start))
	log.Println("Uncompressed size is", humanize.Bytes(uint64(len(inputBuffer))))
	start = time.Now()

	var decoded []byte

	for q := 0; q <= 9; q++ {
		params := enc.NewBrotliParams()
		params.SetQuality(q)

		encoded, err := enc.CompressBuffer(params, inputBuffer, make([]byte, 1))
		if err != nil {
			panic(err)
		}

		log.Println("Compressed (q=", q, ") to", humanize.Bytes(uint64(len(encoded))), "in", time.Since(start))
		start = time.Now()

		decoded, err = dec.DecompressBuffer(encoded, make([]byte, 1))
		if err != nil {
			panic(err)
		}

		log.Println("Decompressed in", time.Since(start))
		start = time.Now()
	}

	if !bytes.Equal(inputBuffer, decoded) {
		log.Println("Decoded output does not match original input")
		return
	}

	log.Println("Compared in", time.Since(start))
	start = time.Now()

	log.Println("Round-trip through brotli successful!")
}
<|endoftext|>


<commit_before>// +build !providerless

/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gce

import (
	"context"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func TestGetLoadBalancer(t *testing.T) {
	t.Parallel()

	vals := DefaultTestClusterValues()
	gce, err := fakeGCECloud(vals)
	require.NoError(t, err)

	apiService := fakeLoadbalancerService("")

	// When a loadbalancer has not been created
	status, found, err := gce.GetLoadBalancer(context.Background(), vals.ClusterName, apiService)
	assert.Nil(t, status)
	assert.False(t, found)
	assert.NoError(t, err)

	nodeNames := []string{"test-node-1"}
	nodes, err := createAndInsertNodes(gce, nodeNames, vals.ZoneName)
	require.NoError(t, err)
	expectedStatus, err := gce.EnsureLoadBalancer(context.Background(), vals.ClusterName, apiService, nodes)
	require.NoError(t, err)

	status, found, err = gce.GetLoadBalancer(context.Background(), vals.ClusterName, apiService)
	assert.Equal(t, expectedStatus, status)
	assert.True(t, found)
	assert.NoError(t, err)
}

func TestEnsureLoadBalancerCreatesExternalLb(t *testing.T) {
	t.Parallel()

	vals := DefaultTestClusterValues()
	gce, err := fakeGCECloud(vals)
	require.NoError(t, err)

	nodeNames := []string{"test-node-1"}
	nodes, err := createAndInsertNodes(gce, nodeNames, vals.ZoneName)
	require.NoError(t, err)

	apiService := fakeLoadbalancerService("")
	status, err := gce.EnsureLoadBalancer(context.Background(), vals.ClusterName, apiService, nodes)
	assert.NoError(t, err)
	assert.NotEmpty(t, status.Ingress)
	assertExternalLbResources(t, gce, apiService, vals, nodeNames)
}

func TestEnsureLoadBalancerCreatesInternalLb(t *testing.T) {
	t.Parallel()

	vals := DefaultTestClusterValues()
	gce, err := fakeGCECloud(vals)
	require.NoError(t, err)

	nodeNames := []string{"test-node-1"}
	nodes, err := createAndInsertNodes(gce, nodeNames, vals.ZoneName)
	require.NoError(t, err)

	apiService := fakeLoadbalancerService(string(LBTypeInternal))
	apiService, err = gce.client.CoreV1().Services(apiService.Namespace).Create(context.TODO(), apiService, metav1.CreateOptions{})
	require.NoError(t, err)
	status, err := gce.EnsureLoadBalancer(context.Background(), vals.ClusterName, apiService, nodes)
	assert.NoError(t, err)
	assert.NotEmpty(t, status.Ingress)
	assertInternalLbResources(t, gce, apiService, vals, nodeNames)
}

func TestEnsureLoadBalancerDeletesExistingInternalLb(t *testing.T) {
	t.Parallel()

	vals := DefaultTestClusterValues()
	gce, err := fakeGCECloud(vals)
	require.NoError(t, err)

	nodeNames := []string{"test-node-1"}
	nodes, err := createAndInsertNodes(gce, nodeNames, vals.ZoneName)
	require.NoError(t, err)

	apiService := fakeLoadbalancerService("")
	createInternalLoadBalancer(gce, apiService, nil, nodeNames, vals.ClusterName, vals.ClusterID, vals.ZoneName)

	status, err := gce.EnsureLoadBalancer(context.Background(), vals.ClusterName, apiService, nodes)
	assert.NoError(t, err)
	assert.NotEmpty(t, status.Ingress)

	assertExternalLbResources(t, gce, apiService, vals, nodeNames)
	assertInternalLbResourcesDeleted(t, gce, apiService, vals, false)
}

func TestEnsureLoadBalancerDeletesExistingExternalLb(t *testing.T) {
	t.Parallel()

	vals := DefaultTestClusterValues()
	gce, err := fakeGCECloud(vals)
	require.NoError(t, err)

	nodeNames := []string{"test-node-1"}
	nodes, err := createAndInsertNodes(gce, nodeNames, vals.ZoneName)
	require.NoError(t, err)

	apiService := fakeLoadbalancerService("")
	createExternalLoadBalancer(gce, apiService, nodeNames, vals.ClusterName, vals.ClusterID, vals.ZoneName)

	apiService = fakeLoadbalancerService(string(LBTypeInternal))
	apiService, err = gce.client.CoreV1().Services(apiService.Namespace).Create(context.TODO(), apiService, metav1.CreateOptions{})
	require.NoError(t, err)
	status, err := gce.EnsureLoadBalancer(context.Background(), vals.ClusterName, apiService, nodes)
	assert.NoError(t, err)
	assert.NotEmpty(t, status.Ingress)

	assertInternalLbResources(t, gce, apiService, vals, nodeNames)
	assertExternalLbResourcesDeleted(t, gce, apiService, vals, false)
}

func TestEnsureLoadBalancerDeletedDeletesExternalLb(t *testing.T) {
	t.Parallel()

	vals := DefaultTestClusterValues()
	gce, err := fakeGCECloud(vals)
	require.NoError(t, err)

	nodeNames := []string{"test-node-1"}
	_, err = createAndInsertNodes(gce, nodeNames, vals.ZoneName)
	require.NoError(t, err)

	apiService := fakeLoadbalancerService("")
	createExternalLoadBalancer(gce, apiService, nodeNames, vals.ClusterName, vals.ClusterID, vals.ZoneName)

	err = gce.EnsureLoadBalancerDeleted(context.Background(), vals.ClusterName, apiService)
	assert.NoError(t, err)
	assertExternalLbResourcesDeleted(t, gce, apiService, vals, true)
}

func TestEnsureLoadBalancerDeletedDeletesInternalLb(t *testing.T) {
	t.Parallel()

	vals := DefaultTestClusterValues()
	gce, err := fakeGCECloud(vals)
	require.NoError(t, err)

	nodeNames := []string{"test-node-1"}
	_, err = createAndInsertNodes(gce, nodeNames, vals.ZoneName)
	require.NoError(t, err)

	apiService := fakeLoadbalancerService(string(LBTypeInternal))
	apiService, err = gce.client.CoreV1().Services(apiService.Namespace).Create(context.TODO(), apiService, metav1.CreateOptions{})
	require.NoError(t, err)
	createInternalLoadBalancer(gce, apiService, nil, nodeNames, vals.ClusterName, vals.ClusterID, vals.ZoneName)

	err = gce.EnsureLoadBalancerDeleted(context.Background(), vals.ClusterName, apiService)
	assert.NoError(t, err)
	assertInternalLbResourcesDeleted(t, gce, apiService, vals, true)
}
<commit_msg>Add a test for verifying compute basepath.<commit_after>// +build !providerless

/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gce

import (
	"context"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func TestGetLoadBalancer(t *testing.T) {
	t.Parallel()

	vals := DefaultTestClusterValues()
	gce, err := fakeGCECloud(vals)
	require.NoError(t, err)

	apiService := fakeLoadbalancerService("")

	// When a loadbalancer has not been created
	status, found, err := gce.GetLoadBalancer(context.Background(), vals.ClusterName, apiService)
	assert.Nil(t, status)
	assert.False(t, found)
	assert.NoError(t, err)

	nodeNames := []string{"test-node-1"}
	nodes, err := createAndInsertNodes(gce, nodeNames, vals.ZoneName)
	require.NoError(t, err)
	expectedStatus, err := gce.EnsureLoadBalancer(context.Background(), vals.ClusterName, apiService, nodes)
	require.NoError(t, err)

	status, found, err = gce.GetLoadBalancer(context.Background(), vals.ClusterName, apiService)
	assert.Equal(t, expectedStatus, status)
	assert.True(t, found)
	assert.NoError(t, err)
}

func TestEnsureLoadBalancerCreatesExternalLb(t *testing.T) {
	t.Parallel()

	vals := DefaultTestClusterValues()
	gce, err := fakeGCECloud(vals)
	require.NoError(t, err)

	nodeNames := []string{"test-node-1"}
	nodes, err := createAndInsertNodes(gce, nodeNames, vals.ZoneName)
	require.NoError(t, err)

	apiService := fakeLoadbalancerService("")
	status, err := gce.EnsureLoadBalancer(context.Background(), vals.ClusterName, apiService, nodes)
	assert.NoError(t, err)
	assert.NotEmpty(t, status.Ingress)
	assertExternalLbResources(t, gce, apiService, vals, nodeNames)
}

func TestEnsureLoadBalancerCreatesInternalLb(t *testing.T) {
	t.Parallel()

	vals := DefaultTestClusterValues()
	gce, err := fakeGCECloud(vals)
	require.NoError(t, err)

	nodeNames := []string{"test-node-1"}
	nodes, err := createAndInsertNodes(gce, nodeNames, vals.ZoneName)
	require.NoError(t, err)

	apiService := fakeLoadbalancerService(string(LBTypeInternal))
	apiService, err = gce.client.CoreV1().Services(apiService.Namespace).Create(context.TODO(), apiService, metav1.CreateOptions{})
	require.NoError(t, err)
	status, err := gce.EnsureLoadBalancer(context.Background(), vals.ClusterName, apiService, nodes)
	assert.NoError(t, err)
	assert.NotEmpty(t, status.Ingress)
	assertInternalLbResources(t, gce, apiService, vals, nodeNames)
}

func TestEnsureLoadBalancerDeletesExistingInternalLb(t *testing.T) {
	t.Parallel()

	vals := DefaultTestClusterValues()
	gce, err := fakeGCECloud(vals)
	require.NoError(t, err)

	nodeNames := []string{"test-node-1"}
	nodes, err := createAndInsertNodes(gce, nodeNames, vals.ZoneName)
	require.NoError(t, err)

	apiService := fakeLoadbalancerService("")
	createInternalLoadBalancer(gce, apiService, nil, nodeNames, vals.ClusterName, vals.ClusterID, vals.ZoneName)

	status, err := gce.EnsureLoadBalancer(context.Background(), vals.ClusterName, apiService, nodes)
	assert.NoError(t, err)
	assert.NotEmpty(t, status.Ingress)

	assertExternalLbResources(t, gce, apiService, vals, nodeNames)
	assertInternalLbResourcesDeleted(t, gce, apiService, vals, false)
}

func TestEnsureLoadBalancerDeletesExistingExternalLb(t *testing.T) {
	t.Parallel()

	vals := DefaultTestClusterValues()
	gce, err := fakeGCECloud(vals)
	require.NoError(t, err)

	nodeNames := []string{"test-node-1"}
	nodes, err := createAndInsertNodes(gce, nodeNames, vals.ZoneName)
	require.NoError(t, err)

	apiService := fakeLoadbalancerService("")
	createExternalLoadBalancer(gce, apiService, nodeNames, vals.ClusterName, vals.ClusterID, vals.ZoneName)

	apiService = fakeLoadbalancerService(string(LBTypeInternal))
	apiService, err = gce.client.CoreV1().Services(apiService.Namespace).Create(context.TODO(), apiService, metav1.CreateOptions{})
	require.NoError(t, err)
	status, err := gce.EnsureLoadBalancer(context.Background(), vals.ClusterName, apiService, nodes)
	assert.NoError(t, err)
	assert.NotEmpty(t, status.Ingress)

	assertInternalLbResources(t, gce, apiService, vals, nodeNames)
	assertExternalLbResourcesDeleted(t, gce, apiService, vals, false)
}

func TestEnsureLoadBalancerDeletedDeletesExternalLb(t *testing.T) {
	t.Parallel()

	vals := DefaultTestClusterValues()
	gce, err := fakeGCECloud(vals)
	require.NoError(t, err)

	nodeNames := []string{"test-node-1"}
	_, err = createAndInsertNodes(gce, nodeNames, vals.ZoneName)
	require.NoError(t, err)

	apiService := fakeLoadbalancerService("")
	createExternalLoadBalancer(gce, apiService, nodeNames, vals.ClusterName, vals.ClusterID, vals.ZoneName)

	err = gce.EnsureLoadBalancerDeleted(context.Background(), vals.ClusterName, apiService)
	assert.NoError(t, err)
	assertExternalLbResourcesDeleted(t, gce, apiService, vals, true)
}

func TestEnsureLoadBalancerDeletedDeletesInternalLb(t *testing.T) {
	t.Parallel()

	vals := DefaultTestClusterValues()
	gce, err := fakeGCECloud(vals)
	require.NoError(t, err)

	nodeNames := []string{"test-node-1"}
	_, err = createAndInsertNodes(gce, nodeNames, vals.ZoneName)
	require.NoError(t, err)

	apiService := fakeLoadbalancerService(string(LBTypeInternal))
	apiService, err = gce.client.CoreV1().Services(apiService.Namespace).Create(context.TODO(), apiService, metav1.CreateOptions{})
	require.NoError(t, err)
	createInternalLoadBalancer(gce, apiService, nil, nodeNames, vals.ClusterName, vals.ClusterID, vals.ZoneName)

	err = gce.EnsureLoadBalancerDeleted(context.Background(), vals.ClusterName, apiService)
	assert.NoError(t, err)
	assertInternalLbResourcesDeleted(t, gce, apiService, vals, true)
}

func TestBasePath(t *testing.T) {
	t.Parallel()
	vals := DefaultTestClusterValues()
	gce, err := fakeGCECloud(vals)
	// Loadbalancer controller code expects basepath to contain the projects string.
	expectBasePath := "https://compute.googleapis.com/compute/v1/projects/"
	require.NoError(t, err)
	if gce.service.BasePath != expectBasePath {
		t.Errorf("Compute basePath has changed. Got %q, want %q", gce.service.BasePath, expectBasePath)
	}
}
<|endoftext|>