<commit_before>// Copyright 2012 Jimmy Zelinskie. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package reddit

import (
	"encoding/json"
	"fmt"
)

// Headline represents an individual post from the perspective
// of a subreddit.
type Headline struct {
	Author       string  `json:"author"`
	Title        string  `json:"title"`
	URL          string  `json:"url"`
	Domain       string  `json:"domain"`
	Subreddit    string  `json:"subreddit"`
	SubredditId  string  `json:"subreddit_id"`
	FullId       string  `json:"name"`
	Id           string  `json:"id"`
	Permalink    string  `json:"permalink"`
	Selftext     string  `json:"selftext"`
	ThumbnailURL string  `json:"thumbnail"`
	DateCreated  float32 `json:"created_utc"`
	NumComments  int     `json:"num_comments"`
	Score        int     `json:"score"`
	// Ups and downs are fake to trick spammers
	Ups        int     `json:"ups"`
	Downs      int     `json:"downs"`
	IsNSFW     bool    `json:"over_18"`
	IsSelf     bool    `json:"is_self"`
	WasClicked bool    `json:"clicked"`
	IsSaved    bool    `json:"saved"`
	BannedBy   *string `json:"banned_by"`
}

// Sort headlines by popularity
type PopularitySort string

const (
	DefaultPopularity      PopularitySort = ""
	HotHeadlines                          = "hot"
	NewHeadlines                          = "new"
	RisingHeadlines                       = "rising"
	TopHeadlines                          = "top"
	ControversialHeadlines                = "controversial"
)

// Sort headlines by age
type AgeSort string

const (
	DefaultAge AgeSort = ""
	ThisHour           = "hour"
	ThisMonth          = "month"
	ThisYear           = "year"
	AllTime            = "all"
)

type Headlines []*Headline

// FullPermalink returns the full URL of a headline.
func (h *Headline) FullPermalink() string {
	return "http://reddit.com" + h.Permalink
}

// String returns the string representation of a headline.
func (h *Headline) String() string {
	plural := ""
	if h.NumComments != 1 {
		plural = "s"
	}
	comments := fmt.Sprintf("%d comment%s", h.NumComments, plural)
	return fmt.Sprintf("%d - %s (%s)", h.Score, h.Title, comments)
}

// DefaultHeadlines returns a slice of headlines on the default reddit frontpage.
func DefaultHeadlines() (Headlines, error) {
	url := "http://www.reddit.com/.json"
	body, err := getResponse(url, nil, nil)
	if err != nil {
		return nil, err
	}

	type Response struct {
		Data struct {
			Children []struct {
				Data *Headline
			}
		}
	}

	r := new(Response)
	err = json.NewDecoder(body).Decode(r)
	if err != nil {
		return nil, err
	}

	headlines := make(Headlines, len(r.Data.Children))
	for i, child := range r.Data.Children {
		headlines[i] = child.Data
	}

	return headlines, nil
}

// SubredditHeadlines returns a slice of headlines on the given subreddit.
func SubredditHeadlines(subreddit string) (Headlines, error) {
	url := fmt.Sprintf("http://www.reddit.com/r/%s.json", subreddit)
	body, err := getResponse(url, nil, nil)
	if err != nil {
		return nil, err
	}

	type Response struct {
		Data struct {
			Children []struct {
				Data *Headline
			}
		}
	}

	r := new(Response)
	err = json.NewDecoder(body).Decode(r)
	if err != nil {
		return nil, err
	}

	headlines := make(Headlines, len(r.Data.Children))
	for i, child := range r.Data.Children {
		headlines[i] = child.Data
	}

	return headlines, nil
}

// SortedHeadlines will return headlines from a subreddit (or homepage if "") by popularity and age
func SortedHeadlines(subreddit string, popularity PopularitySort, age AgeSort) (Headlines, error) {
	if age != DefaultAge {
		switch popularity {
		case NewHeadlines, RisingHeadlines, HotHeadlines:
			return nil, fmt.Errorf("Cannot sort %s by %s", popularity, age)
		}
	}

	url := "http://reddit.com/"

	if subreddit != "" {
		url = fmt.Sprintf("http://%s.reddit.com/", subreddit)
	}

	if popularity != DefaultPopularity {
		if popularity == NewHeadlines || popularity == RisingHeadlines {
			url = fmt.Sprintf("%snew.json?sort=%s", url, popularity)
		} else {
			url = fmt.Sprintf("%s%s.json?sort=%s", url, popularity, popularity)
		}
	} else {
		url = fmt.Sprintf("%s.json", url)
	}

	if age != DefaultAge {
		if popularity != DefaultPopularity {
			url = fmt.Sprintf("%s&t=%s", url, age)
		} else {
			url = fmt.Sprintf("%s?t=%s", url, age)
		}
	}

	body, err := getResponse(url, nil, nil)
	if err != nil {
		return nil, err
	}

	type Response struct {
		Data struct {
			Children []struct {
				Data *Headline
			}
		}
	}

	r := new(Response)
	err = json.NewDecoder(body).Decode(r)
	if err != nil {
		return nil, err
	}

	headlines := make(Headlines, len(r.Data.Children))
	for i, child := range r.Data.Children {
		headlines[i] = child.Data
	}

	return headlines, nil

}
<commit_msg>Modified SortedHeadlines to work with multi<commit_after>// Copyright 2012 Jimmy Zelinskie. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package reddit

import (
	"encoding/json"
	"fmt"
)

// Headline represents an individual post from the perspective
// of a subreddit.
type Headline struct {
	Author       string  `json:"author"`
	Title        string  `json:"title"`
	URL          string  `json:"url"`
	Domain       string  `json:"domain"`
	Subreddit    string  `json:"subreddit"`
	SubredditId  string  `json:"subreddit_id"`
	FullId       string  `json:"name"`
	Id           string  `json:"id"`
	Permalink    string  `json:"permalink"`
	Selftext     string  `json:"selftext"`
	ThumbnailURL string  `json:"thumbnail"`
	DateCreated  float32 `json:"created_utc"`
	NumComments  int     `json:"num_comments"`
	Score        int     `json:"score"`
	// Ups and downs are fake to trick spammers
	Ups        int     `json:"ups"`
	Downs      int     `json:"downs"`
	IsNSFW     bool    `json:"over_18"`
	IsSelf     bool    `json:"is_self"`
	WasClicked bool    `json:"clicked"`
	IsSaved    bool    `json:"saved"`
	BannedBy   *string `json:"banned_by"`
}

// Sort headlines by popularity
type PopularitySort string

const (
	DefaultPopularity      PopularitySort = ""
	HotHeadlines                          = "hot"
	NewHeadlines                          = "new"
	RisingHeadlines                       = "rising"
	TopHeadlines                          = "top"
	ControversialHeadlines                = "controversial"
)

// Sort headlines by age
type AgeSort string

const (
	DefaultAge AgeSort = ""
	ThisHour           = "hour"
	ThisMonth          = "month"
	ThisYear           = "year"
	AllTime            = "all"
)

type Headlines []*Headline

// FullPermalink returns the full URL of a headline.
func (h *Headline) FullPermalink() string {
	return "http://reddit.com" + h.Permalink
}

// String returns the string representation of a headline.
func (h *Headline) String() string {
	plural := ""
	if h.NumComments != 1 {
		plural = "s"
	}
	comments := fmt.Sprintf("%d comment%s", h.NumComments, plural)
	return fmt.Sprintf("%d - %s (%s)", h.Score, h.Title, comments)
}

// DefaultHeadlines returns a slice of headlines on the default reddit frontpage.
func DefaultHeadlines() (Headlines, error) {
	url := "http://www.reddit.com/.json"
	body, err := getResponse(url, nil, nil)
	if err != nil {
		return nil, err
	}

	type Response struct {
		Data struct {
			Children []struct {
				Data *Headline
			}
		}
	}

	r := new(Response)
	err = json.NewDecoder(body).Decode(r)
	if err != nil {
		return nil, err
	}

	headlines := make(Headlines, len(r.Data.Children))
	for i, child := range r.Data.Children {
		headlines[i] = child.Data
	}

	return headlines, nil
}

// SubredditHeadlines returns a slice of headlines on the given subreddit.
func SubredditHeadlines(subreddit string) (Headlines, error) {
	url := fmt.Sprintf("http://www.reddit.com/r/%s.json", subreddit)
	body, err := getResponse(url, nil, nil)
	if err != nil {
		return nil, err
	}

	type Response struct {
		Data struct {
			Children []struct {
				Data *Headline
			}
		}
	}

	r := new(Response)
	err = json.NewDecoder(body).Decode(r)
	if err != nil {
		return nil, err
	}

	headlines := make(Headlines, len(r.Data.Children))
	for i, child := range r.Data.Children {
		headlines[i] = child.Data
	}

	return headlines, nil
}

// SortedHeadlines will return headlines from a subreddit (or homepage if "") by popularity and age
func SortedHeadlines(subreddit string, popularity PopularitySort, age AgeSort) (Headlines, error) {
	if age != DefaultAge {
		switch popularity {
		case NewHeadlines, RisingHeadlines, HotHeadlines:
			return nil, fmt.Errorf("Cannot sort %s by %s", popularity, age)
		}
	}

	url := "http://reddit.com/"

	if subreddit != "" {
		url = fmt.Sprintf("%sr/%s/", url, subreddit)
	}

	if popularity != DefaultPopularity {
		if popularity == NewHeadlines || popularity == RisingHeadlines {
			url = fmt.Sprintf("%snew.json?sort=%s", url, popularity)
		} else {
			url = fmt.Sprintf("%s%s.json?sort=%s", url, popularity, popularity)
		}
	} else {
		url = fmt.Sprintf("%s.json", url)
	}

	if age != DefaultAge {
		if popularity != DefaultPopularity {
			url = fmt.Sprintf("%s&t=%s", url, age)
		} else {
			url = fmt.Sprintf("%s?t=%s", url, age)
		}
	}

	body, err := getResponse(url, nil, nil)
	if err != nil {
		return nil, err
	}

	type Response struct {
		Data struct {
			Children []struct {
				Data *Headline
			}
		}
	}

	r := new(Response)
	err = json.NewDecoder(body).Decode(r)
	if err != nil {
		return nil, err
	}

	headlines := make(Headlines, len(r.Data.Children))
	for i, child := range r.Data.Children {
		headlines[i] = child.Data
	}

	return headlines, nil

}
<|endoftext|>


<commit_before>// Copyright 2017 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build linux

package profiler

import (
	"bytes"
	"context"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"testing"
	"text/template"
	"time"

	"cloud.google.com/go/profiler/proftest"
	"golang.org/x/oauth2/google"
	compute "google.golang.org/api/compute/v1"
)

var runBackoffTest = flag.Bool("run_only_profiler_backoff_test", false, "Enables only the backoff integration test. This integration test requires over 45 mins to run, so it is not run by default.")

const (
	cloudScope        = "https://www.googleapis.com/auth/cloud-platform"
	benchFinishString = "benchmark application(s) complete"
	errorString       = "failed to set up or run the benchmark"
	gceBenchDuration  = 200 * time.Second
	gceTestTimeout    = 25 * time.Minute

	// For any agents to receive backoff, there must be more than 32 agents in
	// the deployment. The initial backoff received will be 33 minutes; each
	// subsequent backoff will be one minute longer. Running 45 benchmarks for
	// 45 minutes will ensure that several agents receive backoff responses and
	// are able to wait for the backoff duration then send another request.
	numBackoffBenchmarks = 45
	backoffBenchDuration = 45 * time.Minute
	backoffTestTimeout   = 60 * time.Minute
)

const startupTemplate = `
{{ define "setup"}}
# Install git
retry apt-get update >/dev/null
retry apt-get -y -q install git >/dev/null

# $GOCACHE is required from Go 1.12. See https://golang.org/doc/go1.11#gocache
# $GOCACHE is explicitly set because $HOME is not set when this code runs
mkdir -p /tmp/gocache
export GOCACHE=/tmp/gocache

# Install gcc, needed to install go master
if [ "{{.GoVersion}}" = "master" ]
then
retry apt-get -y -q install gcc >/dev/null
fi

# Install desired Go version
mkdir -p /tmp/bin
retry curl -sL -o /tmp/bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme
chmod +x /tmp/bin/gimme
export PATH=$PATH:/tmp/bin

retry eval "$(gimme {{.GoVersion}})"

# Set $GOPATH
export GOPATH="$HOME/go"

export GOCLOUD_HOME=$GOPATH/src/cloud.google.com/go
mkdir -p $GOCLOUD_HOME

# Install agent
retry git clone https://code.googlesource.com/gocloud $GOCLOUD_HOME >/dev/null
cd $GOCLOUD_HOME
retry git fetch origin {{.Commit}}
git reset --hard {{.Commit}}

cd $GOCLOUD_HOME/profiler/busybench
retry go get >/dev/null
{{- end }}

{{ define "integration" -}}
{{- template "prologue" . }}
{{- template "setup" . }}

# Run benchmark with agent.
go run busybench.go --service="{{.Service}}" --duration={{.DurationSec}} --mutex_profiling="{{.MutexProfiling}}"

echo "{{.FinishString}}"

{{ template "epilogue" . -}}
{{ end }}

{{ define "integration_backoff" -}}
{{- template "prologue" . }}
{{- template "setup" . }}
# Do not display commands being run to simplify logging output.
set +x

# Run benchmarks with agent.
echo "Starting {{.NumBackoffBenchmarks}} benchmarks."
for (( i = 0; i < {{.NumBackoffBenchmarks}}; i++ )); do
	(go run busybench.go --service="{{.Service}}" --duration={{.DurationSec}} \
		--num_busyworkers=1) |& while read line; \
		do echo "benchmark $i: ${line}"; done &
done
echo "Successfully started {{.NumBackoffBenchmarks}} benchmarks."

wait

# Continue displaying commands being run.
set -x

echo "{{.FinishString}}"

{{ template "epilogue" . -}}
{{ end }}
`

type goGCETestCase struct {
	proftest.InstanceConfig
	name          string
	goVersion     string
	benchDuration time.Duration
	timeout       time.Duration

	backoffTest bool

	// mutexProfiling and wantProfileTypes will not be used when the test
	// is a backoff integration test.
	mutexProfiling   bool
	wantProfileTypes []string
}

func (tc *goGCETestCase) initializeStartupScript(template *template.Template, commit string) error {
	params := struct {
		Service              string
		GoVersion            string
		Commit               string
		ErrorString          string
		FinishString         string
		MutexProfiling       bool
		DurationSec          int
		NumBackoffBenchmarks int
	}{
		Service:        tc.InstanceConfig.Name,
		GoVersion:      tc.goVersion,
		Commit:         commit,
		ErrorString:    errorString,
		FinishString:   benchFinishString,
		MutexProfiling: tc.mutexProfiling,
		DurationSec:    int(tc.benchDuration.Seconds()),
	}

	testTemplate := "integration"
	if tc.backoffTest {
		testTemplate = "integration_backoff"
		params.DurationSec = int(backoffBenchDuration.Seconds())
		params.NumBackoffBenchmarks = numBackoffBenchmarks
	}
	var buf bytes.Buffer
	err := template.Lookup(testTemplate).Execute(&buf, params)
	if err != nil {
		return fmt.Errorf("failed to render startup script for %s: %v", tc.name, err)
	}
	tc.StartupScript = buf.String()
	return nil
}

// gitCommit returns the Git commit of the current directory. The source
// checkout in the test VM will run in the same commit. Note that any local
// changes to the profiler agent won't be tested in the integration test. This
// flow only works with code that has been committed and pushed to the public
// repo (either to master or to a branch).
func gitCommit() (string, error) {
	output, err := exec.Command("git", "rev-parse", "HEAD").CombinedOutput()
	if err != nil {
		return "", err
	}
	return strings.Trim(string(output), "\n"), nil
}

// pstTimeStr returns a string representation of the time in the PST timezone.
func pstTimeStr() (string, error) {
	pst, err := time.LoadLocation("America/Los_Angeles")
	if err != nil {
		return "", fmt.Errorf("failed to initialize PST location: %v", err)
	}
	return strings.Replace(time.Now().In(pst).Format("2006-01-02-15-04-05.000000-0700"), ".", "-", -1), nil
}

func TestAgentIntegration(t *testing.T) {
	// Testing against master requires building go code and may take up to 10 minutes.
	// Allow this test to run in parallel with other top level tests to avoid timeouts.
	t.Parallel()

	if testing.Short() {
		t.Skip("skipping profiler integration tests in short mode")
	}

	projectID := os.Getenv("GCLOUD_TESTS_GOLANG_PROJECT_ID")
	if projectID == "" {
		t.Skip("skipping profiler integration test when GCLOUD_TESTS_GOLANG_PROJECT_ID variable is not set")
	}

	// all us-west1 zones
	zones := []string{"us-west1-a", "us-west1-b", "us-west1-c"}

	commit, err := gitCommit()
	if err != nil {
		t.Fatalf("failed to gather the Git revision of the current source: %v", err)
	}
	t.Logf("using Git commit %q for the profiler integration test", commit)

	runID, err := pstTimeStr()
	if err != nil {
		t.Fatalf("failed to get current time to generate a run ID: %v", err)
	}

	ctx := context.Background()

	client, err := google.DefaultClient(ctx, cloudScope)
	if err != nil {
		t.Fatalf("failed to get default client: %v", err)
	}

	computeService, err := compute.New(client)
	if err != nil {
		t.Fatalf("failed to initialize compute service: %v", err)
	}

	template, err := proftest.BaseStartupTmpl.Parse(startupTemplate)
	if err != nil {
		t.Fatalf("failed to parse startup script template: %v", err)
	}

	tr := proftest.TestRunner{
		Client: client,
	}
	gceTr := proftest.GCETestRunner{
		TestRunner:     tr,
		ComputeService: computeService,
	}

	// Determine go version used by current test run
	goVersion := strings.TrimPrefix(runtime.Version(), "go")
	goVersionName := strings.Replace(goVersion, ".", "", -1)

	testcases := []goGCETestCase{
		{
			InstanceConfig: proftest.InstanceConfig{
				ProjectID:   projectID,
				Name:        fmt.Sprintf("profiler-test-gomaster-%s", runID),
				MachineType: "n1-standard-1",
			},
			name:             "profiler-test-gomaster",
			wantProfileTypes: []string{"CPU", "HEAP", "THREADS", "CONTENTION", "HEAP_ALLOC"},
			goVersion:        "master",
			mutexProfiling:   true,
			timeout:          gceTestTimeout,
			benchDuration:    gceBenchDuration,
		},
		{
			InstanceConfig: proftest.InstanceConfig{
				ProjectID:   projectID,
				Name:        fmt.Sprintf("profiler-test-go%s-%s", goVersionName, runID),
				MachineType: "n1-standard-1",
			},
			name:             fmt.Sprintf("profiler-test-go%s", goVersionName),
			wantProfileTypes: []string{"CPU", "HEAP", "THREADS", "CONTENTION", "HEAP_ALLOC"},
			goVersion:        goVersion,
			mutexProfiling:   true,
			timeout:          gceTestTimeout,
			benchDuration:    gceBenchDuration,
		},
	}

	if *runBackoffTest {
		testcases = []goGCETestCase{
			{
				InstanceConfig: proftest.InstanceConfig{
					ProjectID: projectID,
					Name:      fmt.Sprintf("profiler-backoff-test-go%s-%s", goVersionName, runID),

					// Running many copies of the benchmark requires more
					// memory than is available on an n1-standard-1. Use a
					// machine type with more memory for backoff test.
					MachineType: "n1-highmem-2",
				},
				name:          fmt.Sprintf("profiler-backoff-test-go%s", goVersionName),
				goVersion:     goVersion,
				backoffTest:   true,
				timeout:       backoffTestTimeout,
				benchDuration: backoffBenchDuration,
			},
		}
	}
	// The number of tests run in parallel is the current value of GOMAXPROCS.
	runtime.GOMAXPROCS(len(testcases))
	for _, tc := range testcases {
		tc := tc // capture range variable
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			if err := tc.initializeStartupScript(template, commit); err != nil {
				t.Fatalf("failed to initialize startup script")
			}

			for i := range zones {
				tc.InstanceConfig.Zone = zones[i]
				if err := gceTr.StartInstance(ctx, &tc.InstanceConfig); err != nil {
					if strings.Contains(err.Error(), "failed to create instance") && i < (len(zones)-1) {
						// try other zones if instance failed to create
						continue
					}
					t.Fatal(err)
				}
				break
			}
			defer func() {
				if err := gceTr.DeleteInstance(ctx, &tc.InstanceConfig); err != nil {
					t.Fatal(err)
				}
			}()

			timeoutCtx, cancel := context.WithTimeout(ctx, tc.timeout)
			defer cancel()
			output, err := gceTr.PollAndLogSerialPort(timeoutCtx, &tc.InstanceConfig, benchFinishString, errorString, t.Logf)
			if err != nil {
				t.Fatalf("PollAndLogSerialPort() got error: %v", err)
			}

			if tc.backoffTest {
				if err := proftest.CheckSerialOutputForBackoffs(output, numBackoffBenchmarks, "action throttled, backoff", "creating a new profile via profiler service", "benchmark"); err != nil {
					t.Errorf("failed to check serial output for backoffs: %v", err)
				}
			}

			timeNow := time.Now()
			endTime := timeNow.Format(time.RFC3339)
			startTime := timeNow.Add(-1 * time.Hour).Format(time.RFC3339)
			for _, pType := range tc.wantProfileTypes {
				pr, err := tr.QueryProfilesWithZone(tc.ProjectID, tc.InstanceConfig.Name, startTime, endTime, pType, tc.Zone)
				if err != nil {
					t.Errorf("QueryProfilesWithZone(%s, %s, %s, %s, %s, %s) got error: %v", tc.ProjectID, tc.InstanceConfig.Name, startTime, endTime, pType, tc.Zone, err)
					continue
				}
				if err := pr.HasFunction("busywork"); err != nil {
					t.Errorf("HasFunction(%s, %s, %s, %s, %s) got error: %v", tc.ProjectID, tc.InstanceConfig.Name, startTime, endTime, pType, err)
				}
			}
		})
	}
}
<commit_msg>test(profiler): properly retry gimme installation (#3754)<commit_after>// Copyright 2017 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build linux

package profiler

import (
	"bytes"
	"context"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"testing"
	"text/template"
	"time"

	"cloud.google.com/go/profiler/proftest"
	"golang.org/x/oauth2/google"
	compute "google.golang.org/api/compute/v1"
)

var runBackoffTest = flag.Bool("run_only_profiler_backoff_test", false, "Enables only the backoff integration test. This integration test requires over 45 mins to run, so it is not run by default.")

const (
	cloudScope        = "https://www.googleapis.com/auth/cloud-platform"
	benchFinishString = "benchmark application(s) complete"
	errorString       = "failed to set up or run the benchmark"
	gceBenchDuration  = 200 * time.Second
	gceTestTimeout    = 25 * time.Minute

	// For any agents to receive backoff, there must be more than 32 agents in
	// the deployment. The initial backoff received will be 33 minutes; each
	// subsequent backoff will be one minute longer. Running 45 benchmarks for
	// 45 minutes will ensure that several agents receive backoff responses and
	// are able to wait for the backoff duration then send another request.
	numBackoffBenchmarks = 45
	backoffBenchDuration = 45 * time.Minute
	backoffTestTimeout   = 60 * time.Minute
)

const startupTemplate = `
{{ define "setup"}}
# Install git
retry apt-get update >/dev/null
retry apt-get -y -q install git >/dev/null

# $GOCACHE is required from Go 1.12. See https://golang.org/doc/go1.11#gocache
# $GOCACHE is explicitly set because $HOME is not set when this code runs
mkdir -p /tmp/gocache
export GOCACHE=/tmp/gocache

# Install gcc, needed to install go master
if [ "{{.GoVersion}}" = "master" ]
then
retry apt-get -y -q install gcc >/dev/null
fi

# Install desired Go version
mkdir -p /tmp/bin
retry curl -sL -o /tmp/bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme
chmod +x /tmp/bin/gimme
export PATH=$PATH:/tmp/bin

gimme_retrier() {
  eval "$(gimme {{.GoVersion}})"
  which go # To retry on non-zero code if go not installed.
}
retry gimme_retrier

# Set $GOPATH
export GOPATH="$HOME/go"

export GOCLOUD_HOME=$GOPATH/src/cloud.google.com/go
mkdir -p $GOCLOUD_HOME

# Install agent
retry git clone https://code.googlesource.com/gocloud $GOCLOUD_HOME >/dev/null
cd $GOCLOUD_HOME
retry git fetch origin {{.Commit}}
git reset --hard {{.Commit}}

cd $GOCLOUD_HOME/profiler/busybench
retry go get >/dev/null
{{- end }}

{{ define "integration" -}}
{{- template "prologue" . }}
{{- template "setup" . }}

# Run benchmark with agent.
go run busybench.go --service="{{.Service}}" --duration={{.DurationSec}} --mutex_profiling="{{.MutexProfiling}}"

echo "{{.FinishString}}"

{{ template "epilogue" . -}}
{{ end }}

{{ define "integration_backoff" -}}
{{- template "prologue" . }}
{{- template "setup" . }}
# Do not display commands being run to simplify logging output.
set +x

# Run benchmarks with agent.
echo "Starting {{.NumBackoffBenchmarks}} benchmarks."
for (( i = 0; i < {{.NumBackoffBenchmarks}}; i++ )); do
	(go run busybench.go --service="{{.Service}}" --duration={{.DurationSec}} \
		--num_busyworkers=1) |& while read line; \
		do echo "benchmark $i: ${line}"; done &
done
echo "Successfully started {{.NumBackoffBenchmarks}} benchmarks."

wait

# Continue displaying commands being run.
set -x

echo "{{.FinishString}}"

{{ template "epilogue" . -}}
{{ end }}
`

type goGCETestCase struct {
	proftest.InstanceConfig
	name          string
	goVersion     string
	benchDuration time.Duration
	timeout       time.Duration

	backoffTest bool

	// mutexProfiling and wantProfileTypes will not be used when the test
	// is a backoff integration test.
	mutexProfiling   bool
	wantProfileTypes []string
}

func (tc *goGCETestCase) initializeStartupScript(template *template.Template, commit string) error {
	params := struct {
		Service              string
		GoVersion            string
		Commit               string
		ErrorString          string
		FinishString         string
		MutexProfiling       bool
		DurationSec          int
		NumBackoffBenchmarks int
	}{
		Service:        tc.InstanceConfig.Name,
		GoVersion:      tc.goVersion,
		Commit:         commit,
		ErrorString:    errorString,
		FinishString:   benchFinishString,
		MutexProfiling: tc.mutexProfiling,
		DurationSec:    int(tc.benchDuration.Seconds()),
	}

	testTemplate := "integration"
	if tc.backoffTest {
		testTemplate = "integration_backoff"
		params.DurationSec = int(backoffBenchDuration.Seconds())
		params.NumBackoffBenchmarks = numBackoffBenchmarks
	}
	var buf bytes.Buffer
	err := template.Lookup(testTemplate).Execute(&buf, params)
	if err != nil {
		return fmt.Errorf("failed to render startup script for %s: %v", tc.name, err)
	}
	tc.StartupScript = buf.String()
	return nil
}

// gitCommit returns the Git commit of the current directory. The source
// checkout in the test VM will run in the same commit. Note that any local
// changes to the profiler agent won't be tested in the integration test. This
// flow only works with code that has been committed and pushed to the public
// repo (either to master or to a branch).
func gitCommit() (string, error) {
	output, err := exec.Command("git", "rev-parse", "HEAD").CombinedOutput()
	if err != nil {
		return "", err
	}
	return strings.Trim(string(output), "\n"), nil
}

// pstTimeStr returns a string representation of the time in the PST timezone.
func pstTimeStr() (string, error) {
	pst, err := time.LoadLocation("America/Los_Angeles")
	if err != nil {
		return "", fmt.Errorf("failed to initialize PST location: %v", err)
	}
	return strings.Replace(time.Now().In(pst).Format("2006-01-02-15-04-05.000000-0700"), ".", "-", -1), nil
}

func TestAgentIntegration(t *testing.T) {
	// Testing against master requires building go code and may take up to 10 minutes.
	// Allow this test to run in parallel with other top level tests to avoid timeouts.
	t.Parallel()

	if testing.Short() {
		t.Skip("skipping profiler integration tests in short mode")
	}

	projectID := os.Getenv("GCLOUD_TESTS_GOLANG_PROJECT_ID")
	if projectID == "" {
		t.Skip("skipping profiler integration test when GCLOUD_TESTS_GOLANG_PROJECT_ID variable is not set")
	}

	// all us-west1 zones
	zones := []string{"us-west1-a", "us-west1-b", "us-west1-c"}

	commit, err := gitCommit()
	if err != nil {
		t.Fatalf("failed to gather the Git revision of the current source: %v", err)
	}
	t.Logf("using Git commit %q for the profiler integration test", commit)

	runID, err := pstTimeStr()
	if err != nil {
		t.Fatalf("failed to get current time to generate a run ID: %v", err)
	}

	ctx := context.Background()

	client, err := google.DefaultClient(ctx, cloudScope)
	if err != nil {
		t.Fatalf("failed to get default client: %v", err)
	}

	computeService, err := compute.New(client)
	if err != nil {
		t.Fatalf("failed to initialize compute service: %v", err)
	}

	template, err := proftest.BaseStartupTmpl.Parse(startupTemplate)
	if err != nil {
		t.Fatalf("failed to parse startup script template: %v", err)
	}

	tr := proftest.TestRunner{
		Client: client,
	}
	gceTr := proftest.GCETestRunner{
		TestRunner:     tr,
		ComputeService: computeService,
	}

	// Determine go version used by current test run
	goVersion := strings.TrimPrefix(runtime.Version(), "go")
	goVersionName := strings.Replace(goVersion, ".", "", -1)

	testcases := []goGCETestCase{
		{
			InstanceConfig: proftest.InstanceConfig{
				ProjectID:   projectID,
				Name:        fmt.Sprintf("profiler-test-gomaster-%s", runID),
				MachineType: "n1-standard-1",
			},
			name:             "profiler-test-gomaster",
			wantProfileTypes: []string{"CPU", "HEAP", "THREADS", "CONTENTION", "HEAP_ALLOC"},
			goVersion:        "master",
			mutexProfiling:   true,
			timeout:          gceTestTimeout,
			benchDuration:    gceBenchDuration,
		},
		{
			InstanceConfig: proftest.InstanceConfig{
				ProjectID:   projectID,
				Name:        fmt.Sprintf("profiler-test-go%s-%s", goVersionName, runID),
				MachineType: "n1-standard-1",
			},
			name:             fmt.Sprintf("profiler-test-go%s", goVersionName),
			wantProfileTypes: []string{"CPU", "HEAP", "THREADS", "CONTENTION", "HEAP_ALLOC"},
			goVersion:        goVersion,
			mutexProfiling:   true,
			timeout:          gceTestTimeout,
			benchDuration:    gceBenchDuration,
		},
	}

	if *runBackoffTest {
		testcases = []goGCETestCase{
			{
				InstanceConfig: proftest.InstanceConfig{
					ProjectID: projectID,
					Name:      fmt.Sprintf("profiler-backoff-test-go%s-%s", goVersionName, runID),

					// Running many copies of the benchmark requires more
					// memory than is available on an n1-standard-1. Use a
					// machine type with more memory for backoff test.
					MachineType: "n1-highmem-2",
				},
				name:          fmt.Sprintf("profiler-backoff-test-go%s", goVersionName),
				goVersion:     goVersion,
				backoffTest:   true,
				timeout:       backoffTestTimeout,
				benchDuration: backoffBenchDuration,
			},
		}
	}
	// The number of tests run in parallel is the current value of GOMAXPROCS.
	runtime.GOMAXPROCS(len(testcases))
	for _, tc := range testcases {
		tc := tc // capture range variable
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			if err := tc.initializeStartupScript(template, commit); err != nil {
				t.Fatalf("failed to initialize startup script")
			}

			for i := range zones {
				tc.InstanceConfig.Zone = zones[i]
				if err := gceTr.StartInstance(ctx, &tc.InstanceConfig); err != nil {
					if strings.Contains(err.Error(), "failed to create instance") && i < (len(zones)-1) {
						// try other zones if instance failed to create
						continue
					}
					t.Fatal(err)
				}
				break
			}
			defer func() {
				if err := gceTr.DeleteInstance(ctx, &tc.InstanceConfig); err != nil {
					t.Fatal(err)
				}
			}()

			timeoutCtx, cancel := context.WithTimeout(ctx, tc.timeout)
			defer cancel()
			output, err := gceTr.PollAndLogSerialPort(timeoutCtx, &tc.InstanceConfig, benchFinishString, errorString, t.Logf)
			if err != nil {
				t.Fatalf("PollAndLogSerialPort() got error: %v", err)
			}

			if tc.backoffTest {
				if err := proftest.CheckSerialOutputForBackoffs(output, numBackoffBenchmarks, "action throttled, backoff", "creating a new profile via profiler service", "benchmark"); err != nil {
					t.Errorf("failed to check serial output for backoffs: %v", err)
				}
			}

			timeNow := time.Now()
			endTime := timeNow.Format(time.RFC3339)
			startTime := timeNow.Add(-1 * time.Hour).Format(time.RFC3339)
			for _, pType := range tc.wantProfileTypes {
				pr, err := tr.QueryProfilesWithZone(tc.ProjectID, tc.InstanceConfig.Name, startTime, endTime, pType, tc.Zone)
				if err != nil {
					t.Errorf("QueryProfilesWithZone(%s, %s, %s, %s, %s, %s) got error: %v", tc.ProjectID, tc.InstanceConfig.Name, startTime, endTime, pType, tc.Zone, err)
					continue
				}
				if err := pr.HasFunction("busywork"); err != nil {
					t.Errorf("HasFunction(%s, %s, %s, %s, %s) got error: %v", tc.ProjectID, tc.InstanceConfig.Name, startTime, endTime, pType, err)
				}
			}
		})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"

	"github.com/michiwend/goefa"
)

func main() {
	efa := *goefa.Providers["avv"]

	stop := flag.String("stop", "Königsplatz", "id or (part of the) stop name")
	max_results := flag.Int("results", 5, "how many results to show")
	flag.Parse()

	station, err := efa.FindStation(*stop)
	if err != nil {
		fmt.Println("Stop does not exist or name is not unique!")
		return
	}
	fmt.Printf("Selected stop: %s (%d)\n\n",
		station.IdfdStop.StopName,
		station.IdfdStop.StopID)

	departures, err := efa.Departures(station, *max_results)
	if err != nil {
		fmt.Println("Could not retrieve departure times!")
		return
	}
	for _, departure := range departures {
		plu := " "
		if departure.Countdown != 1 {
			plu = "s"
		}

		fmt.Printf("Route %-5s due in %-2d minute%s --> %s\n",
			departure.ServingLine.Number,
			departure.Countdown,
			plu,
			departure.ServingLine.Direction)
	}
}
<commit_msg>* Fixed departure_monitor.go<commit_after>package main

import (
	"flag"
	"fmt"

	"github.com/michiwend/goefa"
)

func main() {
	efa := goefa.Providers["avv"]

	stop := flag.String("stop", "Königsplatz", "id or (part of the) stop name")
	max_results := flag.Int("results", 5, "how many results to show")
	flag.Parse()

	station, err := efa.FindStop(*stop)
	if err != nil {
		fmt.Println("Stop does not exist or name is not unique!")
		return
	}
	fmt.Printf("Selected stop: %s (%d)\n\n",
		station.IdfdStop.StopName,
		station.IdfdStop.StopID)

	departures, err := efa.Departures(station, *max_results)
	if err != nil {
		fmt.Println("Could not retrieve departure times!")
		return
	}
	for _, departure := range departures {
		plu := " "
		if departure.Countdown != 1 {
			plu = "s"
		}

		fmt.Printf("Route %-5s due in %-2d minute%s --> %s\n",
			departure.ServingLine.Number,
			departure.Countdown,
			plu,
			departure.ServingLine.Direction)
	}
}
<|endoftext|>


<commit_before>//   Copyright 2019 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package reports

import (
	"context"
	"database/sql"
	"strconv"
	"time"

	"github.com/360EntSecGroup-Skylar/excelize"
	"github.com/trackit/jsonlog"

	"github.com/trackit/trackit-server/aws"
	"github.com/trackit/trackit-server/aws/usageReports/history"
	"github.com/trackit/trackit-server/s3/costs"
)

const s3CostReportSheetName = "S3 Cost Report"

var s3CostReportModule = module{
	Name:          "S3 Cost Report",
	SheetName:     s3CostReportSheetName,
	ErrorName:     "s3CostReportError",
	GenerateSheet: generateS3CostReportSheet,
}

func generateS3CostReportSheet(ctx context.Context, aas []aws.AwsAccount, date time.Time, tx *sql.Tx, file *excelize.File) (err error) {
	if date.IsZero() {
		date, _ = history.GetHistoryDate()
	}
	return s3CostReportGenerateSheet(ctx, aas, date, tx, file)
}

func s3CostReportGenerateSheet(ctx context.Context, aas []aws.AwsAccount, date time.Time, tx *sql.Tx, file *excelize.File) (err error) {
	data, err := s3CostReportGetData(ctx, aas, date, tx)
	if err == nil {
		return s3CostReportInsertDataInSheet(ctx, aas, file, data)
	} else {
		return
	}
}

func s3CostReportGetData(ctx context.Context, aas []aws.AwsAccount, date time.Time, tx *sql.Tx) (reports costs.BucketsInfo, err error) {
	logger := jsonlog.LoggerFromContextOrDefault(ctx)

	identities := getAwsIdentities(aas)

	/*user, err := users.GetUserWithId(tx, aas[0].UserId)
	if err != nil {												//user isn't use right now
		return
	}*/

	parameters := costs.S3QueryParams{
		AccountList: identities,
		DateBegin:        date,
		DateEnd:          time.Date(date.Year(), date.Month()+1, 0, 23, 59, 59, 999999999, date.Location()).UTC(),
	}

	logger.Debug("Getting S3 Cost Report for accounts", map[string]interface{}{
		"accounts": aas,
		"date":     date,
	})
	_, reports, err = costs.GetS3CostData(ctx, parameters)
	if err != nil {
		logger.Error("An error occurred while generating an S3 Cost Report", map[string]interface{}{
			"error":    err,
			"accounts": aas,
			"date":     date,
		})
	}
	return
}

func s3CostReportInsertDataInSheet(_ context.Context, aas []aws.AwsAccount, file *excelize.File, data costs.BucketsInfo) (err error) {
	file.NewSheet(s3CostReportSheetName)
	s3CostReportGenerateHeader(file)
	line := 3
	for idx, report := range data {
		account := getAwsAccount(idx/*report.Account*/, aas)
		formattedAccount := idx//report.Account
		if account != nil {
			formattedAccount = formatAwsAccount(*account)
		}
		//instance := report.Instance
		/*name := ""
		if value, ok := instance.Tags["Name"]; ok {
			name = value
		}*/
		//tags := formatTags(instance.Tags)
		cells := cells{
			newCell(formattedAccount, "B"+strconv.Itoa(line)),
			//newCell(instance.Id, "B"+strconv.Itoa(line)),
			newCell(report.GbMonth/*name*/, "C"+strconv.Itoa(line)).addStyles("Gb"),
			newCell(report.StorageCost/*instance.Type*/, "D"+strconv.Itoa(line)).addStyles("price"),
			newCell(report.BandwidthCost/*instance.Region*/, "E"+strconv.Itoa(line)).addStyles("price"),
			newCell(report.RequestsCost/*instance.Purchasing*/, "F"+strconv.Itoa(line)).addStyles("price"),
			//newCell(getTotal(instance.Costs), "G"+strconv.Itoa(line)).addStyles("price"), // TOTAL
			newCell(report.DataIn/*formatMetric(instance.Stats.Network.In)*/, "H"+strconv.Itoa(line)),
			newCell(report.DataOut/*formatMetric(instance.Stats.Network.Out)*/, "I"+strconv.Itoa(line)),
			//newCell(getTotal(instance.Stats.Volumes.Read), "L"+strconv.Itoa(line)),
			//newCell(getTotal(instance.Stats.Volumes.Write), "M"+strconv.Itoa(line)),
			//newCell(instance.KeyPair, "N"+strconv.Itoa(line)),
			//newCell(strings.Join(tags, ";"), "O"+strconv.Itoa(line)),
		}
		cells.addStyles("borders", "centerText").setValues(file, s3CostReportSheetName)
		line++
	}
	return
}

func s3CostReportGenerateHeader(file *excelize.File) {
	header := cells{
		newCell("Account", "A1").mergeTo("A2"),
		newCell("Name", "B1").mergeTo("B2"),
		newCell("Billable Size", "C1").mergeTo("C2"),
		newCell("Cost", "D1").mergeTo("G1"),
		newCell("Storage", "D2") ,
		newCell("Bandwidth", "E2"),
		newCell("Requests", "F2"),
		newCell("Total", "G2"),
		newCell("Data transfers", "H1").mergeTo("I1"),
		newCell("In", "H2"),
		newCell("Out", "I2"),
	}
	header.addStyles("borders", "bold", "centerText").setValues(file, s3CostReportSheetName)
	columns := columnsWidth{
		newColumnWidth("A", 30),
		newColumnWidth("B", 50),
		newColumnWidth("C", 20).toColumn("I"),
	}
	columns.setValues(file, s3CostReportSheetName)
	return
}
<commit_msg>adding total and GB to Billable Size on S3 Cost report<commit_after>//   Copyright 2019 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package reports

import (
	"context"
	"database/sql"
	"strconv"
	"time"

	"github.com/360EntSecGroup-Skylar/excelize"
	"github.com/trackit/jsonlog"

	"github.com/trackit/trackit-server/aws"
	"github.com/trackit/trackit-server/aws/usageReports/history"
	"github.com/trackit/trackit-server/s3/costs"
)

const s3CostReportSheetName = "S3 Cost Report"

var s3CostReportModule = module{
	Name:          "S3 Cost Report",
	SheetName:     s3CostReportSheetName,
	ErrorName:     "s3CostReportError",
	GenerateSheet: generateS3CostReportSheet,
}

func generateS3CostReportSheet(ctx context.Context, aas []aws.AwsAccount, date time.Time, tx *sql.Tx, file *excelize.File) (err error) {
	if date.IsZero() {
		date, _ = history.GetHistoryDate()
	}
	return s3CostReportGenerateSheet(ctx, aas, date, tx, file)
}

func s3CostReportGenerateSheet(ctx context.Context, aas []aws.AwsAccount, date time.Time, tx *sql.Tx, file *excelize.File) (err error) {
	data, err := s3CostReportGetData(ctx, aas, date, tx)
	if err == nil {
		return s3CostReportInsertDataInSheet(ctx, aas, file, data)
	} else {
		return
	}
}

func s3CostReportGetData(ctx context.Context, aas []aws.AwsAccount, date time.Time, tx *sql.Tx) (reports costs.BucketsInfo, err error) {
	logger := jsonlog.LoggerFromContextOrDefault(ctx)

	identities := getAwsIdentities(aas)

	/*user, err := users.GetUserWithId(tx, aas[0].UserId)
	if err != nil {												//user isn't use right now
		return
	}*/

	parameters := costs.S3QueryParams{
		AccountList: identities,
		DateBegin:        date,
		DateEnd:          time.Date(date.Year(), date.Month()+1, 0, 23, 59, 59, 999999999, date.Location()).UTC(),
	}

	logger.Debug("Getting S3 Cost Report for accounts", map[string]interface{}{
		"accounts": aas,
		"date":     date,
	})
	_, reports, err = costs.GetS3CostData(ctx, parameters)
	if err != nil {
		logger.Error("An error occurred while generating an S3 Cost Report", map[string]interface{}{
			"error":    err,
			"accounts": aas,
			"date":     date,
		})
	}
	return
}

func s3CostReportInsertDataInSheet(_ context.Context, aas []aws.AwsAccount, file *excelize.File, data costs.BucketsInfo) (err error) {
	file.NewSheet(s3CostReportSheetName)
	s3CostReportGenerateHeader(file)
	line := 3
	for idx, report := range data {
		account := getAwsAccount(idx, aas)
		formattedAccount := idx
		if account != nil {
			formattedAccount = formatAwsAccount(*account)
		}
		//instance := report.Instance
		/*name := ""
		if value, ok := instance.Tags["Name"]; ok {
			name = value
		}*/
		//tags := formatTags(instance.Tags)
		total := make(map[string]float64)
		total["Storage"] = report.StorageCost
		total["BandwidthCost"] = report.BandwidthCost
		total["RequestCost"] = report.RequestsCost
		cells := cells{
			//newCell(instance.Id, "B"+strconv.Itoa(line)), // ACCOUNT
			newCell(formattedAccount, "B"+strconv.Itoa(line)),
			newCell(report.GbMonth, "C"+strconv.Itoa(line)).addStyles("Gb"),
			newCell(report.StorageCost, "D"+strconv.Itoa(line)).addStyles("price"),
			newCell(report.BandwidthCost, "E"+strconv.Itoa(line)).addStyles("price"),
			newCell(report.RequestsCost, "F"+strconv.Itoa(line)).addStyles("price"),
			newCell(getTotal(total), "G"+strconv.Itoa(line)).addStyles("price"),
			newCell(report.DataIn, "H"+strconv.Itoa(line)),
			newCell(report.DataOut, "I"+strconv.Itoa(line)),
		}
		cells.addStyles("borders", "centerText").setValues(file, s3CostReportSheetName)
		line++
	}
	return
}

func s3CostReportGenerateHeader(file *excelize.File) {
	header := cells{
		newCell("Account", "A1").mergeTo("A2"),
		newCell("Name", "B1").mergeTo("B2"),
		newCell("Billable Size (GB)", "C1").mergeTo("C2"),
		newCell("Cost", "D1").mergeTo("G1"),
		newCell("Storage", "D2") ,
		newCell("Bandwidth", "E2"),
		newCell("Requests", "F2"),
		newCell("Total", "G2"),
		newCell("Data transfers", "H1").mergeTo("I1"),
		newCell("In", "H2"),
		newCell("Out", "I2"),
	}
	header.addStyles("borders", "bold", "centerText").setValues(file, s3CostReportSheetName)
	columns := columnsWidth{
		newColumnWidth("A", 30),
		newColumnWidth("B", 50),
		newColumnWidth("C", 20).toColumn("I"),
	}
	columns.setValues(file, s3CostReportSheetName)
	return
}
<|endoftext|>


<commit_before>// +build windows

/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mount

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"k8s.io/klog/v2"
	utilexec "k8s.io/utils/exec"
	"k8s.io/utils/keymutex"
	utilpath "k8s.io/utils/path"
)

// Mounter provides the default implementation of mount.Interface
// for the windows platform.  This implementation assumes that the
// kubelet is running in the host's root mount namespace.
type Mounter struct {
	mounterPath string
}

// New returns a mount.Interface for the current system.
// It provides options to override the default mounter behavior.
// mounterPath allows using an alternative to `/bin/mount` for mounting.
func New(mounterPath string) Interface {
	return &Mounter{
		mounterPath: mounterPath,
	}
}

// acquire lock for smb mount
var getSMBMountMutex = keymutex.NewHashed(0)

// Mount : mounts source to target with given options.
// currently only supports cifs(smb), bind mount(for disk)
func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error {
	return mounter.MountSensitive(source, target, fstype, options, nil /* sensitiveOptions */)
}

// MountSensitive is the same as Mount() but this method allows
// sensitiveOptions to be passed in a separate parameter from the normal
// mount options and ensures the sensitiveOptions are never logged. This
// method should be used by callers that pass sensitive material (like
// passwords) as mount options.
func (mounter *Mounter) MountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error {
	target = NormalizeWindowsPath(target)
	sanitizedOptionsForLogging := sanitizedOptionsForLogging(options, sensitiveOptions)

	if source == "tmpfs" {
		klog.V(3).Infof("mounting source (%q), target (%q), with options (%q)", source, target, sanitizedOptionsForLogging)
		return os.MkdirAll(target, 0755)
	}

	parentDir := filepath.Dir(target)
	if err := os.MkdirAll(parentDir, 0755); err != nil {
		return err
	}

	klog.V(4).Infof("mount options(%q) source:%q, target:%q, fstype:%q, begin to mount",
		sanitizedOptionsForLogging, source, target, fstype)
	bindSource := source

	if bind, _, _, _ := MakeBindOptsSensitive(options, sensitiveOptions); bind {
		bindSource = NormalizeWindowsPath(source)
	} else {
		allOptions := []string{}
		allOptions = append(allOptions, options...)
		allOptions = append(allOptions, sensitiveOptions...)
		if len(allOptions) < 2 {
			klog.Warningf("mount options(%q) command number(%d) less than 2, source:%q, target:%q, skip mounting",
				sanitizedOptionsForLogging, len(allOptions), source, target)
			return nil
		}

		// currently only cifs mount is supported
		if strings.ToLower(fstype) != "cifs" {
			return fmt.Errorf("only cifs mount is supported now, fstype: %q, mounting source (%q), target (%q), with options (%q)", fstype, source, target, sanitizedOptionsForLogging)
		}

		// lock smb mount for the same source
		getSMBMountMutex.LockKey(source)
		defer getSMBMountMutex.UnlockKey(source)

		if output, err := newSMBMapping(allOptions[0], allOptions[1], source); err != nil {
			if isSMBMappingExist(source) {
				klog.V(2).Infof("SMB Mapping(%s) already exists, now begin to remove and remount", source)
				if output, err := removeSMBMapping(source); err != nil {
					return fmt.Errorf("Remove-SmbGlobalMapping failed: %v, output: %q", err, output)
				}
				if output, err := newSMBMapping(allOptions[0], allOptions[1], source); err != nil {
					return fmt.Errorf("New-SmbGlobalMapping remount failed: %v, output: %q", err, output)
				}
			} else {
				return fmt.Errorf("New-SmbGlobalMapping failed: %v, output: %q", err, output)
			}
		}
	}

	if output, err := exec.Command("cmd", "/c", "mklink", "/D", target, bindSource).CombinedOutput(); err != nil {
		klog.Errorf("mklink failed: %v, source(%q) target(%q) output: %q", err, bindSource, target, string(output))
		return err
	}

	return nil
}

// do the SMB mount with username, password, remotepath
// return (output, error)
func newSMBMapping(username, password, remotepath string) (string, error) {
	if username == "" || password == "" || remotepath == "" {
		return "", fmt.Errorf("invalid parameter(username: %s, password: %s, remoteapth: %s)", username, sensitiveOptionsRemoved, remotepath)
	}

	// use PowerShell Environment Variables to store user input string to prevent command line injection
	// https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_environment_variables?view=powershell-5.1
	cmdLine := `$PWord = ConvertTo-SecureString -String $Env:smbpassword -AsPlainText -Force` +
		`;$Credential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $Env:smbuser, $PWord` +
		`;New-SmbGlobalMapping -RemotePath $Env:smbremotepath -Credential $Credential`
	cmd := exec.Command("powershell", "/c", cmdLine)
	cmd.Env = append(os.Environ(),
		fmt.Sprintf("smbuser=%s", username),
		fmt.Sprintf("smbpassword=%s", password),
		fmt.Sprintf("smbremotepath=%s", remotepath))

	output, err := cmd.CombinedOutput()
	return string(output), err
}

// check whether remotepath is already mounted
func isSMBMappingExist(remotepath string) bool {
	cmd := exec.Command("powershell", "/c", `Get-SmbGlobalMapping -RemotePath $Env:smbremotepath`)
	cmd.Env = append(os.Environ(), fmt.Sprintf("smbremotepath=%s", remotepath))
	_, err := cmd.CombinedOutput()
	return err == nil
}

// remove SMB mapping
func removeSMBMapping(remotepath string) (string, error) {
	cmd := exec.Command("powershell", "/c", `Remove-SmbGlobalMapping -RemotePath $Env:smbremotepath -Force`)
	cmd.Env = append(os.Environ(), fmt.Sprintf("smbremotepath=%s", remotepath))
	output, err := cmd.CombinedOutput()
	return string(output), err
}

// Unmount unmounts the target.
func (mounter *Mounter) Unmount(target string) error {
	klog.V(4).Infof("azureMount: Unmount target (%q)", target)
	target = NormalizeWindowsPath(target)
	if output, err := exec.Command("cmd", "/c", "rmdir", target).CombinedOutput(); err != nil {
		klog.Errorf("rmdir failed: %v, output: %q", err, string(output))
		return err
	}
	return nil
}

// List returns a list of all mounted filesystems. todo
func (mounter *Mounter) List() ([]MountPoint, error) {
	return []MountPoint{}, nil
}

// IsLikelyNotMountPoint determines if a directory is not a mountpoint.
func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
	stat, err := os.Lstat(file)
	if err != nil {
		return true, err
	}
	// If current file is a symlink, then it is a mountpoint.
	if stat.Mode()&os.ModeSymlink != 0 {
		target, err := os.Readlink(file)
		if err != nil {
			return true, fmt.Errorf("readlink error: %v", err)
		}
		exists, err := utilpath.Exists(utilpath.CheckFollowSymlink, target)
		if err != nil {
			return true, err
		}
		return !exists, nil
	}

	return true, nil
}

// GetMountRefs : empty implementation here since there is no place to query all mount points on Windows
func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
	windowsPath := NormalizeWindowsPath(pathname)
	pathExists, pathErr := PathExists(windowsPath)
	if !pathExists {
		return []string{}, nil
	} else if IsCorruptedMnt(pathErr) {
		klog.Warningf("GetMountRefs found corrupted mount at %s, treating as unmounted path", windowsPath)
		return []string{}, nil
	} else if pathErr != nil {
		return nil, fmt.Errorf("error checking path %s: %v", windowsPath, pathErr)
	}
	return []string{pathname}, nil
}

func (mounter *SafeFormatAndMount) formatAndMountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error {
	// Try to mount the disk
	klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, source, target)

	if err := ValidateDiskNumber(source); err != nil {
		klog.Errorf("diskMount: formatAndMount failed, err: %v", err)
		return err
	}

	if len(fstype) == 0 {
		// Use 'NTFS' as the default
		fstype = "NTFS"
	}

	// format disk if it is unformatted(raw)
	cmd := fmt.Sprintf("Get-Disk -Number %s | Where partitionstyle -eq 'raw' | Initialize-Disk -PartitionStyle MBR -PassThru"+
		" | New-Partition -AssignDriveLetter -UseMaximumSize | Format-Volume -FileSystem %s -Confirm:$false", source, fstype)
	if output, err := mounter.Exec.Command("powershell", "/c", cmd).CombinedOutput(); err != nil {
		return fmt.Errorf("diskMount: format disk failed, error: %v, output: %q", err, string(output))
	}
	klog.V(4).Infof("diskMount: Disk successfully formatted, disk: %q, fstype: %q", source, fstype)

	driveLetter, err := getDriveLetterByDiskNumber(source, mounter.Exec)
	if err != nil {
		return err
	}
	driverPath := driveLetter + ":"
	target = NormalizeWindowsPath(target)
	klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, driverPath, target)
	if output, err := mounter.Exec.Command("cmd", "/c", "mklink", "/D", target, driverPath).CombinedOutput(); err != nil {
		klog.Errorf("mklink failed: %v, output: %q", err, string(output))
		return err
	}
	return nil
}

// Get drive letter according to windows disk number
func getDriveLetterByDiskNumber(diskNum string, exec utilexec.Interface) (string, error) {
	cmd := fmt.Sprintf("(Get-Partition -DiskNumber %s).DriveLetter", diskNum)
	output, err := exec.Command("powershell", "/c", cmd).CombinedOutput()
	if err != nil {
		return "", fmt.Errorf("azureMount: Get Drive Letter failed: %v, output: %q", err, string(output))
	}
	if len(string(output)) < 1 {
		return "", fmt.Errorf("azureMount: Get Drive Letter failed, output is empty")
	}
	return string(output)[:1], nil
}

// getAllParentLinks walks all symbolic links and return all the parent targets recursively
func getAllParentLinks(path string) ([]string, error) {
	const maxIter = 255
	links := []string{}
	for {
		links = append(links, path)
		if len(links) > maxIter {
			return links, fmt.Errorf("unexpected length of parent links: %v", links)
		}

		fi, err := os.Lstat(path)
		if err != nil {
			return links, fmt.Errorf("Lstat: %v", err)
		}
		if fi.Mode()&os.ModeSymlink == 0 {
			break
		}

		path, err = os.Readlink(path)
		if err != nil {
			return links, fmt.Errorf("Readlink error: %v", err)
		}
	}

	return links, nil
}
<commit_msg>Remove driver letter assignment during volume format<commit_after>// +build windows

/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mount

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"k8s.io/klog/v2"
	"k8s.io/utils/keymutex"
	utilpath "k8s.io/utils/path"
)

// Mounter provides the default implementation of mount.Interface
// for the windows platform.  This implementation assumes that the
// kubelet is running in the host's root mount namespace.
type Mounter struct {
	mounterPath string
}

// New returns a mount.Interface for the current system.
// It provides options to override the default mounter behavior.
// mounterPath allows using an alternative to `/bin/mount` for mounting.
func New(mounterPath string) Interface {
	return &Mounter{
		mounterPath: mounterPath,
	}
}

// acquire lock for smb mount
var getSMBMountMutex = keymutex.NewHashed(0)

// Mount : mounts source to target with given options.
// currently only supports cifs(smb), bind mount(for disk)
func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error {
	return mounter.MountSensitive(source, target, fstype, options, nil /* sensitiveOptions */)
}

// MountSensitive is the same as Mount() but this method allows
// sensitiveOptions to be passed in a separate parameter from the normal
// mount options and ensures the sensitiveOptions are never logged. This
// method should be used by callers that pass sensitive material (like
// passwords) as mount options.
func (mounter *Mounter) MountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error {
	target = NormalizeWindowsPath(target)
	sanitizedOptionsForLogging := sanitizedOptionsForLogging(options, sensitiveOptions)

	if source == "tmpfs" {
		klog.V(3).Infof("mounting source (%q), target (%q), with options (%q)", source, target, sanitizedOptionsForLogging)
		return os.MkdirAll(target, 0755)
	}

	parentDir := filepath.Dir(target)
	if err := os.MkdirAll(parentDir, 0755); err != nil {
		return err
	}

	klog.V(4).Infof("mount options(%q) source:%q, target:%q, fstype:%q, begin to mount",
		sanitizedOptionsForLogging, source, target, fstype)
	bindSource := source

	if bind, _, _, _ := MakeBindOptsSensitive(options, sensitiveOptions); bind {
		bindSource = NormalizeWindowsPath(source)
	} else {
		allOptions := []string{}
		allOptions = append(allOptions, options...)
		allOptions = append(allOptions, sensitiveOptions...)
		if len(allOptions) < 2 {
			klog.Warningf("mount options(%q) command number(%d) less than 2, source:%q, target:%q, skip mounting",
				sanitizedOptionsForLogging, len(allOptions), source, target)
			return nil
		}

		// currently only cifs mount is supported
		if strings.ToLower(fstype) != "cifs" {
			return fmt.Errorf("only cifs mount is supported now, fstype: %q, mounting source (%q), target (%q), with options (%q)", fstype, source, target, sanitizedOptionsForLogging)
		}

		// lock smb mount for the same source
		getSMBMountMutex.LockKey(source)
		defer getSMBMountMutex.UnlockKey(source)

		if output, err := newSMBMapping(allOptions[0], allOptions[1], source); err != nil {
			if isSMBMappingExist(source) {
				klog.V(2).Infof("SMB Mapping(%s) already exists, now begin to remove and remount", source)
				if output, err := removeSMBMapping(source); err != nil {
					return fmt.Errorf("Remove-SmbGlobalMapping failed: %v, output: %q", err, output)
				}
				if output, err := newSMBMapping(allOptions[0], allOptions[1], source); err != nil {
					return fmt.Errorf("New-SmbGlobalMapping remount failed: %v, output: %q", err, output)
				}
			} else {
				return fmt.Errorf("New-SmbGlobalMapping failed: %v, output: %q", err, output)
			}
		}
	}

	if output, err := exec.Command("cmd", "/c", "mklink", "/D", target, bindSource).CombinedOutput(); err != nil {
		klog.Errorf("mklink failed: %v, source(%q) target(%q) output: %q", err, bindSource, target, string(output))
		return err
	}

	return nil
}

// do the SMB mount with username, password, remotepath
// return (output, error)
func newSMBMapping(username, password, remotepath string) (string, error) {
	if username == "" || password == "" || remotepath == "" {
		return "", fmt.Errorf("invalid parameter(username: %s, password: %s, remoteapth: %s)", username, sensitiveOptionsRemoved, remotepath)
	}

	// use PowerShell Environment Variables to store user input string to prevent command line injection
	// https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_environment_variables?view=powershell-5.1
	cmdLine := `$PWord = ConvertTo-SecureString -String $Env:smbpassword -AsPlainText -Force` +
		`;$Credential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $Env:smbuser, $PWord` +
		`;New-SmbGlobalMapping -RemotePath $Env:smbremotepath -Credential $Credential`
	cmd := exec.Command("powershell", "/c", cmdLine)
	cmd.Env = append(os.Environ(),
		fmt.Sprintf("smbuser=%s", username),
		fmt.Sprintf("smbpassword=%s", password),
		fmt.Sprintf("smbremotepath=%s", remotepath))

	output, err := cmd.CombinedOutput()
	return string(output), err
}

// check whether remotepath is already mounted
func isSMBMappingExist(remotepath string) bool {
	cmd := exec.Command("powershell", "/c", `Get-SmbGlobalMapping -RemotePath $Env:smbremotepath`)
	cmd.Env = append(os.Environ(), fmt.Sprintf("smbremotepath=%s", remotepath))
	_, err := cmd.CombinedOutput()
	return err == nil
}

// remove SMB mapping
func removeSMBMapping(remotepath string) (string, error) {
	cmd := exec.Command("powershell", "/c", `Remove-SmbGlobalMapping -RemotePath $Env:smbremotepath -Force`)
	cmd.Env = append(os.Environ(), fmt.Sprintf("smbremotepath=%s", remotepath))
	output, err := cmd.CombinedOutput()
	return string(output), err
}

// Unmount unmounts the target.
func (mounter *Mounter) Unmount(target string) error {
	klog.V(4).Infof("azureMount: Unmount target (%q)", target)
	target = NormalizeWindowsPath(target)
	if output, err := exec.Command("cmd", "/c", "rmdir", target).CombinedOutput(); err != nil {
		klog.Errorf("rmdir failed: %v, output: %q", err, string(output))
		return err
	}
	return nil
}

// List returns a list of all mounted filesystems. todo
func (mounter *Mounter) List() ([]MountPoint, error) {
	return []MountPoint{}, nil
}

// IsLikelyNotMountPoint determines if a directory is not a mountpoint.
func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
	stat, err := os.Lstat(file)
	if err != nil {
		return true, err
	}
	// If current file is a symlink, then it is a mountpoint.
	if stat.Mode()&os.ModeSymlink != 0 {
		target, err := os.Readlink(file)
		if err != nil {
			return true, fmt.Errorf("readlink error: %v", err)
		}
		exists, err := utilpath.Exists(utilpath.CheckFollowSymlink, target)
		if err != nil {
			return true, err
		}
		return !exists, nil
	}

	return true, nil
}

// GetMountRefs : empty implementation here since there is no place to query all mount points on Windows
func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
	windowsPath := NormalizeWindowsPath(pathname)
	pathExists, pathErr := PathExists(windowsPath)
	if !pathExists {
		return []string{}, nil
	} else if IsCorruptedMnt(pathErr) {
		klog.Warningf("GetMountRefs found corrupted mount at %s, treating as unmounted path", windowsPath)
		return []string{}, nil
	} else if pathErr != nil {
		return nil, fmt.Errorf("error checking path %s: %v", windowsPath, pathErr)
	}
	return []string{pathname}, nil
}

func (mounter *SafeFormatAndMount) formatAndMountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error {
	// Try to mount the disk
	klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, source, target)

	if err := ValidateDiskNumber(source); err != nil {
		klog.Errorf("diskMount: formatAndMount failed, err: %v", err)
		return err
	}

	if len(fstype) == 0 {
		// Use 'NTFS' as the default
		fstype = "NTFS"
	}

	// format disk if it is unformatted(raw)
	cmd := fmt.Sprintf("Get-Disk -Number %s | Where partitionstyle -eq 'raw' | Initialize-Disk -PartitionStyle MBR -PassThru"+
		" | New-Partition -UseMaximumSize | Format-Volume -FileSystem %s -Confirm:$false", source, fstype)
	if output, err := mounter.Exec.Command("powershell", "/c", cmd).CombinedOutput(); err != nil {
		return fmt.Errorf("diskMount: format disk failed, error: %v, output: %q", err, string(output))
	}
	klog.V(4).Infof("diskMount: Disk successfully formatted, disk: %q, fstype: %q", source, fstype)

	volumeIds, err := listVolumesOnDisk(source)
	if err != nil {
		return err
	}
	driverPath := volumeIds[0]
	target = NormalizeWindowsPath(target)
	klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, driverPath, target)
	if output, err := mounter.Exec.Command("cmd", "/c", "mklink", "/D", target, driverPath).CombinedOutput(); err != nil {
		klog.Errorf("mklink failed: %v, output: %q", err, string(output))
		return err
	}
	return nil
}

// ListVolumesOnDisk - returns back list of volumes(volumeIDs) in the disk (requested in diskID).
func listVolumesOnDisk(diskID string) (volumeIDs []string, err error) {
	cmd := fmt.Sprintf("(Get-Disk -DeviceId %s | Get-Partition | Get-Volume).UniqueId", diskID)
	output, err := exec.Command("powershell", "/c", cmd).CombinedOutput()
	klog.V(4).Infof("listVolumesOnDisk id from %s: %s", diskID, string(output))
	if err != nil {
		return []string{}, fmt.Errorf("error list volumes on disk. cmd: %s, output: %s, error: %v", cmd, string(output), err)
	}

	volumeIds := strings.Split(strings.TrimSpace(string(output)), "\r\n")
	return volumeIds, nil
}

// getAllParentLinks walks all symbolic links and return all the parent targets recursively
func getAllParentLinks(path string) ([]string, error) {
	const maxIter = 255
	links := []string{}
	for {
		links = append(links, path)
		if len(links) > maxIter {
			return links, fmt.Errorf("unexpected length of parent links: %v", links)
		}

		fi, err := os.Lstat(path)
		if err != nil {
			return links, fmt.Errorf("Lstat: %v", err)
		}
		if fi.Mode()&os.ModeSymlink == 0 {
			break
		}

		path, err = os.Readlink(path)
		if err != nil {
			return links, fmt.Errorf("Readlink error: %v", err)
		}
	}

	return links, nil
}
<|endoftext|>


<commit_before>package lexer

import "testing"
import "unicode/utf8"
import "os"

func TestStreamFromString(t *testing.T) {
	count := 0

	const input = "abc"

	stream := StreamFromString(input)

	for r := range stream {
		if r != rune(input[count]) {
			t.Errorf("Expected to receive %c over channel at index %d from input %s", r, count, input)
		}
		count++
	}

	if count != len(input) {
		t.Errorf("Expected count (%d) to be equal to %d.", count, len(input))
	}
}

func TestStreamFromFile(t *testing.T) {
	_, err := StreamFromFile("aNonExistingFile.lbd")
	if err == nil {
		t.Error("Expected to get error when opening non existing file, but got nil instead.")
	}

	const fileName = "../example/main.lbd"

	size, err := fileSize(fileName)
	if err != nil {
		t.Errorf("Expected to get no error when opening existing file, but got %s instead.", err.Error())
	}

	s, err := StreamFromFile(fileName)
	if err != nil {
		t.Errorf("Expected to get no error when opening existing file, but got %s instead.", err.Error())
	}
	if s == nil {
		t.Error("Expected stream of existing file to be non nil.")
	}

	i := int64(0)
	for r := range s {
		i += int64(utf8.RuneLen(r))
	}

	if i != size {
		t.Errorf("Expected to get filesize of 85 instead of %d", i)
	}
}

func fileSize(fileName string) (size int64, err error) {
	f, err := os.Open(fileName)
	if err != nil {
		return
	}
	fi, err := f.Stat()
	if err != nil {
		return
	}
	size = fi.Size()
	return
}

func TestCombineStreams(t *testing.T) {
	s1 := StreamFromString("a = 4")
	s2 := StreamFromString("b = 5")
	if s1 == nil || s2 == nil {
		t.Error("Expected both streams to be non nil")
	}

	combined := CombineStreams(s1, s2)

	runes := []rune{'a', ' ', '=', ' ', '4', 'b', ' ', '=', ' ', '5'}

	i := 0
	for r := range combined {
		if r != runes[i] {
			t.Errorf("Expected to get %c at index %d, but got %c instead.", runes[i], i, r)
		}
		i++
	}

	if i != len(runes) {
		t.Errorf("Expected to get %d runes, but got %s instead.", len(runes), i)
	}
}

func TestIsWhitespace(t *testing.T) {
	testCases := []struct {
		input    rune
		expected bool
	}{
		{'a', false},
		{'b', false},
		{'\n', true},
		{' ', true},
		{'\t', true},
	}

	for _, c := range testCases {
		got := isWhitespace(c.input)
		if got != c.expected {
			t.Errorf("Whitespace test fail. Input: %c - Expected: %#v - Got: %#v", c.input, c.expected, got)
		}
	}
}
<commit_msg>go vet<commit_after>package lexer

import "testing"
import "unicode/utf8"
import "os"

func TestStreamFromString(t *testing.T) {
	count := 0

	const input = "abc"

	stream := StreamFromString(input)

	for r := range stream {
		if r != rune(input[count]) {
			t.Errorf("Expected to receive %c over channel at index %d from input %s", r, count, input)
		}
		count++
	}

	if count != len(input) {
		t.Errorf("Expected count (%d) to be equal to %d.", count, len(input))
	}
}

func TestStreamFromFile(t *testing.T) {
	_, err := StreamFromFile("aNonExistingFile.lbd")
	if err == nil {
		t.Error("Expected to get error when opening non existing file, but got nil instead.")
	}

	const fileName = "../example/main.lbd"

	size, err := fileSize(fileName)
	if err != nil {
		t.Errorf("Expected to get no error when opening existing file, but got %s instead.", err.Error())
	}

	s, err := StreamFromFile(fileName)
	if err != nil {
		t.Errorf("Expected to get no error when opening existing file, but got %s instead.", err.Error())
	}
	if s == nil {
		t.Error("Expected stream of existing file to be non nil.")
	}

	i := int64(0)
	for r := range s {
		i += int64(utf8.RuneLen(r))
	}

	if i != size {
		t.Errorf("Expected to get filesize of 85 instead of %d", i)
	}
}

func fileSize(fileName string) (size int64, err error) {
	f, err := os.Open(fileName)
	if err != nil {
		return
	}
	fi, err := f.Stat()
	if err != nil {
		return
	}
	size = fi.Size()
	return
}

func TestCombineStreams(t *testing.T) {
	s1 := StreamFromString("a = 4")
	s2 := StreamFromString("b = 5")
	if s1 == nil || s2 == nil {
		t.Error("Expected both streams to be non nil")
	}

	combined := CombineStreams(s1, s2)

	runes := []rune{'a', ' ', '=', ' ', '4', 'b', ' ', '=', ' ', '5'}

	i := 0
	for r := range combined {
		if r != runes[i] {
			t.Errorf("Expected to get %c at index %d, but got %c instead.", runes[i], i, r)
		}
		i++
	}

	if i != len(runes) {
		t.Errorf("Expected to get %d runes, but got %d instead.", len(runes), i)
	}
}

func TestIsWhitespace(t *testing.T) {
	testCases := []struct {
		input    rune
		expected bool
	}{
		{'a', false},
		{'b', false},
		{'\n', true},
		{' ', true},
		{'\t', true},
	}

	for _, c := range testCases {
		got := isWhitespace(c.input)
		if got != c.expected {
			t.Errorf("Whitespace test fail. Input: %c - Expected: %#v - Got: %#v", c.input, c.expected, got)
		}
	}
}
<|endoftext|>


<commit_before>package http

import (
	"github.com/astaxie/beego/orm"
	"net/http"
	"sort"
	"strconv"
	"strings"
	"time"
)

type Nqm_node struct {
	Id         int
	Nid        string
	Pid        string
	Pname      string
	Pname_abbr string
	Cid        string
	Cname      string
	Cname_abbr string
	Iid        int
	Iname      string
	Status     int
	Note       string
	Addtime    int
}

func getNQMNodes(rw http.ResponseWriter, req *http.Request) {
	var nodes = make(map[string]interface{})
	errors := []string{}
	var result = make(map[string]interface{})
	result["error"] = errors
	items := []interface{}{}
	o := orm.NewOrm()
	o.Using("gz_nqm")
	var NQMNodes []*Nqm_node
	_, err := o.Raw("SELECT nid, pname, cname, status, note FROM gz_nqm.nqm_node ORDER BY nid ASC").QueryRows(&NQMNodes)
	if err != nil {
		setError(err.Error(), result)
	} else {
		for _, node := range NQMNodes {
			idc := map[string]string{
				"idc":      node.Nid,
				"status":   strconv.Itoa(node.Status),
				"province": node.Pname,
				"city":     node.Cname,
				"note":     node.Note,
			}
			items = append(items, idc)
		}
	}
	result["items"] = items
	nodes["result"] = result
	nodes["count"] = len(items)
	setResponse(rw, nodes)
}

func getLatestTimestamp(tableName string, result map[string]interface{}) string {
	timestamp := ""
	o := orm.NewOrm()
	o.Using("gz_nqm")
	sqlcmd := "SELECT mtime FROM gz_nqm." + tableName + " ORDER BY mtime DESC LIMIT 1"
	var rows []orm.Params
	num, err := o.Raw(sqlcmd).Values(&rows)
	if err != nil {
		setError(err.Error(), result)
	} else if num > 0 {
		timestamp = rows[0]["mtime"].(string)
	}
	return timestamp
}

func getSum(slice []float64) float64 {
	sum := float64(0)
	for _, number := range slice {
		sum += number
	}
	return sum
}

func getPacketLossAndAveragePingTime(tableName string, timestamp string, result map[string]interface{}) map[string]interface{} {
	idc := map[string]interface{}{}
	sends := []float64{}
	receives := []float64{}
	averages := []float64{}

	o := orm.NewOrm()
	o.Using("gz_nqm")
	sqlcmd := "SELECT send, receive, avg FROM gz_nqm." + tableName + " WHERE mtime = ?"
	var rows []orm.Params
	num, err := o.Raw(sqlcmd, timestamp).Values(&rows)
	if err != nil {
		setError(err.Error(), result)
	} else if num > 0 {
		for _, row := range rows {
			send, err := strconv.ParseFloat(row["send"].(string), 64)
			if err != nil {
				setError(err.Error(), result)
			} else {
				sends = append(sends, send)
			}
			receive, err := strconv.ParseFloat(row["receive"].(string), 64)
			if err != nil {
				setError(err.Error(), result)
			} else {
				receives = append(receives, receive)
			}
			avg, err := strconv.ParseFloat(row["avg"].(string), 64)
			if err != nil {
				setError(err.Error(), result)
			} else {
				averages = append(averages, avg)
			}
		}
	}
	divider := float64(len(sends))
	packetLossRate := 1 - (getSum(receives) / getSum(sends))
	averagePingTimeMilliseconds := getSum(averages) / divider

	timestampInt, err := strconv.Atoi(timestamp)
	if err != nil {
		setError(err.Error(), result)
	} else {
		idc = map[string]interface{}{
			"packetLossRate":              packetLossRate,
			"averagePingTimeMilliseconds": averagePingTimeMilliseconds,
			"time": time.Unix(int64(timestampInt), 0).Format("2006-01-02 15:04"),
		}
	}
	return idc
}

func getNQMPacketLoss(rw http.ResponseWriter, req *http.Request) {
	var nodes = make(map[string]interface{})
	errors := []string{}
	var result = make(map[string]interface{})
	result["error"] = errors
	items := []interface{}{}

	tablesMap := map[string]interface{}{}
	o := orm.NewOrm()
	o.Using("gz_nqm")
	sqlcmd := "SELECT table_name FROM information_schema.tables WHERE table_name LIKE 'nqm_log_prober_%'"
	var rows []orm.Params
	num, err := o.Raw(sqlcmd).Values(&rows)
	if err != nil {
		setError(err.Error(), result)
	} else if num > 0 {
		for _, row := range rows {
			tablesMap[row["table_name"].(string)] = ""
		}
	}

	nids := []string{}
	pidMap := map[string]interface{}{}
	var NQMNodes []*Nqm_node
	_, err = o.Raw("SELECT nid, pid, status FROM gz_nqm.nqm_node ORDER BY nid ASC").QueryRows(&NQMNodes)
	if err != nil {
		setError(err.Error(), result)
	} else {
		for _, node := range NQMNodes {
			if node.Status > 0 {
				nids = append(nids, node.Nid)
				pidMap[node.Nid] = node.Pid
			}
		}
	}
	idcNames := []string{}
	for _, nid := range nids {
		if _, ok := tablesMap["nqm_log_prober_"+strings.Replace(nid, "-", "_", -1)]; ok {
			idcNames = append(idcNames, nid)
		}
	}
	sort.Strings(idcNames)
	for _, idcName := range idcNames {
		tableName := "nqm_log_prober_" + strings.Replace(idcName, "-", "_", -1)
		timestamp := getLatestTimestamp(tableName, result)
		if timestamp != "" {
			idc := getPacketLossAndAveragePingTime(tableName, timestamp, result)
			idc["nodeName"] = idcName
			idc["pid"] = pidMap[idcName]
			items = append(items, idc)
		}
	}
	result["items"] = items
	nodes["result"] = result
	nodes["count"] = len(items)
	rw.Header().Set("Access-Control-Allow-Origin", "*")
	setResponse(rw, nodes)
}

func configNQMRoutes() {
	http.HandleFunc("/api/nqm/nodes", getNQMNodes)
	http.HandleFunc("/api/nqm/loss", getNQMPacketLoss)
}
<commit_msg>[OWL-1119][query] add func getNearestTimestamp()<commit_after>package http

import (
	"github.com/astaxie/beego/orm"
	"net/http"
	"sort"
	"strconv"
	"strings"
	"time"
)

type Nqm_node struct {
	Id         int
	Nid        string
	Pid        string
	Pname      string
	Pname_abbr string
	Cid        string
	Cname      string
	Cname_abbr string
	Iid        int
	Iname      string
	Status     int
	Note       string
	Addtime    int
}

func getNQMNodes(rw http.ResponseWriter, req *http.Request) {
	var nodes = make(map[string]interface{})
	errors := []string{}
	var result = make(map[string]interface{})
	result["error"] = errors
	items := []interface{}{}
	o := orm.NewOrm()
	o.Using("gz_nqm")
	var NQMNodes []*Nqm_node
	_, err := o.Raw("SELECT nid, pname, cname, status, note FROM gz_nqm.nqm_node ORDER BY nid ASC").QueryRows(&NQMNodes)
	if err != nil {
		setError(err.Error(), result)
	} else {
		for _, node := range NQMNodes {
			idc := map[string]string{
				"idc":      node.Nid,
				"status":   strconv.Itoa(node.Status),
				"province": node.Pname,
				"city":     node.Cname,
				"note":     node.Note,
			}
			items = append(items, idc)
		}
	}
	result["items"] = items
	nodes["result"] = result
	nodes["count"] = len(items)
	setResponse(rw, nodes)
}

func getLatestTimestamp(tableName string, result map[string]interface{}) string {
	timestamp := ""
	o := orm.NewOrm()
	o.Using("gz_nqm")
	sqlcmd := "SELECT mtime FROM gz_nqm." + tableName + " ORDER BY mtime DESC LIMIT 1"
	var rows []orm.Params
	num, err := o.Raw(sqlcmd).Values(&rows)
	if err != nil {
		setError(err.Error(), result)
	} else if num > 0 {
		timestamp = rows[0]["mtime"].(string)
	}
	return timestamp
}

func getNearestTimestamp(tableName string, bound int64, result map[string]interface{}) int64 {
	timestamp := int64(0)
	o := orm.NewOrm()
	o.Using("gz_nqm")
	sqlcmd := "SELECT mtime FROM gz_nqm." + tableName
	sqlcmd += " WHERE mtime <= ? ORDER BY mtime DESC LIMIT 1"
	var rows []orm.Params
	num, err := o.Raw(sqlcmd, bound).Values(&rows)
	if err != nil {
		setError(err.Error(), result)
	} else if num > 0 {
		mtime, err := strconv.Atoi(rows[0]["mtime"].(string))
		if err == nil {
			timestamp = int64(mtime)
		}
	}
	return timestamp
}

func getSum(slice []float64) float64 {
	sum := float64(0)
	for _, number := range slice {
		sum += number
	}
	return sum
}

func getPacketLossAndAveragePingTime(tableName string, timestamp string, result map[string]interface{}) map[string]interface{} {
	idc := map[string]interface{}{}
	sends := []float64{}
	receives := []float64{}
	averages := []float64{}

	o := orm.NewOrm()
	o.Using("gz_nqm")
	sqlcmd := "SELECT send, receive, avg FROM gz_nqm." + tableName + " WHERE mtime = ?"
	var rows []orm.Params
	num, err := o.Raw(sqlcmd, timestamp).Values(&rows)
	if err != nil {
		setError(err.Error(), result)
	} else if num > 0 {
		for _, row := range rows {
			send, err := strconv.ParseFloat(row["send"].(string), 64)
			if err != nil {
				setError(err.Error(), result)
			} else {
				sends = append(sends, send)
			}
			receive, err := strconv.ParseFloat(row["receive"].(string), 64)
			if err != nil {
				setError(err.Error(), result)
			} else {
				receives = append(receives, receive)
			}
			avg, err := strconv.ParseFloat(row["avg"].(string), 64)
			if err != nil {
				setError(err.Error(), result)
			} else {
				averages = append(averages, avg)
			}
		}
	}
	divider := float64(len(sends))
	packetLossRate := 1 - (getSum(receives) / getSum(sends))
	averagePingTimeMilliseconds := getSum(averages) / divider

	timestampInt, err := strconv.Atoi(timestamp)
	if err != nil {
		setError(err.Error(), result)
	} else {
		idc = map[string]interface{}{
			"packetLossRate":              packetLossRate,
			"averagePingTimeMilliseconds": averagePingTimeMilliseconds,
			"time": time.Unix(int64(timestampInt), 0).Format("2006-01-02 15:04"),
		}
	}
	return idc
}

func getNQMPacketLoss(rw http.ResponseWriter, req *http.Request) {
	var nodes = make(map[string]interface{})
	errors := []string{}
	var result = make(map[string]interface{})
	result["error"] = errors
	items := []interface{}{}

	tablesMap := map[string]interface{}{}
	o := orm.NewOrm()
	o.Using("gz_nqm")
	sqlcmd := "SELECT table_name FROM information_schema.tables WHERE table_name LIKE 'nqm_log_prober_%'"
	var rows []orm.Params
	num, err := o.Raw(sqlcmd).Values(&rows)
	if err != nil {
		setError(err.Error(), result)
	} else if num > 0 {
		for _, row := range rows {
			tablesMap[row["table_name"].(string)] = ""
		}
	}

	nids := []string{}
	pidMap := map[string]interface{}{}
	var NQMNodes []*Nqm_node
	_, err = o.Raw("SELECT nid, pid, status FROM gz_nqm.nqm_node ORDER BY nid ASC").QueryRows(&NQMNodes)
	if err != nil {
		setError(err.Error(), result)
	} else {
		for _, node := range NQMNodes {
			if node.Status > 0 {
				nids = append(nids, node.Nid)
				pidMap[node.Nid] = node.Pid
			}
		}
	}
	idcNames := []string{}
	for _, nid := range nids {
		if _, ok := tablesMap["nqm_log_prober_"+strings.Replace(nid, "-", "_", -1)]; ok {
			idcNames = append(idcNames, nid)
		}
	}
	sort.Strings(idcNames)
	for _, idcName := range idcNames {
		tableName := "nqm_log_prober_" + strings.Replace(idcName, "-", "_", -1)
		timestamp := getLatestTimestamp(tableName, result)
		if timestamp != "" {
			idc := getPacketLossAndAveragePingTime(tableName, timestamp, result)
			idc["nodeName"] = idcName
			idc["pid"] = pidMap[idcName]
			items = append(items, idc)
		}
	}
	result["items"] = items
	nodes["result"] = result
	nodes["count"] = len(items)
	rw.Header().Set("Access-Control-Allow-Origin", "*")
	setResponse(rw, nodes)
}

func configNQMRoutes() {
	http.HandleFunc("/api/nqm/nodes", getNQMNodes)
	http.HandleFunc("/api/nqm/loss", getNQMPacketLoss)
}
<|endoftext|>