<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// TODO(jba): test that OnError is getting called appropriately.

package logging_test

import (
	"flag"
	"fmt"
	"log"
	"os"
	"reflect"
	"strings"
	"testing"
	"time"

	"cloud.google.com/go/internal/testutil"
	"cloud.google.com/go/logging"
	"cloud.google.com/go/logging/internal"
	ltesting "cloud.google.com/go/logging/internal/testing"
	"cloud.google.com/go/logging/logadmin"
	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"google.golang.org/api/iterator"
	"google.golang.org/api/option"
	mrpb "google.golang.org/genproto/googleapis/api/monitoredres"
	"google.golang.org/grpc"
)

const testLogIDPrefix = "GO-LOGGING-CLIENT/TEST-LOG"

var (
	client        *logging.Client
	aclient       *logadmin.Client
	testProjectID string
	testLogID     string
	testFilter    string
	errorc        chan error

	// Adjust the fields of a FullEntry received from the production service
	// before comparing it with the expected result. We can't correctly
	// compare certain fields, like times or server-generated IDs.
	clean func(*logging.Entry)

	// Create a new client with the given project ID.
	newClients func(ctx context.Context, projectID string) (*logging.Client, *logadmin.Client)
)

func testNow() time.Time {
	return time.Unix(1000, 0)
}

// If true, this test is using the production service, not a fake.
var integrationTest bool

func TestMain(m *testing.M) {
	flag.Parse() // needed for testing.Short()
	ctx := context.Background()
	testProjectID = testutil.ProjID()
	errorc = make(chan error, 100)
	if testProjectID == "" || testing.Short() {
		integrationTest = false
		if testProjectID != "" {
			log.Print("Integration tests skipped in short mode (using fake instead)")
		}
		testProjectID = "PROJECT_ID"
		clean = func(e *logging.Entry) {
			// Remove the insert ID for consistency with the integration test.
			e.InsertID = ""
		}

		addr, err := ltesting.NewServer()
		if err != nil {
			log.Fatalf("creating fake server: %v", err)
		}
		logging.SetNow(testNow)

		newClients = func(ctx context.Context, projectID string) (*logging.Client, *logadmin.Client) {
			conn, err := grpc.Dial(addr, grpc.WithInsecure())
			if err != nil {
				log.Fatalf("dialing %q: %v", addr, err)
			}
			c, err := logging.NewClient(ctx, projectID, option.WithGRPCConn(conn))
			if err != nil {
				log.Fatalf("creating client for fake at %q: %v", addr, err)
			}
			ac, err := logadmin.NewClient(ctx, projectID, option.WithGRPCConn(conn))
			if err != nil {
				log.Fatalf("creating client for fake at %q: %v", addr, err)
			}
			return c, ac
		}

	} else {
		integrationTest = true
		clean = func(e *logging.Entry) {
			// We cannot compare timestamps, so set them to the test time.
			// Also, remove the insert ID added by the service.
			e.Timestamp = testNow().UTC()
			e.InsertID = ""
		}
		ts := testutil.TokenSource(ctx, logging.AdminScope)
		if ts == nil {
			log.Fatal("The project key must be set. See CONTRIBUTING.md for details")
		}
		log.Printf("running integration tests with project %s", testProjectID)
		newClients = func(ctx context.Context, projectID string) (*logging.Client, *logadmin.Client) {
			c, err := logging.NewClient(ctx, projectID, option.WithTokenSource(ts))
			if err != nil {
				log.Fatalf("creating prod client: %v", err)
			}
			ac, err := logadmin.NewClient(ctx, projectID, option.WithTokenSource(ts))
			if err != nil {
				log.Fatalf("creating prod client: %v", err)
			}
			return c, ac
		}

	}
	client, aclient = newClients(ctx, testProjectID)
	client.OnError = func(e error) { errorc <- e }
	initLogs(ctx)
	testFilter = fmt.Sprintf(`logName = "projects/%s/logs/%s"`, testProjectID,
		strings.Replace(testLogID, "/", "%2F", -1))
	exit := m.Run()
	client.Close()
	os.Exit(exit)
}

func initLogs(ctx context.Context) {
	testLogID = ltesting.UniqueID(testLogIDPrefix)
	// TODO(jba): Clean up from previous aborted tests by deleting old logs; requires ListLogs RPC.
}

// Testing of Logger.Log is done in logadmin_test.go, TestEntries.

func TestLogSync(t *testing.T) {
	ctx := context.Background()
	lg := client.Logger(testLogID)
	defer deleteLog(ctx, testLogID)
	err := lg.LogSync(ctx, logging.Entry{Payload: "hello"})
	if err != nil {
		t.Fatal(err)
	}
	err = lg.LogSync(ctx, logging.Entry{Payload: "goodbye"})
	if err != nil {
		t.Fatal(err)
	}
	// Allow overriding the MonitoredResource.
	err = lg.LogSync(ctx, logging.Entry{Payload: "mr", Resource: &mrpb.MonitoredResource{Type: "global"}})
	if err != nil {
		t.Fatal(err)
	}

	want := []*logging.Entry{
		entryForTesting("hello"),
		entryForTesting("goodbye"),
		entryForTesting("mr"),
	}
	var got []*logging.Entry
	waitFor(func() bool {
		got, err = allTestLogEntries(ctx)
		if err != nil {
			return false
		}
		return len(got) >= len(want)
	})
	if msg, ok := compareEntries(got, want); !ok {
		t.Error(msg)
	}
}

func TestLogAndEntries(t *testing.T) {
	ctx := context.Background()
	payloads := []string{"p1", "p2", "p3", "p4", "p5"}
	lg := client.Logger(testLogID)
	defer deleteLog(ctx, testLogID)
	for _, p := range payloads {
		// Use the insert ID to guarantee iteration order.
		lg.Log(logging.Entry{Payload: p, InsertID: p})
	}
	lg.Flush()
	var want []*logging.Entry
	for _, p := range payloads {
		want = append(want, entryForTesting(p))
	}
	var got []*logging.Entry
	waitFor(func() bool {
		var err error
		got, err = allTestLogEntries(ctx)
		if err != nil {
			return false
		}
		return len(got) >= len(want)
	})
	if msg, ok := compareEntries(got, want); !ok {
		t.Error(msg)
	}
}

func compareEntries(got, want []*logging.Entry) (string, bool) {
	if len(got) != len(want) {
		return fmt.Sprintf("got %d entries, want %d", len(got), len(want)), false
	}
	for i := range got {
		if !reflect.DeepEqual(got[i], want[i]) {
			return fmt.Sprintf("#%d:\ngot  %+v\nwant %+v", i, got[i], want[i]), false
		}
	}
	return "", true
}

func entryForTesting(payload interface{}) *logging.Entry {
	return &logging.Entry{
		Timestamp: testNow().UTC(),
		Payload:   payload,
		LogName:   "projects/" + testProjectID + "/logs/" + testLogID,
		Resource:  &mrpb.MonitoredResource{Type: "global"},
	}
}

func countLogEntries(ctx context.Context, filter string) int {
	it := aclient.Entries(ctx, logadmin.Filter(filter))
	n := 0
	for {
		_, err := it.Next()
		if err == iterator.Done {
			return n
		}
		if err != nil {
			log.Fatalf("counting log entries: %v", err)
		}
		n++
	}
}

func allTestLogEntries(ctx context.Context) ([]*logging.Entry, error) {
	var es []*logging.Entry
	it := aclient.Entries(ctx, logadmin.Filter(testFilter))
	for {
		e, err := cleanNext(it)
		switch err {
		case nil:
			es = append(es, e)
		case iterator.Done:
			return es, nil
		default:
			return nil, err
		}
	}
}

func cleanNext(it *logadmin.EntryIterator) (*logging.Entry, error) {
	e, err := it.Next()
	if err != nil {
		return nil, err
	}
	clean(e)
	return e, nil
}

func TestStandardLogger(t *testing.T) {
	ctx := context.Background()
	lg := client.Logger(testLogID)
	defer deleteLog(ctx, testLogID)
	slg := lg.StandardLogger(logging.Info)

	if slg != lg.StandardLogger(logging.Info) {
		t.Error("There should be only one standard logger at each severity.")
	}
	if slg == lg.StandardLogger(logging.Debug) {
		t.Error("There should be a different standard logger for each severity.")
	}

	slg.Print("info")
	lg.Flush()
	var got []*logging.Entry
	waitFor(func() bool {
		var err error
		got, err = allTestLogEntries(ctx)
		if err != nil {
			return false
		}
		return len(got) >= 1
	})
	if len(got) != 1 {
		t.Fatalf("expected non-nil request with one entry; got:\n%+v", got)
	}
	if got, want := got[0].Payload.(string), "info\n"; got != want {
		t.Errorf("payload: got %q, want %q", got, want)
	}
	if got, want := logging.Severity(got[0].Severity), logging.Info; got != want {
		t.Errorf("severity: got %s, want %s", got, want)
	}
}

func TestSeverity(t *testing.T) {
	if got, want := logging.Info.String(), "Info"; got != want {
		t.Errorf("got %q, want %q", got, want)
	}
	if got, want := logging.Severity(-99).String(), "-99"; got != want {
		t.Errorf("got %q, want %q", got, want)
	}
}

func TestParseSeverity(t *testing.T) {
	for _, test := range []struct {
		in   string
		want logging.Severity
	}{
		{"", logging.Default},
		{"whatever", logging.Default},
		{"Default", logging.Default},
		{"ERROR", logging.Error},
		{"Error", logging.Error},
		{"error", logging.Error},
	} {
		got := logging.ParseSeverity(test.in)
		if got != test.want {
			t.Errorf("%q: got %s, want %s\n", test.in, got, test.want)
		}
	}
}

func TestErrors(t *testing.T) {
	// Drain errors already seen.
loop:
	for {
		select {
		case <-errorc:
		default:
			break loop
		}
	}
	// Try to log something that can't be JSON-marshalled.
	lg := client.Logger(testLogID)
	lg.Log(logging.Entry{Payload: func() {}})
	// Expect an error.
	select {
	case <-errorc: // pass
	case <-time.After(100 * time.Millisecond):
		t.Fatal("expected an error but timed out")
	}
}

type badTokenSource struct{}

func (badTokenSource) Token() (*oauth2.Token, error) {
	return &oauth2.Token{}, nil
}

func TestPing(t *testing.T) {
	// Ping twice, in case the service's InsertID logic messes with the error code.
	ctx := context.Background()
	// The global client should be valid.
	if err := client.Ping(ctx); err != nil {
		t.Errorf("project %s: got %v, expected nil", testProjectID, err)
	}
	if err := client.Ping(ctx); err != nil {
		t.Errorf("project %s, #2: got %v, expected nil", testProjectID, err)
	}
	// nonexistent project
	c, _ := newClients(ctx, testProjectID+"-BAD")
	if err := c.Ping(ctx); err == nil {
		t.Errorf("nonexistent project: want error pinging logging api, got nil")
	}
	if err := c.Ping(ctx); err == nil {
		t.Errorf("nonexistent project, #2: want error pinging logging api, got nil")
	}

	// Bad creds. We cannot test this with the fake, since it doesn't do auth.
	if integrationTest {
		c, err := logging.NewClient(ctx, testProjectID, option.WithTokenSource(badTokenSource{}))
		if err != nil {
			t.Fatal(err)
		}
		if err := c.Ping(ctx); err == nil {
			t.Errorf("bad creds: want error pinging logging api, got nil")
		}
		if err := c.Ping(ctx); err == nil {
			t.Errorf("bad creds, #2: want error pinging logging api, got nil")
		}
		if err := c.Close(); err != nil {
			t.Fatalf("error closing client: %v", err)
		}
	}
}

// deleteLog is used to clean up a log after a test that writes to it.
func deleteLog(ctx context.Context, logID string) {
	aclient.DeleteLog(ctx, logID)
	// DeleteLog can take some time to happen, so we wait for the log to
	// disappear. There is no direct way to determine if a log exists, so we
	// just wait until there are no log entries associated with the ID.
	filter := fmt.Sprintf(`logName = "%s"`, internal.LogPath("projects/"+testProjectID, logID))
	waitFor(func() bool { return countLogEntries(ctx, filter) == 0 })
}

// waitFor calls f repeatedly with exponential backoff, blocking until it returns true.
// It calls log.Fatal after two minutes.
func waitFor(f func() bool) {
	delay := time.Second
	timeout := time.NewTimer(2 * time.Minute)
	for {
		select {
		case <-time.After(delay):
			if f() {
				timeout.Stop()
				return
			}
			delay = delay * 2
		case <-timeout.C:
			if f() {
				return
			}
			log.Fatal("timed out")
		}
	}
}
<commit_msg>logging: increase timeout<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// TODO(jba): test that OnError is getting called appropriately.

package logging_test

import (
	"flag"
	"fmt"
	"log"
	"os"
	"reflect"
	"strings"
	"testing"
	"time"

	"cloud.google.com/go/internal/testutil"
	"cloud.google.com/go/logging"
	"cloud.google.com/go/logging/internal"
	ltesting "cloud.google.com/go/logging/internal/testing"
	"cloud.google.com/go/logging/logadmin"
	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"google.golang.org/api/iterator"
	"google.golang.org/api/option"
	mrpb "google.golang.org/genproto/googleapis/api/monitoredres"
	"google.golang.org/grpc"
)

const testLogIDPrefix = "GO-LOGGING-CLIENT/TEST-LOG"

var (
	client        *logging.Client
	aclient       *logadmin.Client
	testProjectID string
	testLogID     string
	testFilter    string
	errorc        chan error

	// Adjust the fields of a FullEntry received from the production service
	// before comparing it with the expected result. We can't correctly
	// compare certain fields, like times or server-generated IDs.
	clean func(*logging.Entry)

	// Create a new client with the given project ID.
	newClients func(ctx context.Context, projectID string) (*logging.Client, *logadmin.Client)
)

func testNow() time.Time {
	return time.Unix(1000, 0)
}

// If true, this test is using the production service, not a fake.
var integrationTest bool

func TestMain(m *testing.M) {
	flag.Parse() // needed for testing.Short()
	ctx := context.Background()
	testProjectID = testutil.ProjID()
	errorc = make(chan error, 100)
	if testProjectID == "" || testing.Short() {
		integrationTest = false
		if testProjectID != "" {
			log.Print("Integration tests skipped in short mode (using fake instead)")
		}
		testProjectID = "PROJECT_ID"
		clean = func(e *logging.Entry) {
			// Remove the insert ID for consistency with the integration test.
			e.InsertID = ""
		}

		addr, err := ltesting.NewServer()
		if err != nil {
			log.Fatalf("creating fake server: %v", err)
		}
		logging.SetNow(testNow)

		newClients = func(ctx context.Context, projectID string) (*logging.Client, *logadmin.Client) {
			conn, err := grpc.Dial(addr, grpc.WithInsecure())
			if err != nil {
				log.Fatalf("dialing %q: %v", addr, err)
			}
			c, err := logging.NewClient(ctx, projectID, option.WithGRPCConn(conn))
			if err != nil {
				log.Fatalf("creating client for fake at %q: %v", addr, err)
			}
			ac, err := logadmin.NewClient(ctx, projectID, option.WithGRPCConn(conn))
			if err != nil {
				log.Fatalf("creating client for fake at %q: %v", addr, err)
			}
			return c, ac
		}

	} else {
		integrationTest = true
		clean = func(e *logging.Entry) {
			// We cannot compare timestamps, so set them to the test time.
			// Also, remove the insert ID added by the service.
			e.Timestamp = testNow().UTC()
			e.InsertID = ""
		}
		ts := testutil.TokenSource(ctx, logging.AdminScope)
		if ts == nil {
			log.Fatal("The project key must be set. See CONTRIBUTING.md for details")
		}
		log.Printf("running integration tests with project %s", testProjectID)
		newClients = func(ctx context.Context, projectID string) (*logging.Client, *logadmin.Client) {
			c, err := logging.NewClient(ctx, projectID, option.WithTokenSource(ts))
			if err != nil {
				log.Fatalf("creating prod client: %v", err)
			}
			ac, err := logadmin.NewClient(ctx, projectID, option.WithTokenSource(ts))
			if err != nil {
				log.Fatalf("creating prod client: %v", err)
			}
			return c, ac
		}

	}
	client, aclient = newClients(ctx, testProjectID)
	client.OnError = func(e error) { errorc <- e }
	initLogs(ctx)
	testFilter = fmt.Sprintf(`logName = "projects/%s/logs/%s"`, testProjectID,
		strings.Replace(testLogID, "/", "%2F", -1))
	exit := m.Run()
	client.Close()
	os.Exit(exit)
}

func initLogs(ctx context.Context) {
	testLogID = ltesting.UniqueID(testLogIDPrefix)
	// TODO(jba): Clean up from previous aborted tests by deleting old logs; requires ListLogs RPC.
}

// Testing of Logger.Log is done in logadmin_test.go, TestEntries.

func TestLogSync(t *testing.T) {
	ctx := context.Background()
	lg := client.Logger(testLogID)
	defer deleteLog(ctx, testLogID)
	err := lg.LogSync(ctx, logging.Entry{Payload: "hello"})
	if err != nil {
		t.Fatal(err)
	}
	err = lg.LogSync(ctx, logging.Entry{Payload: "goodbye"})
	if err != nil {
		t.Fatal(err)
	}
	// Allow overriding the MonitoredResource.
	err = lg.LogSync(ctx, logging.Entry{Payload: "mr", Resource: &mrpb.MonitoredResource{Type: "global"}})
	if err != nil {
		t.Fatal(err)
	}

	want := []*logging.Entry{
		entryForTesting("hello"),
		entryForTesting("goodbye"),
		entryForTesting("mr"),
	}
	var got []*logging.Entry
	waitFor(func() bool {
		got, err = allTestLogEntries(ctx)
		if err != nil {
			return false
		}
		return len(got) >= len(want)
	})
	if msg, ok := compareEntries(got, want); !ok {
		t.Error(msg)
	}
}

func TestLogAndEntries(t *testing.T) {
	ctx := context.Background()
	payloads := []string{"p1", "p2", "p3", "p4", "p5"}
	lg := client.Logger(testLogID)
	defer deleteLog(ctx, testLogID)
	for _, p := range payloads {
		// Use the insert ID to guarantee iteration order.
		lg.Log(logging.Entry{Payload: p, InsertID: p})
	}
	lg.Flush()
	var want []*logging.Entry
	for _, p := range payloads {
		want = append(want, entryForTesting(p))
	}
	var got []*logging.Entry
	waitFor(func() bool {
		var err error
		got, err = allTestLogEntries(ctx)
		if err != nil {
			return false
		}
		return len(got) >= len(want)
	})
	if msg, ok := compareEntries(got, want); !ok {
		t.Error(msg)
	}
}

func compareEntries(got, want []*logging.Entry) (string, bool) {
	if len(got) != len(want) {
		return fmt.Sprintf("got %d entries, want %d", len(got), len(want)), false
	}
	for i := range got {
		if !reflect.DeepEqual(got[i], want[i]) {
			return fmt.Sprintf("#%d:\ngot  %+v\nwant %+v", i, got[i], want[i]), false
		}
	}
	return "", true
}

func entryForTesting(payload interface{}) *logging.Entry {
	return &logging.Entry{
		Timestamp: testNow().UTC(),
		Payload:   payload,
		LogName:   "projects/" + testProjectID + "/logs/" + testLogID,
		Resource:  &mrpb.MonitoredResource{Type: "global"},
	}
}

func countLogEntries(ctx context.Context, filter string) int {
	it := aclient.Entries(ctx, logadmin.Filter(filter))
	n := 0
	for {
		_, err := it.Next()
		if err == iterator.Done {
			return n
		}
		if err != nil {
			log.Fatalf("counting log entries: %v", err)
		}
		n++
	}
}

func allTestLogEntries(ctx context.Context) ([]*logging.Entry, error) {
	var es []*logging.Entry
	it := aclient.Entries(ctx, logadmin.Filter(testFilter))
	for {
		e, err := cleanNext(it)
		switch err {
		case nil:
			es = append(es, e)
		case iterator.Done:
			return es, nil
		default:
			return nil, err
		}
	}
}

func cleanNext(it *logadmin.EntryIterator) (*logging.Entry, error) {
	e, err := it.Next()
	if err != nil {
		return nil, err
	}
	clean(e)
	return e, nil
}

func TestStandardLogger(t *testing.T) {
	ctx := context.Background()
	lg := client.Logger(testLogID)
	defer deleteLog(ctx, testLogID)
	slg := lg.StandardLogger(logging.Info)

	if slg != lg.StandardLogger(logging.Info) {
		t.Error("There should be only one standard logger at each severity.")
	}
	if slg == lg.StandardLogger(logging.Debug) {
		t.Error("There should be a different standard logger for each severity.")
	}

	slg.Print("info")
	lg.Flush()
	var got []*logging.Entry
	waitFor(func() bool {
		var err error
		got, err = allTestLogEntries(ctx)
		if err != nil {
			return false
		}
		return len(got) >= 1
	})
	if len(got) != 1 {
		t.Fatalf("expected non-nil request with one entry; got:\n%+v", got)
	}
	if got, want := got[0].Payload.(string), "info\n"; got != want {
		t.Errorf("payload: got %q, want %q", got, want)
	}
	if got, want := logging.Severity(got[0].Severity), logging.Info; got != want {
		t.Errorf("severity: got %s, want %s", got, want)
	}
}

func TestSeverity(t *testing.T) {
	if got, want := logging.Info.String(), "Info"; got != want {
		t.Errorf("got %q, want %q", got, want)
	}
	if got, want := logging.Severity(-99).String(), "-99"; got != want {
		t.Errorf("got %q, want %q", got, want)
	}
}

func TestParseSeverity(t *testing.T) {
	for _, test := range []struct {
		in   string
		want logging.Severity
	}{
		{"", logging.Default},
		{"whatever", logging.Default},
		{"Default", logging.Default},
		{"ERROR", logging.Error},
		{"Error", logging.Error},
		{"error", logging.Error},
	} {
		got := logging.ParseSeverity(test.in)
		if got != test.want {
			t.Errorf("%q: got %s, want %s\n", test.in, got, test.want)
		}
	}
}

func TestErrors(t *testing.T) {
	// Drain errors already seen.
loop:
	for {
		select {
		case <-errorc:
		default:
			break loop
		}
	}
	// Try to log something that can't be JSON-marshalled.
	lg := client.Logger(testLogID)
	lg.Log(logging.Entry{Payload: func() {}})
	// Expect an error.
	select {
	case <-errorc: // pass
	case <-time.After(100 * time.Millisecond):
		t.Fatal("expected an error but timed out")
	}
}

type badTokenSource struct{}

func (badTokenSource) Token() (*oauth2.Token, error) {
	return &oauth2.Token{}, nil
}

func TestPing(t *testing.T) {
	// Ping twice, in case the service's InsertID logic messes with the error code.
	ctx := context.Background()
	// The global client should be valid.
	if err := client.Ping(ctx); err != nil {
		t.Errorf("project %s: got %v, expected nil", testProjectID, err)
	}
	if err := client.Ping(ctx); err != nil {
		t.Errorf("project %s, #2: got %v, expected nil", testProjectID, err)
	}
	// nonexistent project
	c, _ := newClients(ctx, testProjectID+"-BAD")
	if err := c.Ping(ctx); err == nil {
		t.Errorf("nonexistent project: want error pinging logging api, got nil")
	}
	if err := c.Ping(ctx); err == nil {
		t.Errorf("nonexistent project, #2: want error pinging logging api, got nil")
	}

	// Bad creds. We cannot test this with the fake, since it doesn't do auth.
	if integrationTest {
		c, err := logging.NewClient(ctx, testProjectID, option.WithTokenSource(badTokenSource{}))
		if err != nil {
			t.Fatal(err)
		}
		if err := c.Ping(ctx); err == nil {
			t.Errorf("bad creds: want error pinging logging api, got nil")
		}
		if err := c.Ping(ctx); err == nil {
			t.Errorf("bad creds, #2: want error pinging logging api, got nil")
		}
		if err := c.Close(); err != nil {
			t.Fatalf("error closing client: %v", err)
		}
	}
}

// deleteLog is used to clean up a log after a test that writes to it.
func deleteLog(ctx context.Context, logID string) {
	aclient.DeleteLog(ctx, logID)
	// DeleteLog can take some time to happen, so we wait for the log to
	// disappear. There is no direct way to determine if a log exists, so we
	// just wait until there are no log entries associated with the ID.
	filter := fmt.Sprintf(`logName = "%s"`, internal.LogPath("projects/"+testProjectID, logID))
	waitFor(func() bool { return countLogEntries(ctx, filter) == 0 })
}

// waitFor calls f repeatedly with exponential backoff, blocking until it returns true.
// It calls log.Fatal after a while.
func waitFor(f func() bool) {
	delay := time.Second
	// TODO(shadams): Find a better way to deflake these tests.
	timeout := time.NewTimer(4 * time.Minute)
	for {
		select {
		case <-time.After(delay):
			if f() {
				timeout.Stop()
				return
			}
			delay = delay * 2
		case <-timeout.C:
			if f() {
				return
			}
			log.Fatal("timed out")
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/spf13/cobra"
)

/*
#ifndef _GNU_SOURCE
#define _GNU_SOURCE 1
#endif
#include <fcntl.h>
#include <libgen.h>
#include <sched.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/capability.h>
#include <sys/fsuid.h>
#include <sys/prctl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/vfs.h>
#include <unistd.h>

#include "include/memory_utils.h"

extern char* advance_arg(bool required);
extern int dosetns(int pid, char *nstype);

static uid_t get_ns_uid(uid_t uid, pid_t pid)
{
        __do_free char *line = NULL;
        __do_fclose FILE *f = NULL;
        size_t sz = 0;
	char path[256];
        uid_t nsid, hostid, range;

	snprintf(path, sizeof(path), "/proc/%d/uid_map", pid);
	f = fopen(path, "re");
	if (!f)
		return -1;

        while (getline(&line, &sz, f) != -1) {
                if (sscanf(line, "%u %u %u", &nsid, &hostid, &range) != 3)
                        continue;

                if (nsid <= uid && nsid + range > uid) {
                        nsid += uid - hostid;
			return nsid;
                }
        }

        return -1;
}

static gid_t get_ns_gid(uid_t gid, pid_t pid)
{
        __do_free char *line = NULL;
        __do_fclose FILE *f = NULL;
        size_t sz = 0;
	char path[256];
        uid_t nsid, hostid, range;

	snprintf(path, sizeof(path), "/proc/%d/gid_map", pid);
	f = fopen(path, "re");
	if (!f)
		return -1;

        while (getline(&line, &sz, f) != -1) {
                if (sscanf(line, "%u %u %u", &nsid, &hostid, &range) != 3)
                        continue;

                if (nsid <= gid && nsid + range > gid) {
                        nsid += gid - hostid;
			return nsid;
                }
        }

        return -1;
}

static inline bool same_fsinfo(struct stat *s1, struct stat *s2,
			       struct statfs *sfs1, struct statfs *sfs2)
{
	return ((sfs1->f_type == sfs2->f_type) && (s1->st_dev == s2->st_dev) && (s1->st_ino == s2->st_ino));
}

static int fstat_fstatfs(int fd, struct stat *s, struct statfs *sfs)
{
	if (fstat(fd, s))
		return -1;

	if (fstatfs(fd, sfs))
		return -1;

	return 0;
}

static bool chdirchroot(pid_t pid)
{
	char path[PATH_MAX];

	snprintf(path, sizeof(path), "/proc/%d/cwd", pid);
	if (chdir(path))
		return false;

	snprintf(path, sizeof(path), "/proc/%d/root", pid);
	if (chroot(path))
		return false;

	return true;
}

// Expects command line to be in the form:
// <PID> <root-uid> <root-gid> <path> <mode> <dev>
static void forkmknod()
{
	__do_close_prot_errno int cwd_fd = -EBADF, host_target_fd = -EBADF, mnt_fd = -EBADF;
	int ret;
	char *cur = NULL, *target = NULL, *target_dir = NULL, *target_host = NULL;
	char path[PATH_MAX];
	mode_t mode = 0;
	dev_t dev = 0;
	pid_t pid = 0;
	uid_t uid = -1;
	gid_t gid = -1;
	struct stat s1, s2;
	struct statfs sfs1, sfs2;
	cap_t caps;
	int chk_perm_only;

	pid = atoi(advance_arg(true));
	target = advance_arg(true);
	mode = atoi(advance_arg(true));
	dev = atoi(advance_arg(true));
	target_host = advance_arg(true);
	uid = atoi(advance_arg(true));
	gid = atoi(advance_arg(true));
	chk_perm_only = atoi(advance_arg(true));

	if (*target == '/') {
		// user has specified an absolute path
		snprintf(path, sizeof(path), "%s", target);
		target_dir = dirname(path);
	} else {
		// user has specified a relative path
		snprintf(path, sizeof(path), "/proc/%d/cwd", pid);
		target_dir = path;
	}
	cwd_fd = open(path, O_PATH | O_RDONLY | O_CLOEXEC);
	if (cwd_fd < 0) {
		fprintf(stderr, "%d", ENOANO);
		_exit(EXIT_FAILURE);
	}

	host_target_fd = open(dirname(target_host), O_PATH | O_RDONLY | O_CLOEXEC);
	if (host_target_fd < 0) {
		fprintf(stderr, "%d", ENOANO);
		_exit(EXIT_FAILURE);
	}

	snprintf(path, sizeof(path), "/proc/%d/ns/mnt", pid);
	mnt_fd = open(path, O_RDONLY | O_CLOEXEC);
	if (mnt_fd < 0) {
		fprintf(stderr, "%d", ENOANO);
		_exit(EXIT_FAILURE);
	}

	if (chdirchroot(pid)) {
		fprintf(stderr, "%d", ENOANO);
		_exit(EXIT_FAILURE);
	}
	if (setns(mnt_fd, CLONE_NEWNS)) {
		fprintf(stderr, "%d", ENOANO);
		_exit(EXIT_FAILURE);
	}

	caps = cap_get_pid(pid);
	if (!caps) {
		fprintf(stderr, "%d", ENOANO);
		_exit(EXIT_FAILURE);
	}

	ret = prctl(PR_SET_KEEPCAPS, 1);
	if (ret) {
		fprintf(stderr, "%d", ENOANO);
		_exit(EXIT_FAILURE);
	}

	ret = setegid(gid);
	if (ret) {
		fprintf(stderr, "%d", ENOANO);
		_exit(EXIT_FAILURE);
	}

	setfsgid(gid);

	ret = seteuid(uid);
	if (ret) {
		fprintf(stderr, "%d", ENOANO);
		_exit(EXIT_FAILURE);
	}

	setfsuid(uid);

	ret = cap_set_proc(caps);
	if (ret) {
		fprintf(stderr, "%d", ENOANO);
		_exit(EXIT_FAILURE);
	}

	ret = fstat_fstatfs(cwd_fd, &s2, &sfs2);
	if (ret) {
		fprintf(stderr, "%d", ENOANO);
		_exit(EXIT_FAILURE);
	}

	if (sfs2.f_flags & MS_NODEV) {
		fprintf(stderr, "%d", EPERM);
		_exit(EXIT_FAILURE);
	}

	ret = fstat_fstatfs(host_target_fd, &s1, &sfs1);
	if (ret) {
		fprintf(stderr, "%d", ENOANO);
		_exit(EXIT_FAILURE);
	}

	if (!same_fsinfo(&s1, &s2, &sfs1, &sfs2)) {
		fprintf(stderr, "%d", ENOMEDIUM);
		_exit(EXIT_FAILURE);
	}

	if (chk_perm_only) {
		fprintf(stderr, "%d", ENOMEDIUM);
		_exit(EXIT_FAILURE);
	}

	// basename() can modify its argument so accessing target_host is
	// invalid from now on.
	ret = mknodat(cwd_fd, target, mode, dev);
	if (ret) {
		fprintf(stderr, "%d", errno);
		_exit(EXIT_FAILURE);
	}
}

void forksyscall()
{
	char *syscall = NULL;

	// Check that we're root
	if (geteuid() != 0)
		_exit(EXIT_FAILURE);

	// Get the subcommand
	syscall = advance_arg(false);
	if (syscall == NULL ||
	    (strcmp(syscall, "--help") == 0 ||
	     strcmp(syscall, "--version") == 0 || strcmp(syscall, "-h") == 0))
		_exit(EXIT_SUCCESS);

	if (strcmp(syscall, "mknod") == 0)
		forkmknod();
	else
		_exit(EXIT_FAILURE);

	_exit(EXIT_SUCCESS);
}
*/
// #cgo CFLAGS: -std=gnu11 -Wvla
// #cgo LDFLAGS: -lcap
import "C"

type cmdForksyscall struct {
	global *cmdGlobal
}

func GetNSUid(uid uint, pid int) int {
	return int(C.get_ns_uid(C.uid_t(uid), C.pid_t(pid)))
}

func GetNSGid(gid uint, pid int) int {
	return int(C.get_ns_gid(C.gid_t(gid), C.pid_t(pid)))
}

func (c *cmdForksyscall) Command() *cobra.Command {
	// Main subcommand
	cmd := &cobra.Command{}
	cmd.Use = "forksyscall <syscall> <PID> <path> <mode> <dev>"
	cmd.Short = "Perform syscall operations"
	cmd.Long = `Description:
  Perform syscall operations

  This set of internal commands are used for all seccom-based container syscall
  operations.
`
	cmd.RunE = c.Run
	cmd.Hidden = true

	return cmd
}

func (c *cmdForksyscall) Run(cmd *cobra.Command, args []string) error {
	return fmt.Errorf("This command should have been intercepted in cgo")
}
<commit_msg>forksyscall: use correct error handling for chdirchroot()<commit_after>package main

import (
	"fmt"

	"github.com/spf13/cobra"
)

/*
#ifndef _GNU_SOURCE
#define _GNU_SOURCE 1
#endif
#include <fcntl.h>
#include <libgen.h>
#include <sched.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/capability.h>
#include <sys/fsuid.h>
#include <sys/prctl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/vfs.h>
#include <unistd.h>

#include "include/memory_utils.h"

extern char* advance_arg(bool required);
extern int dosetns(int pid, char *nstype);

static uid_t get_ns_uid(uid_t uid, pid_t pid)
{
        __do_free char *line = NULL;
        __do_fclose FILE *f = NULL;
        size_t sz = 0;
	char path[256];
        uid_t nsid, hostid, range;

	snprintf(path, sizeof(path), "/proc/%d/uid_map", pid);
	f = fopen(path, "re");
	if (!f)
		return -1;

        while (getline(&line, &sz, f) != -1) {
                if (sscanf(line, "%u %u %u", &nsid, &hostid, &range) != 3)
                        continue;

                if (nsid <= uid && nsid + range > uid) {
                        nsid += uid - hostid;
			return nsid;
                }
        }

        return -1;
}

static gid_t get_ns_gid(uid_t gid, pid_t pid)
{
        __do_free char *line = NULL;
        __do_fclose FILE *f = NULL;
        size_t sz = 0;
	char path[256];
        uid_t nsid, hostid, range;

	snprintf(path, sizeof(path), "/proc/%d/gid_map", pid);
	f = fopen(path, "re");
	if (!f)
		return -1;

        while (getline(&line, &sz, f) != -1) {
                if (sscanf(line, "%u %u %u", &nsid, &hostid, &range) != 3)
                        continue;

                if (nsid <= gid && nsid + range > gid) {
                        nsid += gid - hostid;
			return nsid;
                }
        }

        return -1;
}

static inline bool same_fsinfo(struct stat *s1, struct stat *s2,
			       struct statfs *sfs1, struct statfs *sfs2)
{
	return ((sfs1->f_type == sfs2->f_type) && (s1->st_dev == s2->st_dev) && (s1->st_ino == s2->st_ino));
}

static int fstat_fstatfs(int fd, struct stat *s, struct statfs *sfs)
{
	if (fstat(fd, s))
		return -1;

	if (fstatfs(fd, sfs))
		return -1;

	return 0;
}

static bool chdirchroot(pid_t pid)
{
	char path[PATH_MAX];

	snprintf(path, sizeof(path), "/proc/%d/cwd", pid);
	if (chdir(path))
		return false;

	snprintf(path, sizeof(path), "/proc/%d/root", pid);
	if (chroot(path))
		return false;

	return true;
}

// Expects command line to be in the form:
// <PID> <root-uid> <root-gid> <path> <mode> <dev>
static void forkmknod()
{
	__do_close_prot_errno int cwd_fd = -EBADF, host_target_fd = -EBADF, mnt_fd = -EBADF;
	int ret;
	char *cur = NULL, *target = NULL, *target_dir = NULL, *target_host = NULL;
	char path[PATH_MAX];
	mode_t mode = 0;
	dev_t dev = 0;
	pid_t pid = 0;
	uid_t uid = -1;
	gid_t gid = -1;
	struct stat s1, s2;
	struct statfs sfs1, sfs2;
	cap_t caps;
	int chk_perm_only;

	pid = atoi(advance_arg(true));
	target = advance_arg(true);
	mode = atoi(advance_arg(true));
	dev = atoi(advance_arg(true));
	target_host = advance_arg(true);
	uid = atoi(advance_arg(true));
	gid = atoi(advance_arg(true));
	chk_perm_only = atoi(advance_arg(true));

	if (*target == '/') {
		// user has specified an absolute path
		snprintf(path, sizeof(path), "%s", target);
		target_dir = dirname(path);
	} else {
		// user has specified a relative path
		snprintf(path, sizeof(path), "/proc/%d/cwd", pid);
		target_dir = path;
	}
	cwd_fd = open(path, O_PATH | O_RDONLY | O_CLOEXEC);
	if (cwd_fd < 0) {
		fprintf(stderr, "%d", ENOANO);
		_exit(EXIT_FAILURE);
	}

	host_target_fd = open(dirname(target_host), O_PATH | O_RDONLY | O_CLOEXEC);
	if (host_target_fd < 0) {
		fprintf(stderr, "%d", ENOANO);
		_exit(EXIT_FAILURE);
	}

	snprintf(path, sizeof(path), "/proc/%d/ns/mnt", pid);
	mnt_fd = open(path, O_RDONLY | O_CLOEXEC);
	if (mnt_fd < 0) {
		fprintf(stderr, "%d", ENOANO);
		_exit(EXIT_FAILURE);
	}

	if (!chdirchroot(pid)) {
		fprintf(stderr, "%d", ENOANO);
		_exit(EXIT_FAILURE);
	}

	if (setns(mnt_fd, CLONE_NEWNS)) {
		fprintf(stderr, "%d", ENOANO);
		_exit(EXIT_FAILURE);
	}

	caps = cap_get_pid(pid);
	if (!caps) {
		fprintf(stderr, "%d", ENOANO);
		_exit(EXIT_FAILURE);
	}

	ret = prctl(PR_SET_KEEPCAPS, 1);
	if (ret) {
		fprintf(stderr, "%d", ENOANO);
		_exit(EXIT_FAILURE);
	}

	ret = setegid(gid);
	if (ret) {
		fprintf(stderr, "%d", ENOANO);
		_exit(EXIT_FAILURE);
	}

	setfsgid(gid);

	ret = seteuid(uid);
	if (ret) {
		fprintf(stderr, "%d", ENOANO);
		_exit(EXIT_FAILURE);
	}

	setfsuid(uid);

	ret = cap_set_proc(caps);
	if (ret) {
		fprintf(stderr, "%d", ENOANO);
		_exit(EXIT_FAILURE);
	}

	ret = fstat_fstatfs(cwd_fd, &s2, &sfs2);
	if (ret) {
		fprintf(stderr, "%d", ENOANO);
		_exit(EXIT_FAILURE);
	}

	if (sfs2.f_flags & MS_NODEV) {
		fprintf(stderr, "%d", EPERM);
		_exit(EXIT_FAILURE);
	}

	ret = fstat_fstatfs(host_target_fd, &s1, &sfs1);
	if (ret) {
		fprintf(stderr, "%d", ENOANO);
		_exit(EXIT_FAILURE);
	}

	if (!same_fsinfo(&s1, &s2, &sfs1, &sfs2)) {
		fprintf(stderr, "%d", ENOMEDIUM);
		_exit(EXIT_FAILURE);
	}

	if (chk_perm_only) {
		fprintf(stderr, "%d", ENOMEDIUM);
		_exit(EXIT_FAILURE);
	}

	// basename() can modify its argument so accessing target_host is
	// invalid from now on.
	ret = mknodat(cwd_fd, target, mode, dev);
	if (ret) {
		fprintf(stderr, "%d", errno);
		_exit(EXIT_FAILURE);
	}
}

void forksyscall()
{
	char *syscall = NULL;

	// Check that we're root
	if (geteuid() != 0)
		_exit(EXIT_FAILURE);

	// Get the subcommand
	syscall = advance_arg(false);
	if (syscall == NULL ||
	    (strcmp(syscall, "--help") == 0 ||
	     strcmp(syscall, "--version") == 0 || strcmp(syscall, "-h") == 0))
		_exit(EXIT_SUCCESS);

	if (strcmp(syscall, "mknod") == 0)
		forkmknod();
	else
		_exit(EXIT_FAILURE);

	_exit(EXIT_SUCCESS);
}
*/
// #cgo CFLAGS: -std=gnu11 -Wvla
// #cgo LDFLAGS: -lcap
import "C"

type cmdForksyscall struct {
	global *cmdGlobal
}

func GetNSUid(uid uint, pid int) int {
	return int(C.get_ns_uid(C.uid_t(uid), C.pid_t(pid)))
}

func GetNSGid(gid uint, pid int) int {
	return int(C.get_ns_gid(C.gid_t(gid), C.pid_t(pid)))
}

func (c *cmdForksyscall) Command() *cobra.Command {
	// Main subcommand
	cmd := &cobra.Command{}
	cmd.Use = "forksyscall <syscall> <PID> <path> <mode> <dev>"
	cmd.Short = "Perform syscall operations"
	cmd.Long = `Description:
  Perform syscall operations

  This set of internal commands are used for all seccom-based container syscall
  operations.
`
	cmd.RunE = c.Run
	cmd.Hidden = true

	return cmd
}

func (c *cmdForksyscall) Run(cmd *cobra.Command, args []string) error {
	return fmt.Errorf("This command should have been intercepted in cgo")
}
<|endoftext|>


<commit_before>//   Copyright 2018 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package anomalies

import (
	"context"
	"database/sql"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/trackit/jsonlog"
	"github.com/trackit/trackit-server/aws/s3"
	"github.com/trackit/trackit-server/db"
	"github.com/trackit/trackit-server/es"
	"github.com/trackit/trackit-server/routes"
	"github.com/trackit/trackit-server/users"

	"gopkg.in/olivere/elastic.v5"
)

// AnomalyEsQueryParams will store the parsed query params
type AnomalyEsQueryParams struct {
	DateBegin   time.Time
	DateEnd     time.Time
	AccountList []string
	IndexList   []string
}

// anomalyQueryArgs allows to get required queryArgs params
var anomalyQueryArgs = []routes.QueryArg{
	routes.AwsAccountsOptionalQueryArg,
	routes.DateBeginQueryArg,
	routes.DateEndQueryArg,
}

func init() {
	routes.MethodMuxer{
		http.MethodGet: routes.H(getAnomaliesData).With(
			db.RequestTransaction{Db: db.Db},
			users.RequireAuthenticatedUser{users.ViewerAsParent},
			routes.QueryArgs(anomalyQueryArgs),
			routes.Documentation{
				Summary:     "get the cost anomalies",
				Description: "Responds with the cost anomalies based on the query args passed to it",
			},
		),
	}.H().Register("/costs/anomalies")
}

// makeElasticSearchRequest prepares and run the request to retrieve the cost anomalies.
// It will return the data, an http status code (as int) and an error.
// Because an error can be generated, but is not critical and is not needed to be known by
// the user (e.g if the index does not exists because it was not yet indexed) the error will
// be returned, but instead of having a 500 status code, it will return the provided status code
// with empty data
func makeElasticSearchRequest(ctx context.Context, parsedParams AnomalyEsQueryParams, user users.User) (*elastic.SearchResult, int, error) {
	l := jsonlog.LoggerFromContextOrDefault(ctx)
	index = strings.Join(parsedParams.IndexList, ",")
	searchService := GetElasticSearchParams(
		parsedParams.AccountList,
		parsedParams.DateBegin,
		parsedParams.DateEnd,
		"day",
		es.Client,
		index,
	)
	res, err := searchService.Do(ctx)
	if err != nil {
		if elastic.IsNotFound(err) {
			l.Warning("Query execution failed, ES index does not exists : "+index, err)
			return nil, http.StatusOK, err
		}
		l.Error("Query execution failed : "+err.Error(), nil)
		return nil, http.StatusInternalServerError, fmt.Errorf("could not execute the ElasticSearch query")
	}
	return res, http.StatusOK, nil
}

// getAnomaliesData checks the request and returns AnomaliesData.
func getAnomaliesData(request *http.Request, a routes.Arguments) (int, interface{}) {
	user := a[users.AuthenticatedUser].(users.User)
	parsedParams := AnomalyEsQueryParams{
		AccountList: []string{},
		DateBegin:   a[anomalyQueryArgs[1]].(time.Time),
		DateEnd:     a[anomalyQueryArgs[2]].(time.Time).Add(time.Hour*time.Duration(23) + time.Minute*time.Duration(59) + time.Second*time.Duration(59)),
	}
	if a[anomalyQueryArgs[0]] != nil {
		parsedParams.AccountList = a[anomalyQueryArgs[0]].([]string)
	}
	tx := a[db.Transaction].(*sql.Tx)
	accountsAndIndexes, returnCode, err := es.GetAccountsAndIndexes(parsedParams.AccountList, user, tx, s3.IndexPrefixLineItem)
	if err != nil {
		return returnCode, err
	}
	parsedParams.AccountList = accountsAndIndexes.Accounts
	parsedParams.IndexList = accountsAndIndexes.Indexes
	res, returnCode, err := GetAnomaliesData(request.Context(), parsedParams, user)
	if err != nil {
		if returnCode == http.StatusOK {
			return returnCode, nil
		} else {
			return http.StatusInternalServerError, err
		}
	}
	return http.StatusOK, res
}

// deleteOffset deletes the offset set in createQueryTimeRange.
func deleteOffset(c ProductsCostAnomalies, dateBegin time.Time) {
	for k, costAnomalies := range c {
		var toDelete []int
		for i, an := range costAnomalies {
			if d, err := time.Parse("2006-01-02T15:04:05.000Z", an.Date); err == nil {
				if dateBegin.After(d) && !dateBegin.Equal(d) {
					toDelete = append(toDelete, i)
				}
			}
		}
		for n, i := range toDelete {
			c[k] = append(c[k][:i-n], c[k][i-n+1:]...)
		}
	}
}

// GetAnomaliesData returns the cost anomalies based on the query params, in JSON format.
func GetAnomaliesData(ctx context.Context, params AnomalyEsQueryParams, user users.User) (ProductsCostAnomalies, int, error) {
	sr, returnCode, err := makeElasticSearchRequest(ctx, params, user)
	if err != nil {
		return ProductsCostAnomalies{}, returnCode, err
	}
	res, err := prepareAnomalyData(ctx, sr)
	if err != nil {
		return ProductsCostAnomalies{}, 0, err
	}
	deleteOffset(res, params.DateBegin)
	return res, 0, nil
}
<commit_msg>fixed indexes<commit_after>//   Copyright 2018 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package anomalies

import (
	"context"
	"database/sql"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/trackit/jsonlog"
	"github.com/trackit/trackit-server/aws/s3"
	"github.com/trackit/trackit-server/db"
	"github.com/trackit/trackit-server/es"
	"github.com/trackit/trackit-server/routes"
	"github.com/trackit/trackit-server/users"

	"gopkg.in/olivere/elastic.v5"
)

// AnomalyEsQueryParams will store the parsed query params
type AnomalyEsQueryParams struct {
	DateBegin   time.Time
	DateEnd     time.Time
	AccountList []string
	IndexList   []string
}

// anomalyQueryArgs allows to get required queryArgs params
var anomalyQueryArgs = []routes.QueryArg{
	routes.AwsAccountsOptionalQueryArg,
	routes.DateBeginQueryArg,
	routes.DateEndQueryArg,
}

func init() {
	routes.MethodMuxer{
		http.MethodGet: routes.H(getAnomaliesData).With(
			db.RequestTransaction{Db: db.Db},
			users.RequireAuthenticatedUser{users.ViewerAsParent},
			routes.QueryArgs(anomalyQueryArgs),
			routes.Documentation{
				Summary:     "get the cost anomalies",
				Description: "Responds with the cost anomalies based on the query args passed to it",
			},
		),
	}.H().Register("/costs/anomalies")
}

// makeElasticSearchRequest prepares and run the request to retrieve the cost anomalies.
// It will return the data, an http status code (as int) and an error.
// Because an error can be generated, but is not critical and is not needed to be known by
// the user (e.g if the index does not exists because it was not yet indexed) the error will
// be returned, but instead of having a 500 status code, it will return the provided status code
// with empty data
func makeElasticSearchRequest(ctx context.Context, parsedParams AnomalyEsQueryParams, user users.User) (*elastic.SearchResult, int, error) {
	l := jsonlog.LoggerFromContextOrDefault(ctx)
	index := strings.Join(parsedParams.IndexList, ",")
	searchService := GetElasticSearchParams(
		parsedParams.AccountList,
		parsedParams.DateBegin,
		parsedParams.DateEnd,
		"day",
		es.Client,
		index,
	)
	res, err := searchService.Do(ctx)
	if err != nil {
		if elastic.IsNotFound(err) {
			l.Warning("Query execution failed, ES index does not exists : "+index, err)
			return nil, http.StatusOK, err
		}
		l.Error("Query execution failed : "+err.Error(), nil)
		return nil, http.StatusInternalServerError, fmt.Errorf("could not execute the ElasticSearch query")
	}
	return res, http.StatusOK, nil
}

// getAnomaliesData checks the request and returns AnomaliesData.
func getAnomaliesData(request *http.Request, a routes.Arguments) (int, interface{}) {
	user := a[users.AuthenticatedUser].(users.User)
	parsedParams := AnomalyEsQueryParams{
		AccountList: []string{},
		DateBegin:   a[anomalyQueryArgs[1]].(time.Time),
		DateEnd:     a[anomalyQueryArgs[2]].(time.Time).Add(time.Hour*time.Duration(23) + time.Minute*time.Duration(59) + time.Second*time.Duration(59)),
	}
	if a[anomalyQueryArgs[0]] != nil {
		parsedParams.AccountList = a[anomalyQueryArgs[0]].([]string)
	}
	tx := a[db.Transaction].(*sql.Tx)
	accountsAndIndexes, returnCode, err := es.GetAccountsAndIndexes(parsedParams.AccountList, user, tx, s3.IndexPrefixLineItem)
	if err != nil {
		return returnCode, err
	}
	parsedParams.AccountList = accountsAndIndexes.Accounts
	parsedParams.IndexList = accountsAndIndexes.Indexes
	res, returnCode, err := GetAnomaliesData(request.Context(), parsedParams, user)
	if err != nil {
		if returnCode == http.StatusOK {
			return returnCode, nil
		} else {
			return http.StatusInternalServerError, err
		}
	}
	return http.StatusOK, res
}

// deleteOffset deletes the offset set in createQueryTimeRange.
func deleteOffset(c ProductsCostAnomalies, dateBegin time.Time) {
	for k, costAnomalies := range c {
		var toDelete []int
		for i, an := range costAnomalies {
			if d, err := time.Parse("2006-01-02T15:04:05.000Z", an.Date); err == nil {
				if dateBegin.After(d) && !dateBegin.Equal(d) {
					toDelete = append(toDelete, i)
				}
			}
		}
		for n, i := range toDelete {
			c[k] = append(c[k][:i-n], c[k][i-n+1:]...)
		}
	}
}

// GetAnomaliesData returns the cost anomalies based on the query params, in JSON format.
func GetAnomaliesData(ctx context.Context, params AnomalyEsQueryParams, user users.User) (ProductsCostAnomalies, int, error) {
	sr, returnCode, err := makeElasticSearchRequest(ctx, params, user)
	if err != nil {
		return ProductsCostAnomalies{}, returnCode, err
	}
	res, err := prepareAnomalyData(ctx, sr)
	if err != nil {
		return ProductsCostAnomalies{}, 0, err
	}
	deleteOffset(res, params.DateBegin)
	return res, 0, nil
}
<|endoftext|>


<commit_before>//
// DISCLAIMER
//
// Copyright 2017 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Ewout Prangsma
//

package test

import (
	"context"
	"reflect"
	"testing"
	"time"

	driver "github.com/arangodb/go-driver"
)

type queryTest struct {
	Query             string
	BindVars          map[string]interface{}
	ExpectSuccess     bool
	ExpectedDocuments []interface{}
	DocumentType      reflect.Type
}

type queryTestContext struct {
	Context     context.Context
	ExpectCount bool
}

// TestCreateCursor creates several cursors.
func TestCreateCursor(t *testing.T) {
	ctx := context.Background()
	c := createClientFromEnv(t, true)
	db := ensureDatabase(ctx, c, "cursor_test", nil, t)

	// Create data set
	collectionData := map[string][]interface{}{
		"books": []interface{}{
			Book{Title: "Book 01"},
			Book{Title: "Book 02"},
			Book{Title: "Book 03"},
			Book{Title: "Book 04"},
			Book{Title: "Book 05"},
			Book{Title: "Book 06"},
			Book{Title: "Book 07"},
			Book{Title: "Book 08"},
			Book{Title: "Book 09"},
			Book{Title: "Book 10"},
			Book{Title: "Book 11"},
			Book{Title: "Book 12"},
			Book{Title: "Book 13"},
			Book{Title: "Book 14"},
			Book{Title: "Book 15"},
			Book{Title: "Book 16"},
			Book{Title: "Book 17"},
			Book{Title: "Book 18"},
			Book{Title: "Book 19"},
			Book{Title: "Book 20"},
		},
		"users": []interface{}{
			UserDoc{Name: "John", Age: 13},
			UserDoc{Name: "Jake", Age: 25},
			UserDoc{Name: "Clair", Age: 12},
			UserDoc{Name: "Johnny", Age: 42},
			UserDoc{Name: "Blair", Age: 67},
			UserDoc{Name: "Zz", Age: 12},
		},
	}
	for colName, colDocs := range collectionData {
		col := ensureCollection(ctx, db, colName, nil, t)
		if _, _, err := col.CreateDocuments(ctx, colDocs); err != nil {
			t.Fatalf("Expected success, got %s", describe(err))
		}
	}

	// Setup tests
	tests := []queryTest{
		queryTest{
			Query:             "FOR d IN books SORT d.Title RETURN d",
			ExpectSuccess:     true,
			ExpectedDocuments: collectionData["books"],
			DocumentType:      reflect.TypeOf(Book{}),
		},
		queryTest{
			Query:             "FOR d IN books FILTER d.Title==@title SORT d.Title RETURN d",
			BindVars:          map[string]interface{}{"title": "Book 02"},
			ExpectSuccess:     true,
			ExpectedDocuments: []interface{}{collectionData["books"][1]},
			DocumentType:      reflect.TypeOf(Book{}),
		},
		queryTest{
			Query:         "FOR d IN books FILTER d.Title==@title SORT d.Title RETURN d",
			BindVars:      map[string]interface{}{"somethingelse": "Book 02"},
			ExpectSuccess: false, // Unknown `@title`
		},
		queryTest{
			Query:             "FOR u IN users FILTER u.age>100 SORT u.name RETURN u",
			ExpectSuccess:     true,
			ExpectedDocuments: []interface{}{},
			DocumentType:      reflect.TypeOf(UserDoc{}),
		},
		queryTest{
			Query:             "FOR u IN users FILTER u.age<@maxAge SORT u.name RETURN u",
			BindVars:          map[string]interface{}{"maxAge": 20},
			ExpectSuccess:     true,
			ExpectedDocuments: []interface{}{collectionData["users"][2], collectionData["users"][0], collectionData["users"][5]},
			DocumentType:      reflect.TypeOf(UserDoc{}),
		},
		queryTest{
			Query:         "FOR u IN users FILTER u.age<@maxAge SORT u.name RETURN u",
			BindVars:      map[string]interface{}{"maxage": 20},
			ExpectSuccess: false, // `@maxage` versus `@maxAge`
		},
		queryTest{
			Query:             "FOR u IN users SORT u.age RETURN u.age",
			ExpectedDocuments: []interface{}{12, 12, 13, 25, 42, 67},
			DocumentType:      reflect.TypeOf(12),
			ExpectSuccess:     true,
		},
		queryTest{
			Query:             "FOR p IN users COLLECT a = p.age WITH COUNT INTO c SORT a RETURN [a, c]",
			ExpectedDocuments: []interface{}{[]int{12, 2}, []int{13, 1}, []int{25, 1}, []int{42, 1}, []int{67, 1}},
			DocumentType:      reflect.TypeOf([]int{}),
			ExpectSuccess:     true,
		},
		queryTest{
			Query:             "FOR u IN users SORT u.name RETURN u.name",
			ExpectedDocuments: []interface{}{"Blair", "Clair", "Jake", "John", "Johnny", "Zz"},
			DocumentType:      reflect.TypeOf("foo"),
			ExpectSuccess:     true,
		},
	}

	// Setup context alternatives
	contexts := []queryTestContext{
		queryTestContext{nil, false},
		queryTestContext{context.Background(), false},
		queryTestContext{driver.WithQueryCount(nil), true},
		queryTestContext{driver.WithQueryCount(nil, true), true},
		queryTestContext{driver.WithQueryCount(nil, false), false},
		queryTestContext{driver.WithQueryBatchSize(nil, 1), false},
		queryTestContext{driver.WithQueryCache(nil), false},
		queryTestContext{driver.WithQueryCache(nil, true), false},
		queryTestContext{driver.WithQueryCache(nil, false), false},
		queryTestContext{driver.WithQueryMemoryLimit(nil, 60000), false},
		queryTestContext{driver.WithQueryTTL(nil, time.Minute), false},
		queryTestContext{driver.WithQueryBatchSize(driver.WithQueryCount(nil), 1), true},
		queryTestContext{driver.WithQueryCache(driver.WithQueryCount(driver.WithQueryBatchSize(nil, 2))), true},
	}

	// Run tests for every context alternative
	for _, qctx := range contexts {
		ctx := qctx.Context
		for i, test := range tests {
			cursor, err := db.Query(ctx, test.Query, test.BindVars)
			if err == nil {
				// Close upon exit of the function
				defer cursor.Close()
			}
			if test.ExpectSuccess {
				if err != nil {
					t.Errorf("Expected success in query %d (%s), got '%s'", i, test.Query, describe(err))
					continue
				}
				count := cursor.Count()
				if qctx.ExpectCount {
					if count != int64(len(test.ExpectedDocuments)) {
						t.Errorf("Expected count of %d, got %d in query %d (%s)", len(test.ExpectedDocuments), count, i, test.Query)
					}
				} else {
					if count != 0 {
						t.Errorf("Expected count of 0, got %d in query %d (%s)", count, i, test.Query)
					}
				}
				var result []interface{}
				for {
					hasMore := cursor.HasMore()
					doc := reflect.New(test.DocumentType)
					if _, err := cursor.ReadDocument(ctx, doc.Interface()); driver.IsNoMoreDocuments(err) {
						if hasMore {
							t.Error("HasMore returned true, but ReadDocument returns a IsNoMoreDocuments error")
						}
						break
					} else if err != nil {
						t.Errorf("Failed to result document %d: %s", len(result), describe(err))
					}
					if !hasMore {
						t.Error("HasMore returned false, but ReadDocument returns a document")
					}
					result = append(result, doc.Elem().Interface())
				}
				if len(result) != len(test.ExpectedDocuments) {
					t.Errorf("Expected %d documents, got %d in query %d (%s)", len(test.ExpectedDocuments), len(result), i, test.Query)
				} else {
					for resultIdx, resultDoc := range result {
						if !reflect.DeepEqual(resultDoc, test.ExpectedDocuments[resultIdx]) {
							t.Errorf("Unexpected document in query %d (%s) at index %d: got %+v, expected %+v", i, test.Query, resultIdx, resultDoc, test.ExpectedDocuments[resultIdx])
						}
					}
				}
				// Close anyway (this tests calling Close more than once)
				if err := cursor.Close(); err != nil {
					t.Errorf("Expected success in Close of cursor from query %d (%s), got '%s'", i, test.Query, describe(err))
				}
			} else {
				if err == nil {
					t.Errorf("Expected error in query %d (%s), got '%s'", i, test.Query, describe(err))
					continue
				}
			}
		}
	}
}

// Test stream query cursors. The goroutines are technically only
// relevant for the MMFiles engine, but don't hurt on rocksdb either
func TestCreateStreamCursor(t *testing.T) {
	ctx := context.Background()
	c := createClientFromEnv(t, true)

	version, err := c.Version(nil)
	if err != nil {
		t.Fatalf("Version failed: %s", describe(err))
	}
	if version.Version.CompareTo("3.4") < 0 {
		t.Skip("This test requires version 3.4")
		return
	}

	db := ensureDatabase(ctx, c, "cursor_stream_test", nil, t)
	col := ensureCollection(ctx, db, "cursor_stream_test", nil, t)

	// Query engine info (on rocksdb, JournalSize is always 0)
	info, err := db.EngineInfo(nil)
	if err != nil {
		t.Fatalf("Failed to get engine info: %s", describe(err))
	}

	// This might take a few seconds
	for i := 0; i < 10000; i++ {
		user := UserDoc{Name: "John", Age: i}
		if _, err := col.CreateDocument(ctx, user); err != nil {
			t.Fatalf("Expected success, got %s", describe(err))
		}
	}

	const expectedResults int = 10 * 5000
	query := "FOR doc IN cursor_stream_test RETURN doc"
	ctx2 := driver.WithQueryStream(ctx, true)
	var cursors []driver.Cursor

	// create a bunch of read-only cursors
	for i := 0; i < 10; i++ {
		cursor, err := db.Query(ctx2, query, nil)
		if err == nil {
			// Close upon exit of the function
			defer cursor.Close()
		}
		if err != nil {
			t.Errorf("Expected success in query %d (%s), got '%s'", i, query, describe(err))
			continue
		}
		count := cursor.Count()
		if count != 0 {
			t.Errorf("Expected count of 0, got %d in query %d (%s)", count, i, query)
		}
		stats := cursor.Statistics()
		count = stats.FullCount()
		if count != 0 {
			t.Errorf("Expected fullCount of 0, got %d in query %d (%s)", count, i, query)
		}
		if !cursor.HasMore() {
			t.Errorf("Expected cursor %d to have more documents", i)
		}

		cursors = append(cursors, cursor)
	}

	out := make(chan bool)
	defer close(out)

	// start a write query on the same collection inbetween
	// contrary to normal cursors which are executed right
	// away this will block until all read cursors are resolved
	go func() {
		query = "FOR doc IN 1..5 LET y = SLEEP(0.01) INSERT {name:'Peter', age:0} INTO cursor_stream_test"
		cursor, err := db.Query(ctx2, query, nil) // should not return immediately
		if err == nil {
			// Close upon exit of the function
			defer cursor.Close()
		}
		if err != nil {
			t.Errorf("Expected success in write-query %s, got '%s'", query, describe(err))
		}

		for cursor.HasMore() {
			var data interface{}
			if _, err := cursor.ReadDocument(ctx2, &data); err != nil {
				t.Errorf("Failed to read document, err: %s", describe(err))
			}
		}
		out <- true // signal write done
	}()

	readCount := 0
	go func() {
		// read all cursors until the end, server closes them automatically
		for i, cursor := range cursors {
			for cursor.HasMore() {
				var user UserDoc
				if _, err := cursor.ReadDocument(ctx2, &user); err != nil {
					t.Errorf("Failed to result document %d: %s", i, describe(err))
				}
				readCount++
			}
		}
		out <- false // signal read done
	}()

	writeDone := false
	readDone := false
	for {
		done := <-out
		if done {
			t.Logf("Write done")
			writeDone = true
		} else {
			t.Logf("Read done")
			readDone = true
		}
		// On MMFiles the read-cursors have to finish first
		if writeDone && !readDone && info.Type == driver.EngineTypeMMFiles {
			//t.Error("Write cursor was able to complete before read cursors")
			t.Logf("Write cursor was able to complete before read cursors")
		}

		if writeDone && readDone {
			break
		}
	}

	t.Logf("Read count: %d", readCount)

	if readCount != expectedResults {
		t.Errorf("Expected to read %d documents, instead got %d", expectedResults, readCount)
	}
}
<commit_msg>Fix the expected result number<commit_after>//
// DISCLAIMER
//
// Copyright 2017 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Ewout Prangsma
//

package test

import (
	"context"
	"reflect"
	"testing"
	"time"

	driver "github.com/arangodb/go-driver"
)

type queryTest struct {
	Query             string
	BindVars          map[string]interface{}
	ExpectSuccess     bool
	ExpectedDocuments []interface{}
	DocumentType      reflect.Type
}

type queryTestContext struct {
	Context     context.Context
	ExpectCount bool
}

// TestCreateCursor creates several cursors.
func TestCreateCursor(t *testing.T) {
	ctx := context.Background()
	c := createClientFromEnv(t, true)
	db := ensureDatabase(ctx, c, "cursor_test", nil, t)

	// Create data set
	collectionData := map[string][]interface{}{
		"books": []interface{}{
			Book{Title: "Book 01"},
			Book{Title: "Book 02"},
			Book{Title: "Book 03"},
			Book{Title: "Book 04"},
			Book{Title: "Book 05"},
			Book{Title: "Book 06"},
			Book{Title: "Book 07"},
			Book{Title: "Book 08"},
			Book{Title: "Book 09"},
			Book{Title: "Book 10"},
			Book{Title: "Book 11"},
			Book{Title: "Book 12"},
			Book{Title: "Book 13"},
			Book{Title: "Book 14"},
			Book{Title: "Book 15"},
			Book{Title: "Book 16"},
			Book{Title: "Book 17"},
			Book{Title: "Book 18"},
			Book{Title: "Book 19"},
			Book{Title: "Book 20"},
		},
		"users": []interface{}{
			UserDoc{Name: "John", Age: 13},
			UserDoc{Name: "Jake", Age: 25},
			UserDoc{Name: "Clair", Age: 12},
			UserDoc{Name: "Johnny", Age: 42},
			UserDoc{Name: "Blair", Age: 67},
			UserDoc{Name: "Zz", Age: 12},
		},
	}
	for colName, colDocs := range collectionData {
		col := ensureCollection(ctx, db, colName, nil, t)
		if _, _, err := col.CreateDocuments(ctx, colDocs); err != nil {
			t.Fatalf("Expected success, got %s", describe(err))
		}
	}

	// Setup tests
	tests := []queryTest{
		queryTest{
			Query:             "FOR d IN books SORT d.Title RETURN d",
			ExpectSuccess:     true,
			ExpectedDocuments: collectionData["books"],
			DocumentType:      reflect.TypeOf(Book{}),
		},
		queryTest{
			Query:             "FOR d IN books FILTER d.Title==@title SORT d.Title RETURN d",
			BindVars:          map[string]interface{}{"title": "Book 02"},
			ExpectSuccess:     true,
			ExpectedDocuments: []interface{}{collectionData["books"][1]},
			DocumentType:      reflect.TypeOf(Book{}),
		},
		queryTest{
			Query:         "FOR d IN books FILTER d.Title==@title SORT d.Title RETURN d",
			BindVars:      map[string]interface{}{"somethingelse": "Book 02"},
			ExpectSuccess: false, // Unknown `@title`
		},
		queryTest{
			Query:             "FOR u IN users FILTER u.age>100 SORT u.name RETURN u",
			ExpectSuccess:     true,
			ExpectedDocuments: []interface{}{},
			DocumentType:      reflect.TypeOf(UserDoc{}),
		},
		queryTest{
			Query:             "FOR u IN users FILTER u.age<@maxAge SORT u.name RETURN u",
			BindVars:          map[string]interface{}{"maxAge": 20},
			ExpectSuccess:     true,
			ExpectedDocuments: []interface{}{collectionData["users"][2], collectionData["users"][0], collectionData["users"][5]},
			DocumentType:      reflect.TypeOf(UserDoc{}),
		},
		queryTest{
			Query:         "FOR u IN users FILTER u.age<@maxAge SORT u.name RETURN u",
			BindVars:      map[string]interface{}{"maxage": 20},
			ExpectSuccess: false, // `@maxage` versus `@maxAge`
		},
		queryTest{
			Query:             "FOR u IN users SORT u.age RETURN u.age",
			ExpectedDocuments: []interface{}{12, 12, 13, 25, 42, 67},
			DocumentType:      reflect.TypeOf(12),
			ExpectSuccess:     true,
		},
		queryTest{
			Query:             "FOR p IN users COLLECT a = p.age WITH COUNT INTO c SORT a RETURN [a, c]",
			ExpectedDocuments: []interface{}{[]int{12, 2}, []int{13, 1}, []int{25, 1}, []int{42, 1}, []int{67, 1}},
			DocumentType:      reflect.TypeOf([]int{}),
			ExpectSuccess:     true,
		},
		queryTest{
			Query:             "FOR u IN users SORT u.name RETURN u.name",
			ExpectedDocuments: []interface{}{"Blair", "Clair", "Jake", "John", "Johnny", "Zz"},
			DocumentType:      reflect.TypeOf("foo"),
			ExpectSuccess:     true,
		},
	}

	// Setup context alternatives
	contexts := []queryTestContext{
		queryTestContext{nil, false},
		queryTestContext{context.Background(), false},
		queryTestContext{driver.WithQueryCount(nil), true},
		queryTestContext{driver.WithQueryCount(nil, true), true},
		queryTestContext{driver.WithQueryCount(nil, false), false},
		queryTestContext{driver.WithQueryBatchSize(nil, 1), false},
		queryTestContext{driver.WithQueryCache(nil), false},
		queryTestContext{driver.WithQueryCache(nil, true), false},
		queryTestContext{driver.WithQueryCache(nil, false), false},
		queryTestContext{driver.WithQueryMemoryLimit(nil, 60000), false},
		queryTestContext{driver.WithQueryTTL(nil, time.Minute), false},
		queryTestContext{driver.WithQueryBatchSize(driver.WithQueryCount(nil), 1), true},
		queryTestContext{driver.WithQueryCache(driver.WithQueryCount(driver.WithQueryBatchSize(nil, 2))), true},
	}

	// Run tests for every context alternative
	for _, qctx := range contexts {
		ctx := qctx.Context
		for i, test := range tests {
			cursor, err := db.Query(ctx, test.Query, test.BindVars)
			if err == nil {
				// Close upon exit of the function
				defer cursor.Close()
			}
			if test.ExpectSuccess {
				if err != nil {
					t.Errorf("Expected success in query %d (%s), got '%s'", i, test.Query, describe(err))
					continue
				}
				count := cursor.Count()
				if qctx.ExpectCount {
					if count != int64(len(test.ExpectedDocuments)) {
						t.Errorf("Expected count of %d, got %d in query %d (%s)", len(test.ExpectedDocuments), count, i, test.Query)
					}
				} else {
					if count != 0 {
						t.Errorf("Expected count of 0, got %d in query %d (%s)", count, i, test.Query)
					}
				}
				var result []interface{}
				for {
					hasMore := cursor.HasMore()
					doc := reflect.New(test.DocumentType)
					if _, err := cursor.ReadDocument(ctx, doc.Interface()); driver.IsNoMoreDocuments(err) {
						if hasMore {
							t.Error("HasMore returned true, but ReadDocument returns a IsNoMoreDocuments error")
						}
						break
					} else if err != nil {
						t.Errorf("Failed to result document %d: %s", len(result), describe(err))
					}
					if !hasMore {
						t.Error("HasMore returned false, but ReadDocument returns a document")
					}
					result = append(result, doc.Elem().Interface())
				}
				if len(result) != len(test.ExpectedDocuments) {
					t.Errorf("Expected %d documents, got %d in query %d (%s)", len(test.ExpectedDocuments), len(result), i, test.Query)
				} else {
					for resultIdx, resultDoc := range result {
						if !reflect.DeepEqual(resultDoc, test.ExpectedDocuments[resultIdx]) {
							t.Errorf("Unexpected document in query %d (%s) at index %d: got %+v, expected %+v", i, test.Query, resultIdx, resultDoc, test.ExpectedDocuments[resultIdx])
						}
					}
				}
				// Close anyway (this tests calling Close more than once)
				if err := cursor.Close(); err != nil {
					t.Errorf("Expected success in Close of cursor from query %d (%s), got '%s'", i, test.Query, describe(err))
				}
			} else {
				if err == nil {
					t.Errorf("Expected error in query %d (%s), got '%s'", i, test.Query, describe(err))
					continue
				}
			}
		}
	}
}

// Test stream query cursors. The goroutines are technically only
// relevant for the MMFiles engine, but don't hurt on rocksdb either
func TestCreateStreamCursor(t *testing.T) {
	ctx := context.Background()
	c := createClientFromEnv(t, true)

	version, err := c.Version(nil)
	if err != nil {
		t.Fatalf("Version failed: %s", describe(err))
	}
	if version.Version.CompareTo("3.4") < 0 {
		t.Skip("This test requires version 3.4")
		return
	}

	db := ensureDatabase(ctx, c, "cursor_stream_test", nil, t)
	col := ensureCollection(ctx, db, "cursor_stream_test", nil, t)

	// Query engine info (on rocksdb, JournalSize is always 0)
	info, err := db.EngineInfo(nil)
	if err != nil {
		t.Fatalf("Failed to get engine info: %s", describe(err))
	}

	// This might take a few seconds
	for i := 0; i < 10000; i++ {
		user := UserDoc{Name: "John", Age: i}
		if _, err := col.CreateDocument(ctx, user); err != nil {
			t.Fatalf("Expected success, got %s", describe(err))
		}
	}

	const expectedResults int = 10 * 10000
	query := "FOR doc IN cursor_stream_test RETURN doc"
	ctx2 := driver.WithQueryStream(ctx, true)
	var cursors []driver.Cursor

	// create a bunch of read-only cursors
	for i := 0; i < 10; i++ {
		cursor, err := db.Query(ctx2, query, nil)
		if err == nil {
			// Close upon exit of the function
			defer cursor.Close()
		}
		if err != nil {
			t.Errorf("Expected success in query %d (%s), got '%s'", i, query, describe(err))
			continue
		}
		count := cursor.Count()
		if count != 0 {
			t.Errorf("Expected count of 0, got %d in query %d (%s)", count, i, query)
		}
		stats := cursor.Statistics()
		count = stats.FullCount()
		if count != 0 {
			t.Errorf("Expected fullCount of 0, got %d in query %d (%s)", count, i, query)
		}
		if !cursor.HasMore() {
			t.Errorf("Expected cursor %d to have more documents", i)
		}

		cursors = append(cursors, cursor)
	}

	out := make(chan bool)
	defer close(out)

	// start a write query on the same collection inbetween
	// contrary to normal cursors which are executed right
	// away this will block until all read cursors are resolved
	go func() {
		query = "FOR doc IN 1..5 LET y = SLEEP(0.01) INSERT {name:'Peter', age:0} INTO cursor_stream_test"
		cursor, err := db.Query(ctx2, query, nil) // should not return immediately
		if err == nil {
			// Close upon exit of the function
			defer cursor.Close()
		}
		if err != nil {
			t.Errorf("Expected success in write-query %s, got '%s'", query, describe(err))
		}

		for cursor.HasMore() {
			var data interface{}
			if _, err := cursor.ReadDocument(ctx2, &data); err != nil {
				t.Errorf("Failed to read document, err: %s", describe(err))
			}
		}
		out <- true // signal write done
	}()

	readCount := 0
	go func() {
		// read all cursors until the end, server closes them automatically
		for i, cursor := range cursors {
			for cursor.HasMore() {
				var user UserDoc
				if _, err := cursor.ReadDocument(ctx2, &user); err != nil {
					t.Errorf("Failed to result document %d: %s", i, describe(err))
				}
				readCount++
			}
		}
		out <- false // signal read done
	}()

	writeDone := false
	readDone := false
	for {
		done := <-out
		if done {
			writeDone = true
		} else {
			readDone = true
		}
		// On MMFiles the read-cursors have to finish first
		if writeDone && !readDone && info.Type == driver.EngineTypeMMFiles {
			t.Error("Write cursor was able to complete before read cursors")
		}

		if writeDone && readDone {
			break
		}
	}

	if readCount != expectedResults {
		t.Errorf("Expected to read %d documents, instead got %d", expectedResults, readCount)
	}
}
<|endoftext|>


<commit_before>package httpool

import (
	"context"
	"net/http"
	"time"
)

type Httpool struct {
	client   *http.Client
	size     int
	timeout  time.Duration
	retry    int
	reqQueue chan *HttpoolRequest
}

type HttpoolRequest struct {
	Req *http.Request
	Res chan *HttpoolResponse
}

type HttpoolResponse struct {
	Res *http.Response
	Err error
}

func NewRequest(req *http.Request) *HttpoolRequest {
	return &HttpoolRequest{
		Req: req,
		Res: make(chan *HttpoolResponse, 1),
	}
}

func New(
	client *http.Client,
	size int,
	timeout time.Duration,
	retry int,
) *Httpool {
	if size < 1 {
		size = 1
	}

	httpool := &Httpool{
		client:   client,
		size:     size,
		timeout:  timeout,
		retry:    retry,
		reqQueue: make(chan *HttpoolRequest, size),
	}

	for i := 0; i < size; i++ {
		go func(httpool *Httpool) {
			for httpoolReq := range httpool.reqQueue {
				retry := httpool.retry

			Retry:
				timeoutCtx, _ := context.WithTimeout(
					context.Background(),
					httpool.timeout,
				)
				req := httpoolReq.Req.WithContext(timeoutCtx)
				res, err := client.Do(req)
				if err != nil && retry > 0 {
					retry--
					goto Retry
				}
				httpoolReq.Res <- &HttpoolResponse{
					Res: res,
					Err: err,
				}
			}

		}(httpool)
	}

	return httpool
}

func (p *Httpool) Do(req *http.Request) (*http.Response, error) {
	httpoolReq := NewRequest(req)
	p.reqQueue <- httpoolReq
	httpoolRes := <-httpoolReq.Res
	return httpoolRes.Res, httpoolRes.Err
}

func (p *Httpool) Get(url string) (resp *http.Response, err error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	return p.Do(req)
}
<commit_msg>feat: add context to httpool<commit_after>package httpool

import (
	"context"
	"net/http"
	"time"
)

type Httpool struct {
	client   *http.Client
	size     int
	timeout  time.Duration
	retry    int
	reqQueue chan *HttpoolRequest
}

type HttpoolRequest struct {
	Ctx context.Context
	Req *http.Request
	Res chan *HttpoolResponse
}

type HttpoolResponse struct {
	Res *http.Response
	Err error
}

func NewRequest(ctx context.Context, req *http.Request) *HttpoolRequest {
	return &HttpoolRequest{
		Ctx: ctx,
		Req: req,
		Res: make(chan *HttpoolResponse, 1),
	}
}

func New(
	client *http.Client,
	size int,
	timeout time.Duration,
	retry int,
) *Httpool {
	if size < 1 {
		size = 1
	}

	httpool := &Httpool{
		client:   client,
		size:     size,
		timeout:  timeout,
		retry:    retry,
		reqQueue: make(chan *HttpoolRequest, size),
	}

	for i := 0; i < size; i++ {
		go func(httpool *Httpool) {
			for httpoolReq := range httpool.reqQueue {
				retry := httpool.retry

			Retry:
				ctx, _ := context.WithTimeout(
					httpoolReq.Ctx,
					httpool.timeout,
				)
				req := httpoolReq.Req.WithContext(ctx)
				res, err := client.Do(req)
				if err != nil && retry > 0 {
					retry--
					goto Retry
				}
				httpoolReq.Res <- &HttpoolResponse{
					Res: res,
					Err: err,
				}
			}

		}(httpool)
	}

	return httpool
}

func (p *Httpool) Do(ctx context.Context, req *http.Request) (*http.Response, error) {
	httpoolReq := NewRequest(ctx, req)
	p.reqQueue <- httpoolReq
	select {
	case <-ctx.Done():
		return nil, ctx.Err()
	case httpoolRes := <-httpoolReq.Res:
		return httpoolRes.Res, httpoolRes.Err
	}
}

func (p *Httpool) Get(ctx context.Context, url string) (resp *http.Response, err error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	return p.Do(ctx, req)
}
<|endoftext|>


<commit_before>package httpseverywhere

import (
	"bytes"
	"encoding/gob"
	"regexp"

	"github.com/getlantern/golog"
	"github.com/getlantern/tldextract"
)

var (
	log     = golog.LoggerFor("httpseverywhere")
	extract = tldextract.New()
)

// Rewrite exports the rewrite method for users of this library.
var Rewrite = new()

// rewrite changes an HTTP URL to rewrite.
type rewrite func(url string) (string, bool)

//type domainRoot string

type https struct {
	log     golog.Logger
	targets map[string]*Targets
}

// A rule maps the regular expression to match and the string to change it to.
// It also stores the compiled regular expression for efficiency.
type rule struct {
	from *regexp.Regexp
	From string
	To   string
}

// An exclusion just contains the compiled regular expression exclusion pattern.
type exclusion struct {
	Pattern string
	pattern *regexp.Regexp
}

// Rules is a struct containing rules and exclusions for a given rule set. This
// is public so that we can encode and decode it from GOB format.
type Rules struct {
	Rules      []*rule
	Exclusions []*exclusion
}

// Targets contains the target hosts for the given base domain.
type Targets struct {
	wildcardPrefix []*regexp.Regexp
	wildcardSuffix []*regexp.Regexp

	// We use maps here to filter duplicates.
	WildcardPrefix map[string]bool
	WildcardSuffix map[string]bool
	Plain          map[string]bool

	Rules *Rules
}

// new creates a new rewrite instance from embedded GOB data.
func new() rewrite {
	data := MustAsset("targets.gob")
	buf := bytes.NewBuffer(data)

	dec := gob.NewDecoder(buf)
	targets := make(map[string]*Targets)
	err := dec.Decode(&targets)
	if err != nil {
		log.Errorf("Could not decode: %v", err)
		return nil
	}

	// The compiled regular expressions aren't serialized, so we have to manually
	// compile them.
	for _, v := range targets {
		for _, r := range v.Rules.Rules {
			r.from, _ = regexp.Compile(r.From)
		}

		for _, e := range v.Rules.Exclusions {
			e.pattern, _ = regexp.Compile(e.Pattern)
		}

		v.wildcardPrefix = make([]*regexp.Regexp, 0)
		for pre := range v.WildcardPrefix {
			comp, err := regexp.Compile(pre)
			if err != nil {
				v.wildcardPrefix = append(v.wildcardPrefix, comp)
			}
		}

		v.wildcardSuffix = make([]*regexp.Regexp, 0)
		for suff := range v.WildcardSuffix {
			comp, err := regexp.Compile(suff)
			if err != nil {
				v.wildcardSuffix = append(v.wildcardSuffix, comp)
			}
		}
	}
	return newRewrite(targets)
}

func (t *Targets) rewrite(url, domain string) (string, bool) {
	// We basically want to apply the associated set of rules if any of the
	// targets match the url.
	log.Debugf("Attempting to rewrite %v", domain)
	for k := range t.Plain {
		if domain == k {
			return t.Rules.rewrite(url)
		}
	}

	for _, pre := range t.wildcardPrefix {
		if pre.MatchString(url) {
			return t.Rules.rewrite(url)
		}
	}

	for _, suff := range t.wildcardSuffix {
		log.Debugf("Checking %v against %v", url, suff.String())
		if suff.MatchString(url) {
			log.Debugf("Rewriting %v with %v", url, suff.String())
			return t.Rules.rewrite(url)
		}
	}

	return url, false
}

// rewrite converts the given URL to HTTPS if there is an associated rule for
// it.
func (r *Rules) rewrite(url string) (string, bool) {
	for _, exclude := range r.Exclusions {
		if exclude.pattern.MatchString(url) {
			return url, false
		}
	}
	for _, rule := range r.Rules {
		if rule.from.MatchString(url) {
			log.Debugf("Rewriting with rules from:\n%v\n to:\n %v\nfor URL:\n"+url, rule.From, rule.To)
			return rule.from.ReplaceAllString(url, rule.To), true
		}
	}
	return url, false
}

func newRewrite(targets map[string]*Targets) rewrite {
	return (&https{log: log, targets: targets}).rewrite
}

func (h *https) rewrite(urlStr string) (string, bool) {
	result := extract.Extract(urlStr)

	domain := result.Root + "." + result.Tld
	log.Debugf("Checking domain %v", result.Root)
	//var dr domainRoot = result.Root
	if targets, ok := h.targets[result.Root]; ok {
		return targets.rewrite(urlStr, domain)
	}
	return urlStr, false
}
<commit_msg>Added timing<commit_after>package httpseverywhere

import (
	"bytes"
	"encoding/gob"
	"regexp"
	"time"

	"github.com/getlantern/golog"
	"github.com/getlantern/tldextract"
)

var (
	log     = golog.LoggerFor("httpseverywhere")
	extract = tldextract.New()
)

// Rewrite exports the rewrite method for users of this library.
var Rewrite = new()

// rewrite changes an HTTP URL to rewrite.
type rewrite func(url string) (string, bool)

//type domainRoot string

type https struct {
	log     golog.Logger
	targets map[string]*Targets
}

// A rule maps the regular expression to match and the string to change it to.
// It also stores the compiled regular expression for efficiency.
type rule struct {
	from *regexp.Regexp
	From string
	To   string
}

// An exclusion just contains the compiled regular expression exclusion pattern.
type exclusion struct {
	Pattern string
	pattern *regexp.Regexp
}

// Rules is a struct containing rules and exclusions for a given rule set. This
// is public so that we can encode and decode it from GOB format.
type Rules struct {
	Rules      []*rule
	Exclusions []*exclusion
}

// Targets contains the target hosts for the given base domain.
type Targets struct {
	wildcardPrefix []*regexp.Regexp
	wildcardSuffix []*regexp.Regexp

	// We use maps here to filter duplicates.
	WildcardPrefix map[string]bool
	WildcardSuffix map[string]bool
	Plain          map[string]bool

	Rules *Rules
}

// new creates a new rewrite instance from embedded GOB data.
func new() rewrite {
	start := time.Now()
	data := MustAsset("targets.gob")
	buf := bytes.NewBuffer(data)

	dec := gob.NewDecoder(buf)
	targets := make(map[string]*Targets)
	err := dec.Decode(&targets)
	if err != nil {
		log.Errorf("Could not decode: %v", err)
		return nil
	}
	log.Debugf("Loaded HTTPS Everywhere in %v", time.Now().Sub(start).String())

	// The compiled regular expressions aren't serialized, so we have to manually
	// compile them.
	for _, v := range targets {
		for _, r := range v.Rules.Rules {
			r.from, _ = regexp.Compile(r.From)
		}

		for _, e := range v.Rules.Exclusions {
			e.pattern, _ = regexp.Compile(e.Pattern)
		}

		v.wildcardPrefix = make([]*regexp.Regexp, 0)
		for pre := range v.WildcardPrefix {
			comp, err := regexp.Compile(pre)
			if err != nil {
				v.wildcardPrefix = append(v.wildcardPrefix, comp)
			}
		}

		v.wildcardSuffix = make([]*regexp.Regexp, 0)
		for suff := range v.WildcardSuffix {
			comp, err := regexp.Compile(suff)
			if err != nil {
				v.wildcardSuffix = append(v.wildcardSuffix, comp)
			}
		}
	}
	return newRewrite(targets)
}

func (t *Targets) rewrite(url, domain string) (string, bool) {
	// We basically want to apply the associated set of rules if any of the
	// targets match the url.
	log.Debugf("Attempting to rewrite %v", domain)
	for k := range t.Plain {
		if domain == k {
			return t.Rules.rewrite(url)
		}
	}

	for _, pre := range t.wildcardPrefix {
		if pre.MatchString(url) {
			return t.Rules.rewrite(url)
		}
	}

	for _, suff := range t.wildcardSuffix {
		log.Debugf("Checking %v against %v", url, suff.String())
		if suff.MatchString(url) {
			log.Debugf("Rewriting %v with %v", url, suff.String())
			return t.Rules.rewrite(url)
		}
	}

	return url, false
}

// rewrite converts the given URL to HTTPS if there is an associated rule for
// it.
func (r *Rules) rewrite(url string) (string, bool) {
	for _, exclude := range r.Exclusions {
		if exclude.pattern.MatchString(url) {
			return url, false
		}
	}
	for _, rule := range r.Rules {
		if rule.from.MatchString(url) {
			log.Debugf("Rewriting with rules from:\n%v\n to:\n %v\nfor URL:\n"+url, rule.From, rule.To)
			return rule.from.ReplaceAllString(url, rule.To), true
		}
	}
	return url, false
}

func newRewrite(targets map[string]*Targets) rewrite {
	return (&https{log: log, targets: targets}).rewrite
}

func (h *https) rewrite(urlStr string) (string, bool) {
	result := extract.Extract(urlStr)

	domain := result.Root + "." + result.Tld
	log.Debugf("Checking domain %v", result.Root)
	//var dr domainRoot = result.Root
	if targets, ok := h.targets[result.Root]; ok {
		return targets.rewrite(urlStr, domain)
	}
	return urlStr, false
}
<|endoftext|>


<commit_before>package gitlabclient

import (
	"net/http"
	"log"
	"os"
	"fmt"
)

func SetupK8sIntegrationForGitlabProject(projectId, namespace, token string) {
	k8sUrl := os.Getenv("K8S_API_URL")
	if k8sUrl == "" {
		// abort if K8S_API_URL was not set
		log.Println("K8S_API_URL was not set, skipping setup of K8s integration in Gitlab...")
		return
	}

	url := fmt.Sprintf("%sprojects/%s/services/kubernetes",getGitlabBaseUrl(),projectId)

	req, err := http.NewRequest(http.MethodPut, url, nil)
	if err != nil {
		log.Fatalln(err)
	}

	q := req.URL.Query()
	q.Add("token",token)
	q.Add("namespace", namespace)
	q.Add("api_url", k8sUrl)

	caPem := os.Getenv("K8S_CA_PEM")
	if caPem != "" {
		q.Add("ca_pem", caPem)
	}

	req.URL.RawQuery = q.Encode()

	req.Header.Add("PRIVATE-TOKEN", os.Getenv("GITLAB_PRIVATE_TOKEN"))

	resp, err := http.DefaultClient.Do(req)

	if err != nil {
		log.Println(fmt.Sprintf("Could not set up Kubernetes Integration for project %s . Err was: %s ", projectId, err))
	}

	if resp.StatusCode != http.StatusOK {
		log.Println(fmt.Sprintf("Setting up Kubernetes Integration for project %s failed with errorCode %d", projectId, resp.StatusCode))
	}
}

<commit_msg>implemented environment creation<commit_after>package gitlabclient

import (
	"net/http"
	"log"
	"os"
	"fmt"
	"encoding/json"
	"bytes"
	"io/ioutil"
)

func SetupK8sIntegrationForGitlabProject(projectId, namespace, token string) {
	k8sUrl := os.Getenv("K8S_API_URL")
	if k8sUrl == "" {
		// abort if K8S_API_URL was not set
		log.Println("K8S_API_URL was not set, skipping setup of K8s integration in Gitlab...")
		return
	}

	url := fmt.Sprintf("%sprojects/%s/services/kubernetes",getGitlabBaseUrl(),projectId)

	req, err := http.NewRequest(http.MethodPut, url, nil)
	if err != nil {
		log.Fatalln(err)
	}

	q := req.URL.Query()
	q.Add("token",token)
	q.Add("namespace", namespace)
	q.Add("api_url", k8sUrl)

	caPem := os.Getenv("K8S_CA_PEM")
	if caPem != "" {
		q.Add("ca_pem", caPem)
	}

	req.URL.RawQuery = q.Encode()

	req.Header.Add("PRIVATE-TOKEN", os.Getenv("GITLAB_PRIVATE_TOKEN"))

	resp, err := http.DefaultClient.Do(req)

	if err != nil {
		log.Println(fmt.Sprintf("Could not set up Kubernetes Integration for project %s . Err was: %s ", projectId, err))
	}

	if resp.StatusCode != http.StatusOK {
		log.Println(fmt.Sprintf("Setting up Kubernetes Integration for project %s failed with errorCode %d", projectId, resp.StatusCode))
	}

	setupEnvironment(projectId)
}

type ErrorMessage struct {
	Message Msg
}

type Msg struct {
	Name []string
	Slug []string
}

func setupEnvironment(projectId string){
	envName := "icc-dev"
	url := fmt.Sprintf("%sprojects/%s/environments/kubernetes",getGitlabBaseUrl(),projectId)
	values := map[string]string{"id": projectId, "name": envName}
	jsonValue, err := json.Marshal(values)
	if err != nil {
		log.Fatalln(err)
	}

	req, err := http.NewRequest(http.MethodPost,url, bytes.NewBuffer(jsonValue))
	req.Header.Add("PRIVATE-TOKEN", os.Getenv("GITLAB_PRIVATE_TOKEN"))
	req.Header.Add("Content-Type", "application/json")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		log.Fatalln(err)
	}

	switch resp.StatusCode {
	case http.StatusCreated:
		return

	case http.StatusBadRequest:
		var msg ErrorMessage
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			log.Fatal(err)
		}
		json.Unmarshal(body, &msg)
		if len(msg.Message.Name) > 0 && msg.Message.Name[0] == "has already been taken" {
			return
		}
		log.Println(fmt.Sprintf("Creation of environment failed with http error %s", resp.StatusCode))
	default:
		log.Println(fmt.Sprintf("Creation of environment failed with http error %s", resp.StatusCode))
	}
}

<|endoftext|>