<commit_before>package s3proxy

import (
	"bytes"
	"context"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"path"
	"time"

	"github.com/buchgr/bazel-remote/cache"
	"github.com/buchgr/bazel-remote/config"
	"github.com/minio/minio-go/v7"
	"github.com/minio/minio-go/v7/pkg/credentials"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
)

type uploadReq struct {
	hash string
	size int64
	kind cache.EntryKind
	rc   io.ReadCloser
}

type s3Cache struct {
	mcore        *minio.Core
	prefix       string
	bucket       string
	keyVersion   int
	uploadQueue  chan<- uploadReq
	accessLogger cache.Logger
	errorLogger  cache.Logger
	v2mode       bool
	objectKey    func(hash string, kind cache.EntryKind) string
}

var (
	cacheHits = promauto.NewCounter(prometheus.CounterOpts{
		Name: "bazel_remote_s3_cache_hits",
		Help: "The total number of s3 backend cache hits",
	})
	cacheMisses = promauto.NewCounter(prometheus.CounterOpts{
		Name: "bazel_remote_s3_cache_misses",
		Help: "The total number of s3 backend cache misses",
	})
)

// Used in place of minio's verbose "NoSuchKey" error.
var errNotFound = errors.New("NOT FOUND")

// New returns a new instance of the S3-API based cache
func New(s3Config *config.S3CloudStorageConfig, storageMode string, accessLogger cache.Logger,
	errorLogger cache.Logger, numUploaders, maxQueuedUploads int) cache.Proxy {

	fmt.Println("Using S3 backend.")

	var minioCore *minio.Core
	var err error

	if s3Config.AccessKeyID != "" && s3Config.SecretAccessKey != "" {
		// Initialize minio client object.
		opts := &minio.Options{
			Creds:  credentials.NewStaticV4(s3Config.AccessKeyID, s3Config.SecretAccessKey, ""),
			Secure: !s3Config.DisableSSL,
			Region: s3Config.Region,
		}
		minioCore, err = minio.NewCore(s3Config.Endpoint, opts)
		if err != nil {
			log.Fatalln(err)
		}
	} else {
		// Initialize minio client object with IAM credentials
		opts := &minio.Options{
			// This config value may be empty.
			Creds: credentials.NewIAM(s3Config.IAMRoleEndpoint),

			Region: s3Config.Region,
			Secure: !s3Config.DisableSSL,
		}

		minioClient, err := minio.New(
			s3Config.Endpoint,
			opts,
		)
		if err != nil {
			log.Fatalln(err)
		}

		minioCore = &minio.Core{
			Client: minioClient,
		}
	}

	if storageMode != "zstd" && storageMode != "uncompressed" {
		log.Fatalf("Unsupported storage mode for the s3proxy backend: %q, must be one of \"zstd\" or \"uncompressed\"",
			storageMode)
	}

	c := &s3Cache{
		mcore:        minioCore,
		prefix:       s3Config.Prefix,
		bucket:       s3Config.Bucket,
		keyVersion:   s3Config.KeyVersion,
		accessLogger: accessLogger,
		errorLogger:  errorLogger,
		v2mode:       storageMode == "zstd",
	}

	if c.v2mode {
		c.objectKey = func(hash string, kind cache.EntryKind) string {
			return objectKeyV2(c.prefix, hash, kind)
		}
	} else {
		c.objectKey = func(hash string, kind cache.EntryKind) string {
			return objectKeyV1(c.prefix, hash, kind)
		}
	}

	if maxQueuedUploads > 0 && numUploaders > 0 {
		uploadQueue := make(chan uploadReq, maxQueuedUploads)
		for uploader := 0; uploader < numUploaders; uploader++ {
			go func() {
				for item := range uploadQueue {
					c.uploadFile(item)
				}
			}()
		}

		c.uploadQueue = uploadQueue
	}

	return c
}

func objectKeyV2(prefix string, hash string, kind cache.EntryKind) string {
	var baseKey string
	if kind == cache.CAS {
		// Use "cas.v2" to distinguish new from old format blobs.
		baseKey = path.Join("cas.v2", hash[:2], hash)
	} else {
		baseKey = path.Join(kind.String(), hash[:2], hash)
	}

	if prefix == "" {
		return baseKey
	}

	return path.Join(prefix, baseKey)
}

func objectKeyV1(prefix string, hash string, kind cache.EntryKind) string {
	if prefix == "" {
		return path.Join(kind.String(), hash[:2], hash)
	}

	return path.Join(prefix, kind.String(), hash[:2], hash)
}

// Helper function for logging responses
func logResponse(log cache.Logger, method, bucket, key string, err error) {
	status := "OK"
	if err != nil {
		status = err.Error()
	}

	log.Printf("S3 %s %s %s %s", method, bucket, key, status)
}

func (c *s3Cache) uploadFile(item uploadReq) {
	_, err := c.mcore.PutObject(
		context.Background(),
		c.bucket,                          // bucketName
		c.objectKey(item.hash, item.kind), // objectName
		item.rc,                           // reader
		item.size,                         // objectSize
		"",                                // md5base64
		"",                                // sha256
		minio.PutObjectOptions{
			UserMetadata: map[string]string{
				"Content-Type": "application/octet-stream",
			},
		}, // metadata
	)

	logResponse(c.accessLogger, "UPLOAD", c.bucket, c.objectKey(item.hash, item.kind), err)

	item.rc.Close()
}

func (c *s3Cache) Put(kind cache.EntryKind, hash string, size int64, rc io.ReadCloser) {
	if c.uploadQueue == nil {
		rc.Close()
		return
	}

	select {
	case c.uploadQueue <- uploadReq{
		hash: hash,
		size: size,
		kind: kind,
		rc:   rc,
	}:
	default:
		c.errorLogger.Printf("too many uploads queued\n")
		rc.Close()
	}
}

func (c *s3Cache) Get(kind cache.EntryKind, hash string) (io.ReadCloser, int64, error) {

	object, info, _, err := c.mcore.GetObject(
		context.Background(),
		c.bucket,                 // bucketName
		c.objectKey(hash, kind),  // objectName
		minio.GetObjectOptions{}, // opts
	)
	if err != nil {
		if minio.ToErrorResponse(err).Code == "NoSuchKey" {
			cacheMisses.Inc()
			logResponse(c.accessLogger, "DOWNLOAD", c.bucket, c.objectKey(hash, kind), errNotFound)
			return nil, -1, nil
		}
		cacheMisses.Inc()
		logResponse(c.accessLogger, "DOWNLOAD", c.bucket, c.objectKey(hash, kind), err)
		return nil, -1, err
	}
	cacheHits.Inc()

	logResponse(c.accessLogger, "DOWNLOAD", c.bucket, c.objectKey(hash, kind), nil)

	return object, info.Size, nil
}

func (c *s3Cache) Contains(kind cache.EntryKind, hash string) (bool, int64) {
	size := int64(-1)
	exists := false

	if kind != cache.CAS {
		s, err := c.mcore.StatObject(
			context.Background(),
			c.bucket,                  // bucketName
			c.objectKey(hash, kind),   // objectName
			minio.StatObjectOptions{}, // opts
		)

		exists = (err == nil)
		if err != nil {
			err = errNotFound
		} else {
			size = s.Size
		}

		logResponse(c.accessLogger, "CONTAINS", c.bucket, c.objectKey(hash, kind), err)

		return exists, size
	}

	// Handle the more complicated, compressed CAS blob case.

	// https://github.com/minio/minio-go/issues/1106

	var err error
	var uncompressedSize int64
	var n int
	var req *http.Request
	var rsp *http.Response
	var blobHeader []byte
	var u *url.URL

	u, err = c.mcore.PresignedGetObject(context.Background(), c.bucket,
		c.objectKey(hash, kind), time.Second*20, make(url.Values))
	if err != nil {
		goto end
	}

	// TODO: The following code is essentially duplicated from the
	// httpproxy code. Refactor and reuse it (and drop the goto's)?

	req, err = http.NewRequest("GET", u.String(), nil)
	if err != nil {
		goto end
	}
	req.Header.Add("Range", "bytes=0-7")
	rsp, err = http.DefaultClient.Do(req) // FIXME: use a single http.Client
	if err != nil {
		goto end
	}
	if rsp.StatusCode != http.StatusOK && rsp.StatusCode != http.StatusPartialContent {
		goto end
	}

	blobHeader = make([]byte, 8)
	n, err = io.ReadFull(rsp.Body, blobHeader)
	if err != nil {
		goto end
	}
	defer rsp.Body.Close()
	if n != 8 {
		goto end
	}

	err = binary.Read(bytes.NewReader(blobHeader), binary.LittleEndian,
		&uncompressedSize)
	if err != nil {
		goto end
	}
	exists = true
	size = uncompressedSize

end:
	logResponse(c.accessLogger, "CONTAINS", c.bucket, c.objectKey(hash, kind), err)

	return exists, size
}
<commit_msg>[s3proxy] don't read remote file headers for CAS.v2 Contains size value<commit_after>package s3proxy

import (
	"context"
	"errors"
	"fmt"
	"io"
	"log"
	"path"

	"github.com/buchgr/bazel-remote/cache"
	"github.com/buchgr/bazel-remote/config"
	"github.com/minio/minio-go/v7"
	"github.com/minio/minio-go/v7/pkg/credentials"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
)

type uploadReq struct {
	hash string
	size int64
	kind cache.EntryKind
	rc   io.ReadCloser
}

type s3Cache struct {
	mcore        *minio.Core
	prefix       string
	bucket       string
	keyVersion   int
	uploadQueue  chan<- uploadReq
	accessLogger cache.Logger
	errorLogger  cache.Logger
	v2mode       bool
	objectKey    func(hash string, kind cache.EntryKind) string
}

var (
	cacheHits = promauto.NewCounter(prometheus.CounterOpts{
		Name: "bazel_remote_s3_cache_hits",
		Help: "The total number of s3 backend cache hits",
	})
	cacheMisses = promauto.NewCounter(prometheus.CounterOpts{
		Name: "bazel_remote_s3_cache_misses",
		Help: "The total number of s3 backend cache misses",
	})
)

// Used in place of minio's verbose "NoSuchKey" error.
var errNotFound = errors.New("NOT FOUND")

// New returns a new instance of the S3-API based cache
func New(s3Config *config.S3CloudStorageConfig, storageMode string, accessLogger cache.Logger,
	errorLogger cache.Logger, numUploaders, maxQueuedUploads int) cache.Proxy {

	fmt.Println("Using S3 backend.")

	var minioCore *minio.Core
	var err error

	if s3Config.AccessKeyID != "" && s3Config.SecretAccessKey != "" {
		// Initialize minio client object.
		opts := &minio.Options{
			Creds:  credentials.NewStaticV4(s3Config.AccessKeyID, s3Config.SecretAccessKey, ""),
			Secure: !s3Config.DisableSSL,
			Region: s3Config.Region,
		}
		minioCore, err = minio.NewCore(s3Config.Endpoint, opts)
		if err != nil {
			log.Fatalln(err)
		}
	} else {
		// Initialize minio client object with IAM credentials
		opts := &minio.Options{
			// This config value may be empty.
			Creds: credentials.NewIAM(s3Config.IAMRoleEndpoint),

			Region: s3Config.Region,
			Secure: !s3Config.DisableSSL,
		}

		minioClient, err := minio.New(
			s3Config.Endpoint,
			opts,
		)
		if err != nil {
			log.Fatalln(err)
		}

		minioCore = &minio.Core{
			Client: minioClient,
		}
	}

	if storageMode != "zstd" && storageMode != "uncompressed" {
		log.Fatalf("Unsupported storage mode for the s3proxy backend: %q, must be one of \"zstd\" or \"uncompressed\"",
			storageMode)
	}

	c := &s3Cache{
		mcore:        minioCore,
		prefix:       s3Config.Prefix,
		bucket:       s3Config.Bucket,
		keyVersion:   s3Config.KeyVersion,
		accessLogger: accessLogger,
		errorLogger:  errorLogger,
		v2mode:       storageMode == "zstd",
	}

	if c.v2mode {
		c.objectKey = func(hash string, kind cache.EntryKind) string {
			return objectKeyV2(c.prefix, hash, kind)
		}
	} else {
		c.objectKey = func(hash string, kind cache.EntryKind) string {
			return objectKeyV1(c.prefix, hash, kind)
		}
	}

	if maxQueuedUploads > 0 && numUploaders > 0 {
		uploadQueue := make(chan uploadReq, maxQueuedUploads)
		for uploader := 0; uploader < numUploaders; uploader++ {
			go func() {
				for item := range uploadQueue {
					c.uploadFile(item)
				}
			}()
		}

		c.uploadQueue = uploadQueue
	}

	return c
}

func objectKeyV2(prefix string, hash string, kind cache.EntryKind) string {
	var baseKey string
	if kind == cache.CAS {
		// Use "cas.v2" to distinguish new from old format blobs.
		baseKey = path.Join("cas.v2", hash[:2], hash)
	} else {
		baseKey = path.Join(kind.String(), hash[:2], hash)
	}

	if prefix == "" {
		return baseKey
	}

	return path.Join(prefix, baseKey)
}

func objectKeyV1(prefix string, hash string, kind cache.EntryKind) string {
	if prefix == "" {
		return path.Join(kind.String(), hash[:2], hash)
	}

	return path.Join(prefix, kind.String(), hash[:2], hash)
}

// Helper function for logging responses
func logResponse(log cache.Logger, method, bucket, key string, err error) {
	status := "OK"
	if err != nil {
		status = err.Error()
	}

	log.Printf("S3 %s %s %s %s", method, bucket, key, status)
}

func (c *s3Cache) uploadFile(item uploadReq) {
	_, err := c.mcore.PutObject(
		context.Background(),
		c.bucket,                          // bucketName
		c.objectKey(item.hash, item.kind), // objectName
		item.rc,                           // reader
		item.size,                         // objectSize
		"",                                // md5base64
		"",                                // sha256
		minio.PutObjectOptions{
			UserMetadata: map[string]string{
				"Content-Type": "application/octet-stream",
			},
		}, // metadata
	)

	logResponse(c.accessLogger, "UPLOAD", c.bucket, c.objectKey(item.hash, item.kind), err)

	item.rc.Close()
}

func (c *s3Cache) Put(kind cache.EntryKind, hash string, size int64, rc io.ReadCloser) {
	if c.uploadQueue == nil {
		rc.Close()
		return
	}

	select {
	case c.uploadQueue <- uploadReq{
		hash: hash,
		size: size,
		kind: kind,
		rc:   rc,
	}:
	default:
		c.errorLogger.Printf("too many uploads queued\n")
		rc.Close()
	}
}

func (c *s3Cache) Get(kind cache.EntryKind, hash string) (io.ReadCloser, int64, error) {

	object, info, _, err := c.mcore.GetObject(
		context.Background(),
		c.bucket,                 // bucketName
		c.objectKey(hash, kind),  // objectName
		minio.GetObjectOptions{}, // opts
	)
	if err != nil {
		if minio.ToErrorResponse(err).Code == "NoSuchKey" {
			cacheMisses.Inc()
			logResponse(c.accessLogger, "DOWNLOAD", c.bucket, c.objectKey(hash, kind), errNotFound)
			return nil, -1, nil
		}
		cacheMisses.Inc()
		logResponse(c.accessLogger, "DOWNLOAD", c.bucket, c.objectKey(hash, kind), err)
		return nil, -1, err
	}
	cacheHits.Inc()

	logResponse(c.accessLogger, "DOWNLOAD", c.bucket, c.objectKey(hash, kind), nil)

	return object, info.Size, nil
}

func (c *s3Cache) Contains(kind cache.EntryKind, hash string) (bool, int64) {
	size := int64(-1)
	exists := false

	s, err := c.mcore.StatObject(
		context.Background(),
		c.bucket,                  // bucketName
		c.objectKey(hash, kind),   // objectName
		minio.StatObjectOptions{}, // opts
	)

	exists = (err == nil)
	if err != nil {
		err = errNotFound
	} else if kind != cache.CAS {
		size = s.Size
	}

	logResponse(c.accessLogger, "CONTAINS", c.bucket, c.objectKey(hash, kind), err)

	return exists, size
}
<|endoftext|>


<commit_before>// THIS NO LONGER WORKS!!
// Latitude is using OAuth 2.0 now.
package main

import (
	"flag"
	"fmt"
	"github.com/mrjones/oauth"
	"io/ioutil"
	"log"
)

const (
	CURRENT_LOCATION_URL = "https://www.googleapis.com/latitude/v1/currentLocation"
)

func Usage() {
	fmt.Println("Usage:")
	fmt.Print("go run examples/latitude/latitude.go")
	fmt.Print("  --consumerkey <consumerkey>")
	fmt.Print("  --consumersecret <consumersecret>")
	fmt.Println("  --apikey <apikey>")
	fmt.Println("")
}

func main() {
	var consumerKey *string = flag.String("consumerkey", "", "")
	var consumerSecret *string = flag.String("consumersecret", "", "")
	var apiKey *string = flag.String("apikey", "", "")
	flag.Parse()

	c := oauth.NewConsumer(
		*consumerKey,
		*consumerSecret,
		oauth.ServiceProvider{
			RequestTokenUrl:   "https://www.google.com/accounts/OAuthGetRequestToken",
			AuthorizeTokenUrl: "https://www.google.com/latitude/apps/OAuthAuthorizeToken",
			AccessTokenUrl:    "https://www.google.com/accounts/OAuthGetAccessToken",
		})

	c.AdditionalParams["scope"] = "https://www.googleapis.com/auth/latitude"
	requestToken, url, err := c.GetRequestTokenAndUrl("oob")
	if err != nil {
		log.Fatal(err)
	}

	fmt.Println("(1) Go to: " + url + "&domain=mrjon.es&granularity=best&location=all")
	fmt.Println("(2) Grant access, you should get back a verification code.")
	fmt.Println("(3) Enter that verification code here: ")

	verificationCode := ""
	fmt.Scanln(&verificationCode)

	accessToken, err := c.AuthorizeToken(requestToken, verificationCode)
	if err != nil {
		log.Fatal(err)
	}

	response, err := c.Get(
		CURRENT_LOCATION_URL,
		map[string]string{"key": *apiKey},
		accessToken)

	defer response.Body.Close()

	bits, err := ioutil.ReadAll(response.Body)
	fmt.Println("Your latest location: " + string(bits))
}
<commit_msg>updated latitude example to new interface, even though it's broken<commit_after>// THIS NO LONGER WORKS!!
// Latitude is using OAuth 2.0 now.
package main

import (
	"flag"
	"fmt"
	"github.com/mrjones/oauth"
	"io/ioutil"
	"log"
)

const (
	CURRENT_LOCATION_URL = "https://www.googleapis.com/latitude/v1/currentLocation"
)

func Usage() {
	fmt.Println("Usage:")
	fmt.Print("go run examples/latitude/latitude.go")
	fmt.Print("  --consumerkey <consumerkey>")
	fmt.Print("  --consumersecret <consumersecret>")
	fmt.Println("  --apikey <apikey>")
	fmt.Println("")
}

func main() {
	var consumerKey *string = flag.String("consumerkey", "", "")
	var consumerSecret *string = flag.String("consumersecret", "", "")
	var apiKey *string = flag.String("apikey", "", "")
	flag.Parse()

	c := oauth.NewConsumer(
		*consumerKey,
		*consumerSecret,
		oauth.ServiceProvider{
			RequestTokenUrl:   "https://www.google.com/accounts/OAuthGetRequestToken",
			AuthorizeTokenUrl: "https://www.google.com/latitude/apps/OAuthAuthorizeToken",
			AccessTokenUrl:    "https://www.google.com/accounts/OAuthGetAccessToken",
		})

	c.AdditionalParams["scope"] = "https://www.googleapis.com/auth/latitude"
	requestToken, url, err := c.GetRequestTokenAndUrl("oob")
	if err != nil {
		log.Fatal(err)
	}

	fmt.Println("(1) Go to: " + url + "&domain=mrjon.es&granularity=best&location=all")
	fmt.Println("(2) Grant access, you should get back a verification code.")
	fmt.Println("(3) Enter that verification code here: ")

	verificationCode := ""
	fmt.Scanln(&verificationCode)

	accessToken, err := c.AuthorizeToken(requestToken, verificationCode)
	if err != nil {
		log.Fatal(err)
	}

	client, err := c.MakeHttpClient(accessToken)
	if err != nil {
		log.Fatal(err)
	}

	response, err := client.Get(
		fmt.Sprintf("%s?key=%s", CURRENT_LOCATION_URL, *apiKey))
	if err != nil {
		log.Fatal(err)
	}

	defer response.Body.Close()

	bits, err := ioutil.ReadAll(response.Body)
	fmt.Println("Your latest location: " + string(bits))
}
<|endoftext|>


<commit_before>// Copyright 2011 Google Inc. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

/*
Package capability exposes information about outages and scheduled downtime
for specific API capabilities.

This package does not work in App Engine "flexible environment".

Example:
	if !capability.Enabled(c, "datastore_v3", "write") {
		// show user a different page
	}
*/
package capability // import "google.golang.org/appengine/capability"

import (
	"golang.org/x/net/context"

	"google.golang.org/appengine/internal"
	"google.golang.org/appengine/log"

	pb "google.golang.org/appengine/internal/capability"
)

// Enabled returns whether an API's capabilities are enabled.
// The wildcard "*" capability matches every capability of an API.
// If the underlying RPC fails (if the package is unknown, for example),
// false is returned and information is written to the application log.
func Enabled(ctx context.Context, api, capability string) bool {
	req := &pb.IsEnabledRequest{
		Package:    &api,
		Capability: []string{capability},
	}
	res := &pb.IsEnabledResponse{}
	if err := internal.Call(ctx, "capability_service", "IsEnabled", req, res); err != nil {
		log.Warningf(ctx, "capability.Enabled: RPC failed: %v", err)
		return false
	}
	switch *res.SummaryStatus {
	case pb.IsEnabledResponse_ENABLED,
		pb.IsEnabledResponse_SCHEDULED_FUTURE,
		pb.IsEnabledResponse_SCHEDULED_NOW:
		return true
	case pb.IsEnabledResponse_UNKNOWN:
		log.Errorf(ctx, "capability.Enabled: unknown API capability %s/%s", api, capability)
		return false
	default:
		return false
	}
}
<commit_msg>capability: Simplify the API to remove unnecessary RPC calls (#206)<commit_after>// Copyright 2011 Google Inc. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

/*
Package capability exposes information about outages and scheduled downtime
for specific API capabilities.

This package does not work in App Engine "flexible environment".

Example:
	if !capability.Enabled(c, "datastore_v3", "write") {
		// show user a different page
	}
*/
package capability // import "google.golang.org/appengine/capability"

import (
	"golang.org/x/net/context"

	"google.golang.org/appengine/internal"
	"google.golang.org/appengine/log"

	pb "google.golang.org/appengine/internal/capability"
)

// Enabled returns whether an API's capabilities are enabled.
// The wildcard "*" capability matches every capability of an API.
// If the underlying RPC fails (if the package is unknown, for example),
// false is returned and information is written to the application log.
func Enabled(ctx context.Context, api, capability string) bool {
	// For non datastore*/write requests always return ENABLED
	if !(api == "datastore_v3" && capability == "write") {
		return true
	}

	req := &pb.IsEnabledRequest{
		Package:    &api,
		Capability: []string{capability},
	}
	res := &pb.IsEnabledResponse{}
	if err := internal.Call(ctx, "capability_service", "IsEnabled", req, res); err != nil {
		log.Warningf(ctx, "capability.Enabled: RPC failed: %v", err)
		return false
	}
	return *res.SummaryStatus == pb.IsEnabledResponse_ENABLED
}
<|endoftext|>


<commit_before>package postgres_test

import (
	"os"
	"testing"

	"github.com/omeid/conex"
	"github.com/omeid/conex/postgres"
)

func TestMain(m *testing.M) {
	os.Exit(conex.Run(m))
}

func TestMySQL(t *testing.T) {

	sql, con := postgres.Box(t, &postgres.Config{
		Database: "postgres",
		User:     "postgres",
	})
	defer con.Drop()

	var resp int
	err := sql.QueryRow("SELECT 1").Scan(&resp)

	if err != nil {
		t.Fatal(err)
	}

	if resp != 1 {
		t.Fatal("Unexpected response: %v")
	}

}
<commit_msg>Fix postgres test name<commit_after>package postgres_test

import (
	"os"
	"testing"

	"github.com/omeid/conex"
	"github.com/omeid/conex/postgres"
)

func TestMain(m *testing.M) {
	os.Exit(conex.Run(m))
}

func TestPostgres(t *testing.T) {

	sql, con := postgres.Box(t, &postgres.Config{
		Database: "postgres",
		User:     "postgres",
	})
	defer con.Drop()

	var resp int
	err := sql.QueryRow("SELECT 1").Scan(&resp)

	if err != nil {
		t.Fatal(err)
	}

	if resp != 1 {
		t.Fatal("Unexpected response: %v")
	}

}
<|endoftext|>


<commit_before>package dice

import (
	"encoding/json"
	"fmt"
	"github.com/gamelost/bot3server/server"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
)

type DiceService struct {
}

type DiceResult struct {
	Result       int64  `json:"result"`
	Details      string `json:"details"`
	Code         string `json:"code"`
	Timestamp    int64  `json:"timestamp"`
	Illustration string `json:"illustration"`
}

func (svc *DiceService) NewService() server.BotHandler {
	return &DiceService{}
}

func (svc *DiceService) Handle(botRequest *server.BotRequest, botResponse *server.BotResponse) {

	log.Println("Received Handle for !dice")
	result := svc.RollDice(parseInput(botRequest.RawLine.Text()))
	strResp := fmt.Sprintf("%s: your result: %d, Rolls: %s", botRequest.RawLine.Nick, result.Result, result.Details)
	botResponse.SetSingleLineResponse(strResp)
}

func (svc *DiceService) RollDice(diceString string) *DiceResult {

	log.Println("Rolling dice")
	// download the CAH json file
	resp, err := http.Get("http://rolz.org/api/?" + diceString + ".json")
	if err != nil {
		panic(err)
	}

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	fmt.Printf("Output: %s\n", body)
	result := &DiceResult{}

	err = json.Unmarshal(body, result)
	if err != nil {
		panic(err)
	}

	fmt.Printf("JSON: %s\n", result)
	return result
}

func parseInput(input string) string {

	input = strings.TrimPrefix(input, "!dice")
	diceRequest := strings.TrimSpace(input)
	return diceRequest
}
<commit_msg>adding dice code to output<commit_after>package dice

import (
	"encoding/json"
	"fmt"
	"github.com/gamelost/bot3server/server"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
)

type DiceService struct {
}

type DiceResult struct {
	Result       int64  `json:"result"`
	Details      string `json:"details"`
	Code         string `json:"code"`
	Timestamp    int64  `json:"timestamp"`
	Illustration string `json:"illustration"`
}

func (svc *DiceService) NewService() server.BotHandler {
	return &DiceService{}
}

func (svc *DiceService) Handle(botRequest *server.BotRequest, botResponse *server.BotResponse) {

	log.Println("Received Handle for !dice")
	result := svc.RollDice(parseInput(botRequest.RawLine.Text()))
	strResp := fmt.Sprintf("%s: your roll (%s) result: %d, Rolls: %s", botRequest.RawLine.Nick, result.Code, result.Result, result.Details)
	botResponse.SetSingleLineResponse(strResp)
}

func (svc *DiceService) RollDice(diceString string) *DiceResult {

	log.Println("Rolling dice")
	// download the CAH json file
	resp, err := http.Get("http://rolz.org/api/?" + diceString + ".json")
	if err != nil {
		panic(err)
	}

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	fmt.Printf("Output: %s\n", body)
	result := &DiceResult{}

	err = json.Unmarshal(body, result)
	if err != nil {
		panic(err)
	}

	fmt.Printf("JSON: %s\n", result)
	return result
}

func parseInput(input string) string {

	input = strings.TrimPrefix(input, "!dice")
	diceRequest := strings.TrimSpace(input)
	return diceRequest
}
<|endoftext|>


<commit_before>package llvm

/*
#include <llvm-c/Target.h>
#include <llvm-c/TargetMachine.h>
#include <stdlib.h>
*/
import "C"
import "unsafe"
import "errors"

type (
	TargetData struct {
		C C.LLVMTargetDataRef
	}
	Target struct {
		C C.LLVMTargetRef
	}
	TargetMachine struct {
		C C.LLVMTargetMachineRef
	}
	ByteOrdering    C.enum_LLVMByteOrdering
	RelocMode       C.LLVMRelocMode
	CodeGenOptLevel C.LLVMCodeGenOptLevel
	CodeGenFileType C.LLVMCodeGenFileType
	CodeModel       C.LLVMCodeModel
)

const (
	BigEndian    ByteOrdering = C.LLVMBigEndian
	LittleEndian ByteOrdering = C.LLVMLittleEndian
)

const (
	RelocDefault      RelocMode = C.LLVMRelocDefault
	RelocStatic       RelocMode = C.LLVMRelocStatic
	RelocPIC          RelocMode = C.LLVMRelocPIC
	RelocDynamicNoPic RelocMode = C.LLVMRelocDynamicNoPic
)

const (
	CodeGenLevelNone       CodeGenOptLevel = C.LLVMCodeGenLevelNone
	CodeGenLevelLess       CodeGenOptLevel = C.LLVMCodeGenLevelLess
	CodeGenLevelDefault    CodeGenOptLevel = C.LLVMCodeGenLevelDefault
	CodeGenLevelAggressive CodeGenOptLevel = C.LLVMCodeGenLevelAggressive
)

const (
	CodeModelDefault    CodeModel = C.LLVMCodeModelDefault
	CodeModelJITDefault CodeModel = C.LLVMCodeModelJITDefault
	CodeModelSmall      CodeModel = C.LLVMCodeModelSmall
	CodeModelKernel     CodeModel = C.LLVMCodeModelKernel
	CodeModelMedium     CodeModel = C.LLVMCodeModelMedium
	CodeModelLarge      CodeModel = C.LLVMCodeModelLarge
)

const (
	AssemblyFile CodeGenFileType = C.LLVMAssemblyFile
	ObjectFile   CodeGenFileType = C.LLVMObjectFile
)

// InitializeAllTargetInfos - The main program should call this function if it
// wants access to all available targets that LLVM is configured to support.
func InitializeAllTargetInfos() { C.LLVMInitializeAllTargetInfos() }

// InitializeAllTargets - The main program should call this function if it wants
// to link in all available targets that LLVM is configured to support.
func InitializeAllTargets() { C.LLVMInitializeAllTargets() }

func InitializeAllTargetMCs() { C.LLVMInitializeAllTargetMCs() }

func InitializeAllAsmParsers() { C.LLVMInitializeAllAsmParsers() }

func InitializeAllAsmPrinters() { C.LLVMInitializeAllAsmPrinters() }

var initializeNativeTargetError = errors.New("Failed to initialize native target")

// InitializeNativeTarget - The main program should call this function to
// initialize the native target corresponding to the host. This is useful
// for JIT applications to ensure that the target gets linked in correctly.
func InitializeNativeTarget() error {
	fail := C.LLVMInitializeNativeTarget()
	if fail != 0 {
		return initializeNativeTargetError
	}
	return nil
}

//-------------------------------------------------------------------------
// llvm.TargetData
//-------------------------------------------------------------------------

// Creates target data from a target layout string.
// See the constructor llvm::TargetData::TargetData.
func NewTargetData(rep string) (td TargetData) {
	crep := C.CString(rep)
	td.C = C.LLVMCreateTargetData(crep)
	C.free(unsafe.Pointer(crep))
	return
}

// Adds target data information to a pass manager. This does not take ownership
// of the target data.
// See the method llvm::PassManagerBase::add.
func (pm PassManager) Add(td TargetData) {
	C.LLVMAddTargetData(td.C, pm.C)
}

// Converts target data to a target layout string. The string must be disposed
// with LLVMDisposeMessage.
// See the constructor llvm::TargetData::TargetData.
func (td TargetData) String() (s string) {
	cmsg := C.LLVMCopyStringRepOfTargetData(td.C)
	s = C.GoString(cmsg)
	C.LLVMDisposeMessage(cmsg)
	return
}

// Returns the byte order of a target, either BigEndian or LittleEndian.
// See the method llvm::TargetData::isLittleEndian.
func (td TargetData) ByteOrder() ByteOrdering { return ByteOrdering(C.LLVMByteOrder(td.C)) }

// Returns the pointer size in bytes for a target.
// See the method llvm::TargetData::getPointerSize.
func (td TargetData) PointerSize() int { return int(C.LLVMPointerSize(td.C)) }

// Returns the integer type that is the same size as a pointer on a target.
// See the method llvm::TargetData::getIntPtrType.
func (td TargetData) IntPtrType() (t Type) { t.C = C.LLVMIntPtrType(td.C); return }

// Computes the size of a type in bytes for a target.
// See the method llvm::TargetData::getTypeSizeInBits.
func (td TargetData) TypeSizeInBits(t Type) uint64 {
	return uint64(C.LLVMSizeOfTypeInBits(td.C, t.C))
}

// Computes the storage size of a type in bytes for a target.
// See the method llvm::TargetData::getTypeStoreSize.
func (td TargetData) TypeStoreSize(t Type) uint64 {
	return uint64(C.LLVMStoreSizeOfType(td.C, t.C))
}

// Computes the ABI size of a type in bytes for a target.
// See the method llvm::TargetData::getTypeAllocSize.
func (td TargetData) TypeAllocSize(t Type) uint64 {
	return uint64(C.LLVMABISizeOfType(td.C, t.C))
}

// Computes the ABI alignment of a type in bytes for a target.
// See the method llvm::TargetData::getABITypeAlignment.
func (td TargetData) ABITypeAlignment(t Type) int {
	return int(C.LLVMABIAlignmentOfType(td.C, t.C))
}

// Computes the call frame alignment of a type in bytes for a target.
// See the method llvm::TargetData::getCallFrameTypeAlignment.
func (td TargetData) CallFrameTypeAlignment(t Type) int {
	return int(C.LLVMCallFrameAlignmentOfType(td.C, t.C))
}

// Computes the preferred alignment of a type in bytes for a target.
// See the method llvm::TargetData::getPrefTypeAlignment.
func (td TargetData) PrefTypeAlignment(t Type) int {
	return int(C.LLVMPreferredAlignmentOfType(td.C, t.C))
}

// Computes the preferred alignment of a global variable in bytes for a target.
// See the method llvm::TargetData::getPreferredAlignment.
func (td TargetData) PreferredAlignment(g Value) int {
	return int(C.LLVMPreferredAlignmentOfGlobal(td.C, g.C))
}

// Computes the structure element that contains the byte offset for a target.
// See the method llvm::StructLayout::getElementContainingOffset.
func (td TargetData) ElementContainingOffset(t Type, offset uint64) int {
	return int(C.LLVMElementAtOffset(td.C, t.C, C.ulonglong(offset)))
}

// Computes the byte offset of the indexed struct element for a target.
// See the method llvm::StructLayout::getElementOffset.
func (td TargetData) ElementOffset(t Type, element int) uint64 {
	return uint64(C.LLVMOffsetOfElement(td.C, t.C, C.unsigned(element)))
}

// Deallocates a TargetData.
// See the destructor llvm::TargetData::~TargetData.
func (td TargetData) Dispose() { C.LLVMDisposeTargetData(td.C) }

///////////////////////////////////////////////////////////////////////////////
// Target

func FirstTarget() Target {
	return Target{C.LLVMGetFirstTarget()}
}

func (t Target) NextTarget() Target {
	return Target{C.LLVMGetNextTarget(t.C)}
}

func (t Target) Name() string {
	return C.GoString(C.LLVMGetTargetName(t.C))
}

func (t Target) Description() string {
	return C.GoString(C.LLVMGetTargetDescription(t.C))
}

///////////////////////////////////////////////////////////////////////////////
// TargetMachine

// CreateTargetMachine creates a new TargetMachine.
func (t Target) CreateTargetMachine(Triple string, CPU string, Features string,
	Level CodeGenOptLevel, Reloc RelocMode,
	CodeModel CodeModel) (tm TargetMachine) {
	cTriple := C.CString(Triple)
	cCPU := C.CString(CPU)
	cFeatures := C.CString(Features)
	tm.C = C.LLVMCreateTargetMachine(t.C, cTriple, cCPU, cFeatures,
		C.LLVMCodeGenOptLevel(Level),
		C.LLVMRelocMode(Reloc),
		C.LLVMCodeModel(CodeModel))
	C.free(unsafe.Pointer(cTriple))
	C.free(unsafe.Pointer(cCPU))
	C.free(unsafe.Pointer(cFeatures))
	return
}

// Triple returns the triple describing the machine (arch-vendor-os).
func (tm TargetMachine) Triple() string {
	cstr := C.LLVMGetTargetMachineTriple(tm.C)
	return C.GoString(cstr)
}

// TargetData returns the TargetData for the machine.
func (tm TargetMachine) TargetData() TargetData {
	return TargetData{C.LLVMGetTargetMachineData(tm.C)}
}

// Dispose releases resources related to the TargetMachine.
func (tm TargetMachine) Dispose() {
	C.LLVMDisposeTargetMachine(tm.C)
}

func DefaultTargetTriple() (triple string) {
	cTriple := C.LLVMGetDefaultTargetTriple()
	triple = C.GoString(cTriple)
	C.free(unsafe.Pointer(cTriple))
	return
}
<commit_msg>Introduce llvm.GetTargetFromTriple()<commit_after>package llvm

/*
#include <llvm-c/Target.h>
#include <llvm-c/TargetMachine.h>
#include <stdlib.h>
*/
import "C"
import "unsafe"
import "errors"

type (
	TargetData struct {
		C C.LLVMTargetDataRef
	}
	Target struct {
		C C.LLVMTargetRef
	}
	TargetMachine struct {
		C C.LLVMTargetMachineRef
	}
	ByteOrdering    C.enum_LLVMByteOrdering
	RelocMode       C.LLVMRelocMode
	CodeGenOptLevel C.LLVMCodeGenOptLevel
	CodeGenFileType C.LLVMCodeGenFileType
	CodeModel       C.LLVMCodeModel
)

const (
	BigEndian    ByteOrdering = C.LLVMBigEndian
	LittleEndian ByteOrdering = C.LLVMLittleEndian
)

const (
	RelocDefault      RelocMode = C.LLVMRelocDefault
	RelocStatic       RelocMode = C.LLVMRelocStatic
	RelocPIC          RelocMode = C.LLVMRelocPIC
	RelocDynamicNoPic RelocMode = C.LLVMRelocDynamicNoPic
)

const (
	CodeGenLevelNone       CodeGenOptLevel = C.LLVMCodeGenLevelNone
	CodeGenLevelLess       CodeGenOptLevel = C.LLVMCodeGenLevelLess
	CodeGenLevelDefault    CodeGenOptLevel = C.LLVMCodeGenLevelDefault
	CodeGenLevelAggressive CodeGenOptLevel = C.LLVMCodeGenLevelAggressive
)

const (
	CodeModelDefault    CodeModel = C.LLVMCodeModelDefault
	CodeModelJITDefault CodeModel = C.LLVMCodeModelJITDefault
	CodeModelSmall      CodeModel = C.LLVMCodeModelSmall
	CodeModelKernel     CodeModel = C.LLVMCodeModelKernel
	CodeModelMedium     CodeModel = C.LLVMCodeModelMedium
	CodeModelLarge      CodeModel = C.LLVMCodeModelLarge
)

const (
	AssemblyFile CodeGenFileType = C.LLVMAssemblyFile
	ObjectFile   CodeGenFileType = C.LLVMObjectFile
)

// InitializeAllTargetInfos - The main program should call this function if it
// wants access to all available targets that LLVM is configured to support.
func InitializeAllTargetInfos() { C.LLVMInitializeAllTargetInfos() }

// InitializeAllTargets - The main program should call this function if it wants
// to link in all available targets that LLVM is configured to support.
func InitializeAllTargets() { C.LLVMInitializeAllTargets() }

func InitializeAllTargetMCs() { C.LLVMInitializeAllTargetMCs() }

func InitializeAllAsmParsers() { C.LLVMInitializeAllAsmParsers() }

func InitializeAllAsmPrinters() { C.LLVMInitializeAllAsmPrinters() }

var initializeNativeTargetError = errors.New("Failed to initialize native target")

// InitializeNativeTarget - The main program should call this function to
// initialize the native target corresponding to the host. This is useful
// for JIT applications to ensure that the target gets linked in correctly.
func InitializeNativeTarget() error {
	fail := C.LLVMInitializeNativeTarget()
	if fail != 0 {
		return initializeNativeTargetError
	}
	return nil
}

//-------------------------------------------------------------------------
// llvm.TargetData
//-------------------------------------------------------------------------

// Creates target data from a target layout string.
// See the constructor llvm::TargetData::TargetData.
func NewTargetData(rep string) (td TargetData) {
	crep := C.CString(rep)
	td.C = C.LLVMCreateTargetData(crep)
	C.free(unsafe.Pointer(crep))
	return
}

// Adds target data information to a pass manager. This does not take ownership
// of the target data.
// See the method llvm::PassManagerBase::add.
func (pm PassManager) Add(td TargetData) {
	C.LLVMAddTargetData(td.C, pm.C)
}

// Converts target data to a target layout string. The string must be disposed
// with LLVMDisposeMessage.
// See the constructor llvm::TargetData::TargetData.
func (td TargetData) String() (s string) {
	cmsg := C.LLVMCopyStringRepOfTargetData(td.C)
	s = C.GoString(cmsg)
	C.LLVMDisposeMessage(cmsg)
	return
}

// Returns the byte order of a target, either BigEndian or LittleEndian.
// See the method llvm::TargetData::isLittleEndian.
func (td TargetData) ByteOrder() ByteOrdering { return ByteOrdering(C.LLVMByteOrder(td.C)) }

// Returns the pointer size in bytes for a target.
// See the method llvm::TargetData::getPointerSize.
func (td TargetData) PointerSize() int { return int(C.LLVMPointerSize(td.C)) }

// Returns the integer type that is the same size as a pointer on a target.
// See the method llvm::TargetData::getIntPtrType.
func (td TargetData) IntPtrType() (t Type) { t.C = C.LLVMIntPtrType(td.C); return }

// Computes the size of a type in bytes for a target.
// See the method llvm::TargetData::getTypeSizeInBits.
func (td TargetData) TypeSizeInBits(t Type) uint64 {
	return uint64(C.LLVMSizeOfTypeInBits(td.C, t.C))
}

// Computes the storage size of a type in bytes for a target.
// See the method llvm::TargetData::getTypeStoreSize.
func (td TargetData) TypeStoreSize(t Type) uint64 {
	return uint64(C.LLVMStoreSizeOfType(td.C, t.C))
}

// Computes the ABI size of a type in bytes for a target.
// See the method llvm::TargetData::getTypeAllocSize.
func (td TargetData) TypeAllocSize(t Type) uint64 {
	return uint64(C.LLVMABISizeOfType(td.C, t.C))
}

// Computes the ABI alignment of a type in bytes for a target.
// See the method llvm::TargetData::getABITypeAlignment.
func (td TargetData) ABITypeAlignment(t Type) int {
	return int(C.LLVMABIAlignmentOfType(td.C, t.C))
}

// Computes the call frame alignment of a type in bytes for a target.
// See the method llvm::TargetData::getCallFrameTypeAlignment.
func (td TargetData) CallFrameTypeAlignment(t Type) int {
	return int(C.LLVMCallFrameAlignmentOfType(td.C, t.C))
}

// Computes the preferred alignment of a type in bytes for a target.
// See the method llvm::TargetData::getPrefTypeAlignment.
func (td TargetData) PrefTypeAlignment(t Type) int {
	return int(C.LLVMPreferredAlignmentOfType(td.C, t.C))
}

// Computes the preferred alignment of a global variable in bytes for a target.
// See the method llvm::TargetData::getPreferredAlignment.
func (td TargetData) PreferredAlignment(g Value) int {
	return int(C.LLVMPreferredAlignmentOfGlobal(td.C, g.C))
}

// Computes the structure element that contains the byte offset for a target.
// See the method llvm::StructLayout::getElementContainingOffset.
func (td TargetData) ElementContainingOffset(t Type, offset uint64) int {
	return int(C.LLVMElementAtOffset(td.C, t.C, C.ulonglong(offset)))
}

// Computes the byte offset of the indexed struct element for a target.
// See the method llvm::StructLayout::getElementOffset.
func (td TargetData) ElementOffset(t Type, element int) uint64 {
	return uint64(C.LLVMOffsetOfElement(td.C, t.C, C.unsigned(element)))
}

// Deallocates a TargetData.
// See the destructor llvm::TargetData::~TargetData.
func (td TargetData) Dispose() { C.LLVMDisposeTargetData(td.C) }

///////////////////////////////////////////////////////////////////////////////
// Target

func FirstTarget() Target {
	return Target{C.LLVMGetFirstTarget()}
}

func (t Target) NextTarget() Target {
	return Target{C.LLVMGetNextTarget(t.C)}
}

func GetTargetFromTriple(triple string) (t Target, err error) {
	var errstr *C.char
	ctriple := C.CString(triple)
	fail := C.LLVMGetTargetFromTriple(ctriple, &t.C, &errstr)
	if fail != 0 {
		err = errors.New(C.GoString(errstr))
		C.free(unsafe.Pointer(errstr))
	}
	C.free(unsafe.Pointer(ctriple))
	return
}

func (t Target) Name() string {
	return C.GoString(C.LLVMGetTargetName(t.C))
}

func (t Target) Description() string {
	return C.GoString(C.LLVMGetTargetDescription(t.C))
}

///////////////////////////////////////////////////////////////////////////////
// TargetMachine

// CreateTargetMachine creates a new TargetMachine.
func (t Target) CreateTargetMachine(Triple string, CPU string, Features string,
	Level CodeGenOptLevel, Reloc RelocMode,
	CodeModel CodeModel) (tm TargetMachine) {
	cTriple := C.CString(Triple)
	cCPU := C.CString(CPU)
	cFeatures := C.CString(Features)
	tm.C = C.LLVMCreateTargetMachine(t.C, cTriple, cCPU, cFeatures,
		C.LLVMCodeGenOptLevel(Level),
		C.LLVMRelocMode(Reloc),
		C.LLVMCodeModel(CodeModel))
	C.free(unsafe.Pointer(cTriple))
	C.free(unsafe.Pointer(cCPU))
	C.free(unsafe.Pointer(cFeatures))
	return
}

// Triple returns the triple describing the machine (arch-vendor-os).
func (tm TargetMachine) Triple() string {
	cstr := C.LLVMGetTargetMachineTriple(tm.C)
	return C.GoString(cstr)
}

// TargetData returns the TargetData for the machine.
func (tm TargetMachine) TargetData() TargetData {
	return TargetData{C.LLVMGetTargetMachineData(tm.C)}
}

// Dispose releases resources related to the TargetMachine.
func (tm TargetMachine) Dispose() {
	C.LLVMDisposeTargetMachine(tm.C)
}

func DefaultTargetTriple() (triple string) {
	cTriple := C.LLVMGetDefaultTargetTriple()
	triple = C.GoString(cTriple)
	C.free(unsafe.Pointer(cTriple))
	return
}
<|endoftext|>


<commit_before>// Copyright 2021 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sTest

import (
	"fmt"
	"net"
	"strings"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"
	. "github.com/onsi/gomega"
)

var _ = SkipDescribeIf(func() bool {
	return helpers.RunsOnEKS() || helpers.RunsOnGKE() || helpers.DoesNotRunWithKubeProxyReplacement() || helpers.DoesNotExistNodeWithoutCilium()
}, "K8sEgressGatewayTest", func() {
	var (
		kubectl         *helpers.Kubectl
		ciliumFilename  string
		randomNamespace string

		egressIP  string
		k8s1IP    string
		k8s2IP    string
		outsideIP string

		assignIPYAML string
		echoPodYAML  string
		policyYAML   string

		namespaceSelector string = "ns=cilium-test"
	)

	runEchoServer := func() {
		// Run echo server on outside node
		originalEchoPodPath := helpers.ManifestGet(kubectl.BasePath(), "echoserver-hostnetns.yaml")
		res := kubectl.ExecMiddle("mktemp")
		res.ExpectSuccess()
		echoPodYAML = strings.Trim(res.Stdout(), "\n")
		kubectl.ExecMiddle(fmt.Sprintf("sed 's/NODE_WITHOUT_CILIUM/%s/' %s > %s",
			helpers.GetNodeWithoutCilium(), originalEchoPodPath, echoPodYAML)).ExpectSuccess()
		kubectl.ApplyDefault(echoPodYAML).ExpectSuccess("Cannot install echoserver application")
		Expect(kubectl.WaitforPods(helpers.DefaultNamespace, "-l name=echoserver-hostnetns",
			helpers.HelperTimeout)).Should(BeNil())
	}

	assignEgressIP := func() {
		// Assign egress IP address to k8s2
		originalAssignIPYAML := helpers.ManifestGet(kubectl.BasePath(), "egress-ip-deployment.yaml")
		res := kubectl.ExecMiddle("mktemp")
		res.ExpectSuccess()
		assignIPYAML = strings.Trim(res.Stdout(), "\n")
		kubectl.ExecMiddle(fmt.Sprintf("sed 's/INPUT_EGRESS_IP/%s/' %s > %s",
			egressIP, originalAssignIPYAML, assignIPYAML)).ExpectSuccess()
		res = kubectl.ApplyDefault(assignIPYAML)
		Expect(res).Should(helpers.CMDSuccess(), "unable to apply %s", assignIPYAML)
		Expect(kubectl.WaitforPods(helpers.DefaultNamespace, "-l name=egress-ip-assign",
			helpers.HelperTimeout)).Should(BeNil())

		// Wait egressIP online
		srcPod, _ := fetchPodsWithOffset(kubectl, helpers.DefaultNamespace, "", "name=egress-ip-assign", "", false, 0)
		res = kubectl.ExecPodCmd(helpers.DefaultNamespace, srcPod, helpers.PingWithCount(egressIP, 5))
		res.ExpectSuccess()
	}

	BeforeAll(func() {

		kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)

		_, k8s1IP = kubectl.GetNodeInfo(helpers.K8s1)
		_, k8s2IP = kubectl.GetNodeInfo(helpers.K8s2)
		_, outsideIP = kubectl.GetNodeInfo(helpers.GetNodeWithoutCilium())

		egressIP = getEgressIP(k8s1IP)

		deploymentManager.SetKubectl(kubectl)

		ciliumFilename = helpers.TimestampFilename("cilium.yaml")
		DeployCiliumAndDNS(kubectl, ciliumFilename)

		runEchoServer()
		assignEgressIP()
	})

	AfterAll(func() {
		_ = kubectl.Delete(echoPodYAML)
		_ = kubectl.Delete(assignIPYAML)

		UninstallCiliumFromManifest(kubectl, ciliumFilename)
	})

	AfterFailed(func() {
		// Especially check if there are duplicated address allocated on cilium_host
		kubectl.CiliumReport("ip addr")
	})

	testEgressGateway := func(fromGateway bool) {
		if fromGateway {
			By("Check egress policy from gateway node")
		} else {
			By("Check egress policy from non-gateway node")
		}

		hostIP := k8s1IP
		if fromGateway {
			hostIP = k8s2IP
		}
		srcPod, _ := fetchPodsWithOffset(kubectl, randomNamespace, "client", "zgroup=testDSClient", hostIP, false, 1)

		res := kubectl.ExecPodCmd(randomNamespace, srcPod, helpers.CurlFail("http://%s:80", outsideIP))
		res.ExpectSuccess()
		res.ExpectMatchesRegexp(fmt.Sprintf("client_address=::ffff:%s\n", egressIP))
	}

	testConnectivity := func(fromGateway bool) {
		if fromGateway {
			By("Check connectivity from gateway node")
		} else {
			By("Check connectivity from non-gateway node")
		}
		hostIP := k8s1IP
		if fromGateway {
			hostIP = k8s2IP
		}
		srcPod, _ := fetchPodsWithOffset(kubectl, randomNamespace, "client", "zgroup=testDSClient", hostIP, false, 1)

		// Pod-to-node connectivity should work
		res := kubectl.ExecPodCmd(randomNamespace, srcPod, helpers.PingWithCount(k8s1IP, 1))
		res.ExpectSuccess()

		res = kubectl.ExecPodCmd(randomNamespace, srcPod, helpers.PingWithCount(k8s2IP, 1))
		res.ExpectSuccess()

		// DNS query should work (pod-to-pod connectivity)
		res = kubectl.ExecPodCmd(randomNamespace, srcPod, "dig kubernetes +time=2")
		res.ExpectSuccess()
	}

	applyEgressPolicy := func() {
		// Apply egress policy yaml
		originalPolicyYAML := helpers.ManifestGet(kubectl.BasePath(), "egress-nat-policy.yaml")
		res := kubectl.ExecMiddle("mktemp")
		res.ExpectSuccess()
		policyYAML = strings.Trim(res.Stdout(), "\n")
		kubectl.ExecMiddle(fmt.Sprintf("sed 's/INPUT_EGRESS_IP/%s/' %s > %s",
			egressIP, originalPolicyYAML, policyYAML)).ExpectSuccess()
		kubectl.ExecMiddle(fmt.Sprintf("sed 's/INPUT_OUTSIDE_NODE_IP/%s/' -i %s",
			outsideIP, policyYAML)).ExpectSuccess()
		res = kubectl.ApplyDefault(policyYAML)
		Expect(res).Should(helpers.CMDSuccess(), "unable to apply %s", policyYAML)
	}

	Context("tunnel disabled with endpoint routes enabled", func() {
		BeforeAll(func() {
			DeployCiliumOptionsAndDNS(kubectl, ciliumFilename, map[string]string{
				"egressGateway.enabled":  "true",
				"tunnel":                 "disabled",
				"autoDirectNodeRoutes":   "true",
				"bpf.masquerade":         "true",
				"endpointRoutes.enabled": "true",
			})

			randomNamespace = deploymentManager.DeployRandomNamespaceShared(DemoDaemonSet)
			kubectl.NamespaceLabel(randomNamespace, namespaceSelector)
			deploymentManager.WaitUntilReady()

		})

		AfterAll(func() {
			deploymentManager.DeleteAll()
			DeployCiliumAndDNS(kubectl, ciliumFilename)
		})

		It("Checks connectivity works without policy", func() {
			testConnectivity(false)
			testConnectivity(true)
		})

		It("Checks egress policy and basic connectivity both work", func() {
			applyEgressPolicy()
			kubectl.WaitForEgressPolicyEntry(k8s1IP, outsideIP)
			kubectl.WaitForEgressPolicyEntry(k8s2IP, outsideIP)

			defer kubectl.Delete(policyYAML)

			testEgressGateway(true)
			testEgressGateway(false)
			testConnectivity(true)
			testConnectivity(false)
		})

	})

	Context("tunnel disabled with endpoint routes disabled", func() {

		BeforeAll(func() {
			DeployCiliumOptionsAndDNS(kubectl, ciliumFilename, map[string]string{
				"egressGateway.enabled":  "true",
				"tunnel":                 "disabled",
				"autoDirectNodeRoutes":   "true",
				"bpf.masquerade":         "true",
				"endpointRoutes.enabled": "false",
			})

			randomNamespace = deploymentManager.DeployRandomNamespaceShared(DemoDaemonSet)
			kubectl.NamespaceLabel(randomNamespace, namespaceSelector)
			deploymentManager.WaitUntilReady()
		})

		AfterAll(func() {
			deploymentManager.DeleteAll()
			DeployCiliumAndDNS(kubectl, ciliumFilename)
		})

		It("Checks connectivity works without policy", func() {
			testConnectivity(false)
			testConnectivity(true)
		})

		It("Checks egress policy and basic connectivity both work", func() {
			applyEgressPolicy()
			kubectl.WaitForEgressPolicyEntry(k8s1IP, outsideIP)
			kubectl.WaitForEgressPolicyEntry(k8s2IP, outsideIP)

			defer kubectl.Delete(policyYAML)

			testEgressGateway(false)
			testEgressGateway(true)
			testConnectivity(false)
			testConnectivity(true)
		})

	})

	Context("tunnel vxlan", func() {

		BeforeAll(func() {
			DeployCiliumOptionsAndDNS(kubectl, ciliumFilename, map[string]string{
				"egressGateway.enabled": "true",
				"bpf.masquerade":        "true",
				"tunnel":                "vxlan",
			})

			randomNamespace = deploymentManager.DeployRandomNamespaceShared(DemoDaemonSet)
			kubectl.NamespaceLabel(randomNamespace, namespaceSelector)
			deploymentManager.WaitUntilReady()
		})

		AfterAll(func() {
			deploymentManager.DeleteAll()
			DeployCiliumAndDNS(kubectl, ciliumFilename)
		})

		It("Checks connectivity works without policy", func() {
			testConnectivity(false)
			testConnectivity(true)
		})

		It("Checks egress policy and basic connectivity both work", func() {
			applyEgressPolicy()
			kubectl.WaitForEgressPolicyEntry(k8s1IP, outsideIP)
			kubectl.WaitForEgressPolicyEntry(k8s2IP, outsideIP)

			defer kubectl.Delete(policyYAML)

			testEgressGateway(false)
			testEgressGateway(true)
			testConnectivity(false)
			testConnectivity(true)
		})

	})

})

// Use x.x.x.100 as the egress IP
func getEgressIP(nodeIP string) string {
	ip := net.ParseIP(nodeIP).To4()
	ip[3] = 100
	return ip.String()
}
<commit_msg>k8sT/Egress: comment about deploying cilium twice<commit_after>// Copyright 2021 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sTest

import (
	"fmt"
	"net"
	"strings"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"
	. "github.com/onsi/gomega"
)

var _ = SkipDescribeIf(func() bool {
	return helpers.RunsOnEKS() || helpers.RunsOnGKE() || helpers.DoesNotRunWithKubeProxyReplacement() || helpers.DoesNotExistNodeWithoutCilium()
}, "K8sEgressGatewayTest", func() {
	var (
		kubectl         *helpers.Kubectl
		ciliumFilename  string
		randomNamespace string

		egressIP  string
		k8s1IP    string
		k8s2IP    string
		outsideIP string

		assignIPYAML string
		echoPodYAML  string
		policyYAML   string

		namespaceSelector string = "ns=cilium-test"
	)

	runEchoServer := func() {
		// Run echo server on outside node
		originalEchoPodPath := helpers.ManifestGet(kubectl.BasePath(), "echoserver-hostnetns.yaml")
		res := kubectl.ExecMiddle("mktemp")
		res.ExpectSuccess()
		echoPodYAML = strings.Trim(res.Stdout(), "\n")
		kubectl.ExecMiddle(fmt.Sprintf("sed 's/NODE_WITHOUT_CILIUM/%s/' %s > %s",
			helpers.GetNodeWithoutCilium(), originalEchoPodPath, echoPodYAML)).ExpectSuccess()
		kubectl.ApplyDefault(echoPodYAML).ExpectSuccess("Cannot install echoserver application")
		Expect(kubectl.WaitforPods(helpers.DefaultNamespace, "-l name=echoserver-hostnetns",
			helpers.HelperTimeout)).Should(BeNil())
	}

	assignEgressIP := func() {
		// Assign egress IP address to k8s2
		originalAssignIPYAML := helpers.ManifestGet(kubectl.BasePath(), "egress-ip-deployment.yaml")
		res := kubectl.ExecMiddle("mktemp")
		res.ExpectSuccess()
		assignIPYAML = strings.Trim(res.Stdout(), "\n")
		kubectl.ExecMiddle(fmt.Sprintf("sed 's/INPUT_EGRESS_IP/%s/' %s > %s",
			egressIP, originalAssignIPYAML, assignIPYAML)).ExpectSuccess()
		res = kubectl.ApplyDefault(assignIPYAML)
		Expect(res).Should(helpers.CMDSuccess(), "unable to apply %s", assignIPYAML)
		Expect(kubectl.WaitforPods(helpers.DefaultNamespace, "-l name=egress-ip-assign",
			helpers.HelperTimeout)).Should(BeNil())

		// Wait egressIP online
		srcPod, _ := fetchPodsWithOffset(kubectl, helpers.DefaultNamespace, "", "name=egress-ip-assign", "", false, 0)
		res = kubectl.ExecPodCmd(helpers.DefaultNamespace, srcPod, helpers.PingWithCount(egressIP, 5))
		res.ExpectSuccess()
	}

	BeforeAll(func() {

		kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)

		_, k8s1IP = kubectl.GetNodeInfo(helpers.K8s1)
		_, k8s2IP = kubectl.GetNodeInfo(helpers.K8s2)
		_, outsideIP = kubectl.GetNodeInfo(helpers.GetNodeWithoutCilium())

		egressIP = getEgressIP(k8s1IP)

		deploymentManager.SetKubectl(kubectl)

		// We deploy cilium, to run the echo server and assign egress IP, and redeploy with
		// different configurations for the tests.
		ciliumFilename = helpers.TimestampFilename("cilium.yaml")
		DeployCiliumAndDNS(kubectl, ciliumFilename)

		runEchoServer()
		assignEgressIP()
	})

	AfterAll(func() {
		_ = kubectl.Delete(echoPodYAML)
		_ = kubectl.Delete(assignIPYAML)

		UninstallCiliumFromManifest(kubectl, ciliumFilename)
	})

	AfterFailed(func() {
		// Especially check if there are duplicated address allocated on cilium_host
		kubectl.CiliumReport("ip addr")
	})

	testEgressGateway := func(fromGateway bool) {
		if fromGateway {
			By("Check egress policy from gateway node")
		} else {
			By("Check egress policy from non-gateway node")
		}

		hostIP := k8s1IP
		if fromGateway {
			hostIP = k8s2IP
		}
		srcPod, _ := fetchPodsWithOffset(kubectl, randomNamespace, "client", "zgroup=testDSClient", hostIP, false, 1)

		res := kubectl.ExecPodCmd(randomNamespace, srcPod, helpers.CurlFail("http://%s:80", outsideIP))
		res.ExpectSuccess()
		res.ExpectMatchesRegexp(fmt.Sprintf("client_address=::ffff:%s\n", egressIP))
	}

	testConnectivity := func(fromGateway bool) {
		if fromGateway {
			By("Check connectivity from gateway node")
		} else {
			By("Check connectivity from non-gateway node")
		}
		hostIP := k8s1IP
		if fromGateway {
			hostIP = k8s2IP
		}
		srcPod, _ := fetchPodsWithOffset(kubectl, randomNamespace, "client", "zgroup=testDSClient", hostIP, false, 1)

		// Pod-to-node connectivity should work
		res := kubectl.ExecPodCmd(randomNamespace, srcPod, helpers.PingWithCount(k8s1IP, 1))
		res.ExpectSuccess()

		res = kubectl.ExecPodCmd(randomNamespace, srcPod, helpers.PingWithCount(k8s2IP, 1))
		res.ExpectSuccess()

		// DNS query should work (pod-to-pod connectivity)
		res = kubectl.ExecPodCmd(randomNamespace, srcPod, "dig kubernetes +time=2")
		res.ExpectSuccess()
	}

	applyEgressPolicy := func() {
		// Apply egress policy yaml
		originalPolicyYAML := helpers.ManifestGet(kubectl.BasePath(), "egress-nat-policy.yaml")
		res := kubectl.ExecMiddle("mktemp")
		res.ExpectSuccess()
		policyYAML = strings.Trim(res.Stdout(), "\n")
		kubectl.ExecMiddle(fmt.Sprintf("sed 's/INPUT_EGRESS_IP/%s/' %s > %s",
			egressIP, originalPolicyYAML, policyYAML)).ExpectSuccess()
		kubectl.ExecMiddle(fmt.Sprintf("sed 's/INPUT_OUTSIDE_NODE_IP/%s/' -i %s",
			outsideIP, policyYAML)).ExpectSuccess()
		res = kubectl.ApplyDefault(policyYAML)
		Expect(res).Should(helpers.CMDSuccess(), "unable to apply %s", policyYAML)
	}

	Context("tunnel disabled with endpoint routes enabled", func() {
		BeforeAll(func() {
			DeployCiliumOptionsAndDNS(kubectl, ciliumFilename, map[string]string{
				"egressGateway.enabled":  "true",
				"tunnel":                 "disabled",
				"autoDirectNodeRoutes":   "true",
				"bpf.masquerade":         "true",
				"endpointRoutes.enabled": "true",
			})

			randomNamespace = deploymentManager.DeployRandomNamespaceShared(DemoDaemonSet)
			kubectl.NamespaceLabel(randomNamespace, namespaceSelector)
			deploymentManager.WaitUntilReady()

		})

		AfterAll(func() {
			deploymentManager.DeleteAll()
			DeployCiliumAndDNS(kubectl, ciliumFilename)
		})

		It("Checks connectivity works without policy", func() {
			testConnectivity(false)
			testConnectivity(true)
		})

		It("Checks egress policy and basic connectivity both work", func() {
			applyEgressPolicy()
			kubectl.WaitForEgressPolicyEntry(k8s1IP, outsideIP)
			kubectl.WaitForEgressPolicyEntry(k8s2IP, outsideIP)

			defer kubectl.Delete(policyYAML)

			testEgressGateway(true)
			testEgressGateway(false)
			testConnectivity(true)
			testConnectivity(false)
		})

	})

	Context("tunnel disabled with endpoint routes disabled", func() {

		BeforeAll(func() {
			DeployCiliumOptionsAndDNS(kubectl, ciliumFilename, map[string]string{
				"egressGateway.enabled":  "true",
				"tunnel":                 "disabled",
				"autoDirectNodeRoutes":   "true",
				"bpf.masquerade":         "true",
				"endpointRoutes.enabled": "false",
			})

			randomNamespace = deploymentManager.DeployRandomNamespaceShared(DemoDaemonSet)
			kubectl.NamespaceLabel(randomNamespace, namespaceSelector)
			deploymentManager.WaitUntilReady()
		})

		AfterAll(func() {
			deploymentManager.DeleteAll()
			DeployCiliumAndDNS(kubectl, ciliumFilename)
		})

		It("Checks connectivity works without policy", func() {
			testConnectivity(false)
			testConnectivity(true)
		})

		It("Checks egress policy and basic connectivity both work", func() {
			applyEgressPolicy()
			kubectl.WaitForEgressPolicyEntry(k8s1IP, outsideIP)
			kubectl.WaitForEgressPolicyEntry(k8s2IP, outsideIP)

			defer kubectl.Delete(policyYAML)

			testEgressGateway(false)
			testEgressGateway(true)
			testConnectivity(false)
			testConnectivity(true)
		})

	})

	Context("tunnel vxlan", func() {

		BeforeAll(func() {
			DeployCiliumOptionsAndDNS(kubectl, ciliumFilename, map[string]string{
				"egressGateway.enabled": "true",
				"bpf.masquerade":        "true",
				"tunnel":                "vxlan",
			})

			randomNamespace = deploymentManager.DeployRandomNamespaceShared(DemoDaemonSet)
			kubectl.NamespaceLabel(randomNamespace, namespaceSelector)
			deploymentManager.WaitUntilReady()
		})

		AfterAll(func() {
			deploymentManager.DeleteAll()
			DeployCiliumAndDNS(kubectl, ciliumFilename)
		})

		It("Checks connectivity works without policy", func() {
			testConnectivity(false)
			testConnectivity(true)
		})

		It("Checks egress policy and basic connectivity both work", func() {
			applyEgressPolicy()
			kubectl.WaitForEgressPolicyEntry(k8s1IP, outsideIP)
			kubectl.WaitForEgressPolicyEntry(k8s2IP, outsideIP)

			defer kubectl.Delete(policyYAML)

			testEgressGateway(false)
			testEgressGateway(true)
			testConnectivity(false)
			testConnectivity(true)
		})

	})

})

// Use x.x.x.100 as the egress IP
func getEgressIP(nodeIP string) string {
	ip := net.ParseIP(nodeIP).To4()
	ip[3] = 100
	return ip.String()
}
<|endoftext|>


<commit_before>package tarpan

import (
	"errors"
	"os"
	"strconv"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	g "github.com/soniah/gosnmp"
)

// Tarpan constants
const (
	ConcurrentProcesses = 5
)

// Exit code
const (
	ExitCodeOK = iota
	ExitCodeConfError
	ExitCodeParseFlagError
)

// SNMP constants
const (
	COMMUNITY = "public"
	PORT      = 161
	TIMEOUT   = 2
	RETRY     = 3
	//MaximumPDUBytes = 512
)

// JSON structure definition
type DataSet struct {
	Global  Global   `json:"global"`
	Targets []Target `json:"targets"`
}

type Global struct {
	Port      uint16 `json:"port"`
	Version   string `json:"version"`
	Community string `json:"community"`
}

type Target struct {
	Name      string `json:"name"`
	Address   string `json:"address"`
	Port      uint16 `json:"port"`
	Version   string `json:"version"`
	Community string `json:"community"`
	OIDs      []OID  `json:"oids"`
}

type OID struct {
	OID         string `json:"oid"`
	Description string `json:"description"`
}

// Tarpan result structures
type TarpanResult struct {
	Name      string    `json:"name"`
	Address   string    `json:"address"`
	Port      uint16    `json:"port"`
	Version   string    `json:"version"`
	Community string    `json:"community"`
	VarBinds  []VarBind `json:"varbinds"`
}

type TarpanResults []*TarpanResult

type VarBind struct {
	Description string      `json:"description"`
	OID         string      `json:"oid"`
	Type        string      `json:"type"`
	Value       interface{} `json:"value"`
	Time        int64       `json:"time"`
}

type Tarpan interface {
	setTarget(ds *DataSet, target_index int)
	Get(oids []string)
	setParams(p *RequestParams)
	getTargetOIDDescription(oid string)
	makeTarpanResult(sp *g.SnmpPacket)
	Run()
}

type TarpanManager struct {
	target *Target
	snmp   *g.GoSNMP
}

type TarpanManagers []*TarpanManager

type RequestParams struct {
	managerIndex int
	address      string
	community    string
	version      string
	port         uint16
	timeout      uint8
	retry        uint8
}

type Channels struct {
	semaphoe chan int
	results  chan TarpanResult
}

func (m *TarpanManager) setTarget(ds *DataSet, target_index int) error {
	params, param_err := getRequestParams(ds, target_index)
	if param_err != nil {
		log.Error(param_err)
		errors.New(param_err.Error())
	}
	m.setParams(params)
	m.target = &ds.Targets[target_index]

	return nil
}

func (m *TarpanManager) Get(oids []string) (TarpanResult, error) {
	connection_err := m.snmp.Connect()
	defer m.snmp.Conn.Close()
	if connection_err != nil {
		log.Error("Connection error: %")
		return TarpanResult{}, errors.New(connection_err.Error())
	}
	snmpPacket, request_err := m.snmp.Get(oids)
	if request_err != nil {
		return TarpanResult{}, errors.New(request_err.Error())
	}
	tarpanResult := m.makeTarpanResult(snmpPacket)

	return tarpanResult, nil
}

func (m *TarpanManager) setParams(p *RequestParams) {
	if p.address != "" {
		m.snmp.Target = p.address
	}
	if p.port != 0 {
		m.snmp.Port = p.port
	} else {
		m.snmp.Port = uint16(PORT)
	}
	if p.community != "" {
		m.snmp.Community = p.community
	} else {
		m.snmp.Community = COMMUNITY
	}
	if p.version != "" {
		if p.version == "2c" {
			m.snmp.Version = g.Version2c
		}
	} else {
		m.snmp.Version = g.Version2c
	}
	if p.timeout != 0 {
		m.snmp.Timeout = time.Duration(p.timeout) * time.Second
	} else {
		m.snmp.Timeout = time.Duration(TIMEOUT) * time.Second
	}
	if p.retry != 0 {
		m.snmp.Retries = int(p.retry)
	} else {
		m.snmp.Retries = RETRY
	}
}

func getRequestParams(ds *DataSet, idx int) (*RequestParams, error) {
	var address string
	err := validateIP(ds.Targets[idx].Address)
	if err == nil {
		address = ds.Targets[idx].Address
	} else {
		return &RequestParams{}, err
	}
	community := ds.Targets[idx].Community
	if community == "" {
		community = ds.Global.Community
	}
	version := ds.Targets[idx].Version
	if version == "" {
		version = ds.Global.Version
	}
	port := ds.Targets[idx].Port
	if port == 0 {
		port = ds.Global.Port
	}

	requestParams := &RequestParams{
		managerIndex: idx,
		address:      address,
		community:    community,
		version:      version,
		port:         port,
	}

	return requestParams, nil
}

func makeChannel(result_buffer_size int) *Channels {
	channel := &Channels{
		semaphoe: make(chan int, ConcurrentProcesses),
		results:  make(chan TarpanResult, result_buffer_size),
	}

	return channel
}

func (m *TarpanManager) getTargetOIDDescription(oid string) (string, error) {
	for _, o := range m.target.OIDs {
		if o.OID == oid {
			return o.Description, nil
		}
	}

	return "", errors.New("oid not found")
}

func removeNilResult(tarpanResults []*TarpanResult) []*TarpanResult {
	var responseResult []*TarpanResult
	for _, v := range tarpanResults {
		if v != nil {
			responseResult = append(responseResult, v)
		}
	}

	return responseResult
}

func (m *TarpanManager) makeTarpanResult(sp *g.SnmpPacket) TarpanResult {
	now := time.Now()
	var varbinds []VarBind
	for _, val := range sp.Variables {
		//n := strings.TrimLeft(val.Name, ".")
		desc, _ := m.getTargetOIDDescription(val.Name)
		asn1ber_name, _ := getAsn1BERName(val.Type)
		//value := formatSnmpValue(val.Type, val.Value.(string))
		v := VarBind{
			Description: desc,
			OID:         val.Name,
			Type:        asn1ber_name,
			Value:       val.Value,
			Time:        now.Unix(),
		}
		varbinds = append(varbinds, v)
	}
	version := getSnmpVersionString(m.snmp.Version)
	tarpanResult := TarpanResult{
		Name:      m.target.Name,
		Address:   m.target.Address,
		Port:      m.snmp.Port,
		Version:   version,
		Community: m.snmp.Community,
		VarBinds:  varbinds,
	}

	return tarpanResult
}

func makeTarpanResults(c *Channels) []*TarpanResult {
	result_length := len(c.results)
	tarpanResults := make(TarpanResults, result_length)
	for i := 0; i < result_length; i++ {
		tarpanResult := <-c.results
		tarpanResults[i] = &tarpanResult
	}

	return removeNilResult(tarpanResults)
}

func makeManagers(ds *DataSet) []*TarpanManager {
	var manager *TarpanManager
	var managers TarpanManagers

	// Set Target
	for t_idx := range ds.Targets {
		manager = &TarpanManager{
			snmp: &g.GoSNMP{},
		}
		err := manager.setTarget(ds, t_idx)
		if err != nil {
			log.Error("set target error")
		}
		managers = append(managers, manager)
	}

	return managers
}

func Collect(dataset *DataSet) []*TarpanResult {
	var waitGroup sync.WaitGroup
	var oids []string
	var tarpanResults []*TarpanResult

	managers := makeManagers(dataset)
	channels := makeChannel(len(dataset.Targets))
	for t_idx := range dataset.Targets {

		// TODO: split oid slice depending on PDU size
		oids = []string{}
		for o_idx := range dataset.Targets[t_idx].OIDs {
			oids = append(oids, dataset.Targets[t_idx].OIDs[o_idx].OID)
		}

		waitGroup.Add(1)
		go func(m *TarpanManager, o []string, c *Channels) {
			c.semaphoe <- 0
			defer func() {
				waitGroup.Done()
				<-c.semaphoe
			}()
			result, err := m.Get(o)
			if err != nil {
				log.Error(err)
				return
			}
			c.results <- result
		}(managers[t_idx], oids, channels)
	}
	waitGroup.Wait()
	tarpanResults = makeTarpanResults(channels)

	return tarpanResults
}

func Run(target string, output_type string, debug bool) (int, error) {
	var err error

	// setup logger
	log.SetOutput(os.Stderr)
	if debug == true {
		log.SetLevel(log.DebugLevel)
	} else {
		log.SetLevel(log.WarnLevel)
	}

	// load targets
	dataset, loadErr := loadConfig(target)
	if loadErr != nil {
		log.Error(loadErr)
		return ExitCodeConfError, loadErr
	}

	// collect data
	tr := Collect(dataset)

	// output
	Write(output_type, tr)

	return ExitCodeOK, err
}
<commit_msg>remove strconv<commit_after>package tarpan

import (
	"errors"
	"os"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	g "github.com/soniah/gosnmp"
)

// Tarpan constants
const (
	ConcurrentProcesses = 5
)

// Exit code
const (
	ExitCodeOK = iota
	ExitCodeConfError
	ExitCodeParseFlagError
)

// SNMP constants
const (
	COMMUNITY = "public"
	PORT      = 161
	TIMEOUT   = 2
	RETRY     = 3
	//MaximumPDUBytes = 512
)

// JSON structure definition
type DataSet struct {
	Global  Global   `json:"global"`
	Targets []Target `json:"targets"`
}

type Global struct {
	Port      uint16 `json:"port"`
	Version   string `json:"version"`
	Community string `json:"community"`
}

type Target struct {
	Name      string `json:"name"`
	Address   string `json:"address"`
	Port      uint16 `json:"port"`
	Version   string `json:"version"`
	Community string `json:"community"`
	OIDs      []OID  `json:"oids"`
}

type OID struct {
	OID         string `json:"oid"`
	Description string `json:"description"`
}

// Tarpan result structures
type TarpanResult struct {
	Name      string    `json:"name"`
	Address   string    `json:"address"`
	Port      uint16    `json:"port"`
	Version   string    `json:"version"`
	Community string    `json:"community"`
	VarBinds  []VarBind `json:"varbinds"`
}

type TarpanResults []*TarpanResult

type VarBind struct {
	Description string      `json:"description"`
	OID         string      `json:"oid"`
	Type        string      `json:"type"`
	Value       interface{} `json:"value"`
	Time        int64       `json:"time"`
}

type Tarpan interface {
	setTarget(ds *DataSet, target_index int)
	Get(oids []string)
	setParams(p *RequestParams)
	getTargetOIDDescription(oid string)
	makeTarpanResult(sp *g.SnmpPacket)
	Run()
}

type TarpanManager struct {
	target *Target
	snmp   *g.GoSNMP
}

type TarpanManagers []*TarpanManager

type RequestParams struct {
	managerIndex int
	address      string
	community    string
	version      string
	port         uint16
	timeout      uint8
	retry        uint8
}

type Channels struct {
	semaphoe chan int
	results  chan TarpanResult
}

func (m *TarpanManager) setTarget(ds *DataSet, target_index int) error {
	params, param_err := getRequestParams(ds, target_index)
	if param_err != nil {
		log.Error(param_err)
		errors.New(param_err.Error())
	}
	m.setParams(params)
	m.target = &ds.Targets[target_index]

	return nil
}

func (m *TarpanManager) Get(oids []string) (TarpanResult, error) {
	connection_err := m.snmp.Connect()
	defer m.snmp.Conn.Close()
	if connection_err != nil {
		log.Error("Connection error: %")
		return TarpanResult{}, errors.New(connection_err.Error())
	}
	snmpPacket, request_err := m.snmp.Get(oids)
	if request_err != nil {
		return TarpanResult{}, errors.New(request_err.Error())
	}
	tarpanResult := m.makeTarpanResult(snmpPacket)

	return tarpanResult, nil
}

func (m *TarpanManager) setParams(p *RequestParams) {
	if p.address != "" {
		m.snmp.Target = p.address
	}
	if p.port != 0 {
		m.snmp.Port = p.port
	} else {
		m.snmp.Port = uint16(PORT)
	}
	if p.community != "" {
		m.snmp.Community = p.community
	} else {
		m.snmp.Community = COMMUNITY
	}
	if p.version != "" {
		if p.version == "2c" {
			m.snmp.Version = g.Version2c
		}
	} else {
		m.snmp.Version = g.Version2c
	}
	if p.timeout != 0 {
		m.snmp.Timeout = time.Duration(p.timeout) * time.Second
	} else {
		m.snmp.Timeout = time.Duration(TIMEOUT) * time.Second
	}
	if p.retry != 0 {
		m.snmp.Retries = int(p.retry)
	} else {
		m.snmp.Retries = RETRY
	}
}

func getRequestParams(ds *DataSet, idx int) (*RequestParams, error) {
	var address string
	err := validateIP(ds.Targets[idx].Address)
	if err == nil {
		address = ds.Targets[idx].Address
	} else {
		return &RequestParams{}, err
	}
	community := ds.Targets[idx].Community
	if community == "" {
		community = ds.Global.Community
	}
	version := ds.Targets[idx].Version
	if version == "" {
		version = ds.Global.Version
	}
	port := ds.Targets[idx].Port
	if port == 0 {
		port = ds.Global.Port
	}

	requestParams := &RequestParams{
		managerIndex: idx,
		address:      address,
		community:    community,
		version:      version,
		port:         port,
	}

	return requestParams, nil
}

func makeChannel(result_buffer_size int) *Channels {
	channel := &Channels{
		semaphoe: make(chan int, ConcurrentProcesses),
		results:  make(chan TarpanResult, result_buffer_size),
	}

	return channel
}

func (m *TarpanManager) getTargetOIDDescription(oid string) (string, error) {
	for _, o := range m.target.OIDs {
		if o.OID == oid {
			return o.Description, nil
		}
	}

	return "", errors.New("oid not found")
}

func removeNilResult(tarpanResults []*TarpanResult) []*TarpanResult {
	var responseResult []*TarpanResult
	for _, v := range tarpanResults {
		if v != nil {
			responseResult = append(responseResult, v)
		}
	}

	return responseResult
}

func (m *TarpanManager) makeTarpanResult(sp *g.SnmpPacket) TarpanResult {
	now := time.Now()
	var varbinds []VarBind
	for _, val := range sp.Variables {
		//n := strings.TrimLeft(val.Name, ".")
		desc, _ := m.getTargetOIDDescription(val.Name)
		asn1ber_name, _ := getAsn1BERName(val.Type)
		//value := formatSnmpValue(val.Type, val.Value.(string))
		v := VarBind{
			Description: desc,
			OID:         val.Name,
			Type:        asn1ber_name,
			Value:       val.Value,
			Time:        now.Unix(),
		}
		varbinds = append(varbinds, v)
	}
	version := getSnmpVersionString(m.snmp.Version)
	tarpanResult := TarpanResult{
		Name:      m.target.Name,
		Address:   m.target.Address,
		Port:      m.snmp.Port,
		Version:   version,
		Community: m.snmp.Community,
		VarBinds:  varbinds,
	}

	return tarpanResult
}

func makeTarpanResults(c *Channels) []*TarpanResult {
	result_length := len(c.results)
	tarpanResults := make(TarpanResults, result_length)
	for i := 0; i < result_length; i++ {
		tarpanResult := <-c.results
		tarpanResults[i] = &tarpanResult
	}

	return removeNilResult(tarpanResults)
}

func makeManagers(ds *DataSet) []*TarpanManager {
	var manager *TarpanManager
	var managers TarpanManagers

	// Set Target
	for t_idx := range ds.Targets {
		manager = &TarpanManager{
			snmp: &g.GoSNMP{},
		}
		err := manager.setTarget(ds, t_idx)
		if err != nil {
			log.Error("set target error")
		}
		managers = append(managers, manager)
	}

	return managers
}

func Collect(dataset *DataSet) []*TarpanResult {
	var waitGroup sync.WaitGroup
	var oids []string
	var tarpanResults []*TarpanResult

	managers := makeManagers(dataset)
	channels := makeChannel(len(dataset.Targets))
	for t_idx := range dataset.Targets {

		// TODO: split oid slice depending on PDU size
		oids = []string{}
		for o_idx := range dataset.Targets[t_idx].OIDs {
			oids = append(oids, dataset.Targets[t_idx].OIDs[o_idx].OID)
		}

		waitGroup.Add(1)
		go func(m *TarpanManager, o []string, c *Channels) {
			c.semaphoe <- 0
			defer func() {
				waitGroup.Done()
				<-c.semaphoe
			}()
			result, err := m.Get(o)
			if err != nil {
				log.Error(err)
				return
			}
			c.results <- result
		}(managers[t_idx], oids, channels)
	}
	waitGroup.Wait()
	tarpanResults = makeTarpanResults(channels)

	return tarpanResults
}

func Run(target string, output_type string, debug bool) (int, error) {
	var err error

	// setup logger
	log.SetOutput(os.Stderr)
	if debug == true {
		log.SetLevel(log.DebugLevel)
	} else {
		log.SetLevel(log.WarnLevel)
	}

	// load targets
	dataset, loadErr := loadConfig(target)
	if loadErr != nil {
		log.Error(loadErr)
		return ExitCodeConfError, loadErr
	}

	// collect data
	tr := Collect(dataset)

	// output
	Write(output_type, tr)

	return ExitCodeOK, err
}
<|endoftext|>


<commit_before>// SPDX-License-Identifier: Apache-2.0
// Copyright 2018-2019 Authors of Cilium

package k8sTest

import (
	"context"
	"fmt"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"

	. "github.com/onsi/gomega"
)

var _ = Describe("K8sHealthTest", func() {
	SkipContextIf(func() bool {
		return helpers.DoesNotRunOnGKE() && helpers.DoesNotRunOnEKS()
	}, "cilium-health", func() {
		var (
			kubectl        *helpers.Kubectl
			ciliumFilename string
		)

		BeforeAll(func() {
			kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)

			ciliumFilename = helpers.TimestampFilename("cilium.yaml")
			DeployCiliumAndDNS(kubectl, ciliumFilename)
		})

		AfterFailed(func() {
			kubectl.CiliumReport("cilium endpoint list")
		})

		JustAfterEach(func() {
			kubectl.ValidateNoErrorsInLogs(CurrentGinkgoTestDescription().Duration)
		})

		AfterEach(func() {
			ExpectAllPodsTerminated(kubectl)
		})

		AfterAll(func() {
			UninstallCiliumFromManifest(kubectl, ciliumFilename)
			kubectl.CloseSSHClient()
		})

		getCilium := func(node string) (pod, ip string) {
			pod, err := kubectl.GetCiliumPodOnNode(node)
			Expect(err).Should(BeNil())

			res, err := kubectl.Get(
				helpers.CiliumNamespace,
				fmt.Sprintf("pod %s", pod)).Filter("{.status.podIP}")
			Expect(err).Should(BeNil())
			ip = res.String()

			return pod, ip
		}

		checkIP := func(pod, ip string) {
			jsonpath := "{.nodes[*].host.primary-address.ip}"
			ciliumCmd := fmt.Sprintf("cilium-health status -o jsonpath='%s'", jsonpath)

			err := kubectl.CiliumExecUntilMatch(pod, ciliumCmd, ip)
			ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Never saw cilium-health ip %s in pod %s", ip, pod)
		}

		It("Checks status between nodes", func() {
			cilium1, cilium1IP := getCilium(helpers.K8s1)
			cilium2, cilium2IP := getCilium(helpers.K8s2)

			By("checking that cilium API exposes health instances")
			checkIP(cilium1, cilium1IP)
			checkIP(cilium1, cilium2IP)
			checkIP(cilium2, cilium1IP)
			checkIP(cilium2, cilium2IP)

			By("checking that `cilium-health --probe` succeeds")
			healthCmd := "cilium-health status --probe -o json"
			status := kubectl.CiliumExecMustSucceed(context.TODO(), cilium1, healthCmd)
			Expect(status.Stdout()).ShouldNot(ContainSubstring("error"))

			apiPaths := []string{
				"endpoint.icmp",
				"endpoint.http",
				"host.primary-address.icmp",
				"host.primary-address.http",
			}
			for node := 0; node <= 1; node++ {
				healthCmd := "cilium-health status -o json"
				status := kubectl.CiliumExecMustSucceed(context.TODO(), cilium1, healthCmd, "Cannot retrieve health status")
				for _, path := range apiPaths {
					filter := fmt.Sprintf("{.nodes[%d].%s}", node, path)
					By("checking API response for %q", filter)
					data, err := status.Filter(filter)
					Expect(err).To(BeNil(), "cannot retrieve filter %q from health output", filter)
					Expect(data.String()).Should(Not((BeEmpty())))
					statusFilter := fmt.Sprintf("{.nodes[%d].%s.status}", node, path)
					By("checking API status response for %q", statusFilter)
					data, err = status.Filter(statusFilter)
					Expect(err).To(BeNil(), "cannot retrieve filter %q from health output", statusFilter)
					Expect(data.String()).Should(BeEmpty())
				}
			}
		}, 30)
	})
})
<commit_msg>ci: Set ClusterHealthPort in K8sHealth<commit_after>// SPDX-License-Identifier: Apache-2.0
// Copyright 2018-2019 Authors of Cilium

package k8sTest

import (
	"context"
	"fmt"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"

	. "github.com/onsi/gomega"
)

var _ = Describe("K8sHealthTest", func() {
	SkipContextIf(func() bool {
		return helpers.DoesNotRunOnGKE() && helpers.DoesNotRunOnEKS()
	}, "cilium-health", func() {
		var (
			kubectl        *helpers.Kubectl
			ciliumFilename string
		)

		BeforeAll(func() {
			kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)

			ciliumFilename = helpers.TimestampFilename("cilium.yaml")
			DeployCiliumOptionsAndDNS(kubectl, ciliumFilename, map[string]string{
				"clusterHealthPort": "9940", // tests use of custom port
			})
		})

		AfterFailed(func() {
			kubectl.CiliumReport("cilium endpoint list")
		})

		JustAfterEach(func() {
			kubectl.ValidateNoErrorsInLogs(CurrentGinkgoTestDescription().Duration)
		})

		AfterEach(func() {
			ExpectAllPodsTerminated(kubectl)
		})

		AfterAll(func() {
			UninstallCiliumFromManifest(kubectl, ciliumFilename)
			kubectl.CloseSSHClient()
		})

		getCilium := func(node string) (pod, ip string) {
			pod, err := kubectl.GetCiliumPodOnNode(node)
			Expect(err).Should(BeNil())

			res, err := kubectl.Get(
				helpers.CiliumNamespace,
				fmt.Sprintf("pod %s", pod)).Filter("{.status.podIP}")
			Expect(err).Should(BeNil())
			ip = res.String()

			return pod, ip
		}

		checkIP := func(pod, ip string) {
			jsonpath := "{.nodes[*].host.primary-address.ip}"
			ciliumCmd := fmt.Sprintf("cilium-health status -o jsonpath='%s'", jsonpath)

			err := kubectl.CiliumExecUntilMatch(pod, ciliumCmd, ip)
			ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Never saw cilium-health ip %s in pod %s", ip, pod)
		}

		It("Checks status between nodes", func() {
			cilium1, cilium1IP := getCilium(helpers.K8s1)
			cilium2, cilium2IP := getCilium(helpers.K8s2)

			By("checking that cilium API exposes health instances")
			checkIP(cilium1, cilium1IP)
			checkIP(cilium1, cilium2IP)
			checkIP(cilium2, cilium1IP)
			checkIP(cilium2, cilium2IP)

			By("checking that `cilium-health --probe` succeeds")
			healthCmd := "cilium-health status --probe -o json"
			status := kubectl.CiliumExecMustSucceed(context.TODO(), cilium1, healthCmd)
			Expect(status.Stdout()).ShouldNot(ContainSubstring("error"))

			apiPaths := []string{
				"endpoint.icmp",
				"endpoint.http",
				"host.primary-address.icmp",
				"host.primary-address.http",
			}
			for node := 0; node <= 1; node++ {
				healthCmd := "cilium-health status -o json"
				status := kubectl.CiliumExecMustSucceed(context.TODO(), cilium1, healthCmd, "Cannot retrieve health status")
				for _, path := range apiPaths {
					filter := fmt.Sprintf("{.nodes[%d].%s}", node, path)
					By("checking API response for %q", filter)
					data, err := status.Filter(filter)
					Expect(err).To(BeNil(), "cannot retrieve filter %q from health output", filter)
					Expect(data.String()).Should(Not((BeEmpty())))
					statusFilter := fmt.Sprintf("{.nodes[%d].%s.status}", node, path)
					By("checking API status response for %q", statusFilter)
					data, err = status.Filter(statusFilter)
					Expect(err).To(BeNil(), "cannot retrieve filter %q from health output", statusFilter)
					Expect(data.String()).Should(BeEmpty())
				}
			}
		}, 30)
	})
})
<|endoftext|>