<commit_before>package sdl

// #include "SDL.h"
import "C"

import (
	"image"
	"image/color"
	"unsafe"
)

// Surface is a rectangular array of pixels.
type Surface struct {
	s C.SDL_Surface
}

// PixelFormat returns the surface's pixel format.
func (surface *Surface) PixelFormat() *PixelFormat {
	return &PixelFormat{
		Format:        PixelFormatEnum(surface.s.format.format),
		BitsPerPixel:  uint8(surface.s.format.BitsPerPixel),
		BytesPerPixel: uint8(surface.s.format.BytesPerPixel),
		Rmask:         uint32(surface.s.format.Rmask),
		Gmask:         uint32(surface.s.format.Gmask),
		Bmask:         uint32(surface.s.format.Bmask),
		Amask:         uint32(surface.s.format.Amask),
	}
}

// Size returns the surface's width and height.
func (surface *Surface) Size() Point {
	return Point{int(surface.s.w), int(surface.s.h)}
}

// PixelData locks the surface and returns a PixelData value,
// which can be used to access and modify the surface's pixels.
// The returned PixelData must be closed before the surface can be
// used again.
func (surface *Surface) PixelData() (PixelData, error) {
	if result := C.SDL_LockSurface(&surface.s); result < 0 {
		return PixelData{}, GetError()
	}
	return PixelData{s: &surface.s}, nil
}

// Destroy destroys the surface.  The surface should not be used after
// a call to Destroy.
func (surface *Surface) Destroy() {
	C.SDL_FreeSurface(&surface.s)
}

// PixelData is a mutable view of a surface's pixels.  The data is only
// available while a surface is locked, so pixel data should be closed to
// allow the surface to be used again.
//
// PixelData implements the image.Image and draw.Image interfaces.
// See: http://golang.org/pkg/image/#Image and http://golang.org/pkg/image/draw/#Image.
type PixelData struct {
	s *C.SDL_Surface
}

// Expand out a component of a color from a pixel representing the color.
// TODO(adam): unit tests
func expandColor(pixel uint32, mask C.Uint32, shift, loss C.Uint8) uint8 {
	temp := pixel & uint32(mask)
	temp = temp >> uint8(shift)
	return uint8(temp << uint8(loss))
}

// getPixelPointer returns the address of the pixel at (x, y) relative
// to a base pointer.  pitch is the number of bytes in a horizontal row.
func getPixelPointer(pixels unsafe.Pointer, x, y int, bytesPerPixel, pitch uintptr) unsafe.Pointer {
	offset := uintptr(x)*bytesPerPixel + uintptr(y)*pitch
	return unsafe.Pointer(uintptr(pixels) + offset)
}

// At returns the pixel at the given position.
func (pix PixelData) At(x, y int) color.Color {
	format := pix.s.format
	bytesPerPixel := uintptr(format.BytesPerPixel)

	ptr := getPixelPointer(pix.s.pixels, x, y, bytesPerPixel, uintptr(pix.s.pitch))
	pixel := *(*uint32)(ptr)

	// TODO(adam): not necesarily NRGBA (which would be an entirely different codepath)
	var col color.NRGBA

	switch bytesPerPixel {
	case 1:
		// TODO(adam): look up the color in color palette
	case 2, 3, 4:
		col.R = expandColor(pixel, format.Rmask, format.Rshift, format.Rloss)
		col.G = expandColor(pixel, format.Gmask, format.Gshift, format.Gloss)
		col.B = expandColor(pixel, format.Bmask, format.Bshift, format.Bloss)
		// If the alpha mask is 0, there's no alpha component, so set it to 1
		if format.Amask == 0 {
			col.A = 1
		} else {
			col.A = expandColor(pixel, format.Amask, format.Ashift, format.Aloss)
		}
	}

	return col
}

// ColorModel returns the color model of the pixel data.
func (pix PixelData) ColorModel() color.Model {
	// TODO(adam): this is a guess
	return color.NRGBAModel
}

// Bounds returns a rectangle of (0,0) => (w,h).
func (pix PixelData) Bounds() image.Rectangle {
	return image.Rectangle{image.Point{0, 0}, image.Point{int(pix.s.w), int(pix.s.h)}}
}

// Collapse a component of the color into a pointer at a pixel representing the color.
// TODO(adam): unit tests
func collapseColor(pixel *uint32, color uint8, shift, loss C.Uint8) {
	temp := uint32(color >> uint8(loss))
	temp = temp << uint8(shift)
	*pixel = *pixel & temp
}

// Set sets the color at an x, y position in the PixelData to a given color.
func (pix PixelData) Set(x, y int, c color.Color) {
	format := pix.s.format
	bytesPerPixel := uintptr(format.BytesPerPixel)

	switch bytesPerPixel {
	case 1:
		// TODO(adam): look up the color in color palette
	case 2, 3, 4:
		// if necessary, convert color model to NRGBA
		col := pix.ColorModel().Convert(c).(color.NRGBA)

		// put that in a uint32 that I can slap into the void* of pixel data (also helper function?)
		var pixel *uint32
		collapseColor(pixel, col.R, format.Rshift, format.Rloss)
		collapseColor(pixel, col.G, format.Gshift, format.Gloss)
		collapseColor(pixel, col.B, format.Bshift, format.Bloss)
		// If the alpha mask is 0, there's no alpha component, so set it to 1
		if format.Amask == 0 {
			col.A = 1
		}
		collapseColor(pixel, col.A, format.Ashift, format.Aloss)

		// get pixel offset that's (x, y)
		ptr := getPixelPointer(pix.s.pixels, x, y, bytesPerPixel, uintptr(pix.s.pitch))

		// slap it in
		*(*uint32)(ptr) = *pixel
	}
}

// Destroy unlocks the underlying surface.  pix should not be used after
// calling Destroy.
func (pix PixelData) Destroy() {
	C.SDL_UnlockSurface(pix.s)
}
<commit_msg>Clean up PixelData code, including some potential bug fixes<commit_after>package sdl

// #include "SDL.h"
import "C"

import (
	"image"
	"image/color"
	"unsafe"
)

// Surface is a rectangular array of pixels.
type Surface struct {
	s C.SDL_Surface
}

// PixelFormat returns the surface's pixel format.
func (surface *Surface) PixelFormat() *PixelFormat {
	return &PixelFormat{
		Format:        PixelFormatEnum(surface.s.format.format),
		BitsPerPixel:  uint8(surface.s.format.BitsPerPixel),
		BytesPerPixel: uint8(surface.s.format.BytesPerPixel),
		Rmask:         uint32(surface.s.format.Rmask),
		Gmask:         uint32(surface.s.format.Gmask),
		Bmask:         uint32(surface.s.format.Bmask),
		Amask:         uint32(surface.s.format.Amask),
	}
}

// Size returns the surface's width and height.
func (surface *Surface) Size() Point {
	return Point{int(surface.s.w), int(surface.s.h)}
}

// PixelData locks the surface and returns a PixelData value,
// which can be used to access and modify the surface's pixels.
// The returned PixelData must be closed before the surface can be
// used again.
func (surface *Surface) PixelData() (PixelData, error) {
	if result := C.SDL_LockSurface(&surface.s); result < 0 {
		return PixelData{}, GetError()
	}
	return PixelData{s: &surface.s}, nil
}

// Destroy destroys the surface.  The surface should not be used after
// a call to Destroy.
func (surface *Surface) Destroy() {
	C.SDL_FreeSurface(&surface.s)
}

// PixelData is a mutable view of a surface's pixels.  The data is only
// available while a surface is locked, so pixel data should be closed to
// allow the surface to be used again.
//
// PixelData implements the image.Image and draw.Image interfaces.
// See: http://golang.org/pkg/image/#Image and http://golang.org/pkg/image/draw/#Image.
type PixelData struct {
	s *C.SDL_Surface
}

// Expand out a component of a color from a pixel representing the color.
// TODO(adam): unit tests
func expandColor(pixel uint32, mask C.Uint32, shift, loss C.Uint8) uint8 {
	temp := pixel & uint32(mask)
	temp = temp >> uint8(shift)
	return uint8(temp << uint8(loss))
}

// pixel returns the address of the pixel at (x, y).
func (pix PixelData) pixel(x, y int) unsafe.Pointer {
	offset := uintptr(y)*uintptr(pix.s.pitch) + uintptr(x)*uintptr(pix.s.format.BytesPerPixel)
	return unsafe.Pointer(uintptr(pix.s.pixels) + offset)
}

// At returns the pixel at the given position.
func (pix PixelData) At(x, y int) color.Color {
	format := pix.s.format
	ptr := pix.pixel(x, y)
	// TODO(adam): not necesarily NRGBA (which would be an entirely different codepath)
	var col color.NRGBA

	switch format.BytesPerPixel {
	case 4:
		pixel := *(*uint32)(ptr)
		col.R = expandColor(pixel, format.Rmask, format.Rshift, format.Rloss)
		col.G = expandColor(pixel, format.Gmask, format.Gshift, format.Gloss)
		col.B = expandColor(pixel, format.Bmask, format.Bshift, format.Bloss)
		// If the alpha mask is 0, there's no alpha component, so set it opaque.
		if format.Amask == 0 {
			col.A = ^uint8(0)
		} else {
			col.A = expandColor(pixel, format.Amask, format.Ashift, format.Aloss)
		}
	default:
		// TODO(#22): handle all pixel formats
		panic("pixel format not handled")
	}
	return col
}

// ColorModel returns the color model of the pixel data.
func (pix PixelData) ColorModel() color.Model {
	// TODO(adam): this is a guess
	return color.NRGBAModel
}

// Bounds returns a rectangle of (0,0) => (w,h).
func (pix PixelData) Bounds() image.Rectangle {
	return image.Rect(0, 0, int(pix.s.w), int(pix.s.h))
}

// collapseColor collapses a component of the color into an OR'able mask.
// TODO(adam): unit tests
func collapseColor(color uint8, shift, loss C.Uint8) uint32 {
	return uint32(color) >> loss << shift
}

// Set sets the color at an x, y position in the PixelData to a given color.
func (pix PixelData) Set(x, y int, c color.Color) {
	format := pix.s.format
	switch format.BytesPerPixel {
	case 4:
		col := pix.ColorModel().Convert(c).(color.NRGBA)

		p := (*uint32)(pix.pixel(x, y))
		*p = collapseColor(col.R, format.Rshift, format.Rloss)
		*p |= collapseColor(col.G, format.Gshift, format.Gloss)
		*p |= collapseColor(col.B, format.Bshift, format.Bloss)
		if format.Amask == 0 {
			// If the alpha mask is 0, there's no alpha component, so set it opaque.
			col.A = ^uint8(0)
		}
		*p |= collapseColor(col.A, format.Ashift, format.Aloss)
	default:
		// TODO(#22): handle all pixel formats
		panic("pixel format not handled")
	}
}

// Destroy unlocks the underlying surface.  pix should not be used after
// calling Destroy.
func (pix PixelData) Destroy() {
	C.SDL_UnlockSurface(pix.s)
}
<|endoftext|>


<commit_before>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storagetransfer

import (
	"bytes"
	"context"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"testing"

	"cloud.google.com/go/iam"
	"cloud.google.com/go/storage"
	storagetransfer "cloud.google.com/go/storagetransfer/apiv1"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	storagetransferpb "google.golang.org/genproto/googleapis/storagetransfer/v1"

	"github.com/GoogleCloudPlatform/golang-samples/internal/testutil"
	"github.com/aws/aws-sdk-go/service/s3"
)

var sc *storage.Client
var sts *storagetransfer.Client
var s3Bucket string
var gcsSourceBucket string
var gcsSinkBucket string

func TestMain(m *testing.M) {
	// Initialize global vars
	tc, _ := testutil.ContextMain(m)

	ctx := context.Background()
	c, err := storage.NewClient(ctx)
	if err != nil {
		log.Fatalf("storage.NewClient: %v", err)
	}
	sc = c
	defer sc.Close()

	gcsSourceBucket = testutil.UniqueBucketName("gcssourcebucket")
	source := sc.Bucket(gcsSourceBucket)
	err = source.Create(ctx, tc.ProjectID, nil)
	if err != nil {
		log.Fatalf("couldn't create GCS Source bucket: %v", err)
	}

	gcsSinkBucket = testutil.UniqueBucketName("gcssinkbucket")
	sink := sc.Bucket(gcsSinkBucket)
	err = sink.Create(ctx, tc.ProjectID, nil)
	if err != nil {
		log.Fatalf("couldn't create GCS Sink bucket: %v", err)
	}

	sts, err = storagetransfer.NewClient(ctx)
	if err != nil {
		log.Fatalf("storagetransfer.NewClient: %v", err)
	}
	defer sts.Close()

	grantSTSPermissions(gcsSourceBucket, tc.ProjectID, sts, sc)
	grantSTSPermissions(gcsSinkBucket, tc.ProjectID, sts, sc)

	s3Bucket = testutil.UniqueBucketName("stss3bucket")
	sess, err := session.NewSession(&aws.Config{
		Region: aws.String("us-west-2")},
	)
	s3c := s3.New(sess)
	_, err = s3c.CreateBucket(&s3.CreateBucketInput{
		Bucket: aws.String(s3Bucket),
	})
	if err != nil {
		log.Fatalf("couldn't create S3 bucket: %v", err)
	}

	// Run tests
	exit := m.Run()

	err = sink.Delete(ctx)
	if err != nil {
		log.Printf("couldn't delete GCS Sink bucket: %v", err)
	}

	err = source.Delete(ctx)
	if err != nil {
		log.Printf("couldn't delete GCS Source bucket: %v", err)
	}
	s3manager.NewDeleteListIterator(s3c, &s3.ListObjectsInput{
		Bucket: aws.String(s3Bucket),
	})
	_, err = s3c.DeleteBucket(&s3.DeleteBucketInput{
		Bucket: aws.String(s3Bucket),
	})
	if err != nil {
		log.Printf("couldn't delete S3 bucket: %v", err)
	}

	os.Exit(exit)
}

func TestQuickstart(t *testing.T) {
	tc := testutil.SystemTest(t)

	buf := new(bytes.Buffer)
	resp, err := quickstart(buf, tc.ProjectID, gcsSourceBucket, gcsSinkBucket)
	defer cleanupSTSJob(resp.Name, tc.ProjectID)

	if err != nil {
		t.Errorf("quickstart: %#v", err)
	}

	got := buf.String()
	if want := "transferJobs/"; !strings.Contains(got, want) {
		t.Errorf("quickstart: got %q, want %q", got, want)
	}
}

func TestTransferFromAws(t *testing.T) {
	tc := testutil.SystemTest(t)

	buf := new(bytes.Buffer)

	resp, err := transferFromAws(buf, tc.ProjectID, s3Bucket, gcsSinkBucket)
	defer cleanupSTSJob(resp.Name, tc.ProjectID)

	if err != nil {
		t.Errorf("transfer_from_aws: %#v", err)
	}

	got := buf.String()
	if want := "transferJobs/"; !strings.Contains(got, want) {
		t.Errorf("transfer_from_aws: got %q, want %q", got, want)
	}
}

func TestTransferToNearline(t *testing.T) {
	tc := testutil.SystemTest(t)

	buf := new(bytes.Buffer)

	resp, err := transferToNearline(buf, tc.ProjectID, gcsSourceBucket, gcsSinkBucket)
	defer cleanupSTSJob(resp.Name, tc.ProjectID)

	if err != nil {
		t.Errorf("transfer_from_aws: %#v", err)
	}

	got := buf.String()
	if want := "transferJobs/"; !strings.Contains(got, want) {
		t.Errorf("transfer_to_nearline: got %q, want %q", got, want)
	}
}

func TestGetLatestTransferOperation(t *testing.T) {
	tc := testutil.SystemTest(t)

	buf := new(bytes.Buffer)

	job, err := transferToNearline(buf, tc.ProjectID, gcsSourceBucket, gcsSinkBucket)
	defer cleanupSTSJob(job.Name, tc.ProjectID)

	op, err := checkLatestTransferOperation(buf, tc.ProjectID, job.Name)

	if err != nil {
		t.Errorf("check_latest_transfer_operation: %#v", err)
	}
	if !strings.Contains(op.Name, "transferOperations/") {
		t.Errorf("check_latest_transfer_operation: Operation returned didn't have a valid operation name: %q", op.Name)
	}

	got := buf.String()
	if want := op.Name; !strings.Contains(got, want) {
		t.Errorf("check_latest_transfer_operation: got %q, want %q", got, want)
	}
}

func TestDownloadToPosix(t *testing.T) {
	tc := testutil.SystemTest(t)

	buf := new(bytes.Buffer)

	rootDirectory, err := ioutil.TempDir("", "download-to-posix-test")
	if err != nil {
		t.Fatalf("download_to_posix: %#v", err)
	}
	defer os.RemoveAll(rootDirectory)

	sinkAgentPoolName := "" //use default agent pool
	gcsSourcePath := rootDirectory + "/"

	resp, err := downloadToPosix(buf, tc.ProjectID, sinkAgentPoolName, gcsSinkBucket, gcsSourcePath, rootDirectory)
	defer cleanupSTSJob(resp.Name, tc.ProjectID)

	if err != nil {
		t.Errorf("download_to_posix: %#v", err)
	}

	got := buf.String()
	if want := "transferJobs/"; !strings.Contains(got, want) {
		t.Errorf("download_to_posix: got %q, want %q", got, want)
	}
}

func TestTransferFromPosix(t *testing.T) {
	tc := testutil.SystemTest(t)

	buf := new(bytes.Buffer)

	rootDirectory, err := ioutil.TempDir("", "transfer-from-posix-test")
	if err != nil {
		t.Fatalf("transfer_from_posix: %#v", err)
	}
	defer os.RemoveAll(rootDirectory)

	sourceAgentPoolName := "" //use default agent pool

	resp, err := transferFromPosix(buf, tc.ProjectID, sourceAgentPoolName, rootDirectory, gcsSinkBucket)
	defer cleanupSTSJob(resp.Name, tc.ProjectID)

	if err != nil {
		t.Errorf("transfer_from_posix: %#v", err)
	}

	got := buf.String()
	if want := "transferJobs/"; !strings.Contains(got, want) {
		t.Errorf("transfer_from_posix: got %q, want %q", got, want)
	}
}

func TestTransferBetweenPosix(t *testing.T) {
	tc := testutil.SystemTest(t)

	buf := new(bytes.Buffer)

	rootDirectory, err := ioutil.TempDir("", "transfer-between-posix-test-source")
	if err != nil {
		t.Fatalf("transfer_between_posix: %#v", err)
	}
	defer os.RemoveAll(rootDirectory)

	destinationDirectory, err := ioutil.TempDir("", "transfer-between-posix-test-sink")
	if err != nil {
		t.Fatalf("transfer_between_posix: %#v", err)
	}
	defer os.RemoveAll(destinationDirectory)

	sourceAgentPoolName := "" //use default agent pool
	sinkAgentPoolName := ""   //use default agent pool

	resp, err := transferBetweenPosix(buf, tc.ProjectID, sourceAgentPoolName, sinkAgentPoolName, rootDirectory, destinationDirectory, gcsSinkBucket)
	if err != nil {
		t.Errorf("transfer_between_posix: %#v", err)
	}
	defer cleanupSTSJob(resp.Name, tc.ProjectID)

	got := buf.String()
	if want := "transferJobs/"; !strings.Contains(got, want) {
		t.Errorf("transfer_between_posix: got %q, want %q", got, want)
	}
}

func TestTransferUsingManifest(t *testing.T) {
	tc := testutil.SystemTest(t)

	buf := new(bytes.Buffer)

	rootDirectory, err := ioutil.TempDir("", "transfer-using-manifest-test")
	if err != nil {
		t.Fatalf("transfer_using_manifest: %#v", err)
	}
	defer os.RemoveAll(rootDirectory)

	sourceAgentPoolName := "" //use default agent pool
	object := sc.Bucket(gcsSourceBucket).Object("manifest.csv")
	defer object.Delete(context.Background())

	resp, err := transferUsingManifest(buf, tc.ProjectID, sourceAgentPoolName, rootDirectory, gcsSinkBucket, gcsSourceBucket, "manifest.csv")
	defer cleanupSTSJob(resp.Name, tc.ProjectID)

	if err != nil {
		t.Errorf("transfer_using_manifest: %#v", err)
	}

	got := buf.String()
	if want := "transferJobs/"; !strings.Contains(got, want) {
		t.Errorf("transfer_using_manifest: got %q, want %q", got, want)
	}
}

func grantSTSPermissions(bucketName string, projectID string, sts *storagetransfer.Client, str *storage.Client) {
	ctx := context.Background()

	req := &storagetransferpb.GetGoogleServiceAccountRequest{
		ProjectId: projectID,
	}

	resp, err := sts.GetGoogleServiceAccount(ctx, req)
	if err != nil {
		log.Fatalf("error getting service account")
	}
	email := resp.AccountEmail

	identity := "serviceAccount:" + email

	bucket := str.Bucket(bucketName)
	policy, err := bucket.IAM().Policy(ctx)
	if err != nil {
		log.Fatalf("Bucket(%q).IAM().Policy: %v", bucketName, err)
	}

	var objectViewer iam.RoleName = "roles/storage.objectViewer"
	var bucketReader iam.RoleName = "roles/storage.legacyBucketReader"
	var bucketWriter iam.RoleName = "roles/storage.legacyBucketWriter"

	policy.Add(identity, objectViewer)
	policy.Add(identity, bucketReader)
	policy.Add(identity, bucketWriter)

	if err := bucket.IAM().SetPolicy(ctx, policy); err != nil {
		log.Fatalf("bucket(%q).IAM().SetPolicy: %v", bucketName, err)
	}
}

func cleanupSTSJob(jobName string, projectID string) {
	ctx := context.Background()

	tj := &storagetransferpb.TransferJob{
		Name:   jobName,
		Status: storagetransferpb.TransferJob_DELETED,
	}
	sts.UpdateTransferJob(ctx, &storagetransferpb.UpdateTransferJobRequest{
		JobName:     jobName,
		ProjectId:   projectID,
		TransferJob: tj,
	})
}
<commit_msg>test(storagetransfer): fix test cleanup panic (#2656)<commit_after>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storagetransfer

import (
	"bytes"
	"context"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"testing"

	"cloud.google.com/go/iam"
	"cloud.google.com/go/storage"
	storagetransfer "cloud.google.com/go/storagetransfer/apiv1"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	storagetransferpb "google.golang.org/genproto/googleapis/storagetransfer/v1"

	"github.com/GoogleCloudPlatform/golang-samples/internal/testutil"
	"github.com/aws/aws-sdk-go/service/s3"
)

var sc *storage.Client
var sts *storagetransfer.Client
var s3Bucket string
var gcsSourceBucket string
var gcsSinkBucket string

func TestMain(m *testing.M) {
	// Initialize global vars
	tc, _ := testutil.ContextMain(m)

	ctx := context.Background()
	c, err := storage.NewClient(ctx)
	if err != nil {
		log.Fatalf("storage.NewClient: %v", err)
	}
	sc = c
	defer sc.Close()

	gcsSourceBucket = testutil.UniqueBucketName("gcssourcebucket")
	source := sc.Bucket(gcsSourceBucket)
	err = source.Create(ctx, tc.ProjectID, nil)
	if err != nil {
		log.Fatalf("couldn't create GCS Source bucket: %v", err)
	}

	gcsSinkBucket = testutil.UniqueBucketName("gcssinkbucket")
	sink := sc.Bucket(gcsSinkBucket)
	err = sink.Create(ctx, tc.ProjectID, nil)
	if err != nil {
		log.Fatalf("couldn't create GCS Sink bucket: %v", err)
	}

	sts, err = storagetransfer.NewClient(ctx)
	if err != nil {
		log.Fatalf("storagetransfer.NewClient: %v", err)
	}
	defer sts.Close()

	grantSTSPermissions(gcsSourceBucket, tc.ProjectID, sts, sc)
	grantSTSPermissions(gcsSinkBucket, tc.ProjectID, sts, sc)

	s3Bucket = testutil.UniqueBucketName("stss3bucket")
	sess, err := session.NewSession(&aws.Config{
		Region: aws.String("us-west-2")},
	)
	s3c := s3.New(sess)
	_, err = s3c.CreateBucket(&s3.CreateBucketInput{
		Bucket: aws.String(s3Bucket),
	})
	if err != nil {
		log.Fatalf("couldn't create S3 bucket: %v", err)
	}

	// Run tests
	exit := m.Run()

	err = sink.Delete(ctx)
	if err != nil {
		log.Printf("couldn't delete GCS Sink bucket: %v", err)
	}

	err = source.Delete(ctx)
	if err != nil {
		log.Printf("couldn't delete GCS Source bucket: %v", err)
	}
	s3manager.NewDeleteListIterator(s3c, &s3.ListObjectsInput{
		Bucket: aws.String(s3Bucket),
	})
	_, err = s3c.DeleteBucket(&s3.DeleteBucketInput{
		Bucket: aws.String(s3Bucket),
	})
	if err != nil {
		log.Printf("couldn't delete S3 bucket: %v", err)
	}

	os.Exit(exit)
}

func TestQuickstart(t *testing.T) {
	tc := testutil.SystemTest(t)

	buf := new(bytes.Buffer)
	resp, err := quickstart(buf, tc.ProjectID, gcsSourceBucket, gcsSinkBucket)
	defer cleanupSTSJob(resp, tc.ProjectID)

	if err != nil {
		t.Errorf("quickstart: %#v", err)
	}

	got := buf.String()
	if want := "transferJobs/"; !strings.Contains(got, want) {
		t.Errorf("quickstart: got %q, want %q", got, want)
	}
}

func TestTransferFromAws(t *testing.T) {
	tc := testutil.SystemTest(t)

	buf := new(bytes.Buffer)

	resp, err := transferFromAws(buf, tc.ProjectID, s3Bucket, gcsSinkBucket)
	defer cleanupSTSJob(resp, tc.ProjectID)

	if err != nil {
		t.Errorf("transfer_from_aws: %#v", err)
	}

	got := buf.String()
	if want := "transferJobs/"; !strings.Contains(got, want) {
		t.Errorf("transfer_from_aws: got %q, want %q", got, want)
	}
}

func TestTransferToNearline(t *testing.T) {
	tc := testutil.SystemTest(t)

	buf := new(bytes.Buffer)

	resp, err := transferToNearline(buf, tc.ProjectID, gcsSourceBucket, gcsSinkBucket)
	defer cleanupSTSJob(resp, tc.ProjectID)

	if err != nil {
		t.Errorf("transfer_from_aws: %#v", err)
	}

	got := buf.String()
	if want := "transferJobs/"; !strings.Contains(got, want) {
		t.Errorf("transfer_to_nearline: got %q, want %q", got, want)
	}
}

func TestGetLatestTransferOperation(t *testing.T) {
	tc := testutil.SystemTest(t)

	buf := new(bytes.Buffer)

	job, err := transferToNearline(buf, tc.ProjectID, gcsSourceBucket, gcsSinkBucket)
	defer cleanupSTSJob(job, tc.ProjectID)

	op, err := checkLatestTransferOperation(buf, tc.ProjectID, job.Name)

	if err != nil {
		t.Errorf("check_latest_transfer_operation: %#v", err)
	}
	if !strings.Contains(op.Name, "transferOperations/") {
		t.Errorf("check_latest_transfer_operation: Operation returned didn't have a valid operation name: %q", op.Name)
	}

	got := buf.String()
	if want := op.Name; !strings.Contains(got, want) {
		t.Errorf("check_latest_transfer_operation: got %q, want %q", got, want)
	}
}

func TestDownloadToPosix(t *testing.T) {
	tc := testutil.SystemTest(t)

	buf := new(bytes.Buffer)

	rootDirectory, err := ioutil.TempDir("", "download-to-posix-test")
	if err != nil {
		t.Fatalf("download_to_posix: %#v", err)
	}
	defer os.RemoveAll(rootDirectory)

	sinkAgentPoolName := "" //use default agent pool
	gcsSourcePath := rootDirectory + "/"

	resp, err := downloadToPosix(buf, tc.ProjectID, sinkAgentPoolName, gcsSinkBucket, gcsSourcePath, rootDirectory)
	defer cleanupSTSJob(resp, tc.ProjectID)

	if err != nil {
		t.Errorf("download_to_posix: %#v", err)
	}

	got := buf.String()
	if want := "transferJobs/"; !strings.Contains(got, want) {
		t.Errorf("download_to_posix: got %q, want %q", got, want)
	}
}

func TestTransferFromPosix(t *testing.T) {
	tc := testutil.SystemTest(t)

	buf := new(bytes.Buffer)

	rootDirectory, err := ioutil.TempDir("", "transfer-from-posix-test")
	if err != nil {
		t.Fatalf("transfer_from_posix: %#v", err)
	}
	defer os.RemoveAll(rootDirectory)

	sourceAgentPoolName := "" //use default agent pool

	resp, err := transferFromPosix(buf, tc.ProjectID, sourceAgentPoolName, rootDirectory, gcsSinkBucket)
	defer cleanupSTSJob(resp, tc.ProjectID)

	if err != nil {
		t.Errorf("transfer_from_posix: %#v", err)
	}

	got := buf.String()
	if want := "transferJobs/"; !strings.Contains(got, want) {
		t.Errorf("transfer_from_posix: got %q, want %q", got, want)
	}
}

func TestTransferBetweenPosix(t *testing.T) {
	tc := testutil.SystemTest(t)

	buf := new(bytes.Buffer)

	rootDirectory, err := ioutil.TempDir("", "transfer-between-posix-test-source")
	if err != nil {
		t.Fatalf("transfer_between_posix: %#v", err)
	}
	defer os.RemoveAll(rootDirectory)

	destinationDirectory, err := ioutil.TempDir("", "transfer-between-posix-test-sink")
	if err != nil {
		t.Fatalf("transfer_between_posix: %#v", err)
	}
	defer os.RemoveAll(destinationDirectory)

	sourceAgentPoolName := "" //use default agent pool
	sinkAgentPoolName := ""   //use default agent pool

	resp, err := transferBetweenPosix(buf, tc.ProjectID, sourceAgentPoolName, sinkAgentPoolName, rootDirectory, destinationDirectory, gcsSinkBucket)
	if err != nil {
		t.Errorf("transfer_between_posix: %#v", err)
	}
	defer cleanupSTSJob(resp, tc.ProjectID)

	got := buf.String()
	if want := "transferJobs/"; !strings.Contains(got, want) {
		t.Errorf("transfer_between_posix: got %q, want %q", got, want)
	}
}

func TestTransferUsingManifest(t *testing.T) {
	tc := testutil.SystemTest(t)

	buf := new(bytes.Buffer)

	rootDirectory, err := ioutil.TempDir("", "transfer-using-manifest-test")
	if err != nil {
		t.Fatalf("transfer_using_manifest: %#v", err)
	}
	defer os.RemoveAll(rootDirectory)

	sourceAgentPoolName := "" //use default agent pool
	object := sc.Bucket(gcsSourceBucket).Object("manifest.csv")
	defer object.Delete(context.Background())

	resp, err := transferUsingManifest(buf, tc.ProjectID, sourceAgentPoolName, rootDirectory, gcsSinkBucket, gcsSourceBucket, "manifest.csv")
	defer cleanupSTSJob(resp, tc.ProjectID)

	if err != nil {
		t.Errorf("transfer_using_manifest: %#v", err)
	}

	got := buf.String()
	if want := "transferJobs/"; !strings.Contains(got, want) {
		t.Errorf("transfer_using_manifest: got %q, want %q", got, want)
	}
}

func grantSTSPermissions(bucketName string, projectID string, sts *storagetransfer.Client, str *storage.Client) {
	ctx := context.Background()

	req := &storagetransferpb.GetGoogleServiceAccountRequest{
		ProjectId: projectID,
	}

	resp, err := sts.GetGoogleServiceAccount(ctx, req)
	if err != nil {
		log.Fatalf("error getting service account")
	}
	email := resp.AccountEmail

	identity := "serviceAccount:" + email

	bucket := str.Bucket(bucketName)
	policy, err := bucket.IAM().Policy(ctx)
	if err != nil {
		log.Fatalf("Bucket(%q).IAM().Policy: %v", bucketName, err)
	}

	var objectViewer iam.RoleName = "roles/storage.objectViewer"
	var bucketReader iam.RoleName = "roles/storage.legacyBucketReader"
	var bucketWriter iam.RoleName = "roles/storage.legacyBucketWriter"

	policy.Add(identity, objectViewer)
	policy.Add(identity, bucketReader)
	policy.Add(identity, bucketWriter)

	if err := bucket.IAM().SetPolicy(ctx, policy); err != nil {
		log.Fatalf("bucket(%q).IAM().SetPolicy: %v", bucketName, err)
	}
}

func cleanupSTSJob(job *storagetransferpb.TransferJob, projectID string) {
	if job == nil {
		return
	}

	ctx := context.Background()

	tj := &storagetransferpb.TransferJob{
		Name:   job.Name,
		Status: storagetransferpb.TransferJob_DELETED,
	}
	sts.UpdateTransferJob(ctx, &storagetransferpb.UpdateTransferJobRequest{
		JobName:     job.Name,
		ProjectId:   projectID,
		TransferJob: tj,
	})
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"io/ioutil"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/go-errors/errors"
	"github.com/privacybydesign/gabi"
	"github.com/privacybydesign/gabi/big"
	irma "github.com/privacybydesign/irmago"
	"github.com/privacybydesign/irmago/internal/fs"
	"github.com/sirupsen/logrus"
)

// Configuration contains configuration for the irmaserver library and irmad.
type Configuration struct {
	// irma_configuration. If not given, this will be popupated using SchemesPath.
	IrmaConfiguration *irma.Configuration `json:"-"`
	// Path to IRMA schemes to parse into IrmaConfiguration (only used if IrmaConfiguration == nil).
	// If left empty, default value is taken using DefaultSchemesPath().
	// If an empty folder is specified, default schemes (irma-demo and pbdf) are downloaded into it.
	SchemesPath string `json:"schemes_path" mapstructure:"schemes_path"`
	// If specified, schemes found here are copied into SchemesPath (only used if IrmaConfiguration == nil)
	SchemesAssetsPath string `json:"schemes_assets_path" mapstructure:"schemes_assets_path"`
	// Disable scheme updating
	DisableSchemesUpdate bool `json:"disable_schemes_update" mapstructure:"disable_schemes_update"`
	// Update all schemes every x minutes (default value 0 means 60) (use DisableSchemesUpdate to disable)
	SchemesUpdateInterval int `json:"schemes_update" mapstructure:"schemes_update"`
	// Path to issuer private keys to parse
	IssuerPrivateKeysPath string `json:"privkeys" mapstructure:"privkeys"`
	// Issuer private keys
	IssuerPrivateKeys map[irma.IssuerIdentifier]map[uint]*gabi.PrivateKey `json:"-"`
	// URL at which the IRMA app can reach this server during sessions
	URL string `json:"url" mapstructure:"url"`
	// Required to be set to true if URL does not begin with https:// in production mode.
	// In this case, the server would communicate with IRMA apps over plain HTTP. You must otherwise
	// ensure (using eg a reverse proxy with TLS enabled) that the attributes are protected in transit.
	DisableTLS bool `json:"disable_tls" mapstructure:"disable_tls"`
	// (Optional) email address of server admin, for incidental notifications such as breaking API changes
	// See https://github.com/privacybydesign/irmago/tree/master/server#specifying-an-email-address
	// for more information
	Email string `json:"email" mapstructure:"email"`
	// Enable server sent events for status updates (experimental; tends to hang when a reverse proxy is used)
	EnableSSE bool `json:"enable_sse" mapstructure:"enable_sse"`

	// Logging verbosity level: 0 is normal, 1 includes DEBUG level, 2 includes TRACE level
	Verbose int `json:"verbose" mapstructure:"verbose"`
	// Don't log anything at all
	Quiet bool `json:"quiet" mapstructure:"quiet"`
	// Output structured log in JSON format
	LogJSON bool `json:"log_json" mapstructure:"log_json"`
	// Custom logger instance. If specified, Verbose, Quiet and LogJSON are ignored.
	Logger *logrus.Logger `json:"-"`

	// Connection string for revocation database
	RevocationDBConnStr string `json:"revocation_db_str" mapstructure:"revocation_db_str"`
	// Database type for revocation database, supported: postgres, mysql
	RevocationDBType string `json:"revocation_db_type" mapstructure:"revocation_db_type"`
	// Credentials types for which revocation database should be hosted
	RevocationSettings map[irma.CredentialTypeIdentifier]*irma.RevocationSetting `json:"revocation_settings" mapstructure:"revocation_settings"`

	// Production mode: enables safer and stricter defaults and config checking
	Production bool `json:"production" mapstructure:"production"`
}

// Check ensures that the Configuration is loaded, usable and free of errors.
func (conf *Configuration) Check() error {
	if conf.Logger == nil {
		conf.Logger = NewLogger(conf.Verbose, conf.Quiet, conf.LogJSON)
	}
	Logger = conf.Logger
	irma.Logger = conf.Logger

	// loop to avoid repetetive err != nil line triplets
	for _, f := range []func() error{
		conf.verifyIrmaConf, conf.verifyPrivateKeys, conf.verifyURL, conf.verifyEmail, conf.verifyRevocation,
	} {
		if err := f(); err != nil {
			_ = LogError(err)
			if conf.IrmaConfiguration != nil {
				if e := conf.IrmaConfiguration.Revocation.Close(); e != nil {
					_ = LogError(e)
				}
			}
			return err
		}
	}

	return nil
}

func (conf *Configuration) HavePrivateKeys() bool {
	var err error
	for id := range conf.IrmaConfiguration.Issuers {
		if _, err = conf.IrmaConfiguration.PrivateKeyLatest(id); err == nil {
			return true
		}
	}
	return false
}

// helpers

func (conf *Configuration) verifyIrmaConf() error {
	if conf.IrmaConfiguration == nil {
		var (
			err    error
			exists bool
		)
		if conf.SchemesPath == "" {
			conf.SchemesPath = irma.DefaultSchemesPath() // Returns an existing path
		}
		if exists, err = fs.PathExists(conf.SchemesPath); err != nil {
			return err
		}
		if !exists {
			return errors.Errorf("Nonexisting schemes_path provided: %s", conf.SchemesPath)
		}
		conf.Logger.WithField("schemes_path", conf.SchemesPath).Info("Determined schemes path")
		conf.IrmaConfiguration, err = irma.NewConfiguration(conf.SchemesPath, irma.ConfigurationOptions{
			Assets:              conf.SchemesAssetsPath,
			RevocationDBType:    conf.RevocationDBType,
			RevocationDBConnStr: conf.RevocationDBConnStr,
			RevocationSettings:  conf.RevocationSettings,
		})
		if err != nil {
			return err
		}
		if err = conf.IrmaConfiguration.ParseFolder(); err != nil {
			return err
		}
	}

	// Put private keys into conf.IrmaConfiguration so we can use conf.IrmaConfiguration.PrivateKey()
	if len(conf.IssuerPrivateKeys) > 0 {
		conf.IrmaConfiguration.PrivateKeys = conf.IssuerPrivateKeys
	}

	if len(conf.IrmaConfiguration.SchemeManagers) == 0 {
		conf.Logger.Infof("No schemes found in %s, downloading default (irma-demo and pbdf)", conf.SchemesPath)
		if err := conf.IrmaConfiguration.DownloadDefaultSchemes(); err != nil {
			return err
		}
	}
	if conf.SchemesUpdateInterval == 0 {
		conf.SchemesUpdateInterval = 60
	}
	if !conf.DisableSchemesUpdate {
		conf.IrmaConfiguration.AutoUpdateSchemes(uint(conf.SchemesUpdateInterval))
	}

	return nil
}

func (conf *Configuration) verifyPrivateKeys() error {
	if conf.IssuerPrivateKeys == nil {
		conf.IssuerPrivateKeys = make(map[irma.IssuerIdentifier]map[uint]*gabi.PrivateKey)
	}
	if conf.IssuerPrivateKeysPath != "" {
		files, err := ioutil.ReadDir(conf.IssuerPrivateKeysPath)
		if err != nil {
			return err
		}
		for _, file := range files {
			filename := file.Name()
			dotcount := strings.Count(filename, ".")
			if filepath.Ext(filename) != ".xml" || filename[0] == '.' || dotcount < 2 || dotcount > 3 {
				conf.Logger.WithField("file", filename).Infof("Skipping non-private key file encountered in private keys path")
				continue
			}
			base := strings.TrimSuffix(filename, filepath.Ext(filename))
			counter := -1
			var err error
			if dotcount == 3 {
				index := strings.LastIndex(base, ".")
				counter, err = strconv.Atoi(base[index+1:])
				if err != nil {
					return err
				}
				base = base[:index]
			}

			issid := irma.NewIssuerIdentifier(base) // strip .xml
			if _, ok := conf.IrmaConfiguration.Issuers[issid]; !ok {
				return errors.Errorf("Private key %s belongs to an unknown issuer", filename)
			}
			sk, err := gabi.NewPrivateKeyFromFile(filepath.Join(conf.IssuerPrivateKeysPath, filename))
			if err != nil {
				return err
			}
			if counter >= 0 && uint(counter) != sk.Counter {
				return errors.Errorf("private key %s has wrong counter %d in filename, should be %d", filename, counter, sk.Counter)
			}
			if len(conf.IssuerPrivateKeys[issid]) == 0 {
				conf.IssuerPrivateKeys[issid] = map[uint]*gabi.PrivateKey{}
			}
			conf.IssuerPrivateKeys[issid][sk.Counter] = sk
		}
	}
	for issid := range conf.IssuerPrivateKeys {
		for _, sk := range conf.IssuerPrivateKeys[issid] {
			pk, err := conf.IrmaConfiguration.PublicKey(issid, sk.Counter)
			if err != nil {
				return err
			}
			if pk == nil {
				return errors.Errorf("Missing public key belonging to private key %s-%d", issid.String(), sk.Counter)
			}
			if new(big.Int).Mul(sk.P, sk.Q).Cmp(pk.N) != 0 {
				return errors.Errorf("Private key %s-%d does not belong to corresponding public key", issid.String(), sk.Counter)
			}
		}
	}

	return nil
}

func (conf *Configuration) prepareRevocation(credid irma.CredentialTypeIdentifier) error {
	sks, err := conf.IrmaConfiguration.PrivateKeyIndices(credid.IssuerIdentifier())
	if err != nil {
		return errors.WrapPrefix(err, "failed to load private key indices for revocation", 0)
	}
	if len(sks) == 0 {
		return errors.Errorf("revocation server mode enabled for %s but no private key installed", credid)
	}

	rev := conf.IrmaConfiguration.Revocation
	for _, skcounter := range sks {
		isk, err := conf.IrmaConfiguration.PrivateKey(credid.IssuerIdentifier(), skcounter)
		if err != nil {
			return errors.WrapPrefix(err, fmt.Sprintf("failed to load private key %s-%d for revocation", credid, skcounter), 0)
		}
		if !isk.RevocationSupported() {
			continue
		}
		sk, err := isk.RevocationKey()
		if err != nil {
			return errors.WrapPrefix(err, fmt.Sprintf("failed to load revocation private key %s-%d", credid, skcounter), 0)
		}
		exists, err := rev.Exists(credid, skcounter)
		if err != nil {
			return errors.WrapPrefix(err, fmt.Sprintf("failed to check if accumulator exists for %s-%d", credid, skcounter), 0)
		}
		if !exists {
			conf.Logger.Warnf("Creating initial accumulator for %s-%d", credid, skcounter)
			if err := conf.IrmaConfiguration.Revocation.EnableRevocation(credid, sk); err != nil {
				return errors.WrapPrefix(err, fmt.Sprintf("failed create initial accumulator for %s-%d", credid, skcounter), 0)
			}
		}
	}

	return nil
}

func (conf *Configuration) verifyRevocation() error {
	rev := conf.IrmaConfiguration.Revocation

	for credid, settings := range conf.RevocationSettings {
		if _, known := conf.IrmaConfiguration.CredentialTypes[credid]; !known {
			return errors.Errorf("unknown credential type %s in revocation settings", credid)
		}
		if settings.Mode == irma.RevocationModeServer {
			conf.Logger.Info("revocation server mode enabled for " + credid.String())
			conf.Logger.Info("Being the revocation server for a credential type comes with special responsibilities, a.o. that this server is always reachable online for any IRMA participant, and that the contents of the database is never deleted. Failure will lead to all IRMA apps being unable to disclose credentials of this type. Read more at https://irma.app/docs/revocation/#issuer-responsibilities.")
			if err := conf.prepareRevocation(credid); err != nil {
				return err
			}
		}
	}

	for credid, credtype := range conf.IrmaConfiguration.CredentialTypes {
		if !credtype.RevocationSupported() {
			continue
		}
		_, err := rev.Keys.PrivateKeyLatest(credid.IssuerIdentifier())
		haveSK := err == nil
		settings := conf.RevocationSettings[credid]
		if haveSK && settings == nil || (settings.RevocationServerURL == "" && settings.Mode != irma.RevocationModeServer) {
			return errors.Errorf("private key installed for %s, but no revocation server is configured: revocation-enabled issuance sessions will always fail", credid)
		}
	}

	return nil
}

func (conf *Configuration) verifyURL() error {
	if conf.URL != "" {
		if !strings.HasSuffix(conf.URL, "/") {
			conf.URL = conf.URL + "/"
		}
		if !strings.HasPrefix(conf.URL, "https://") {
			if !conf.Production || conf.DisableTLS {
				conf.DisableTLS = true
				conf.Logger.Warnf("TLS is not enabled on the url \"%s\" to which the IRMA app will connect. "+
					"Ensure that attributes are encrypted in transit by either enabling TLS or adding TLS in a reverse proxy.", conf.URL)
			} else {
				return errors.Errorf("Running without TLS in production mode is unsafe without a reverse proxy. " +
					"Either use a https:// URL or explicitly disable TLS.")
			}
		}
	} else {
		conf.Logger.Warn("No url parameter specified in configuration; unless an url is elsewhere prepended in the QR, the IRMA client will not be able to connect")
	}
	return nil
}

func (conf *Configuration) verifyEmail() error {
	if conf.Email != "" {
		// Very basic sanity checks
		if !strings.Contains(conf.Email, "@") || strings.Contains(conf.Email, "\n") {
			return errors.New("Invalid email address specified")
		}
		t := irma.NewHTTPTransport("https://metrics.privacybydesign.foundation/history")
		t.SetHeader("User-Agent", "irmaserver")
		var x string
		_ = t.Post("email", &x, conf.Email)
	}
	return nil
}
<commit_msg>fix: make revocation-related irma server misconfiguration nonfatal in case of demo schemes<commit_after>package server

import (
	"fmt"
	"io/ioutil"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/go-errors/errors"
	"github.com/privacybydesign/gabi"
	"github.com/privacybydesign/gabi/big"
	irma "github.com/privacybydesign/irmago"
	"github.com/privacybydesign/irmago/internal/fs"
	"github.com/sirupsen/logrus"
)

// Configuration contains configuration for the irmaserver library and irmad.
type Configuration struct {
	// irma_configuration. If not given, this will be popupated using SchemesPath.
	IrmaConfiguration *irma.Configuration `json:"-"`
	// Path to IRMA schemes to parse into IrmaConfiguration (only used if IrmaConfiguration == nil).
	// If left empty, default value is taken using DefaultSchemesPath().
	// If an empty folder is specified, default schemes (irma-demo and pbdf) are downloaded into it.
	SchemesPath string `json:"schemes_path" mapstructure:"schemes_path"`
	// If specified, schemes found here are copied into SchemesPath (only used if IrmaConfiguration == nil)
	SchemesAssetsPath string `json:"schemes_assets_path" mapstructure:"schemes_assets_path"`
	// Disable scheme updating
	DisableSchemesUpdate bool `json:"disable_schemes_update" mapstructure:"disable_schemes_update"`
	// Update all schemes every x minutes (default value 0 means 60) (use DisableSchemesUpdate to disable)
	SchemesUpdateInterval int `json:"schemes_update" mapstructure:"schemes_update"`
	// Path to issuer private keys to parse
	IssuerPrivateKeysPath string `json:"privkeys" mapstructure:"privkeys"`
	// Issuer private keys
	IssuerPrivateKeys map[irma.IssuerIdentifier]map[uint]*gabi.PrivateKey `json:"-"`
	// URL at which the IRMA app can reach this server during sessions
	URL string `json:"url" mapstructure:"url"`
	// Required to be set to true if URL does not begin with https:// in production mode.
	// In this case, the server would communicate with IRMA apps over plain HTTP. You must otherwise
	// ensure (using eg a reverse proxy with TLS enabled) that the attributes are protected in transit.
	DisableTLS bool `json:"disable_tls" mapstructure:"disable_tls"`
	// (Optional) email address of server admin, for incidental notifications such as breaking API changes
	// See https://github.com/privacybydesign/irmago/tree/master/server#specifying-an-email-address
	// for more information
	Email string `json:"email" mapstructure:"email"`
	// Enable server sent events for status updates (experimental; tends to hang when a reverse proxy is used)
	EnableSSE bool `json:"enable_sse" mapstructure:"enable_sse"`

	// Logging verbosity level: 0 is normal, 1 includes DEBUG level, 2 includes TRACE level
	Verbose int `json:"verbose" mapstructure:"verbose"`
	// Don't log anything at all
	Quiet bool `json:"quiet" mapstructure:"quiet"`
	// Output structured log in JSON format
	LogJSON bool `json:"log_json" mapstructure:"log_json"`
	// Custom logger instance. If specified, Verbose, Quiet and LogJSON are ignored.
	Logger *logrus.Logger `json:"-"`

	// Connection string for revocation database
	RevocationDBConnStr string `json:"revocation_db_str" mapstructure:"revocation_db_str"`
	// Database type for revocation database, supported: postgres, mysql
	RevocationDBType string `json:"revocation_db_type" mapstructure:"revocation_db_type"`
	// Credentials types for which revocation database should be hosted
	RevocationSettings map[irma.CredentialTypeIdentifier]*irma.RevocationSetting `json:"revocation_settings" mapstructure:"revocation_settings"`

	// Production mode: enables safer and stricter defaults and config checking
	Production bool `json:"production" mapstructure:"production"`
}

// Check ensures that the Configuration is loaded, usable and free of errors.
func (conf *Configuration) Check() error {
	if conf.Logger == nil {
		conf.Logger = NewLogger(conf.Verbose, conf.Quiet, conf.LogJSON)
	}
	Logger = conf.Logger
	irma.Logger = conf.Logger

	// loop to avoid repetetive err != nil line triplets
	for _, f := range []func() error{
		conf.verifyIrmaConf, conf.verifyPrivateKeys, conf.verifyURL, conf.verifyEmail, conf.verifyRevocation,
	} {
		if err := f(); err != nil {
			_ = LogError(err)
			if conf.IrmaConfiguration != nil {
				if e := conf.IrmaConfiguration.Revocation.Close(); e != nil {
					_ = LogError(e)
				}
			}
			return err
		}
	}

	return nil
}

func (conf *Configuration) HavePrivateKeys() bool {
	var err error
	for id := range conf.IrmaConfiguration.Issuers {
		if _, err = conf.IrmaConfiguration.PrivateKeyLatest(id); err == nil {
			return true
		}
	}
	return false
}

// helpers

func (conf *Configuration) verifyIrmaConf() error {
	if conf.IrmaConfiguration == nil {
		var (
			err    error
			exists bool
		)
		if conf.SchemesPath == "" {
			conf.SchemesPath = irma.DefaultSchemesPath() // Returns an existing path
		}
		if exists, err = fs.PathExists(conf.SchemesPath); err != nil {
			return err
		}
		if !exists {
			return errors.Errorf("Nonexisting schemes_path provided: %s", conf.SchemesPath)
		}
		conf.Logger.WithField("schemes_path", conf.SchemesPath).Info("Determined schemes path")
		conf.IrmaConfiguration, err = irma.NewConfiguration(conf.SchemesPath, irma.ConfigurationOptions{
			Assets:              conf.SchemesAssetsPath,
			RevocationDBType:    conf.RevocationDBType,
			RevocationDBConnStr: conf.RevocationDBConnStr,
			RevocationSettings:  conf.RevocationSettings,
		})
		if err != nil {
			return err
		}
		if err = conf.IrmaConfiguration.ParseFolder(); err != nil {
			return err
		}
	}

	// Put private keys into conf.IrmaConfiguration so we can use conf.IrmaConfiguration.PrivateKey()
	if len(conf.IssuerPrivateKeys) > 0 {
		conf.IrmaConfiguration.PrivateKeys = conf.IssuerPrivateKeys
	}

	if len(conf.IrmaConfiguration.SchemeManagers) == 0 {
		conf.Logger.Infof("No schemes found in %s, downloading default (irma-demo and pbdf)", conf.SchemesPath)
		if err := conf.IrmaConfiguration.DownloadDefaultSchemes(); err != nil {
			return err
		}
	}
	if conf.SchemesUpdateInterval == 0 {
		conf.SchemesUpdateInterval = 60
	}
	if !conf.DisableSchemesUpdate {
		conf.IrmaConfiguration.AutoUpdateSchemes(uint(conf.SchemesUpdateInterval))
	}

	return nil
}

func (conf *Configuration) verifyPrivateKeys() error {
	if conf.IssuerPrivateKeys == nil {
		conf.IssuerPrivateKeys = make(map[irma.IssuerIdentifier]map[uint]*gabi.PrivateKey)
	}
	if conf.IssuerPrivateKeysPath != "" {
		files, err := ioutil.ReadDir(conf.IssuerPrivateKeysPath)
		if err != nil {
			return err
		}
		for _, file := range files {
			filename := file.Name()
			dotcount := strings.Count(filename, ".")
			if filepath.Ext(filename) != ".xml" || filename[0] == '.' || dotcount < 2 || dotcount > 3 {
				conf.Logger.WithField("file", filename).Infof("Skipping non-private key file encountered in private keys path")
				continue
			}
			base := strings.TrimSuffix(filename, filepath.Ext(filename))
			counter := -1
			var err error
			if dotcount == 3 {
				index := strings.LastIndex(base, ".")
				counter, err = strconv.Atoi(base[index+1:])
				if err != nil {
					return err
				}
				base = base[:index]
			}

			issid := irma.NewIssuerIdentifier(base) // strip .xml
			if _, ok := conf.IrmaConfiguration.Issuers[issid]; !ok {
				return errors.Errorf("Private key %s belongs to an unknown issuer", filename)
			}
			sk, err := gabi.NewPrivateKeyFromFile(filepath.Join(conf.IssuerPrivateKeysPath, filename))
			if err != nil {
				return err
			}
			if counter >= 0 && uint(counter) != sk.Counter {
				return errors.Errorf("private key %s has wrong counter %d in filename, should be %d", filename, counter, sk.Counter)
			}
			if len(conf.IssuerPrivateKeys[issid]) == 0 {
				conf.IssuerPrivateKeys[issid] = map[uint]*gabi.PrivateKey{}
			}
			conf.IssuerPrivateKeys[issid][sk.Counter] = sk
		}
	}
	for issid := range conf.IssuerPrivateKeys {
		for _, sk := range conf.IssuerPrivateKeys[issid] {
			pk, err := conf.IrmaConfiguration.PublicKey(issid, sk.Counter)
			if err != nil {
				return err
			}
			if pk == nil {
				return errors.Errorf("Missing public key belonging to private key %s-%d", issid.String(), sk.Counter)
			}
			if new(big.Int).Mul(sk.P, sk.Q).Cmp(pk.N) != 0 {
				return errors.Errorf("Private key %s-%d does not belong to corresponding public key", issid.String(), sk.Counter)
			}
		}
	}

	return nil
}

func (conf *Configuration) prepareRevocation(credid irma.CredentialTypeIdentifier) error {
	sks, err := conf.IrmaConfiguration.PrivateKeyIndices(credid.IssuerIdentifier())
	if err != nil {
		return errors.WrapPrefix(err, "failed to load private key indices for revocation", 0)
	}
	if len(sks) == 0 {
		return errors.Errorf("revocation server mode enabled for %s but no private key installed", credid)
	}

	rev := conf.IrmaConfiguration.Revocation
	for _, skcounter := range sks {
		isk, err := conf.IrmaConfiguration.PrivateKey(credid.IssuerIdentifier(), skcounter)
		if err != nil {
			return errors.WrapPrefix(err, fmt.Sprintf("failed to load private key %s-%d for revocation", credid, skcounter), 0)
		}
		if !isk.RevocationSupported() {
			continue
		}
		sk, err := isk.RevocationKey()
		if err != nil {
			return errors.WrapPrefix(err, fmt.Sprintf("failed to load revocation private key %s-%d", credid, skcounter), 0)
		}
		exists, err := rev.Exists(credid, skcounter)
		if err != nil {
			return errors.WrapPrefix(err, fmt.Sprintf("failed to check if accumulator exists for %s-%d", credid, skcounter), 0)
		}
		if !exists {
			conf.Logger.Warnf("Creating initial accumulator for %s-%d", credid, skcounter)
			if err := conf.IrmaConfiguration.Revocation.EnableRevocation(credid, sk); err != nil {
				return errors.WrapPrefix(err, fmt.Sprintf("failed create initial accumulator for %s-%d", credid, skcounter), 0)
			}
		}
	}

	return nil
}

func (conf *Configuration) verifyRevocation() error {
	rev := conf.IrmaConfiguration.Revocation

	for credid, settings := range conf.RevocationSettings {
		if _, known := conf.IrmaConfiguration.CredentialTypes[credid]; !known {
			return errors.Errorf("unknown credential type %s in revocation settings", credid)
		}
		if settings.Mode == irma.RevocationModeServer {
			conf.Logger.Info("revocation server mode enabled for " + credid.String())
			conf.Logger.Info("Being the revocation server for a credential type comes with special responsibilities, a.o. that this server is always reachable online for any IRMA participant, and that the contents of the database is never deleted. Failure will lead to all IRMA apps being unable to disclose credentials of this type. Read more at https://irma.app/docs/revocation/#issuer-responsibilities.")
			if err := conf.prepareRevocation(credid); err != nil {
				return err
			}
		}
	}

	for credid, credtype := range conf.IrmaConfiguration.CredentialTypes {
		if !credtype.RevocationSupported() {
			continue
		}
		_, err := rev.Keys.PrivateKeyLatest(credid.IssuerIdentifier())
		haveSK := err == nil
		settings := conf.RevocationSettings[credid]
		if haveSK && settings == nil || (settings.RevocationServerURL == "" && settings.Mode != irma.RevocationModeServer) {
			message := "Revocation-supporting private key installed for %s, but no revocation server is configured: issuance sessions will always fail"
			if conf.IrmaConfiguration.SchemeManagers[credid.IssuerIdentifier().SchemeManagerIdentifier()].Demo {
				conf.Logger.Warnf(message, credid)
			} else {
				return errors.Errorf(message, credid)
			}
		}
	}

	return nil
}

func (conf *Configuration) verifyURL() error {
	if conf.URL != "" {
		if !strings.HasSuffix(conf.URL, "/") {
			conf.URL = conf.URL + "/"
		}
		if !strings.HasPrefix(conf.URL, "https://") {
			if !conf.Production || conf.DisableTLS {
				conf.DisableTLS = true
				conf.Logger.Warnf("TLS is not enabled on the url \"%s\" to which the IRMA app will connect. "+
					"Ensure that attributes are encrypted in transit by either enabling TLS or adding TLS in a reverse proxy.", conf.URL)
			} else {
				return errors.Errorf("Running without TLS in production mode is unsafe without a reverse proxy. " +
					"Either use a https:// URL or explicitly disable TLS.")
			}
		}
	} else {
		conf.Logger.Warn("No url parameter specified in configuration; unless an url is elsewhere prepended in the QR, the IRMA client will not be able to connect")
	}
	return nil
}

func (conf *Configuration) verifyEmail() error {
	if conf.Email != "" {
		// Very basic sanity checks
		if !strings.Contains(conf.Email, "@") || strings.Contains(conf.Email, "\n") {
			return errors.New("Invalid email address specified")
		}
		t := irma.NewHTTPTransport("https://metrics.privacybydesign.foundation/history")
		t.SetHeader("User-Agent", "irmaserver")
		var x string
		_ = t.Post("email", &x, conf.Email)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"net/http"
	"os"

	"github.com/golang/protobuf/jsonpb"
	"github.com/gorilla/mux"
	p "github.com/tiziano88/linc/server/proto"
)

var (
	m = &jsonpb.Marshaler{}
)

func main() {
	r := mux.NewRouter()
	r.HandleFunc("/", RootHandler)
	r.HandleFunc("/LoadFile", GetFile)
	r.HandleFunc("/SaveFile", UpdateFile)
	http.ListenAndServe(":8080", r)
}

func RootHandler(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "./client/out/index.html")
}

func GetFile(w http.ResponseWriter, r *http.Request) {
	rm := &p.GetFileRequest{}
	jsonpb.Unmarshal(r.Body, rm)
	rm.Path = "/tmp/src.json"
	data, err := ioutil.ReadFile(rm.Path)
	if err != nil {
		panic(err)
	}
	rw := &p.GetFileResponse{
		JsonContent: string(data),
	}
	err = m.Marshal(w, rw)
	if err != nil {
		panic(err)
	}
}

func UpdateFile(w http.ResponseWriter, r *http.Request) {
	rm := &p.UpdateFileRequest{}
	jsonpb.Unmarshal(r.Body, rm)
	rm.Path = "/tmp/src.json"
	err := ioutil.WriteFile(rm.Path, []byte(rm.JsonContent), os.ModePerm)
	if err != nil {
		panic(err)
	}
}
<commit_msg>update proto<commit_after>package main

import (
	"io/ioutil"
	"net/http"
	"os"

	"github.com/golang/protobuf/jsonpb"
	"github.com/gorilla/mux"
	p "github.com/tiziano88/linc/server/proto"
)

var (
	m = &jsonpb.Marshaler{}
)

func main() {
	r := mux.NewRouter()
	r.HandleFunc("/", RootHandler)
	r.HandleFunc("/LoadFile", GetFile)
	r.HandleFunc("/SaveFile", UpdateFile)
	panic(http.ListenAndServe(":8080", r))
}

func RootHandler(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "./client/out/index.html")
}

func GetFile(w http.ResponseWriter, r *http.Request) {
	rm := &p.GetFileRequest{}
	jsonpb.Unmarshal(r.Body, rm)
	rm.Path = "/tmp/src.json"
	data, err := ioutil.ReadFile(rm.Path)
	if err != nil {
		panic(err)
	}
	rw := &p.GetFileResponse{
		JsonContent: string(data),
	}
	err = m.Marshal(w, rw)
	if err != nil {
		panic(err)
	}
}

func UpdateFile(w http.ResponseWriter, r *http.Request) {
	rm := &p.UpdateFileRequest{}
	jsonpb.Unmarshal(r.Body, rm)
	rm.Path = "/tmp/src.json"
	err := ioutil.WriteFile(rm.Path, []byte(rm.JsonContent), os.ModePerm)
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"

	"github.com/admiraldolphin/govhack2017/server/game"
	"github.com/admiraldolphin/govhack2017/server/load"
)

var (
	gamePort = flag.Int("game_port", 23456, "Port for the game to listen on")
	httpPort = flag.Int("http_port", 23480, "Port the webserver listens on")

	cardsJSON  = flag.String("cards", "data/cards.json", "File to load traits from")
	peopleJSON = flag.String("people", "data/person.json", "File to load people from")
)

func main() {
	flag.Parse()

	ppl, err := load.People(*peopleJSON)
	if err != nil {
		log.Printf("Couldn't load people, continuing: %v", err)
	}
	log.Printf("Loaded %d people", len(ppl))

	cts, err := load.Traits(*cardsJSON)
	if err != nil {
		log.Printf("Couldn't load cards.json, continuing: %v", err)
	}
	log.Print("Loaded traits")

	deck := game.Deck(testDeck)
	if cts != nil && ppl != nil {
		deck = CreateDeck(cts, ppl)
	}

	s := server{state: game.New(deck)}

	// Set up HTTP handlers
	http.HandleFunc("/helloz", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, "Hello, GovHack 2017!\n")
	})
	http.HandleFunc("/statusz", func(w http.ResponseWriter, r *http.Request) {
		s.state.Dump(w)
	})

	// Start listening on game port; don't block.
	if err := s.listenAndServe(fmt.Sprintf(":%d", *gamePort)); err != nil {
		log.Fatalf("Coudn't serve game: %v", err)
	}

	// Start listening on HTTP port; block.
	if err := http.ListenAndServe(fmt.Sprintf(":%d", *httpPort), nil); err != nil {
		log.Fatalf("Couldn't serve HTTP: %v", err)
	}
}
<commit_msg>Allow CORS for all<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"

	"github.com/admiraldolphin/govhack2017/server/game"
	"github.com/admiraldolphin/govhack2017/server/load"
)

var (
	gamePort = flag.Int("game_port", 23456, "Port for the game to listen on")
	httpPort = flag.Int("http_port", 23480, "Port the webserver listens on")

	cardsJSON  = flag.String("cards", "data/cards.json", "File to load traits from")
	peopleJSON = flag.String("people", "data/person.json", "File to load people from")
)

func main() {
	flag.Parse()

	ppl, err := load.People(*peopleJSON)
	if err != nil {
		log.Printf("Couldn't load people, continuing: %v", err)
	}
	log.Printf("Loaded %d people", len(ppl))

	cts, err := load.Traits(*cardsJSON)
	if err != nil {
		log.Printf("Couldn't load cards.json, continuing: %v", err)
	}
	log.Print("Loaded traits")

	deck := game.Deck(testDeck)
	if cts != nil && ppl != nil {
		deck = CreateDeck(cts, ppl)
	}

	s := server{state: game.New(deck)}

	// Set up HTTP handlers
	http.HandleFunc("/helloz", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, "Hello, GovHack 2017!\n")
	})
	http.HandleFunc("/statusz", func(w http.ResponseWriter, r *http.Request) {
		h := w.Header()
		// Allow CORS because whatevs
		h.Set("Access-Control-Allow-Origin", "*://*/*")
		s.state.Dump(w)
	})

	// Start listening on game port; don't block.
	if err := s.listenAndServe(fmt.Sprintf(":%d", *gamePort)); err != nil {
		log.Fatalf("Coudn't serve game: %v", err)
	}

	// Start listening on HTTP port; block.
	if err := http.ListenAndServe(fmt.Sprintf(":%d", *httpPort), nil); err != nil {
		log.Fatalf("Couldn't serve HTTP: %v", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"

	"github.com/zero-boilerplate/go-api-helpers/service"

	"github.com/golang-devops/go-psexec/shared"
)

const (
	TempVersion      = "0.0.4" //Until we integrate with travis
	CURRENT_USER_VAL = "use_current"
)

var (
	//These flags is not run as service but will exit after completion
	genpemFlag        = flag.String("genpem", "", "The full path where to generate the pem file containing the private (and public) key")
	genpubFromPemFlag = flag.String("pub_from_pem", "", "Generate the public key from the input pem file")
)

var (
	serviceUsernameFlag       = flag.String("service_username", "", "The username of the installed service (use '"+CURRENT_USER_VAL+"' without quotes to use the current user running the install service command.")
	servicePasswordFlag       = flag.String("service_password", "", "The password of the installed service")
	addressFlag               = flag.String("address", ":62677", "The full host and port to listen on")
	allowedPublicKeysFileFlag = flag.String("allowed_public_keys_file", "", "The path to the allowed public keys file")
	serverPemFlag             = flag.String("server_pem", "", "The file path for the server pem (private+public) key file")
)

func main() {
	fmt.Println("Version " + TempVersion)
	flag.Parse()

	if len(*genpemFlag) > 0 {
		err := shared.GenerateKeyPairPemFile(*genpemFlag)
		if err != nil {
			log.Fatalf("Unable to generate key pair pem file, error: %s", err.Error())
		}
		return
	}

	if len(*genpubFromPemFlag) > 0 {
		err := shared.PrintPemFilePublicKeyAsHex(*genpubFromPemFlag)
		if err != nil {
			log.Fatalf("Unable to generate public key from pem file, error: %s", err.Error())
		}
		return
	}

	var additionalArgs []string = []string{}

	if len(*service.ServiceFlag) == 0 ||
		(*service.ServiceFlag != "uninstall" && *service.ServiceFlag != "stop" && *service.ServiceFlag != "start") {

		if len(*serverPemFlag) == 0 {
			flag.Usage()
			log.Fatalln("The server pem flag is required.")
		}
		if len(*allowedPublicKeysFileFlag) == 0 {
			flag.Usage()
			log.Fatalln("No allowed public keys file specified, no keys will be allowed.")
		}

		additionalArgs = []string{
			"-address",
			*addressFlag,
			"-server_pem",
			*serverPemFlag,
			"-allowed_public_keys_file",
			*allowedPublicKeysFileFlag,
		}
	}

	a := &app{
		debugMode:    true,
		accessLogger: true,
	}

	builder := service.NewServiceRunnerBuilder("GoPsExec", a).WithOnStopHandler(a).WithAdditionalArguments(additionalArgs...)

	if len(*serviceUsernameFlag) > 0 {
		if *serviceUsernameFlag == CURRENT_USER_VAL {
			builder = builder.WithServiceUserName_AsCurrentUser()
		} else {
			builder = builder.WithServiceUserName(*serviceUsernameFlag)
		}
	}

	if len(*servicePasswordFlag) > 0 {
		builder = builder.WithServicePassword(*servicePasswordFlag)
	}

	builder.Run()
}
<commit_msg>Bump to version 0.0.5<commit_after>package main

import (
	"flag"
	"fmt"
	"log"

	"github.com/zero-boilerplate/go-api-helpers/service"

	"github.com/golang-devops/go-psexec/shared"
)

const (
	TempVersion      = "0.0.5" //Until we integrate with travis
	CURRENT_USER_VAL = "use_current"
)

var (
	//These flags is not run as service but will exit after completion
	genpemFlag        = flag.String("genpem", "", "The full path where to generate the pem file containing the private (and public) key")
	genpubFromPemFlag = flag.String("pub_from_pem", "", "Generate the public key from the input pem file")
)

var (
	serviceUsernameFlag       = flag.String("service_username", "", "The username of the installed service (use '"+CURRENT_USER_VAL+"' without quotes to use the current user running the install service command.")
	servicePasswordFlag       = flag.String("service_password", "", "The password of the installed service")
	addressFlag               = flag.String("address", ":62677", "The full host and port to listen on")
	allowedPublicKeysFileFlag = flag.String("allowed_public_keys_file", "", "The path to the allowed public keys file")
	serverPemFlag             = flag.String("server_pem", "", "The file path for the server pem (private+public) key file")
)

func main() {
	fmt.Println("Version " + TempVersion)
	flag.Parse()

	if len(*genpemFlag) > 0 {
		err := shared.GenerateKeyPairPemFile(*genpemFlag)
		if err != nil {
			log.Fatalf("Unable to generate key pair pem file, error: %s", err.Error())
		}
		return
	}

	if len(*genpubFromPemFlag) > 0 {
		err := shared.PrintPemFilePublicKeyAsHex(*genpubFromPemFlag)
		if err != nil {
			log.Fatalf("Unable to generate public key from pem file, error: %s", err.Error())
		}
		return
	}

	var additionalArgs []string = []string{}

	if len(*service.ServiceFlag) == 0 ||
		(*service.ServiceFlag != "uninstall" && *service.ServiceFlag != "stop" && *service.ServiceFlag != "start") {

		if len(*serverPemFlag) == 0 {
			flag.Usage()
			log.Fatalln("The server pem flag is required.")
		}
		if len(*allowedPublicKeysFileFlag) == 0 {
			flag.Usage()
			log.Fatalln("No allowed public keys file specified, no keys will be allowed.")
		}

		additionalArgs = []string{
			"-address",
			*addressFlag,
			"-server_pem",
			*serverPemFlag,
			"-allowed_public_keys_file",
			*allowedPublicKeysFileFlag,
		}
	}

	a := &app{
		debugMode:    true,
		accessLogger: true,
	}

	builder := service.NewServiceRunnerBuilder("GoPsExec", a).WithOnStopHandler(a).WithAdditionalArguments(additionalArgs...)

	if len(*serviceUsernameFlag) > 0 {
		if *serviceUsernameFlag == CURRENT_USER_VAL {
			builder = builder.WithServiceUserName_AsCurrentUser()
		} else {
			builder = builder.WithServiceUserName(*serviceUsernameFlag)
		}
	}

	if len(*servicePasswordFlag) > 0 {
		builder = builder.WithServicePassword(*servicePasswordFlag)
	}

	builder.Run()
}
<|endoftext|>


<commit_before>package petrel

import (
	"net"
	"os"
	"strings"
	"testing"
	"time"
)

// create and destroy an idle petrel instance
func TestServStartStop(t *testing.T) {
	// fail to instantiate petrel by using a terrible filename
	c := &ServerConfig{Sockname: "zzz/zzz/zzz/zzz", Msglvl: All}
	as, err := UnixServ(c, 700)
	if err == nil {
		t.Error("that should have failed, but didn't")
	}

	// instantiate petrel
	c = &ServerConfig{Sockname: "/tmp/test00.sock", Msglvl: All}
	as, err = UnixServ(c, 700)
	if err != nil {
		t.Errorf("Couldn't create socket: %v", err)
	}
	// stat it
	fi, err := os.Stat(as.s)
	if err != nil {
		t.Errorf("Couldn't stat socket: %v", err)
	}
	fm := fi.Mode()
	if fm&os.ModeSocket == 1 {
		t.Errorf("'Socket' is not a socket %v", fm)
	}
	as.Quit()
}

// create petrel. connect to it with a client which does
// nothing but wait 1/10 second before disconnecting. tear down
// petrel.
func TestServConnServer(t *testing.T) {
	// instantiate petrel
	c := &ServerConfig{
		Sockname: "/tmp/test01.sock",
		Msglvl: All,
	}
	as, err := UnixServ(c, 700)
	if err != nil {
		t.Errorf("Couldn't create socket: %v", err)
	}
	// launch fakeclient. we should get a message about the
	// connection.
	go fakeclient(as.s, t)
	msg := <-as.Msgr
	if msg.Err != nil {
		t.Errorf("connection creation returned error: %v", msg.Err)
	}
	if !strings.HasPrefix(msg.Txt, "client connected") {
		t.Errorf("unexpected msg.Txt: %v", msg.Txt)
	}
	if msg.Conn != 1 {
		t.Errorf("msg.Conn should be 1 but got: %v", msg.Conn)
	}
	if msg.Req != 0 {
		t.Errorf("msg.Req should be 0 but got: %v", msg.Req)
	}
	if msg.Code != 100 {
		t.Errorf("msg.Code should be 100 but got: %v", msg.Code)
	}
	// wait for disconnect Msg
	msg = <-as.Msgr
	if msg.Err == nil {
		t.Errorf("connection drop should be an err, but got nil")
	}
	if msg.Txt != "client disconnected" {
		t.Errorf("unexpected msg.Txt: %v", msg.Txt)
	}
	if msg.Code != 198 {
		t.Errorf("msg.Code should be 198 but got: %v", msg.Code)
	}
	// shut down petrel
	as.Quit()
}

// these tests check for petrel.Msg implementing the Error interface
// properly.
func TestServMsgError(t *testing.T) {
	c := &ServerConfig{Sockname: "/tmp/test13.sock", Msglvl: All}
	as, err := UnixServ(c, 700)
	if err != nil {
		t.Errorf("Couldn't create socket: %v", err)
	}

	// first Msg: bare bones
	as.genMsg(1, 1, perrs["success"], "", nil)
	m := <-as.Msgr
	s := m.Error()
	if s != "conn 1 req 1 status 200 (reply sent)" {
		t.Errorf("Expected 'conn 1 req 1 status 200 (reply sent)' but got '%v'", s)
	}

	// now with Msg.Txt
	as.genMsg(1, 1, perrs["success"], "foo", nil)
	m = <-as.Msgr
	s = m.Error()
	if s != "conn 1 req 1 status 200 (reply sent: [foo])" {
		t.Errorf("Expected 'conn 1 req 1 status 200 (reply sent: [foo])' but got '%v'", s)
	}

	// and an error
	e := errors.New("something bad")
	as.genMsg(1, 1, perrs["success"], "foo", e)
	m = <-as.Msgr
	s = m.Error()
	if s != "conn 1 req 1 status 200 (reply sent: [foo]); err: something bad" {
		t.Errorf("Expected 'conn 1 req 1 status 200 (reply sent: [foo]); err: something bad' but got '%v'", s)
	}
	as.Quit()
}

// we need a fake client in order to test here. but it can be really,
// really fake. we're not even going to test send/recv yet.
func fakeclient(sn string, t *testing.T) {
	conn, err := net.Dial("unix", sn)
	defer conn.Close()
	if err != nil {
		t.Errorf("Couldn't connect to %v: %v", sn, err)
	}
	time.Sleep(100 * time.Millisecond)
}
<commit_msg>needed to add errors to imports<commit_after>package petrel

import (
	"errors"
	"net"
	"os"
	"strings"
	"testing"
	"time"
)

// create and destroy an idle petrel instance
func TestServStartStop(t *testing.T) {
	// fail to instantiate petrel by using a terrible filename
	c := &ServerConfig{Sockname: "zzz/zzz/zzz/zzz", Msglvl: All}
	as, err := UnixServ(c, 700)
	if err == nil {
		t.Error("that should have failed, but didn't")
	}

	// instantiate petrel
	c = &ServerConfig{Sockname: "/tmp/test00.sock", Msglvl: All}
	as, err = UnixServ(c, 700)
	if err != nil {
		t.Errorf("Couldn't create socket: %v", err)
	}
	// stat it
	fi, err := os.Stat(as.s)
	if err != nil {
		t.Errorf("Couldn't stat socket: %v", err)
	}
	fm := fi.Mode()
	if fm&os.ModeSocket == 1 {
		t.Errorf("'Socket' is not a socket %v", fm)
	}
	as.Quit()
}

// create petrel. connect to it with a client which does
// nothing but wait 1/10 second before disconnecting. tear down
// petrel.
func TestServConnServer(t *testing.T) {
	// instantiate petrel
	c := &ServerConfig{
		Sockname: "/tmp/test01.sock",
		Msglvl: All,
	}
	as, err := UnixServ(c, 700)
	if err != nil {
		t.Errorf("Couldn't create socket: %v", err)
	}
	// launch fakeclient. we should get a message about the
	// connection.
	go fakeclient(as.s, t)
	msg := <-as.Msgr
	if msg.Err != nil {
		t.Errorf("connection creation returned error: %v", msg.Err)
	}
	if !strings.HasPrefix(msg.Txt, "client connected") {
		t.Errorf("unexpected msg.Txt: %v", msg.Txt)
	}
	if msg.Conn != 1 {
		t.Errorf("msg.Conn should be 1 but got: %v", msg.Conn)
	}
	if msg.Req != 0 {
		t.Errorf("msg.Req should be 0 but got: %v", msg.Req)
	}
	if msg.Code != 100 {
		t.Errorf("msg.Code should be 100 but got: %v", msg.Code)
	}
	// wait for disconnect Msg
	msg = <-as.Msgr
	if msg.Err == nil {
		t.Errorf("connection drop should be an err, but got nil")
	}
	if msg.Txt != "client disconnected" {
		t.Errorf("unexpected msg.Txt: %v", msg.Txt)
	}
	if msg.Code != 198 {
		t.Errorf("msg.Code should be 198 but got: %v", msg.Code)
	}
	// shut down petrel
	as.Quit()
}

// these tests check for petrel.Msg implementing the Error interface
// properly.
func TestServMsgError(t *testing.T) {
	c := &ServerConfig{Sockname: "/tmp/test13.sock", Msglvl: All}
	as, err := UnixServ(c, 700)
	if err != nil {
		t.Errorf("Couldn't create socket: %v", err)
	}

	// first Msg: bare bones
	as.genMsg(1, 1, perrs["success"], "", nil)
	m := <-as.Msgr
	s := m.Error()
	if s != "conn 1 req 1 status 200 (reply sent)" {
		t.Errorf("Expected 'conn 1 req 1 status 200 (reply sent)' but got '%v'", s)
	}

	// now with Msg.Txt
	as.genMsg(1, 1, perrs["success"], "foo", nil)
	m = <-as.Msgr
	s = m.Error()
	if s != "conn 1 req 1 status 200 (reply sent: [foo])" {
		t.Errorf("Expected 'conn 1 req 1 status 200 (reply sent: [foo])' but got '%v'", s)
	}

	// and an error
	e := errors.New("something bad")
	as.genMsg(1, 1, perrs["success"], "foo", e)
	m = <-as.Msgr
	s = m.Error()
	if s != "conn 1 req 1 status 200 (reply sent: [foo]); err: something bad" {
		t.Errorf("Expected 'conn 1 req 1 status 200 (reply sent: [foo]); err: something bad' but got '%v'", s)
	}
	as.Quit()
}

// we need a fake client in order to test here. but it can be really,
// really fake. we're not even going to test send/recv yet.
func fakeclient(sn string, t *testing.T) {
	conn, err := net.Dial("unix", sn)
	defer conn.Close()
	if err != nil {
		t.Errorf("Couldn't connect to %v: %v", sn, err)
	}
	time.Sleep(100 * time.Millisecond)
}
<|endoftext|>


<commit_before>package xmlrpc

import (
    "testing"
    "bytes"
)

func a () bool {
    return true
}

func TestServer(tst *testing.T) {
    h := NewHandler()
    bf := func (p string) string { return p }
    cf := func (p, q string) int { return len(p + q) }
    h.RegFunc(a, "", nil)
    h.RegFunc(bf, "B", nil)
    h.RegFunc(cf, "", nil)
    tst.Logf("method list = %v", h.GetMethodList())
}

func TestWriteFault(tst *testing.T) {
    w := bytes.NewBufferString("")
    writeFault(w, 123, "fault msg string")
}
<commit_msg>add more testcase<commit_after>package xmlrpc

import (
    "testing"
    "bytes"
    "net/http"
    "net/http/httptest"
)

func a () bool {
    return true
}

func TestServer(tst *testing.T) {
    h := NewHandler()
    bf := func (p string) string { return p }
    cf := func (p, q string) int { return len(p + q) }
    h.RegFunc(a, "", nil)
    h.RegFunc(bf, "B", nil)
    h.RegFunc(cf, "", nil)
    tst.Logf("method list = %v", h.GetMethodList())
}

func TestSetLog(tst *testing.T) {
    h := NewHandler()
    h.SetLogf(func(r *http.Request, l int, n string) {})
}

type A struct {
    http.Request
    i int
}
//func (a *A)Add(b int) int { return int(*a) + b }
func (a *A)Add(b int) int { return a.i + b }
func (a *A)Del(b int) int { return a.i - b }

func TestRegister(tst *testing.T) {
    h := NewHandler()
    h.Register(func (){}, func(n string)string{return n}, true)
    h.Register(new(A), func(n string)string{if n == "Del" {return ""}; return n}, true)
    h.Register(new(A), nil, true)
}

func TestWriteFault(tst *testing.T) {
    w := bytes.NewBufferString("")
    writeFault(w, 123, "fault msg string")
}

func TestServeHTTP(tst *testing.T) {
    h := NewHandler()
    h.SetLogf(func(r *http.Request, l int, n string) {})
    // bad xml format
    buf := bytes.NewBufferString("")
    buf.Write([]byte(`<?xml version="1.0"?><ethodResponse`))
    req, err := http.NewRequest("GET", "/rpc", buf)
    if err != nil {
        tst.Error(err)
    }
    w := httptest.NewRecorder()
    h.ServeHTTP(w, req)
    b := w.Body.String()
    tst.Logf("code=%d, body=%s", w.Code, b)

    buf = bytes.NewBufferString("")
    //err := xmlrpc.Marshal(buf, "update458", "rule458.txt")
    err = Marshal(buf, "funcName", "data")
    if err != nil {
        tst.Error(err)
    }
    req, err = http.NewRequest("GET", "/rpc", buf)
    if err != nil {
        tst.Error(err)
    }
    w = httptest.NewRecorder()

    h.ServeHTTP(w, req)

    b = w.Body.String()
    tst.Logf("code=%d, body=%s", w.Code, b)

}
<|endoftext|>


<commit_before>package tests_test

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"os/exec"
	"regexp"
	"strings"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	v1 "k8s.io/api/core/v1"
	k8sv1 "k8s.io/api/storage/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"

	cdiv1 "kubevirt.io/containerized-data-importer/pkg/apis/core/v1alpha1"
	"kubevirt.io/containerized-data-importer/pkg/common"
	"kubevirt.io/containerized-data-importer/pkg/controller"
	"kubevirt.io/containerized-data-importer/tests"
	"kubevirt.io/containerized-data-importer/tests/framework"
	"kubevirt.io/containerized-data-importer/tests/utils"
)

const (
	namespacePrefix                  = "importer"
	assertionPollInterval            = 2 * time.Second
	controllerSkipPVCCompleteTimeout = 90 * time.Second
	invalidEndpoint                  = "http://gopats.com/who-is-the-goat.iso"
	CompletionTimeout                = 60 * time.Second
	BlankImageMD5                    = "cd573cfaace07e7949bc0c46028904ff"
	BlockDeviceMD5                   = "7c55761d39e6428fa27c21d8710a3d19"
)

var _ = Describe("[rfe_id:1115][crit:high][vendor:cnv-qe@redhat.com][level:component]Importer Test Suite", func() {
	var (
		ns string
		f  = framework.NewFrameworkOrDie(namespacePrefix)
		c  = f.K8sClient
	)

	BeforeEach(func() {
		ns = f.Namespace.Name
	})

	It("Should not perform CDI operations on PVC without annotations", func() {
		// Make sure the PVC name is unique, we have no guarantee on order and we are not
		// deleting the PVC at the end of the test, so if another runs first we will fail.
		pvc, err := f.CreatePVCFromDefinition(utils.NewPVCDefinition("no-import-ann", "1G", nil, nil))
		By("Verifying PVC with no annotation remains empty")
		Eventually(func() bool {
			log, err := tests.RunKubectlCommand(f, "logs", f.ControllerPod.Name, "-n", f.CdiInstallNs)
			Expect(err).NotTo(HaveOccurred())
			return strings.Contains(log, "pvc annotation \""+controller.AnnEndpoint+"\" not found, skipping pvc \""+ns+"/no-import-ann\"")
		}, controllerSkipPVCCompleteTimeout, assertionPollInterval).Should(BeTrue())
		Expect(err).ToNot(HaveOccurred())
		// Wait a while to see if CDI puts anything in the PVC.
		isEmpty, err := framework.VerifyPVCIsEmpty(f, pvc)
		Expect(err).ToNot(HaveOccurred())
		Expect(isEmpty).To(BeTrue())
		// Not deleting PVC as it will be removed with the NS removal.
	})

	It("[posneg:negative]Import pod status should be Fail on unavailable endpoint", func() {
		pvc, err := f.CreatePVCFromDefinition(utils.NewPVCDefinition(
			"no-import-noendpoint",
			"1G",
			map[string]string{controller.AnnEndpoint: invalidEndpoint},
			nil))
		Expect(err).ToNot(HaveOccurred())

		importer, err := utils.FindPodByPrefix(c, ns, common.ImporterPodName, common.CDILabelSelector)
		Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Unable to get importer pod %q", ns+"/"+common.ImporterPodName))
		utils.WaitTimeoutForPodStatus(c, importer.Name, importer.Namespace, v1.PodFailed, utils.PodWaitForTime)

		By("Verify the pod status is Failed on the target PVC")
		_, phaseAnnotation, err := utils.WaitForPVCAnnotation(f.K8sClient, f.Namespace.Name, pvc, controller.AnnPodPhase)
		Expect(phaseAnnotation).To(BeTrue())
		Expect(err).NotTo(HaveOccurred())

		By("deleting PVC")
		err = utils.DeletePVC(f.K8sClient, pvc.Namespace, pvc)
		Expect(err).ToNot(HaveOccurred())

		By("verifying pod was deleted")
		deleted, err := utils.WaitPodDeleted(f.K8sClient, importer.Name, f.Namespace.Name, timeout)
		Expect(deleted).To(BeTrue())
		Expect(err).ToNot(HaveOccurred())

		By("verifying pvc was deleted")
		deleted, err = utils.WaitPVCDeleted(f.K8sClient, pvc.Name, f.Namespace.Name, timeout)
		Expect(deleted).To(BeTrue())
		Expect(err).ToNot(HaveOccurred())
	})

	It("Should create import pod for blank raw image", func() {
		pvc, err := f.CreatePVCFromDefinition(utils.NewPVCDefinition(
			"create-image",
			"1G",
			map[string]string{controller.AnnSource: controller.SourceNone, controller.AnnContentType: string(cdiv1.DataVolumeKubeVirt)},
			nil))
		Expect(err).ToNot(HaveOccurred())

		By("Verify the pod status is succeeded on the target PVC")
		found, err := utils.WaitPVCPodStatusSucceeded(f.K8sClient, pvc)
		Expect(err).ToNot(HaveOccurred())
		Expect(found).To(BeTrue())

		By("Verify the image contents")
		same, err := f.VerifyTargetPVCContentMD5(f.Namespace, pvc, utils.DefaultImagePath, BlankImageMD5)
		Expect(err).ToNot(HaveOccurred())
		Expect(same).To(BeTrue())
	})
})

var _ = Describe("[rfe_id:1118][crit:high][vendor:cnv-qe@redhat.com][level:component]Importer Test Suite-prometheus", func() {
	var prometheusURL string
	var portForwardCmd *exec.Cmd
	var err error
	client := &http.Client{
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		},
	}
	f := framework.NewFrameworkOrDie(namespacePrefix)

	BeforeEach(func() {
		_, err := f.CreatePrometheusServiceInNs(f.Namespace.Name)
		Expect(err).NotTo(HaveOccurred(), "Error creating prometheus service")
	})

	AfterEach(func() {
		By("Stop port forwarding")
		if portForwardCmd != nil {
			err = portForwardCmd.Process.Kill()
			Expect(err).ToNot(HaveOccurred())
			portForwardCmd.Wait()
			portForwardCmd = nil
		}
	})

	It("Import pod should have prometheus stats available while importing", func() {
		c := f.K8sClient
		ns := f.Namespace.Name
		httpEp := fmt.Sprintf("http://%s:%d", utils.FileHostName+"."+f.CdiInstallNs, utils.HTTPRateLimitPort)
		pvcAnn := map[string]string{
			controller.AnnEndpoint: httpEp + "/tinyCore.qcow2",
			controller.AnnSecret:   "",
		}

		By("Verifying no end points exist before pvc is created")
		endpoint, err := c.CoreV1().Endpoints(ns).Get("kubevirt-prometheus-metrics", metav1.GetOptions{})
		Expect(err).To(HaveOccurred())

		By(fmt.Sprintf("Creating PVC with endpoint annotation %q", httpEp+"/tinyCore.qcow2"))
		pvc, err := utils.CreatePVCFromDefinition(c, ns, utils.NewPVCDefinition("import-e2e", "20M", pvcAnn, nil))
		Expect(err).NotTo(HaveOccurred(), "Error creating PVC")

		importer, err := utils.FindPodByPrefix(c, ns, common.ImporterPodName, common.CDILabelSelector)
		Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Unable to get importer pod %q", ns+"/"+common.ImporterPodName))

		l, err := labels.Parse(common.PrometheusLabel)
		Expect(err).ToNot(HaveOccurred())
		Eventually(func() int {
			endpoint, err = c.CoreV1().Endpoints(ns).Get("kubevirt-prometheus-metrics", metav1.GetOptions{})
			Expect(err).NotTo(HaveOccurred())
			_, err := c.CoreV1().Pods(ns).List(metav1.ListOptions{LabelSelector: l.String()})
			Expect(err).ToNot(HaveOccurred())
			return len(endpoint.Subsets)
		}, 60, 1).Should(Equal(1))

		By("Set up port forwarding")
		prometheusURL, portForwardCmd, err = startPrometheusPortForward(f)
		Expect(err).ToNot(HaveOccurred())

		By("checking if the endpoint contains the metrics port and only one matching subset")
		Expect(endpoint.Subsets[0].Ports).To(HaveLen(1))
		Expect(endpoint.Subsets[0].Ports[0].Name).To(Equal("metrics"))
		Expect(endpoint.Subsets[0].Ports[0].Port).To(Equal(int32(8443)))

		if importer.OwnerReferences[0].UID == pvc.GetUID() {
			var importRegExp = regexp.MustCompile("progress\\{ownerUID\\=\"" + string(pvc.GetUID()) + "\"\\} (\\d{1,3}\\.?\\d*)")
			Eventually(func() bool {
				fmt.Fprintf(GinkgoWriter, "INFO: Connecting to URL: %s\n", prometheusURL+"/metrics")
				resp, err := client.Get(prometheusURL + "/metrics")
				if err == nil {
					defer resp.Body.Close()
					if resp.StatusCode == http.StatusOK {
						bodyBytes, err := ioutil.ReadAll(resp.Body)
						Expect(err).NotTo(HaveOccurred())
						match := importRegExp.FindStringSubmatch(string(bodyBytes))
						if match != nil {
							return true
						}
					} else {
						fmt.Fprintf(GinkgoWriter, "INFO: received status code: %d\n", resp.StatusCode)
					}
				} else {
					fmt.Fprintf(GinkgoWriter, "INFO: collecting metrics failed: %v\n", err)
				}
				return false
			}, 90, 1).Should(BeTrue())
		} else {
			Fail("importer owner reference doesn't match PVC")
		}
	})
})

func startPrometheusPortForward(f *framework.Framework) (string, *exec.Cmd, error) {
	lp := "28443"
	pm := lp + ":8443"
	url := "https://127.0.0.1:" + lp

	cmd := tests.CreateKubectlCommand(f, "-n", f.Namespace.Name, "port-forward", "svc/kubevirt-prometheus-metrics", pm)
	err := cmd.Start()
	if err != nil {
		return "", nil, err
	}

	return url, cmd, nil
}

var _ = Describe("Importer Test Suite-Block_device", func() {
	f := framework.NewFrameworkOrDie(namespacePrefix)
	var pv *v1.PersistentVolume
	var storageClass *k8sv1.StorageClass
	var pod *v1.Pod
	var err error

	BeforeEach(func() {
		err = f.ClearBlockPV()
		Expect(err).NotTo(HaveOccurred())

		pod, err = utils.FindPodByPrefix(f.K8sClient, f.CdiInstallNs, "cdi-block-device", "kubevirt.io=cdi-block-device")
		Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Unable to get pod %q", f.CdiInstallNs+"/"+"cdi-block-device"))

		nodeName := pod.Spec.NodeName

		By(fmt.Sprintf("Creating storageClass for Block PV"))
		storageClass, err = f.CreateStorageClassFromDefinition(utils.NewStorageClassForBlockPVDefinition("manual"))
		Expect(err).ToNot(HaveOccurred())

		By(fmt.Sprintf("Creating Block PV"))
		pv, err = f.CreatePVFromDefinition(utils.NewBlockPVDefinition("local-volume", "500M", nil, "manual", nodeName))
		Expect(err).ToNot(HaveOccurred())

		By("Verify that PV's phase is Available")
		err = f.WaitTimeoutForPVReady(pv.Name, 60*time.Second)
		Expect(err).ToNot(HaveOccurred())
	})

	AfterEach(func() {
		err := utils.DeletePV(f.K8sClient, pv)
		Expect(err).ToNot(HaveOccurred())

		err = utils.DeleteStorageClass(f.K8sClient, storageClass)
		Expect(err).ToNot(HaveOccurred())
	})

	It("Should create import pod for block pv", func() {
		httpEp := fmt.Sprintf("http://%s:%d", utils.FileHostName+"."+f.CdiInstallNs, utils.HTTPNoAuthPort)
		pvcAnn := map[string]string{
			controller.AnnEndpoint: httpEp + "/tinyCore.iso",
		}

		By(fmt.Sprintf("Creating PVC with endpoint annotation %q", httpEp+"/tinyCore.iso"))

		pvc, err := f.CreatePVCFromDefinition(utils.NewBlockPVCDefinition(
			"import-image-to-block-pvc",
			"500M",
			pvcAnn,
			nil,
			"manual"))
		Expect(err).ToNot(HaveOccurred())

		By("Verify the pod status is succeeded on the target PVC")
		Eventually(func() string {
			status, phaseAnnotation, err := utils.WaitForPVCAnnotation(f.K8sClient, f.Namespace.Name, pvc, controller.AnnPodPhase)
			Expect(err).ToNot(HaveOccurred())
			Expect(phaseAnnotation).To(BeTrue())
			return status
		}, CompletionTimeout, assertionPollInterval).Should(BeEquivalentTo(v1.PodSucceeded))

		By("Verify content")
		same, err := f.VerifyTargetPVCContentMD5(f.Namespace, pvc, "/pvc", BlockDeviceMD5)
		Expect(err).ToNot(HaveOccurred())
		Expect(same).To(BeTrue())

	})
})

var _ = Describe("Importer Archive ContentType", func() {
	f := framework.NewFrameworkOrDie(namespacePrefix)

	It("Should import archive content type tar file", func() {
		c := f.K8sClient
		ns := f.Namespace.Name
		httpEp := fmt.Sprintf("http://%s:%d", utils.FileHostName+"."+utils.FileHostNs, utils.HTTPNoAuthPort)
		pvcAnn := map[string]string{
			controller.AnnEndpoint:    httpEp + "/archive.tar",
			controller.AnnContentType: "archive",
		}

		By(fmt.Sprintf("Creating PVC with endpoint annotation %q", httpEp+"/archive.tar"))
		pvc, err := utils.CreatePVCFromDefinition(c, ns, utils.NewPVCDefinition("import-archive", "100M", pvcAnn, nil))
		Expect(err).NotTo(HaveOccurred(), "Error creating PVC")

		By("Verify the pod status is succeeded on the target PVC")
		found, err := utils.WaitPVCPodStatusSucceeded(c, pvc)
		Expect(err).ToNot(HaveOccurred())
		Expect(found).To(BeTrue())

		By("Verify the target PVC contents")
		same, err := f.VerifyTargetPVCArchiveContent(f.Namespace, pvc, "3")
		Expect(err).ToNot(HaveOccurred())
		Expect(same).To(BeTrue())
	})
})
<commit_msg>Fix archive test to properly use cdi namespace variable.<commit_after>package tests_test

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"os/exec"
	"regexp"
	"strings"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	v1 "k8s.io/api/core/v1"
	k8sv1 "k8s.io/api/storage/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"

	cdiv1 "kubevirt.io/containerized-data-importer/pkg/apis/core/v1alpha1"
	"kubevirt.io/containerized-data-importer/pkg/common"
	"kubevirt.io/containerized-data-importer/pkg/controller"
	"kubevirt.io/containerized-data-importer/tests"
	"kubevirt.io/containerized-data-importer/tests/framework"
	"kubevirt.io/containerized-data-importer/tests/utils"
)

const (
	namespacePrefix                  = "importer"
	assertionPollInterval            = 2 * time.Second
	controllerSkipPVCCompleteTimeout = 90 * time.Second
	invalidEndpoint                  = "http://gopats.com/who-is-the-goat.iso"
	CompletionTimeout                = 60 * time.Second
	BlankImageMD5                    = "cd573cfaace07e7949bc0c46028904ff"
	BlockDeviceMD5                   = "7c55761d39e6428fa27c21d8710a3d19"
)

var _ = Describe("[rfe_id:1115][crit:high][vendor:cnv-qe@redhat.com][level:component]Importer Test Suite", func() {
	var (
		ns string
		f  = framework.NewFrameworkOrDie(namespacePrefix)
		c  = f.K8sClient
	)

	BeforeEach(func() {
		ns = f.Namespace.Name
	})

	It("Should not perform CDI operations on PVC without annotations", func() {
		// Make sure the PVC name is unique, we have no guarantee on order and we are not
		// deleting the PVC at the end of the test, so if another runs first we will fail.
		pvc, err := f.CreatePVCFromDefinition(utils.NewPVCDefinition("no-import-ann", "1G", nil, nil))
		By("Verifying PVC with no annotation remains empty")
		Eventually(func() bool {
			log, err := tests.RunKubectlCommand(f, "logs", f.ControllerPod.Name, "-n", f.CdiInstallNs)
			Expect(err).NotTo(HaveOccurred())
			return strings.Contains(log, "pvc annotation \""+controller.AnnEndpoint+"\" not found, skipping pvc \""+ns+"/no-import-ann\"")
		}, controllerSkipPVCCompleteTimeout, assertionPollInterval).Should(BeTrue())
		Expect(err).ToNot(HaveOccurred())
		// Wait a while to see if CDI puts anything in the PVC.
		isEmpty, err := framework.VerifyPVCIsEmpty(f, pvc)
		Expect(err).ToNot(HaveOccurred())
		Expect(isEmpty).To(BeTrue())
		// Not deleting PVC as it will be removed with the NS removal.
	})

	It("[posneg:negative]Import pod status should be Fail on unavailable endpoint", func() {
		pvc, err := f.CreatePVCFromDefinition(utils.NewPVCDefinition(
			"no-import-noendpoint",
			"1G",
			map[string]string{controller.AnnEndpoint: invalidEndpoint},
			nil))
		Expect(err).ToNot(HaveOccurred())

		importer, err := utils.FindPodByPrefix(c, ns, common.ImporterPodName, common.CDILabelSelector)
		Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Unable to get importer pod %q", ns+"/"+common.ImporterPodName))
		utils.WaitTimeoutForPodStatus(c, importer.Name, importer.Namespace, v1.PodFailed, utils.PodWaitForTime)

		By("Verify the pod status is Failed on the target PVC")
		_, phaseAnnotation, err := utils.WaitForPVCAnnotation(f.K8sClient, f.Namespace.Name, pvc, controller.AnnPodPhase)
		Expect(phaseAnnotation).To(BeTrue())
		Expect(err).NotTo(HaveOccurred())

		By("deleting PVC")
		err = utils.DeletePVC(f.K8sClient, pvc.Namespace, pvc)
		Expect(err).ToNot(HaveOccurred())

		By("verifying pod was deleted")
		deleted, err := utils.WaitPodDeleted(f.K8sClient, importer.Name, f.Namespace.Name, timeout)
		Expect(deleted).To(BeTrue())
		Expect(err).ToNot(HaveOccurred())

		By("verifying pvc was deleted")
		deleted, err = utils.WaitPVCDeleted(f.K8sClient, pvc.Name, f.Namespace.Name, timeout)
		Expect(deleted).To(BeTrue())
		Expect(err).ToNot(HaveOccurred())
	})

	It("Should create import pod for blank raw image", func() {
		pvc, err := f.CreatePVCFromDefinition(utils.NewPVCDefinition(
			"create-image",
			"1G",
			map[string]string{controller.AnnSource: controller.SourceNone, controller.AnnContentType: string(cdiv1.DataVolumeKubeVirt)},
			nil))
		Expect(err).ToNot(HaveOccurred())

		By("Verify the pod status is succeeded on the target PVC")
		found, err := utils.WaitPVCPodStatusSucceeded(f.K8sClient, pvc)
		Expect(err).ToNot(HaveOccurred())
		Expect(found).To(BeTrue())

		By("Verify the image contents")
		same, err := f.VerifyTargetPVCContentMD5(f.Namespace, pvc, utils.DefaultImagePath, BlankImageMD5)
		Expect(err).ToNot(HaveOccurred())
		Expect(same).To(BeTrue())
	})
})

var _ = Describe("[rfe_id:1118][crit:high][vendor:cnv-qe@redhat.com][level:component]Importer Test Suite-prometheus", func() {
	var prometheusURL string
	var portForwardCmd *exec.Cmd
	var err error
	client := &http.Client{
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		},
	}
	f := framework.NewFrameworkOrDie(namespacePrefix)

	BeforeEach(func() {
		_, err := f.CreatePrometheusServiceInNs(f.Namespace.Name)
		Expect(err).NotTo(HaveOccurred(), "Error creating prometheus service")
	})

	AfterEach(func() {
		By("Stop port forwarding")
		if portForwardCmd != nil {
			err = portForwardCmd.Process.Kill()
			Expect(err).ToNot(HaveOccurred())
			portForwardCmd.Wait()
			portForwardCmd = nil
		}
	})

	It("Import pod should have prometheus stats available while importing", func() {
		c := f.K8sClient
		ns := f.Namespace.Name
		httpEp := fmt.Sprintf("http://%s:%d", utils.FileHostName+"."+f.CdiInstallNs, utils.HTTPRateLimitPort)
		pvcAnn := map[string]string{
			controller.AnnEndpoint: httpEp + "/tinyCore.qcow2",
			controller.AnnSecret:   "",
		}

		By("Verifying no end points exist before pvc is created")
		endpoint, err := c.CoreV1().Endpoints(ns).Get("kubevirt-prometheus-metrics", metav1.GetOptions{})
		Expect(err).To(HaveOccurred())

		By(fmt.Sprintf("Creating PVC with endpoint annotation %q", httpEp+"/tinyCore.qcow2"))
		pvc, err := utils.CreatePVCFromDefinition(c, ns, utils.NewPVCDefinition("import-e2e", "20M", pvcAnn, nil))
		Expect(err).NotTo(HaveOccurred(), "Error creating PVC")

		importer, err := utils.FindPodByPrefix(c, ns, common.ImporterPodName, common.CDILabelSelector)
		Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Unable to get importer pod %q", ns+"/"+common.ImporterPodName))

		l, err := labels.Parse(common.PrometheusLabel)
		Expect(err).ToNot(HaveOccurred())
		Eventually(func() int {
			endpoint, err = c.CoreV1().Endpoints(ns).Get("kubevirt-prometheus-metrics", metav1.GetOptions{})
			Expect(err).NotTo(HaveOccurred())
			_, err := c.CoreV1().Pods(ns).List(metav1.ListOptions{LabelSelector: l.String()})
			Expect(err).ToNot(HaveOccurred())
			return len(endpoint.Subsets)
		}, 60, 1).Should(Equal(1))

		By("Set up port forwarding")
		prometheusURL, portForwardCmd, err = startPrometheusPortForward(f)
		Expect(err).ToNot(HaveOccurred())

		By("checking if the endpoint contains the metrics port and only one matching subset")
		Expect(endpoint.Subsets[0].Ports).To(HaveLen(1))
		Expect(endpoint.Subsets[0].Ports[0].Name).To(Equal("metrics"))
		Expect(endpoint.Subsets[0].Ports[0].Port).To(Equal(int32(8443)))

		if importer.OwnerReferences[0].UID == pvc.GetUID() {
			var importRegExp = regexp.MustCompile("progress\\{ownerUID\\=\"" + string(pvc.GetUID()) + "\"\\} (\\d{1,3}\\.?\\d*)")
			Eventually(func() bool {
				fmt.Fprintf(GinkgoWriter, "INFO: Connecting to URL: %s\n", prometheusURL+"/metrics")
				resp, err := client.Get(prometheusURL + "/metrics")
				if err == nil {
					defer resp.Body.Close()
					if resp.StatusCode == http.StatusOK {
						bodyBytes, err := ioutil.ReadAll(resp.Body)
						Expect(err).NotTo(HaveOccurred())
						match := importRegExp.FindStringSubmatch(string(bodyBytes))
						if match != nil {
							return true
						}
					} else {
						fmt.Fprintf(GinkgoWriter, "INFO: received status code: %d\n", resp.StatusCode)
					}
				} else {
					fmt.Fprintf(GinkgoWriter, "INFO: collecting metrics failed: %v\n", err)
				}
				return false
			}, 90, 1).Should(BeTrue())
		} else {
			Fail("importer owner reference doesn't match PVC")
		}
	})
})

func startPrometheusPortForward(f *framework.Framework) (string, *exec.Cmd, error) {
	lp := "28443"
	pm := lp + ":8443"
	url := "https://127.0.0.1:" + lp

	cmd := tests.CreateKubectlCommand(f, "-n", f.Namespace.Name, "port-forward", "svc/kubevirt-prometheus-metrics", pm)
	err := cmd.Start()
	if err != nil {
		return "", nil, err
	}

	return url, cmd, nil
}

var _ = Describe("Importer Test Suite-Block_device", func() {
	f := framework.NewFrameworkOrDie(namespacePrefix)
	var pv *v1.PersistentVolume
	var storageClass *k8sv1.StorageClass
	var pod *v1.Pod
	var err error

	BeforeEach(func() {
		err = f.ClearBlockPV()
		Expect(err).NotTo(HaveOccurred())

		pod, err = utils.FindPodByPrefix(f.K8sClient, f.CdiInstallNs, "cdi-block-device", "kubevirt.io=cdi-block-device")
		Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Unable to get pod %q", f.CdiInstallNs+"/"+"cdi-block-device"))

		nodeName := pod.Spec.NodeName

		By(fmt.Sprintf("Creating storageClass for Block PV"))
		storageClass, err = f.CreateStorageClassFromDefinition(utils.NewStorageClassForBlockPVDefinition("manual"))
		Expect(err).ToNot(HaveOccurred())

		By(fmt.Sprintf("Creating Block PV"))
		pv, err = f.CreatePVFromDefinition(utils.NewBlockPVDefinition("local-volume", "500M", nil, "manual", nodeName))
		Expect(err).ToNot(HaveOccurred())

		By("Verify that PV's phase is Available")
		err = f.WaitTimeoutForPVReady(pv.Name, 60*time.Second)
		Expect(err).ToNot(HaveOccurred())
	})

	AfterEach(func() {
		err := utils.DeletePV(f.K8sClient, pv)
		Expect(err).ToNot(HaveOccurred())

		err = utils.DeleteStorageClass(f.K8sClient, storageClass)
		Expect(err).ToNot(HaveOccurred())
	})

	It("Should create import pod for block pv", func() {
		httpEp := fmt.Sprintf("http://%s:%d", utils.FileHostName+"."+f.CdiInstallNs, utils.HTTPNoAuthPort)
		pvcAnn := map[string]string{
			controller.AnnEndpoint: httpEp + "/tinyCore.iso",
		}

		By(fmt.Sprintf("Creating PVC with endpoint annotation %q", httpEp+"/tinyCore.iso"))

		pvc, err := f.CreatePVCFromDefinition(utils.NewBlockPVCDefinition(
			"import-image-to-block-pvc",
			"500M",
			pvcAnn,
			nil,
			"manual"))
		Expect(err).ToNot(HaveOccurred())

		By("Verify the pod status is succeeded on the target PVC")
		Eventually(func() string {
			status, phaseAnnotation, err := utils.WaitForPVCAnnotation(f.K8sClient, f.Namespace.Name, pvc, controller.AnnPodPhase)
			Expect(err).ToNot(HaveOccurred())
			Expect(phaseAnnotation).To(BeTrue())
			return status
		}, CompletionTimeout, assertionPollInterval).Should(BeEquivalentTo(v1.PodSucceeded))

		By("Verify content")
		same, err := f.VerifyTargetPVCContentMD5(f.Namespace, pvc, "/pvc", BlockDeviceMD5)
		Expect(err).ToNot(HaveOccurred())
		Expect(same).To(BeTrue())

	})
})

var _ = Describe("Importer Archive ContentType", func() {
	f := framework.NewFrameworkOrDie(namespacePrefix)

	It("Should import archive content type tar file", func() {
		c := f.K8sClient
		ns := f.Namespace.Name
		httpEp := fmt.Sprintf("http://%s:%d", utils.FileHostName+"."+f.CdiInstallNs, utils.HTTPNoAuthPort)
		pvcAnn := map[string]string{
			controller.AnnEndpoint:    httpEp + "/archive.tar",
			controller.AnnContentType: "archive",
		}

		By(fmt.Sprintf("Creating PVC with endpoint annotation %q", httpEp+"/archive.tar"))
		pvc, err := utils.CreatePVCFromDefinition(c, ns, utils.NewPVCDefinition("import-archive", "100M", pvcAnn, nil))
		Expect(err).NotTo(HaveOccurred(), "Error creating PVC")

		By("Verify the pod status is succeeded on the target PVC")
		found, err := utils.WaitPVCPodStatusSucceeded(c, pvc)
		Expect(err).ToNot(HaveOccurred())
		Expect(found).To(BeTrue())

		By("Verify the target PVC contents")
		same, err := f.VerifyTargetPVCArchiveContent(f.Namespace, pvc, "3")
		Expect(err).ToNot(HaveOccurred())
		Expect(same).To(BeTrue())
	})
})
<|endoftext|>