<commit_before>// +build integration

package kinesumer

import (
	"fmt"
	"math/rand"
	"os"
	"testing"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/service/kinesis"
	_ "github.com/joho/godotenv/autoload"
	"github.com/pborman/uuid"
	"github.com/remind101/kinesumer/checkpointers/redis"
	"github.com/remind101/kinesumer/provisioners/redis"
	"github.com/remind101/kinesumer/redispool"
)

func consecAll(n string, char rune) bool {
	for _, c := range n {
		if c != char {
			return false
		}
	}
	return true
}

func consecHelper(n, sn string) bool {
	if len(n) == 0 || len(sn) == 0 {
		return false
	} else if n[0] == sn[0] {
		return consecHelper(n[1:], sn[1:])
	} else if n[0]+1 == sn[0] {
		return consecAll(n[1:], '9') && consecAll(sn[1:], '0')
	} else {
		return false
	}
}

func consec(n, sn string) bool {
	if len(n)*len(sn) == 0 {
		return false
	} else if len(n)+1 == len(sn) && sn[0] == '1' {
		return consecAll(n, '9') && consecAll(sn[1:], '0')
	} else {
		return consecHelper(n, sn)
	}
}

func TestConsec(t *testing.T) {
	if consec("123", "1234") {
		t.Fail()
	}

	if !consec("123", "124") {
		t.Fail()
	}

	if !consec("1233999", "1234000") {
		t.Fail()
	}

	if !consec("9", "10") {
		t.Fail()
	}
}

func TestIntegration(t *testing.T) {
	rand.Seed(94133)

	fmt.Println("Creating Kinesis")
	kin := kinesis.New(&aws.Config{
		Credentials: credentials.NewStaticCredentials(os.Getenv("AWS_ACCESS_KEY_ID"),
			os.Getenv("AWS_SECRET_ACCESS_KEY"), ""),
		Region: aws.String(os.Getenv("AWS_REGION")),
	})

	stream := os.Getenv("AWS_KINESIS_STREAM")

	redisPool, err := redispool.NewRedisPool(os.Getenv("REDIS_URL"))
	if err != nil {
		panic(err)
	}
	redisPrefix := os.Getenv("REDIS_PREFIX")

	cpOpt := redischeckpointer.Options{
		SavePeriod:  8 * time.Second,
		RedisPool:   redisPool,
		RedisPrefix: redisPrefix,
	}
	cp, err := redischeckpointer.New(&cpOpt)
	if err != nil {
		panic(err)
	}

	provOpt := redisprovisioner.Options{
		TTL:         8 * time.Second,
		RedisPool:   redisPool,
		RedisPrefix: redisPrefix,
	}
	provOpt2 := provOpt
	prov, err := redisprovisioner.New(&provOpt)
	if err != nil {
		panic(err)
	}

	kinOpt := Options{
		ListStreamsLimit:    25,
		DescribeStreamLimit: 25,
		GetRecordsLimit:     25,
		PollTime:            1,
		MaxShardWorkers:     2,
		Handlers:            DefaultHandlers{},
		DefaultIteratorType: "TRIM_HORIZON",
	}

	fmt.Println("Creating Kinesumer")
	k, err := New(kin, cp, prov, nil, stream, &kinOpt)
	if err != nil {
		panic(err)
	}

	exists, err := k.StreamExists()
	if err != nil {
		panic(err)
	}

	if exists {
		fmt.Println("Deleting stream")
		_, err := kin.DeleteStream(&kinesis.DeleteStreamInput{
			StreamName: aws.String(stream),
		})
		if err != nil {
			panic(err)
		}

		for i := 0; i < 60; i++ {
			time.Sleep(time.Second)
			if exists, err := k.StreamExists(); err != nil {
				panic(err)
			} else if !exists {
				goto cont1
			}
		}
		panic("Could not delete stream")
	}
cont1:

	fmt.Println("Creating stream")
	_, err = kin.CreateStream(&kinesis.CreateStreamInput{
		ShardCount: aws.Int64(3),
		StreamName: aws.String(stream),
	})
	if err != nil {
		panic(err)
	}

	for i := 0; i < 60; i++ {
		time.Sleep(time.Second)
		if exists, err := k.StreamExists(); err != nil {
			panic(err)
		} else if exists {
			goto cont2
		}
	}
	panic("Could not create stream")
cont2:

	time.Sleep(time.Second)
	workers, err := k.Begin()
	if err != nil {
		panic(err)
	}

	time.Sleep(8 * time.Second)

	if len(workers) != 2 {
		panic(fmt.Sprintf("Expected 2 workers to be started by k. Workers: %v",
			workers,
		))
	}

	cp2, err := redischeckpointer.New(&cpOpt)
	if err != nil {
		panic(err)
	}

	prov2, err := redisprovisioner.New(&provOpt2)
	if err != nil {
		panic(err)
	}

	kinOpt2 := kinOpt
	kinOpt2.MaxShardWorkers = 3
	k2, err := New(kin, cp2, prov2, nil, stream, &kinOpt2)
	if err != nil {
		panic(err)
	}

	workers2, err := k2.Begin()
	if err != nil {
		panic(err)
	}

	if len(workers2) != 1 {
		panic(fmt.Sprintf("Expected 1 worker to be started by k2. Workers: %v", workers2))
	}

	records := make([]*kinesis.PutRecordsRequestEntry, 100)
	values := make(map[string]bool)
	for i := 0; i < 100; i++ {
		item := uuid.New()
		values[item] = true
		records[i] = &kinesis.PutRecordsRequestEntry{
			Data:         []byte(item),
			PartitionKey: aws.String(item),
		}
	}

	res, err := kin.PutRecords(&kinesis.PutRecordsInput{
		Records:    records,
		StreamName: aws.String(stream),
	})
	if err != nil {
		panic(err)
	}
	if aws.Int64Value(res.FailedRecordCount) != 0 {
		panic(fmt.Sprintf("Failed to put records: %v", res.Records))
	}

	timeout := time.NewTimer(time.Minute)
	for count := 0; count < 100; count++ {
		select {
		case rec := <-k.Records():
			if !values[string(rec.Data())] {
				panic(fmt.Sprintf("Received unexpected record %v", rec))
			} else {
				delete(values, string(rec.Data()))
			}
		case rec := <-k2.Records():
			if !values[string(rec.Data())] {
				panic(fmt.Sprintf("Received unexpected record %v", rec))
			} else {
				delete(values, string(rec.Data()))
			}
		case <-timeout.C:
			panic(fmt.Sprintf("Timed out fetching records. Missing: %v", values))
		}
	}

	if len(values) > 0 {
		panic(fmt.Sprintf("Did not receive all expected records. Missing: %v", values))
	}

	fmt.Println("Basic functionality works")

	stopC := make(chan struct{}, 2)

	smallRecords := records[:10]

	go func() {
		for {
			select {
			case <-stopC:
			default:
				time.Sleep(time.Second)
				res, err := kin.PutRecords(&kinesis.PutRecordsInput{
					Records:    smallRecords,
					StreamName: aws.String(stream),
				})
				if err != nil {
					panic(err)
				}
				if aws.Int64Value(res.FailedRecordCount) != 0 {
					panic(fmt.Sprintf("Failed to put records: %v", res.Records))
				}
			}
		}
	}()

	go func() {
		for {
			select {
			case <-k.Records():
			case <-k2.Records():
			case <-stopC:
				return
			}
		}
	}()

	shards, err := k.GetShards()
	if err != nil {
		panic(err)
	}
	if len(shards) != 3 {
		panic("Expected 3 shards")
	}

	type pair struct {
		begin, end int
	}
	pairs := []pair{
		{0, 1},
		{0, 2},
		{1, 2},
		{2, 1},
	}
	for _, pair := range pairs {
		if consec(*shards[pair.begin].HashKeyRange.EndingHashKey,
			*shards[pair.end].HashKeyRange.StartingHashKey) {
			_, err := kin.MergeShards(&kinesis.MergeShardsInput{
				ShardToMerge:         shards[pair.begin].ShardID,
				AdjacentShardToMerge: shards[pair.end].ShardID,
				StreamName:           aws.String(stream),
			})
			if err != nil {
				panic(err)
			}
			goto cont3
		}
	}
	panic(func() string {
		s := "Could not find shard to close. Shards: "
		shards, err := k.GetShards()
		if err != nil {
			return err.Error()
		}
		for _, shard := range shards {
			s += shard.GoString()
		}
		return s
	}(),
	)
cont3:

	timeout.Reset(time.Minute)
	select {
	case <-timeout.C:
		panic("Shard worker did not stop after shard closed")
	case <-k.stopped:
		k.stopped <- Unit{}
	case <-k2.stopped:
		k2.stopped <- Unit{}
	}

	stopC <- struct{}{}
	stopC <- struct{}{}

	k.End()
	k2.End()
}
<commit_msg>Added explanation<commit_after>// +build integration

package kinesumer

import (
	"fmt"
	"math/rand"
	"os"
	"testing"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/service/kinesis"
	_ "github.com/joho/godotenv/autoload"
	"github.com/pborman/uuid"
	"github.com/remind101/kinesumer/checkpointers/redis"
	"github.com/remind101/kinesumer/provisioners/redis"
	"github.com/remind101/kinesumer/redispool"
)

func isConsecutiveAllSame(n string, char rune) bool {
	for _, c := range n {
		if c != char {
			return false
		}
	}
	return true
}

func isConsecutiveHelper(n, sn string) bool {
	if len(n) == 0 || len(sn) == 0 {
		return false
	} else if n[0] == sn[0] {
		return isConsecutiveHelper(n[1:], sn[1:])
	} else if n[0]+1 == sn[0] {
		return isConsecutiveAllSame(n[1:], '9') && isConsecutiveAllSame(sn[1:], '0')
	} else {
		return false
	}
}

// We need this to determine if two hash ranges are adjacent because we try to merge two shards
// and only adjacent shards can be merged. Shard hash ranges are 128 bit and come from AWS as
// strings.
func isConsecutive(n, sn string) bool {
	if len(n)*len(sn) == 0 {
		return false
	} else if len(n)+1 == len(sn) && sn[0] == '1' {
		return isConsecutiveAllSame(n, '9') && isConsecutiveAllSame(sn[1:], '0')
	} else {
		return isConsecutiveHelper(n, sn)
	}
}

func TestConsec(t *testing.T) {
	if isConsecutive("123", "1234") {
		t.Fail()
	}

	if !isConsecutive("123", "124") {
		t.Fail()
	}

	if !isConsecutive("1233999", "1234000") {
		t.Fail()
	}

	if !isConsecutive("9", "10") {
		t.Fail()
	}
}

func TestIntegration(t *testing.T) {
	rand.Seed(94133)

	fmt.Println("Creating Kinesis")
	kin := kinesis.New(&aws.Config{
		Credentials: credentials.NewStaticCredentials(os.Getenv("AWS_ACCESS_KEY_ID"),
			os.Getenv("AWS_SECRET_ACCESS_KEY"), ""),
		Region: aws.String(os.Getenv("AWS_REGION")),
	})

	stream := os.Getenv("AWS_KINESIS_STREAM")

	redisPool, err := redispool.NewRedisPool(os.Getenv("REDIS_URL"))
	if err != nil {
		panic(err)
	}
	redisPrefix := os.Getenv("REDIS_PREFIX")

	cpOpt := redischeckpointer.Options{
		SavePeriod:  8 * time.Second,
		RedisPool:   redisPool,
		RedisPrefix: redisPrefix,
	}
	cp, err := redischeckpointer.New(&cpOpt)
	if err != nil {
		panic(err)
	}

	provOpt := redisprovisioner.Options{
		TTL:         8 * time.Second,
		RedisPool:   redisPool,
		RedisPrefix: redisPrefix,
	}
	provOpt2 := provOpt
	prov, err := redisprovisioner.New(&provOpt)
	if err != nil {
		panic(err)
	}

	kinOpt := Options{
		ListStreamsLimit:    25,
		DescribeStreamLimit: 25,
		GetRecordsLimit:     25,
		PollTime:            1,
		MaxShardWorkers:     2,
		Handlers:            DefaultHandlers{},
		DefaultIteratorType: "TRIM_HORIZON",
	}

	fmt.Println("Creating Kinesumer")
	k, err := New(kin, cp, prov, nil, stream, &kinOpt)
	if err != nil {
		panic(err)
	}

	exists, err := k.StreamExists()
	if err != nil {
		panic(err)
	}

	if exists {
		fmt.Println("Deleting stream")
		_, err := kin.DeleteStream(&kinesis.DeleteStreamInput{
			StreamName: aws.String(stream),
		})
		if err != nil {
			panic(err)
		}

		for i := 0; i < 60; i++ {
			time.Sleep(time.Second)
			if exists, err := k.StreamExists(); err != nil {
				panic(err)
			} else if !exists {
				goto cont1
			}
		}
		panic("Could not delete stream")
	}
cont1:

	fmt.Println("Creating stream")
	_, err = kin.CreateStream(&kinesis.CreateStreamInput{
		ShardCount: aws.Int64(3),
		StreamName: aws.String(stream),
	})
	if err != nil {
		panic(err)
	}

	for i := 0; i < 60; i++ {
		time.Sleep(time.Second)
		if exists, err := k.StreamExists(); err != nil {
			panic(err)
		} else if exists {
			goto cont2
		}
	}
	panic("Could not create stream")
cont2:

	time.Sleep(time.Second)
	workers, err := k.Begin()
	if err != nil {
		panic(err)
	}

	time.Sleep(8 * time.Second)

	if len(workers) != 2 {
		panic(fmt.Sprintf("Expected 2 workers to be started by k. Workers: %v",
			workers,
		))
	}

	cp2, err := redischeckpointer.New(&cpOpt)
	if err != nil {
		panic(err)
	}

	prov2, err := redisprovisioner.New(&provOpt2)
	if err != nil {
		panic(err)
	}

	kinOpt2 := kinOpt
	kinOpt2.MaxShardWorkers = 3
	k2, err := New(kin, cp2, prov2, nil, stream, &kinOpt2)
	if err != nil {
		panic(err)
	}

	workers2, err := k2.Begin()
	if err != nil {
		panic(err)
	}

	if len(workers2) != 1 {
		panic(fmt.Sprintf("Expected 1 worker to be started by k2. Workers: %v", workers2))
	}

	records := make([]*kinesis.PutRecordsRequestEntry, 100)
	values := make(map[string]bool)
	for i := 0; i < 100; i++ {
		item := uuid.New()
		values[item] = true
		records[i] = &kinesis.PutRecordsRequestEntry{
			Data:         []byte(item),
			PartitionKey: aws.String(item),
		}
	}

	res, err := kin.PutRecords(&kinesis.PutRecordsInput{
		Records:    records,
		StreamName: aws.String(stream),
	})
	if err != nil {
		panic(err)
	}
	if aws.Int64Value(res.FailedRecordCount) != 0 {
		panic(fmt.Sprintf("Failed to put records: %v", res.Records))
	}

	timeout := time.NewTimer(time.Minute)
	for count := 0; count < 100; count++ {
		select {
		case rec := <-k.Records():
			if !values[string(rec.Data())] {
				panic(fmt.Sprintf("Received unexpected record %v", rec))
			} else {
				delete(values, string(rec.Data()))
			}
		case rec := <-k2.Records():
			if !values[string(rec.Data())] {
				panic(fmt.Sprintf("Received unexpected record %v", rec))
			} else {
				delete(values, string(rec.Data()))
			}
		case <-timeout.C:
			panic(fmt.Sprintf("Timed out fetching records. Missing: %v", values))
		}
	}

	if len(values) > 0 {
		panic(fmt.Sprintf("Did not receive all expected records. Missing: %v", values))
	}

	fmt.Println("Basic functionality works")

	stopC := make(chan struct{}, 2)

	smallRecords := records[:10]

	go func() {
		for {
			select {
			case <-stopC:
			default:
				time.Sleep(time.Second)
				res, err := kin.PutRecords(&kinesis.PutRecordsInput{
					Records:    smallRecords,
					StreamName: aws.String(stream),
				})
				if err != nil {
					panic(err)
				}
				if aws.Int64Value(res.FailedRecordCount) != 0 {
					panic(fmt.Sprintf("Failed to put records: %v", res.Records))
				}
			}
		}
	}()

	go func() {
		for {
			select {
			case <-k.Records():
			case <-k2.Records():
			case <-stopC:
				return
			}
		}
	}()

	shards, err := k.GetShards()
	if err != nil {
		panic(err)
	}
	if len(shards) != 3 {
		panic("Expected 3 shards")
	}

	type pair struct {
		begin, end int
	}
	pairs := []pair{
		{0, 1},
		{0, 2},
		{1, 2},
		{2, 1},
	}
	for _, pair := range pairs {
		if isConsecutive(*shards[pair.begin].HashKeyRange.EndingHashKey,
			*shards[pair.end].HashKeyRange.StartingHashKey) {
			_, err := kin.MergeShards(&kinesis.MergeShardsInput{
				ShardToMerge:         shards[pair.begin].ShardID,
				AdjacentShardToMerge: shards[pair.end].ShardID,
				StreamName:           aws.String(stream),
			})
			if err != nil {
				panic(err)
			}
			goto cont3
		}
	}
	panic(func() string {
		s := "Could not find shard to close. Shards: "
		shards, err := k.GetShards()
		if err != nil {
			return err.Error()
		}
		for _, shard := range shards {
			s += shard.GoString()
		}
		return s
	}(),
	)
cont3:

	timeout.Reset(time.Minute)
	select {
	case <-timeout.C:
		panic("Shard worker did not stop after shard closed")
	case <-k.stopped:
		k.stopped <- Unit{}
	case <-k2.stopped:
		k2.stopped <- Unit{}
	}

	stopC <- struct{}{}
	stopC <- struct{}{}

	k.End()
	k2.End()
}
<|endoftext|>


<commit_before>package statefile

import (
	version "github.com/hashicorp/go-version"

	"github.com/hashicorp/terraform/states"
	tfversion "github.com/hashicorp/terraform/version"
)

// File is the in-memory representation of a state file. It includes the state
// itself along with various metadata used to track changing state files for
// the same configuration over time.
type File struct {
	// TerraformVersion is the version of Terraform that wrote this state file.
	TerraformVersion *version.Version

	// Serial is incremented on any operation that modifies
	// the State file. It is used to detect potentially conflicting
	// updates.
	Serial uint64

	// Lineage is set when a new, blank state file is created and then
	// never updated. This allows us to determine whether the serials
	// of two states can be meaningfully compared.
	// Apart from the guarantee that collisions between two lineages
	// are very unlikely, this value is opaque and external callers
	// should only compare lineage strings byte-for-byte for equality.
	Lineage string

	// State is the actual state represented by this file.
	State *states.State
}

func New(state *states.State, lineage string, serial uint64) *File {
	return &File{
		TerraformVersion: tfversion.SemVer,
		State:            state,
		Lineage:          lineage,
		Serial:           serial,
	}
}

// DeepCopy is a convenience method to create a new File object whose state
// is a deep copy of the receiver's, as implemented by states.State.DeepCopy.
func (f *File) DeepCopy() *File {
	if f == nil {
		return nil
	}
	return &File{
		TerraformVersion: f.TerraformVersion,
		Serial:           f.Serial,
		Lineage:          f.Lineage,
		State:            f.State.DeepCopy(),
	}
}
<commit_msg>states/statefile: Tolerate nil state in statefile.New<commit_after>package statefile

import (
	version "github.com/hashicorp/go-version"

	"github.com/hashicorp/terraform/states"
	tfversion "github.com/hashicorp/terraform/version"
)

// File is the in-memory representation of a state file. It includes the state
// itself along with various metadata used to track changing state files for
// the same configuration over time.
type File struct {
	// TerraformVersion is the version of Terraform that wrote this state file.
	TerraformVersion *version.Version

	// Serial is incremented on any operation that modifies
	// the State file. It is used to detect potentially conflicting
	// updates.
	Serial uint64

	// Lineage is set when a new, blank state file is created and then
	// never updated. This allows us to determine whether the serials
	// of two states can be meaningfully compared.
	// Apart from the guarantee that collisions between two lineages
	// are very unlikely, this value is opaque and external callers
	// should only compare lineage strings byte-for-byte for equality.
	Lineage string

	// State is the actual state represented by this file.
	State *states.State
}

func New(state *states.State, lineage string, serial uint64) *File {
	// To make life easier on callers, we'll accept a nil state here and just
	// allocate an empty one, which is required for this file to be successfully
	// written out.
	if state == nil {
		state = states.NewState()
	}

	return &File{
		TerraformVersion: tfversion.SemVer,
		State:            state,
		Lineage:          lineage,
		Serial:           serial,
	}
}

// DeepCopy is a convenience method to create a new File object whose state
// is a deep copy of the receiver's, as implemented by states.State.DeepCopy.
func (f *File) DeepCopy() *File {
	if f == nil {
		return nil
	}
	return &File{
		TerraformVersion: f.TerraformVersion,
		Serial:           f.Serial,
		Lineage:          f.Lineage,
		State:            f.State.DeepCopy(),
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	sseclient "astuart.co/go-sse"
	"github.com/dgrijalva/jwt-go"
	"github.com/go-errors/errors"
	"github.com/mdp/qrterminal"
	"github.com/privacybydesign/irmago"
	"github.com/privacybydesign/irmago/internal/common"
	"github.com/privacybydesign/irmago/server"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
)

// requestCmd represents the request command
var requestCmd = &cobra.Command{
	Use:   "request",
	Short: "Generate an IRMA session request",
	Run: func(cmd *cobra.Command, args []string) {
		request, _, err := configureRequest(cmd)
		if err != nil {
			die("", err)
		}

		flags := cmd.Flags()
		authmethod, _ := flags.GetString("authmethod")
		var output string
		if authmethod == "none" || authmethod == "token" {
			output = prettyprint(request)
		} else {
			key, _ := flags.GetString("key")
			name, _ := flags.GetString("name")
			if output, err = signRequest(request, name, authmethod, key); err != nil {
				die("Failed to sign request", err)
			}
		}

		fmt.Println(output)
	},
}

func configureJWTKey(authmethod, key string) (interface{}, jwt.SigningMethod, error) {
	var (
		err    error
		sk     interface{}
		jwtalg jwt.SigningMethod
		bts    []byte
	)
	// If the key refers to an existing file, use contents of the file as key
	if bts, err = common.ReadKey("", key); err != nil {
		bts = []byte(key)
	}
	switch authmethod {
	case "hmac":
		jwtalg = jwt.SigningMethodHS256
		if sk, err = common.Base64Decode(bts); err != nil {
			return nil, nil, err
		}
	case "rsa":
		jwtalg = jwt.SigningMethodRS256
		if sk, err = jwt.ParseRSAPrivateKeyFromPEM(bts); err != nil {
			return nil, nil, err
		}
	default:
		return nil, nil, errors.Errorf("Unsupported signing algorithm: '%s'", authmethod)
	}

	return sk, jwtalg, nil
}

func signRequest(request irma.RequestorRequest, name, authmethod, key string) (string, error) {
	sk, jwtalg, err := configureJWTKey(authmethod, key)
	if err != nil {
		return "", err
	}
	return irma.SignRequestorRequest(request, jwtalg, sk, name)
}

func configureRequest(cmd *cobra.Command) (irma.RequestorRequest, *irma.Configuration, error) {
	irmaconfigPath, err := cmd.Flags().GetString("schemes-path")
	if err != nil {
		return nil, nil, err
	}
	irmaconfig, err := irma.NewConfiguration(irmaconfigPath, irma.ConfigurationOptions{})
	if err != nil {
		return nil, nil, err
	}
	if err = irmaconfig.ParseFolder(); err != nil {
		return nil, nil, err
	}
	if len(irmaconfig.SchemeManagers) == 0 {
		if err = irmaconfig.DownloadDefaultSchemes(); err != nil {
			return nil, nil, err
		}
	}

	request, err := constructSessionRequest(cmd, irmaconfig)
	if err != nil {
		return nil, nil, err
	}

	return request, irmaconfig, nil
}

// Helper functions

func wait(initialStatus server.Status, transport *irma.HTTPTransport, statuschan chan server.Status) {
	events := make(chan *sseclient.Event)

	go func() {
		for {
			if e := <-events; e != nil && e.Type != "open" {
				status := server.Status(strings.Trim(string(e.Data), `"`))
				statuschan <- status
				if status.Finished() {
					return
				}
			}
		}
	}()

	if err := sseclient.Notify(nil, transport.Server+"statusevents", true, events); err != nil {
		fmt.Println("SSE failed, fallback to polling", err)
		close(events)
		poll(initialStatus, transport, statuschan)
		return
	}
}

// poll recursively polls the session status until a final status is received.
func poll(initialStatus server.Status, transport *irma.HTTPTransport, statuschan chan server.Status) {
	// First we wait
	<-time.NewTimer(pollInterval).C

	// Get session status
	var s string
	if err := transport.Get("status", &s); err != nil {
		_ = server.LogFatal(err)
	}
	status := server.Status(strings.Trim(s, `"`))

	// report if status changed
	if status != initialStatus {
		statuschan <- status
	}

	if status.Finished() {
		return
	}
	go poll(status, transport, statuschan)
}

func constructSessionRequest(cmd *cobra.Command, conf *irma.Configuration) (irma.RequestorRequest, error) {
	disclose, _ := cmd.Flags().GetStringArray("disclose")
	issue, _ := cmd.Flags().GetStringArray("issue")
	sign, _ := cmd.Flags().GetStringArray("sign")
	message, _ := cmd.Flags().GetString("message")
	jsonrequest, _ := cmd.Flags().GetString("request")

	if len(disclose) == 0 && len(issue) == 0 && len(sign) == 0 && message == "" {
		if jsonrequest == "" {
			return nil, errors.New("Provide either a complete session request using --request or construct one using the other flags")
		}
		request, err := server.ParseSessionRequest(jsonrequest)
		if err != nil {
			return nil, err
		}
		return request, nil
	}

	if jsonrequest != "" {
		return nil, errors.New("Provide either a complete session request using --request or construct one using the other flags")
	}

	if len(sign) != 0 {
		if len(disclose) != 0 {
			return nil, errors.New("cannot combine disclosure and signature sessions, use either --disclose or --sign")
		}
		if len(issue) != 0 {
			return nil, errors.New("cannot combine issuance and signature sessions, use either --issue or --sign")
		}
		if message == "" {
			return nil, errors.New("signature sessions require a message to be signed using --message")
		}
	}

	var request irma.RequestorRequest
	if len(disclose) != 0 {
		disclose, err := parseAttrs(disclose, conf)
		if err != nil {
			return nil, err
		}
		request = &irma.ServiceProviderRequest{
			Request: irma.NewDisclosureRequest(),
		}
		request.SessionRequest().(*irma.DisclosureRequest).Disclose = disclose
	}
	if len(sign) != 0 {
		disclose, err := parseAttrs(sign, conf)
		if err != nil {
			return nil, err
		}
		request = &irma.SignatureRequestorRequest{
			Request: irma.NewSignatureRequest(message),
		}
		request.SessionRequest().(*irma.SignatureRequest).Disclose = disclose
	}
	if len(issue) != 0 {
		creds, err := parseCredentials(issue, conf)
		if err != nil {
			return nil, err
		}
		disclose, err := parseAttrs(disclose, conf)
		if err != nil {
			return nil, err
		}
		request = &irma.IdentityProviderRequest{
			Request: irma.NewIssuanceRequest(creds),
		}
		request.SessionRequest().(*irma.IssuanceRequest).Disclose = disclose
	}

	return request, nil
}

func parseCredentials(credentialsStr []string, conf *irma.Configuration) ([]*irma.CredentialRequest, error) {
	list := make([]*irma.CredentialRequest, 0, len(credentialsStr))

	for _, credStr := range credentialsStr {
		parts := strings.Split(credStr, "=")
		if len(parts) != 2 {
			return nil, errors.New("--issue argument must contain exactly 1 = sign")
		}
		credIdStr, attrsStr := parts[0], parts[1]
		credtype := conf.CredentialTypes[irma.NewCredentialTypeIdentifier(credIdStr)]
		if credtype == nil {
			return nil, errors.New("unknown credential type: " + credIdStr)
		}

		attrsSlice := strings.Split(attrsStr, ",")
		if len(attrsSlice) != len(credtype.AttributeTypes) {
			return nil, errors.Errorf("%d attributes required but %d provided for %s", len(credtype.AttributeTypes), len(attrsSlice), credIdStr)
		}

		attrs := make(map[string]string, len(attrsSlice))
		for i, typ := range credtype.AttributeTypes {
			attrs[typ.ID] = attrsSlice[i]
		}
		list = append(list, &irma.CredentialRequest{
			CredentialTypeID: irma.NewCredentialTypeIdentifier(credIdStr),
			Attributes:       attrs,
		})
	}

	return list, nil
}

func parseAttrs(attrsStr []string, conf *irma.Configuration) (irma.AttributeConDisCon, error) {
	list := make(irma.AttributeConDisCon, 0, len(attrsStr))
	for _, disjunctionStr := range attrsStr {
		disjunction := irma.AttributeDisCon{}
		attrids := strings.Split(disjunctionStr, ",")
		for _, attridStr := range attrids {
			attrid := irma.NewAttributeTypeIdentifier(attridStr)
			if conf.AttributeTypes[attrid] == nil {
				return nil, errors.New("unknown attribute: " + attridStr)
			}
			disjunction = append(disjunction, irma.AttributeCon{irma.AttributeRequest{Type: attrid}})
		}
		list = append(list, disjunction)
	}
	return list, nil
}

func startServer(port int) {
	mux := http.NewServeMux()
	mux.HandleFunc("/", irmaServer.HandlerFunc())
	httpServer = &http.Server{Addr: ":" + strconv.Itoa(port), Handler: mux}
	go func() {
		err := httpServer.ListenAndServe()
		if err != nil && err != http.ErrServerClosed {
			die("Failed to start server", err)
		}
	}()
}

func printQr(qr *irma.Qr, noqr bool) error {
	qrBts, err := json.Marshal(qr)
	if err != nil {
		return err
	}
	if noqr {
		fmt.Println(string(qrBts))
	} else {
		qrterminal.GenerateWithConfig(string(qrBts), qrterminal.Config{
			Level:     qrterminal.L,
			Writer:    os.Stdout,
			BlackChar: qrterminal.BLACK,
			WhiteChar: qrterminal.WHITE,
		})
	}
	return nil
}

func printSessionResult(result *server.SessionResult) {
	fmt.Println("Session result:")
	fmt.Println(prettyprint(result))
}

func init() {
	RootCmd.AddCommand(requestCmd)

	flags := requestCmd.Flags()
	flags.SortFlags = false

	addRequestFlags(flags)
}

func authmethodAlias(f *pflag.FlagSet, name string) pflag.NormalizedName {
	switch name {
	case "authmethod":
		name = "auth-method"
		break
	}
	return pflag.NormalizedName(name)
}

func addRequestFlags(flags *pflag.FlagSet) {
	flags.StringP("schemes-path", "s", irma.DefaultSchemesPath(), "path to irma_configuration")
	flags.StringP("auth-method", "a", "none", "Authentication method to server (none, token, rsa, hmac)")
	flags.SetNormalizeFunc(authmethodAlias)
	flags.String("key", "", "Key to sign request with")
	flags.String("name", "", "Requestor name")
	flags.StringArray("disclose", nil, "Add an attribute disjunction (comma-separated)")
	flags.StringArray("issue", nil, "Add a credential to issue")
	flags.StringArray("sign", nil, "Add an attribute disjunction to signature session")
	flags.String("message", "", "Message to sign in signature session")
}
<commit_msg>fix: double disclosure parsing when specifying combined issuance-disclosure sessions using commandline flags<commit_after>package cmd

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	sseclient "astuart.co/go-sse"
	"github.com/dgrijalva/jwt-go"
	"github.com/go-errors/errors"
	"github.com/mdp/qrterminal"
	"github.com/privacybydesign/irmago"
	"github.com/privacybydesign/irmago/internal/common"
	"github.com/privacybydesign/irmago/server"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
)

// requestCmd represents the request command
var requestCmd = &cobra.Command{
	Use:   "request",
	Short: "Generate an IRMA session request",
	Run: func(cmd *cobra.Command, args []string) {
		request, _, err := configureRequest(cmd)
		if err != nil {
			die("", err)
		}

		flags := cmd.Flags()
		authmethod, _ := flags.GetString("authmethod")
		var output string
		if authmethod == "none" || authmethod == "token" {
			output = prettyprint(request)
		} else {
			key, _ := flags.GetString("key")
			name, _ := flags.GetString("name")
			if output, err = signRequest(request, name, authmethod, key); err != nil {
				die("Failed to sign request", err)
			}
		}

		fmt.Println(output)
	},
}

func configureJWTKey(authmethod, key string) (interface{}, jwt.SigningMethod, error) {
	var (
		err    error
		sk     interface{}
		jwtalg jwt.SigningMethod
		bts    []byte
	)
	// If the key refers to an existing file, use contents of the file as key
	if bts, err = common.ReadKey("", key); err != nil {
		bts = []byte(key)
	}
	switch authmethod {
	case "hmac":
		jwtalg = jwt.SigningMethodHS256
		if sk, err = common.Base64Decode(bts); err != nil {
			return nil, nil, err
		}
	case "rsa":
		jwtalg = jwt.SigningMethodRS256
		if sk, err = jwt.ParseRSAPrivateKeyFromPEM(bts); err != nil {
			return nil, nil, err
		}
	default:
		return nil, nil, errors.Errorf("Unsupported signing algorithm: '%s'", authmethod)
	}

	return sk, jwtalg, nil
}

func signRequest(request irma.RequestorRequest, name, authmethod, key string) (string, error) {
	sk, jwtalg, err := configureJWTKey(authmethod, key)
	if err != nil {
		return "", err
	}
	return irma.SignRequestorRequest(request, jwtalg, sk, name)
}

func configureRequest(cmd *cobra.Command) (irma.RequestorRequest, *irma.Configuration, error) {
	irmaconfigPath, err := cmd.Flags().GetString("schemes-path")
	if err != nil {
		return nil, nil, err
	}
	irmaconfig, err := irma.NewConfiguration(irmaconfigPath, irma.ConfigurationOptions{})
	if err != nil {
		return nil, nil, err
	}
	if err = irmaconfig.ParseFolder(); err != nil {
		return nil, nil, err
	}
	if len(irmaconfig.SchemeManagers) == 0 {
		if err = irmaconfig.DownloadDefaultSchemes(); err != nil {
			return nil, nil, err
		}
	}

	request, err := constructSessionRequest(cmd, irmaconfig)
	if err != nil {
		return nil, nil, err
	}

	return request, irmaconfig, nil
}

// Helper functions

func wait(initialStatus server.Status, transport *irma.HTTPTransport, statuschan chan server.Status) {
	events := make(chan *sseclient.Event)

	go func() {
		for {
			if e := <-events; e != nil && e.Type != "open" {
				status := server.Status(strings.Trim(string(e.Data), `"`))
				statuschan <- status
				if status.Finished() {
					return
				}
			}
		}
	}()

	if err := sseclient.Notify(nil, transport.Server+"statusevents", true, events); err != nil {
		fmt.Println("SSE failed, fallback to polling", err)
		close(events)
		poll(initialStatus, transport, statuschan)
		return
	}
}

// poll recursively polls the session status until a final status is received.
func poll(initialStatus server.Status, transport *irma.HTTPTransport, statuschan chan server.Status) {
	// First we wait
	<-time.NewTimer(pollInterval).C

	// Get session status
	var s string
	if err := transport.Get("status", &s); err != nil {
		_ = server.LogFatal(err)
	}
	status := server.Status(strings.Trim(s, `"`))

	// report if status changed
	if status != initialStatus {
		statuschan <- status
	}

	if status.Finished() {
		return
	}
	go poll(status, transport, statuschan)
}

func constructSessionRequest(cmd *cobra.Command, conf *irma.Configuration) (irma.RequestorRequest, error) {
	disclose, _ := cmd.Flags().GetStringArray("disclose")
	issue, _ := cmd.Flags().GetStringArray("issue")
	sign, _ := cmd.Flags().GetStringArray("sign")
	message, _ := cmd.Flags().GetString("message")
	jsonrequest, _ := cmd.Flags().GetString("request")

	if len(disclose) == 0 && len(issue) == 0 && len(sign) == 0 && message == "" {
		if jsonrequest == "" {
			return nil, errors.New("Provide either a complete session request using --request or construct one using the other flags")
		}
		request, err := server.ParseSessionRequest(jsonrequest)
		if err != nil {
			return nil, err
		}
		return request, nil
	}

	if jsonrequest != "" {
		return nil, errors.New("Provide either a complete session request using --request or construct one using the other flags")
	}

	if len(sign) != 0 {
		if len(disclose) != 0 {
			return nil, errors.New("cannot combine disclosure and signature sessions, use either --disclose or --sign")
		}
		if len(issue) != 0 {
			return nil, errors.New("cannot combine issuance and signature sessions, use either --issue or --sign")
		}
		if message == "" {
			return nil, errors.New("signature sessions require a message to be signed using --message")
		}
	}

	var request irma.RequestorRequest
	if len(disclose) != 0 && len(issue) == 0 {
		disclose, err := parseAttrs(disclose, conf)
		if err != nil {
			return nil, err
		}
		request = &irma.ServiceProviderRequest{
			Request: irma.NewDisclosureRequest(),
		}
		request.SessionRequest().(*irma.DisclosureRequest).Disclose = disclose
	}
	if len(sign) != 0 {
		disclose, err := parseAttrs(sign, conf)
		if err != nil {
			return nil, err
		}
		request = &irma.SignatureRequestorRequest{
			Request: irma.NewSignatureRequest(message),
		}
		request.SessionRequest().(*irma.SignatureRequest).Disclose = disclose
	}
	if len(issue) != 0 {
		creds, err := parseCredentials(issue, conf)
		if err != nil {
			return nil, err
		}
		disclose, err := parseAttrs(disclose, conf)
		if err != nil {
			return nil, err
		}
		request = &irma.IdentityProviderRequest{
			Request: irma.NewIssuanceRequest(creds),
		}
		request.SessionRequest().(*irma.IssuanceRequest).Disclose = disclose
	}

	return request, nil
}

func parseCredentials(credentialsStr []string, conf *irma.Configuration) ([]*irma.CredentialRequest, error) {
	list := make([]*irma.CredentialRequest, 0, len(credentialsStr))

	for _, credStr := range credentialsStr {
		parts := strings.Split(credStr, "=")
		if len(parts) != 2 {
			return nil, errors.New("--issue argument must contain exactly 1 = sign")
		}
		credIdStr, attrsStr := parts[0], parts[1]
		credtype := conf.CredentialTypes[irma.NewCredentialTypeIdentifier(credIdStr)]
		if credtype == nil {
			return nil, errors.New("unknown credential type: " + credIdStr)
		}

		attrsSlice := strings.Split(attrsStr, ",")
		if len(attrsSlice) != len(credtype.AttributeTypes) {
			return nil, errors.Errorf("%d attributes required but %d provided for %s", len(credtype.AttributeTypes), len(attrsSlice), credIdStr)
		}

		attrs := make(map[string]string, len(attrsSlice))
		for i, typ := range credtype.AttributeTypes {
			attrs[typ.ID] = attrsSlice[i]
		}
		list = append(list, &irma.CredentialRequest{
			CredentialTypeID: irma.NewCredentialTypeIdentifier(credIdStr),
			Attributes:       attrs,
		})
	}

	return list, nil
}

func parseAttrs(attrsStr []string, conf *irma.Configuration) (irma.AttributeConDisCon, error) {
	list := make(irma.AttributeConDisCon, 0, len(attrsStr))
	for _, disjunctionStr := range attrsStr {
		disjunction := irma.AttributeDisCon{}
		attrids := strings.Split(disjunctionStr, ",")
		for _, attridStr := range attrids {
			attrid := irma.NewAttributeTypeIdentifier(attridStr)
			if conf.AttributeTypes[attrid] == nil {
				return nil, errors.New("unknown attribute: " + attridStr)
			}
			disjunction = append(disjunction, irma.AttributeCon{irma.AttributeRequest{Type: attrid}})
		}
		list = append(list, disjunction)
	}
	return list, nil
}

func startServer(port int) {
	mux := http.NewServeMux()
	mux.HandleFunc("/", irmaServer.HandlerFunc())
	httpServer = &http.Server{Addr: ":" + strconv.Itoa(port), Handler: mux}
	go func() {
		err := httpServer.ListenAndServe()
		if err != nil && err != http.ErrServerClosed {
			die("Failed to start server", err)
		}
	}()
}

func printQr(qr *irma.Qr, noqr bool) error {
	qrBts, err := json.Marshal(qr)
	if err != nil {
		return err
	}
	if noqr {
		fmt.Println(string(qrBts))
	} else {
		qrterminal.GenerateWithConfig(string(qrBts), qrterminal.Config{
			Level:     qrterminal.L,
			Writer:    os.Stdout,
			BlackChar: qrterminal.BLACK,
			WhiteChar: qrterminal.WHITE,
		})
	}
	return nil
}

func printSessionResult(result *server.SessionResult) {
	fmt.Println("Session result:")
	fmt.Println(prettyprint(result))
}

func init() {
	RootCmd.AddCommand(requestCmd)

	flags := requestCmd.Flags()
	flags.SortFlags = false

	addRequestFlags(flags)
}

func authmethodAlias(f *pflag.FlagSet, name string) pflag.NormalizedName {
	switch name {
	case "authmethod":
		name = "auth-method"
		break
	}
	return pflag.NormalizedName(name)
}

func addRequestFlags(flags *pflag.FlagSet) {
	flags.StringP("schemes-path", "s", irma.DefaultSchemesPath(), "path to irma_configuration")
	flags.StringP("auth-method", "a", "none", "Authentication method to server (none, token, rsa, hmac)")
	flags.SetNormalizeFunc(authmethodAlias)
	flags.String("key", "", "Key to sign request with")
	flags.String("name", "", "Requestor name")
	flags.StringArray("disclose", nil, "Add an attribute disjunction (comma-separated)")
	flags.StringArray("issue", nil, "Add a credential to issue")
	flags.StringArray("sign", nil, "Add an attribute disjunction to signature session")
	flags.String("message", "", "Message to sign in signature session")
}
<|endoftext|>


<commit_before>// Package gooq contains functions for working with databases
package gOOQ

import "testing"

// Test331gooqAsSQLBuilder tests Section 3.3.1 jOOQ as a SQL Builder.
// Found in jOOQ 3.6 User Manual.
// http://www.jooq.org/doc/3.6/manual-single-page/#jooq-as-a-standalone-sql-builder
func Test331gooqAsSQLBuilder(t *testing.T) {
	// Fetch a SQL string from a jOOQ Query in order to manually execute it with another tool.
	//String sql = create.select(field("BOOK.TITLE"), field("AUTHOR.FIRST_NAME"), field("AUTHOR.LAST_NAME"))
	//                   .from(table("BOOK"))
	//                   .join(table("AUTHOR"))
	//                   .on(field("BOOK.AUTHOR_ID").equal(field("AUTHOR.ID")))
	//                   .where(field("BOOK.PUBLISHED_IN").equal(1948))
	//                   .getSQL();
	got := gooq.Select(gooq.Field("BOOK.TITLE"), gooq.Field("AUTHOR.FIRST_NAME"), gooq.Field("AUTHOR.LAST_NAME"))
						.From(gooq.Table("BOOK"))
						.Join(gooq.Table("AUTHOR"))
						.On(gooq.Field("BOOK.AUTHOR_ID").Equal(gooq.Field("AUTHOR_ID")))
						.Where(gooq.Field("BOOK.PUBLISHED_IN").Equal(1948))
						.GetSQL()
	want := "" // having trouble getting this, probably because I'm trying to do it in scala.
	if got != want {
		t.Errorf("gooq.Select == %q, want %q", got, want)
	}
}
<commit_msg>Add Output from scala version of the manual.<commit_after>// Package gooq contains functions for working with databases
package gOOQ

import "testing"

// Test331gooqAsSQLBuilder tests Section 3.3.1 jOOQ as a SQL Builder.
// Found in jOOQ 3.6 User Manual.
// http://www.jooq.org/doc/3.6/manual-single-page/#jooq-as-a-standalone-sql-builder
func Test331gooqAsSQLBuilder(t *testing.T) {
	// Fetch a SQL string from a jOOQ Query in order to manually execute it with another tool.
	//String sql = create.select(field("BOOK.TITLE"), field("AUTHOR.FIRST_NAME"), field("AUTHOR.LAST_NAME"))
	//                   .from(table("BOOK"))
	//                   .join(table("AUTHOR"))
	//                   .on(field("BOOK.AUTHOR_ID").equal(field("AUTHOR.ID")))
	//                   .where(field("BOOK.PUBLISHED_IN").equal(1948))
	//                   .getSQL();
	got := gooq.Select(gooq.Field("BOOK.TITLE"), gooq.Field("AUTHOR.FIRST_NAME"), gooq.Field("AUTHOR.LAST_NAME"))
						.From(gooq.Table("BOOK"))
						.Join(gooq.Table("AUTHOR"))
						.On(gooq.Field("BOOK.AUTHOR_ID").Equal(gooq.Field("AUTHOR_ID")))
						.Where(gooq.Field("BOOK.PUBLISHED_IN").Equal(1948))
						.GetSQL()
	want := "select BOOK.TITLE, AUTHOR.FIRST_NAME, AUTHOR.LAST_NAME from BOOK join AUTHOR on BOOK.AUTHOR_ID = AUTHOR.ID where BOOK.PUBLISHED_IN = 1948"
	if got != want {
		t.Errorf("gooq.Select == %q, want %q", got, want)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The Gosl Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package oblas
<commit_msg>Remove unused file<commit_after><|endoftext|>


<commit_before>/*
Copyright 2017 The Fission Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package storagesvc

import (
	"errors"
	"io"
	"mime/multipart"
	"os"
	"time"

	"github.com/graymeta/stow"
	_ "github.com/graymeta/stow/local"
	"github.com/satori/go.uuid"
	log "github.com/sirupsen/logrus"
)

type (
	StorageType string

	storageConfig struct {
		storageType   StorageType
		localPath     string
		containerName string
		// other stuff, such as google or s3 credentials, bucket names etc
	}

	StowClient struct {
		config    *storageConfig
		location  stow.Location
		container stow.Container
	}
)

const (
	StorageTypeLocal StorageType = "local"
	PaginationSize   int         = 10
)

var (
	ErrNotFound                = errors.New("not found")
	ErrRetrievingItem          = errors.New("unable to retrieve item")
	ErrOpeningItem             = errors.New("unable to open item")
	ErrWritingFile             = errors.New("unable to write file")
	ErrWritingFileIntoResponse = errors.New("unable to copy item into http response")
)

func MakeStowClient(storageType StorageType, storagePath string, containerName string) (*StowClient, error) {
	if storageType != StorageTypeLocal {
		return nil, errors.New("Storage types other than 'local' are not implemented")
	}

	config := &storageConfig{
		storageType:   storageType,
		localPath:     storagePath,
		containerName: containerName,
	}

	stowClient := &StowClient{
		config: config,
	}

	cfg := stow.ConfigMap{"path": config.localPath}
	loc, err := stow.Dial("local", cfg)
	if err != nil {
		log.WithError(err).Error("Error initializing storage")
		return nil, err
	}
	stowClient.location = loc

	con, err := loc.CreateContainer(config.containerName)
	if os.IsExist(err) {
		var cons []stow.Container
		var cursor string

		// use location.Containers to find containers that match the prefix (container name)
		cons, cursor, err = loc.Containers(config.containerName, stow.CursorStart, 1)
		if err == nil {
			if !stow.IsCursorEnd(cursor) {
				// Should only have one storage container
				err = errors.New("Found more than one matched storage containers")
			} else {
				con = cons[0]
			}
		}
	}
	if err != nil {
		log.WithError(err).Error("Error initializing storage")
		return nil, err
	}
	stowClient.container = con

	return stowClient, nil
}

// putFile writes the file on the storage
func (client *StowClient) putFile(file multipart.File, fileSize int64) (string, error) {
	// This is not the item ID (that's returned by Put)
	// should we just use handler.Filename? what are the constraints here?
	uploadName := uuid.NewV4().String()

	// save the file to the storage backend
	item, err := client.container.Put(uploadName, file, int64(fileSize), nil)
	if err != nil {
		log.WithError(err).Errorf("Error writing file: %s on storage", uploadName)
		return "", ErrWritingFile
	}

	log.Debugf("Successfully wrote file:%s on storage", uploadName)
	return item.ID(), nil
}

// copyFileToStream gets the file contents into a stream
func (client *StowClient) copyFileToStream(fileId string, w io.Writer) error {
	item, err := client.container.Item(fileId)
	if err != nil {
		if err == stow.ErrNotFound {
			return ErrNotFound
		} else {
			return ErrRetrievingItem
		}
	}

	f, err := item.Open()
	if err != nil {
		return ErrOpeningItem
	}
	defer f.Close()

	_, err = io.Copy(w, f)
	if err != nil {
		return ErrWritingFileIntoResponse
	}

	log.Debugf("successfully wrote file: %s into httpresponse", fileId)
	return nil
}

// removeFileByID deletes the file from storage
func (client *StowClient) removeFileByID(itemID string) error {
	return client.container.RemoveItem(itemID)
}

// filter defines an interface to filter out items from a set of items
type filter func(stow.Item, interface{}) bool

// This method returns all items in a container, filtering out items based on the filter function passed to it
func (client *StowClient) getItemIDsWithFilter(filterFunc filter, filterFuncParam interface{}) ([]string, error) {
	cursor := stow.CursorStart
	var items []stow.Item
	var err error

	archiveIDList := make([]string, 0)

	for {
		items, cursor, err = client.container.Items(stow.NoPrefix, cursor, PaginationSize)
		if err != nil {
			log.WithError(err).Error("Error getting items from container")
			return nil, err
		}

		for _, item := range items {
			isItemFilterable := filterFunc(item, filterFuncParam)
			if isItemFilterable {
				continue
			}
			archiveIDList = append(archiveIDList, item.ID())
		}

		if stow.IsCursorEnd(cursor) {
			break
		}
	}

	return archiveIDList, nil
}

// filterItemCreatedAMinuteAgo is one type of filter function that filters out items created less than a minute ago.
// More filter functions can be written if needed, as long as they are of type filter
func filterItemCreatedAMinuteAgo(item stow.Item, currentTime interface{}) bool {
	itemLastModTime, _ := item.LastMod()
	if currentTime.(time.Time).Sub(itemLastModTime) < 1*time.Minute {
		log.Debugf("item: %s created less than a minute ago: %v", item.ID(), itemLastModTime)
		return true
	}
	return false
}
<commit_msg>Adding an extra info.<commit_after>/*
Copyright 2017 The Fission Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package storagesvc

import (
	"errors"
	"io"
	"mime/multipart"
	"os"
	"time"

	"github.com/graymeta/stow"
	_ "github.com/graymeta/stow/local"
	"github.com/satori/go.uuid"
	log "github.com/sirupsen/logrus"
)

type (
	StorageType string

	storageConfig struct {
		storageType   StorageType
		localPath     string
		containerName string
		// other stuff, such as google or s3 credentials, bucket names etc
	}

	StowClient struct {
		config    *storageConfig
		location  stow.Location
		container stow.Container
	}
)

const (
	StorageTypeLocal StorageType = "local"
	PaginationSize   int         = 10
)

var (
	ErrNotFound                = errors.New("not found")
	ErrRetrievingItem          = errors.New("unable to retrieve item")
	ErrOpeningItem             = errors.New("unable to open item")
	ErrWritingFile             = errors.New("unable to write file")
	ErrWritingFileIntoResponse = errors.New("unable to copy item into http response")
)

func MakeStowClient(storageType StorageType, storagePath string, containerName string) (*StowClient, error) {
	log.Infof("start : MakeStowClient")
	if storageType != StorageTypeLocal {
		return nil, errors.New("Storage types other than 'local' are not implemented")
	}

	config := &storageConfig{
		storageType:   storageType,
		localPath:     storagePath,
		containerName: containerName,
	}

	stowClient := &StowClient{
		config: config,
	}

	cfg := stow.ConfigMap{"path": config.localPath}
	loc, err := stow.Dial("local", cfg)
	if err != nil {
		log.WithError(err).Error("Error initializing storage")
		return nil, err
	}
	stowClient.location = loc

	con, err := loc.CreateContainer(config.containerName)
	if os.IsExist(err) {
		var cons []stow.Container
		var cursor string

		// use location.Containers to find containers that match the prefix (container name)
		cons, cursor, err = loc.Containers(config.containerName, stow.CursorStart, 1)
		if err == nil {
			if !stow.IsCursorEnd(cursor) {
				// Should only have one storage container
				err = errors.New("Found more than one matched storage containers")
			} else {
				con = cons[0]
			}
		}
	}
	if err != nil {
		log.WithError(err).Error("Error initializing storage")
		return nil, err
	}
	stowClient.container = con

	log.Infof("end : MakeStowClient")
	return stowClient, nil
}

// putFile writes the file on the storage
func (client *StowClient) putFile(file multipart.File, fileSize int64) (string, error) {
	// This is not the item ID (that's returned by Put)
	// should we just use handler.Filename? what are the constraints here?
	uploadName := uuid.NewV4().String()

	// save the file to the storage backend
	item, err := client.container.Put(uploadName, file, int64(fileSize), nil)
	if err != nil {
		log.WithError(err).Errorf("Error writing file: %s on storage", uploadName)
		return "", ErrWritingFile
	}

	log.Debugf("Successfully wrote file:%s on storage", uploadName)
	return item.ID(), nil
}

// copyFileToStream gets the file contents into a stream
func (client *StowClient) copyFileToStream(fileId string, w io.Writer) error {
	item, err := client.container.Item(fileId)
	if err != nil {
		if err == stow.ErrNotFound {
			return ErrNotFound
		} else {
			return ErrRetrievingItem
		}
	}

	f, err := item.Open()
	if err != nil {
		return ErrOpeningItem
	}
	defer f.Close()

	_, err = io.Copy(w, f)
	if err != nil {
		return ErrWritingFileIntoResponse
	}

	log.Debugf("successfully wrote file: %s into httpresponse", fileId)
	return nil
}

// removeFileByID deletes the file from storage
func (client *StowClient) removeFileByID(itemID string) error {
	return client.container.RemoveItem(itemID)
}

// filter defines an interface to filter out items from a set of items
type filter func(stow.Item, interface{}) bool

// This method returns all items in a container, filtering out items based on the filter function passed to it
func (client *StowClient) getItemIDsWithFilter(filterFunc filter, filterFuncParam interface{}) ([]string, error) {
	cursor := stow.CursorStart
	var items []stow.Item
	var err error

	archiveIDList := make([]string, 0)

	for {
		items, cursor, err = client.container.Items(stow.NoPrefix, cursor, PaginationSize)
		if err != nil {
			log.WithError(err).Error("Error getting items from container")
			return nil, err
		}

		for _, item := range items {
			isItemFilterable := filterFunc(item, filterFuncParam)
			if isItemFilterable {
				continue
			}
			archiveIDList = append(archiveIDList, item.ID())
		}

		if stow.IsCursorEnd(cursor) {
			break
		}
	}

	return archiveIDList, nil
}

// filterItemCreatedAMinuteAgo is one type of filter function that filters out items created less than a minute ago.
// More filter functions can be written if needed, as long as they are of type filter
func filterItemCreatedAMinuteAgo(item stow.Item, currentTime interface{}) bool {
	itemLastModTime, _ := item.LastMod()
	if currentTime.(time.Time).Sub(itemLastModTime) < 1*time.Minute {
		log.Debugf("item: %s created less than a minute ago: %v", item.ID(), itemLastModTime)
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package datastore

import (
	"database/sql"
	"os"
	"time"

	"github.com/drone/drone/shared/envconfig"
	"github.com/drone/drone/store"
	"github.com/drone/drone/store/datastore/ddl"
	_ "github.com/go-sql-driver/mysql"
	_ "github.com/lib/pq"
	_ "github.com/mattn/go-sqlite3"
	"github.com/rubenv/sql-migrate"
	"github.com/russross/meddler"

	"github.com/Sirupsen/logrus"
)

// datastore is an implementation of a model.Store built on top
// of the sql/database driver with a relational database backend.
type datastore struct {
	*sql.DB
}

// Load opens a new database connection with the specified driver
// and connection string specified in the environment variables.
func Load(env envconfig.Env) store.Store {
	var (
		driver = env.String("DATABASE_DRIVER", "sqlite3")
		config = env.String("DATABASE_CONFIG", "drone.sqlite")
	)

	logrus.Infof("using database driver %s", driver)
	logrus.Infof("using database config %s", config)

	return New(driver, config)
}

// New creates a database connection for the given driver and datasource
// and returns a new Store.
func New(driver, config string) store.Store {
	return From(
		open(driver, config),
	)
}

// From returns a Store using an existing database connection.
func From(db *sql.DB) store.Store {
	return &datastore{db}
}

// open opens a new database connection with the specified
// driver and connection string and returns a store.
func open(driver, config string) *sql.DB {
	db, err := sql.Open(driver, config)
	if err != nil {
		logrus.Errorln(err)
		logrus.Fatalln("database connection failed")
	}
	if driver == "mysql" {
		// per issue https://github.com/go-sql-driver/mysql/issues/257
		db.SetMaxIdleConns(0)
	}

	setupMeddler(driver)

	if err := pingDatabase(db); err != nil {
		logrus.Errorln(err)
		logrus.Fatalln("database ping attempts failed")
	}

	if err := setupDatabase(driver, db); err != nil {
		logrus.Errorln(err)
		logrus.Fatalln("migration failed")
	}
	cleanupDatabase(db)
	return db
}

// OpenTest opens a new database connection for testing purposes.
// The database driver and connection string are provided by
// environment variables, with fallback to in-memory sqlite.
func openTest() *sql.DB {
	var (
		driver = "sqlite3"
		config = ":memory:"
	)
	if os.Getenv("DATABASE_DRIVER") != "" {
		driver = os.Getenv("DATABASE_DRIVER")
		config = os.Getenv("DATABASE_DATASOURCE")
	}
	return open(driver, config)
}

// helper function to ping the database with backoff to ensure
// a connection can be established before we proceed with the
// database setup and migration.
func pingDatabase(db *sql.DB) (err error) {
	for i := 0; i < 30; i++ {
		err = db.Ping()
		if err == nil {
			return
		}
		logrus.Infof("database ping failed. retry in 1s")
		time.Sleep(time.Second)
	}
	return
}

// helper function to setup the databsae by performing
// automated database migration steps.
func setupDatabase(driver string, db *sql.DB) error {
	var migrations = &migrate.AssetMigrationSource{
		Asset:    ddl.Asset,
		AssetDir: ddl.AssetDir,
		Dir:      driver,
	}
	_, err := migrate.Exec(db, driver, migrations, migrate.Up)
	return err
}

// helper function to avoid stuck jobs when Drone unexpectedly
// restarts. This is a temp fix for https://github.com/drone/drone/issues/1195
func cleanupDatabase(db *sql.DB) {
	db.Exec("update builds set build_status = 'error' where build_status IN ('pending','running')")
	db.Exec("update jobs set job_status = 'error' where job_status IN ('pending','running')")
}

// helper function to setup the meddler default driver
// based on the selected driver name.
func setupMeddler(driver string) {
	switch driver {
	case "sqlite3":
		meddler.Default = meddler.SQLite
	case "mysql":
		meddler.Default = meddler.MySQL
	case "postgres":
		meddler.Default = meddler.PostgreSQL
	}
}
<commit_msg>fixed issue w/ test database driver<commit_after>package datastore

import (
	"database/sql"
	"os"
	"time"

	"github.com/drone/drone/shared/envconfig"
	"github.com/drone/drone/store"
	"github.com/drone/drone/store/datastore/ddl"
	_ "github.com/go-sql-driver/mysql"
	_ "github.com/lib/pq"
	_ "github.com/mattn/go-sqlite3"
	"github.com/rubenv/sql-migrate"
	"github.com/russross/meddler"

	"github.com/Sirupsen/logrus"
)

// datastore is an implementation of a model.Store built on top
// of the sql/database driver with a relational database backend.
type datastore struct {
	*sql.DB
}

// Load opens a new database connection with the specified driver
// and connection string specified in the environment variables.
func Load(env envconfig.Env) store.Store {
	var (
		driver = env.String("DATABASE_DRIVER", "sqlite3")
		config = env.String("DATABASE_CONFIG", "drone.sqlite")
	)

	logrus.Infof("using database driver %s", driver)
	logrus.Infof("using database config %s", config)

	return New(driver, config)
}

// New creates a database connection for the given driver and datasource
// and returns a new Store.
func New(driver, config string) store.Store {
	return From(
		open(driver, config),
	)
}

// From returns a Store using an existing database connection.
func From(db *sql.DB) store.Store {
	return &datastore{db}
}

// open opens a new database connection with the specified
// driver and connection string and returns a store.
func open(driver, config string) *sql.DB {
	db, err := sql.Open(driver, config)
	if err != nil {
		logrus.Errorln(err)
		logrus.Fatalln("database connection failed")
	}
	if driver == "mysql" {
		// per issue https://github.com/go-sql-driver/mysql/issues/257
		db.SetMaxIdleConns(0)
	}

	setupMeddler(driver)

	if err := pingDatabase(db); err != nil {
		logrus.Errorln(err)
		logrus.Fatalln("database ping attempts failed")
	}

	if err := setupDatabase(driver, db); err != nil {
		logrus.Errorln(err)
		logrus.Fatalln("migration failed")
	}
	cleanupDatabase(db)
	return db
}

// OpenTest opens a new database connection for testing purposes.
// The database driver and connection string are provided by
// environment variables, with fallback to in-memory sqlite.
func openTest() *sql.DB {
	var (
		driver = "sqlite3"
		config = ":memory:"
	)
	if os.Getenv("DATABASE_DRIVER") != "" {
		driver = os.Getenv("DATABASE_DRIVER")
		config = os.Getenv("DATABASE_CONFIG")
	}
	return open(driver, config)
}

// helper function to ping the database with backoff to ensure
// a connection can be established before we proceed with the
// database setup and migration.
func pingDatabase(db *sql.DB) (err error) {
	for i := 0; i < 30; i++ {
		err = db.Ping()
		if err == nil {
			return
		}
		logrus.Infof("database ping failed. retry in 1s")
		time.Sleep(time.Second)
	}
	return
}

// helper function to setup the databsae by performing
// automated database migration steps.
func setupDatabase(driver string, db *sql.DB) error {
	var migrations = &migrate.AssetMigrationSource{
		Asset:    ddl.Asset,
		AssetDir: ddl.AssetDir,
		Dir:      driver,
	}
	_, err := migrate.Exec(db, driver, migrations, migrate.Up)
	return err
}

// helper function to avoid stuck jobs when Drone unexpectedly
// restarts. This is a temp fix for https://github.com/drone/drone/issues/1195
func cleanupDatabase(db *sql.DB) {
	db.Exec("update builds set build_status = 'error' where build_status IN ('pending','running')")
	db.Exec("update jobs set job_status = 'error' where job_status IN ('pending','running')")
}

// helper function to setup the meddler default driver
// based on the selected driver name.
func setupMeddler(driver string) {
	switch driver {
	case "sqlite3":
		meddler.Default = meddler.SQLite
	case "mysql":
		meddler.Default = meddler.MySQL
	case "postgres":
		meddler.Default = meddler.PostgreSQL
	}
}
<|endoftext|>


<commit_before>package main

import "bufio"
import "flag"
import "fmt"
import "image"
import "image/png"
import "io/ioutil"
import "math"
import "os"
import "strconv"
import "strings"
import "time"

var b, f int = 72, 22
var left_theta, right_theta, left_alpha, right_alpha float64 = 1.5708, 1.01129, 0.23022, 0.33389
var frame_count int = 96
var image_size_x, image_size_y = 640, 480
var sub_sampling_rate, expected_x_deviation = 8, 100

func scan_dir(dir_name string, calibration_dir string) {
	fmt.Println("Processing scans in directory: " + dir_name)
	print_divider()

	var files, _ = ioutil.ReadDir(dir_name)
	if len(files) == 0 {
		fmt.Println("Directory is empty, make sure there are scans in here.")
	} else {
		var cal_array [][]int = nil
		if calibration_dir != "" {
			cal_array = calibrate_from_dir(calibration_dir)
		}

		z_array := make([][]float64, image_size_x)
		for i := range z_array {
			z_array[i] = make([]float64, image_size_y)
		}

		for x := 0; x < image_size_x; x++ {
			for y := 0; y < image_size_y; y++ {
				z_array[x][y] = -99999
			}
		}

		fmt.Println("Scanning images and creating z array...")
		print_divider()

		start_time := time.Now()

		scan_number := 0
		for i := 0; i < len(files); i++ {
			filename := files[i]
			if strings.Contains(filename.Name(), "png") {
				scan_image_by_dev(dir_name + filename.Name(), scan_number, z_array, cal_array)
				fmt.Println("scanning image " + filename.Name())
				scan_number++
			}
		}

		time_elapsed := time.Since(start_time)
		fmt.Println("scan finished in " + time_elapsed.String())

		output_pcd(z_array)
	}
}

func calibrate_from_dir(dir_name string) [][]int {
	fmt.Println("Calibrating scanner with frames from directory: " + dir_name)
	print_divider()

	var files, _ = ioutil.ReadDir(dir_name)
	if len(files) == 0 {
		fmt.Println("Calibration directory is empty, make sure there are scans in here.")
	} else {
		cal_array := make([][]int, image_size_x)
		for i := range cal_array {
			cal_array[i] = make([]int, image_size_y)
		}

		for x := 0; x < image_size_x; x++ {
			for y := 0; y < image_size_y; y++ {
				cal_array[x][y] = -99999
			}
		}

		fmt.Println("Scanning images and creating calibration array...")
		print_divider()

		scan_number := 0
		for i := 0; i < len(files); i++ {
			filename := files[i]
			if strings.Contains(filename.Name(), "png") {
				calibrate_with_image(dir_name + filename.Name(), scan_number, cal_array)
				scan_number++
			}
		}

		fmt.Println("Finished calibration.")
		print_divider()

		return cal_array
	}

	return nil
}

func find_projection_in_line(scan image.Image, y_pix int, est_location int) int {
	enter_white, exit_white := 0, 0
	var min_x, max_x int

	if est_location == -1 || est_location < expected_x_deviation || est_location > (image_size_x - expected_x_deviation) {
		min_x, max_x = 0, image_size_x
	} else {
		min_x, max_x = est_location - expected_x_deviation, est_location + expected_x_deviation
	}

	for x := min_x; x < max_x; x++ {
		color := scan.At(x, y_pix)
		r, g, b, _ := color.RGBA()
		if is_white(r, g, b) {
			enter_white = x
			break
		}
	}

	for x := enter_white; x < max_x; x++ {
		color := scan.At(x, y_pix)
		r, g, b, _ := color.RGBA()
		if !is_white(r, g, b) {
			exit_white = x - 1
			break
		}
	}

	if enter_white != 0 && exit_white != 0 {
		return int((enter_white + exit_white) / 2)
	}

	return -99999;
}

func calibrate_with_image(filename string, scan_number int, cal_array [][]int) {
	fmt.Println("Opening file " + filename)

	file, err := os.Open(filename)
	if err != nil { panic(err) }
	r := bufio.NewReader(file)
	scan, err := png.Decode(r)

	for y := 0; y < image_size_y; y++ {
		line_location := find_projection_in_line(scan, y, -1)
		if line_location != -99999 {
			cal_array[scan_number][y] = line_location
		}
	}
}

func scan_video(filename string) {
	fmt.Println("going to scan video file: " + filename)
}

func scan_image_with_parallax(filename string, scan_number int, z_array [][]float64) {
	fmt.Println("Opening file " + filename + ". scan number " + strconv.Itoa(scan_number))

	file, err := os.Open(filename)
	if err != nil { panic(err) }
	r := bufio.NewReader(file)
	scan, err := png.Decode(r)

	start_time := time.Now()

	for y := 0; y < image_size_y; y += sub_sampling_rate {
		line_location := find_projection_in_line(scan, y, -1)
		if line_location != -99999 {
			z_array[line_location][y] = z_triangulation(line_location, y, scan_number)
		}
	}

	time_elapsed := time.Since(start_time)
	fmt.Println("scanned image in " + time_elapsed.String())
}

func z_triangulation(x int, y int, scan_number int) float64 {
	y = image_size_y - y
	var theta float64 = translate(float64(scan_number), 0.0, float64(frame_count - 1), left_theta, right_theta)
	var alpha float64 = translate(float64(scan_number), 0.0, float64(frame_count - 1), left_alpha, right_alpha)
	var z float64 = float64(b) * (math.Sin(theta) / math.Sin(alpha + theta))
	return z * (-float64(f))
}

func scan_image_by_dev(filename string, scan_number int, z_array [][]float64, cal_array [][]int) {
	fmt.Println("Opening file " + filename + ". scan number " + strconv.Itoa(scan_number))

	file, err := os.Open(filename)
	if err != nil { panic(err) }
	r := bufio.NewReader(file)
	scan, err := png.Decode(r)

	start_time := time.Now()

	first_line_location := -1
	previous_line_location := -1
	for y := 0; y < image_size_y; y += sub_sampling_rate {
		line_location := find_projection_in_line(scan, y, previous_line_location)
		if line_location != -99999 {
			previous_line_location = line_location
			if cal_array != nil {
				expected_location := cal_array[scan_number][y]
				if expected_location != -99999 {
					xdiff := line_location - expected_location
					z_array[line_location][y] = float64(xdiff)
				}
			} else {
				if first_line_location == -1 {
					z_array[line_location][y] = float64(0)
					first_line_location = line_location
				} else {
					xdiff := line_location - first_line_location
					z_array[line_location][y] = float64(xdiff)
				}
			}
		}
	}

	time_elapsed := time.Since(start_time)
	fmt.Println("scanned image in " + time_elapsed.String())
}

func translate(value float64, left_min float64, left_max float64, right_min float64, right_max float64) float64 {
	var left_span float64 = left_max - left_min
	var right_span float64 = right_max - right_min

	var value_scaled = (value - left_min) / (left_span)

	return right_min + (value_scaled * float64(right_span))
}

func output_pcd(z_array [][]float64) {
	point_count := 0
	for x := 0; x < image_size_x; x++ {
		for y := 0; y < image_size_y; y++ {
			if (z_array[x][y] != -99999) {
				point_count++
			}
		}
	}

	pcd_string := "# .PCD v.7 -- file generated by structured_light_scan.go\n"
	pcd_string += "VERSION .7\n"
	pcd_string += "FIELDS x y z\n"
	pcd_string += "SIZE 8 8 8\n"
	pcd_string += "TYPE F F F\n"
	pcd_string += "COUNT 1 1 1\n"
	pcd_string += "WIDTH " + strconv.Itoa(point_count) + "\n"
	pcd_string += "HEIGHT 1\n"
	pcd_string += "VIEWPOINT 0 0 0 1 0 0 0\n"
	pcd_string += "POINTS " + strconv.Itoa(point_count) + "\n"
	pcd_string += "DATA ascii\n"

	for x := 0; x < image_size_x; x++ {
		for y := 0; y < image_size_y; y++ {
			if (z_array[x][y] != -99999) {
				pcd_string += strconv.Itoa(x) + " " + strconv.Itoa(image_size_y - y) + " " + strconv.FormatFloat(z_array[x][y], 'f', -1, 64) + "\n"
			}
		}
	}

	out_file, _ := os.Create("output.pcd")
	out_file.WriteString(pcd_string)
	out_file.Close()
}

func is_white(r uint32, g uint32, b uint32) bool {
	if r > 60000 && g > 60000 && b > 60000 {
		return true
	}

	return false
}

func print_divider() {
	fmt.Println("-----------------------------------------------------")
}

func fix_dir_name(dir_name string) string {
	if strings.HasSuffix(dir_name, "/") {
		return dir_name
	}

	return dir_name + "/"
}

func main() {
	scan := flag.Bool("s", false, "scan the given directories")
	dir_name := flag.String("d", "", "directory of scan files")
	cal_dir := flag.String("c", "", "directory of calibration scan files")
	flag.Parse()

	if *dir_name == "" {
		fmt.Println("need to specify a scan directory")
	} else {
		fixed_dir := fix_dir_name(*dir_name)
		if *scan {
			if *cal_dir != "" {
				scan_dir(fixed_dir, *cal_dir)
			} else {
				scan_dir(fixed_dir, "")
			}
		}
	}
}<commit_msg>finished updating the go scanner, and boy is it fast! each frame takes around 500 - 1000 microseconds.<commit_after>package main

import "bufio"
import "flag"
import "fmt"
import "image"
import "image/png"
import "io/ioutil"
import "math"
import "os"
import "strconv"
import "strings"
import "time"

var b, f int = 72, 22
var left_theta, right_theta, left_alpha, right_alpha float64 = 1.5708, 1.01129, 0.23022, 0.33389
var frame_count int = 96
var image_size_x, image_size_y = 640, 480
var sub_sampling_rate, expected_x_deviation, maximum_line_size = 8, 100, 15

func scan_dir(dir_name string, calibration_dir string) {
	fmt.Println("Processing scans in directory: " + dir_name)
	print_divider()

	var files, _ = ioutil.ReadDir(dir_name)
	if len(files) == 0 {
		fmt.Println("Directory is empty, make sure there are scans in here.")
	} else {
		var cal_array [][]int = nil
		if calibration_dir != "" {
			cal_array = calibrate_from_dir(calibration_dir)
		}

		z_array := make([][]float64, image_size_x)
		for i := range z_array {
			z_array[i] = make([]float64, image_size_y)
		}

		for x := 0; x < image_size_x; x++ {
			for y := 0; y < image_size_y; y++ {
				z_array[x][y] = -99999
			}
		}

		fmt.Println("Scanning images and creating z array...")
		print_divider()

		start_time := time.Now()

		scan_number := 0
		for i := 0; i < len(files); i++ {
			filename := files[i]
			if strings.Contains(filename.Name(), "png") {
				scan_image_by_dev(dir_name + filename.Name(), scan_number, z_array, cal_array)
				fmt.Println("scanning image " + filename.Name())
				scan_number++
			}
		}

		time_elapsed := time.Since(start_time)
		fmt.Println("scan finished in " + time_elapsed.String())

		output_pcd(z_array)
	}
}

func calibrate_from_dir(dir_name string) [][]int {
	fmt.Println("Calibrating scanner with frames from directory: " + dir_name)
	print_divider()

	var files, _ = ioutil.ReadDir(dir_name)
	if len(files) == 0 {
		fmt.Println("Calibration directory is empty, make sure there are scans in here.")
	} else {
		cal_array := make([][]int, image_size_x)
		for i := range cal_array {
			cal_array[i] = make([]int, image_size_y)
		}

		for x := 0; x < image_size_x; x++ {
			for y := 0; y < image_size_y; y++ {
				cal_array[x][y] = -99999
			}
		}

		fmt.Println("Scanning images and creating calibration array...")
		print_divider()

		scan_number := 0
		for i := 0; i < len(files); i++ {
			filename := files[i]
			if strings.Contains(filename.Name(), "png") {
				calibrate_with_image(dir_name + filename.Name(), scan_number, cal_array)
				scan_number++
			}
		}

		fmt.Println("Finished calibration.")
		print_divider()

		return cal_array
	}

	return nil
}

func find_projection_in_line(scan image.Image, y_pix int, est_location int) int {
	enter_white, exit_white := 0, 0
	var min_x, max_x int

	if est_location == -1 || est_location < expected_x_deviation || est_location > (image_size_x - expected_x_deviation) {
		min_x, max_x = 0, image_size_x
	} else {
		min_x, max_x = est_location - expected_x_deviation, est_location + expected_x_deviation
	}

	for x := min_x; x < max_x; x++ {
		color := scan.At(x, y_pix)
		r, g, b, _ := color.RGBA()
		if is_white(r, g, b) {
			enter_white = x
			break
		}
	}

	for x := enter_white; x < max_x; x++ {
		color := scan.At(x, y_pix)
		r, g, b, _ := color.RGBA()
		if !is_white(r, g, b) {
			exit_white = x - 1
			break
		}
	}

	if enter_white != 0 && exit_white != 0 && (exit_white - enter_white) < maximum_line_size {
		return int((enter_white + exit_white) / 2)
	}

	return -99999;
}

func calibrate_with_image(filename string, scan_number int, cal_array [][]int) {
	fmt.Println("Opening file " + filename)

	file, err := os.Open(filename)
	if err != nil { panic(err) }
	r := bufio.NewReader(file)
	scan, err := png.Decode(r)

	previous_line_location := -1
	for y := 0; y < image_size_y; y++ {
		line_location := find_projection_in_line(scan, y, previous_line_location)
		if line_location != -99999 {
			previous_line_location = line_location
			cal_array[scan_number][y] = line_location
		}
	}
}

func scan_video(filename string) {
	fmt.Println("going to scan video file: " + filename)
}

func scan_image_with_parallax(filename string, scan_number int, z_array [][]float64) {
	fmt.Println("Opening file " + filename + ". scan number " + strconv.Itoa(scan_number))

	file, err := os.Open(filename)
	if err != nil { panic(err) }
	r := bufio.NewReader(file)
	scan, err := png.Decode(r)

	start_time := time.Now()

	for y := 0; y < image_size_y; y += sub_sampling_rate {
		line_location := find_projection_in_line(scan, y, -1)
		if line_location != -99999 {
			z_array[line_location][y] = z_triangulation(line_location, y, scan_number)
		}
	}

	time_elapsed := time.Since(start_time)
	fmt.Println("scanned image in " + time_elapsed.String())
}

func z_triangulation(x int, y int, scan_number int) float64 {
	y = image_size_y - y
	var theta float64 = translate(float64(scan_number), 0.0, float64(frame_count - 1), left_theta, right_theta)
	var alpha float64 = translate(float64(scan_number), 0.0, float64(frame_count - 1), left_alpha, right_alpha)
	var z float64 = float64(b) * (math.Sin(theta) / math.Sin(alpha + theta))
	return z * (-float64(f))
}

func scan_image_by_dev(filename string, scan_number int, z_array [][]float64, cal_array [][]int) {
	fmt.Println("Opening file " + filename + ". scan number " + strconv.Itoa(scan_number))

	file, err := os.Open(filename)
	if err != nil { panic(err) }
	r := bufio.NewReader(file)
	scan, err := png.Decode(r)

	start_time := time.Now()

	first_line_location := -1
	previous_line_location := -1
	for y := 0; y < image_size_y; y += sub_sampling_rate {
		line_location := find_projection_in_line(scan, y, previous_line_location)
		if line_location != -99999 {
			previous_line_location = line_location
			if cal_array != nil {
				expected_location := cal_array[scan_number][y]
				if expected_location != -99999 {
					xdiff := line_location - expected_location
					z_array[line_location][y] = float64(xdiff)
				}
			} else {
				if first_line_location == -1 {
					z_array[line_location][y] = float64(0)
					first_line_location = line_location
				} else {
					xdiff := line_location - first_line_location
					z_array[line_location][y] = float64(xdiff)
				}
			}
		}
	}

	time_elapsed := time.Since(start_time)
	fmt.Println("scanned image in " + time_elapsed.String())
}

func translate(value float64, left_min float64, left_max float64, right_min float64, right_max float64) float64 {
	var left_span float64 = left_max - left_min
	var right_span float64 = right_max - right_min

	var value_scaled = (value - left_min) / (left_span)

	return right_min + (value_scaled * float64(right_span))
}

func output_pcd(z_array [][]float64) {
	point_count := 0
	for x := 0; x < image_size_x; x++ {
		for y := 0; y < image_size_y; y++ {
			if (z_array[x][y] != -99999) {
				point_count++
			}
		}
	}

	pcd_string := "# .PCD v.7 -- file generated by structured_light_scan.go\n"
	pcd_string += "VERSION .7\n"
	pcd_string += "FIELDS x y z\n"
	pcd_string += "SIZE 8 8 8\n"
	pcd_string += "TYPE F F F\n"
	pcd_string += "COUNT 1 1 1\n"
	pcd_string += "WIDTH " + strconv.Itoa(point_count) + "\n"
	pcd_string += "HEIGHT 1\n"
	pcd_string += "VIEWPOINT 0 0 0 1 0 0 0\n"
	pcd_string += "POINTS " + strconv.Itoa(point_count) + "\n"
	pcd_string += "DATA ascii\n"

	for x := 0; x < image_size_x; x++ {
		for y := 0; y < image_size_y; y++ {
			if (z_array[x][y] != -99999) {
				pcd_string += strconv.Itoa(x) + " " + strconv.Itoa(image_size_y - y) + " " + strconv.FormatFloat(z_array[x][y], 'f', -1, 64) + "\n"
			}
		}
	}

	out_file, _ := os.Create("output.pcd")
	out_file.WriteString(pcd_string)
	out_file.Close()
}

func is_white(r uint32, g uint32, b uint32) bool {
	if r > 60000 && g > 60000 && b > 60000 {
		return true
	}

	return false
}

func print_divider() {
	fmt.Println("-----------------------------------------------------")
}

func fix_dir_name(dir_name string) string {
	if strings.HasSuffix(dir_name, "/") {
		return dir_name
	}

	return dir_name + "/"
}

func main() {
	scan := flag.Bool("s", false, "scan the given directories")
	dir_name := flag.String("d", "", "directory of scan files")
	cal_dir := flag.String("c", "", "directory of calibration scan files")
	flag.Parse()

	if *dir_name == "" {
		fmt.Println("need to specify a scan directory")
	} else {
		fixed_dir := fix_dir_name(*dir_name)
		if *scan {
			if *cal_dir != "" {
				scan_dir(fixed_dir, *cal_dir)
			} else {
				scan_dir(fixed_dir, "")
			}
		}
	}
}<|endoftext|>


<commit_before>/*
Copyright 2020 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
	"testing"

	"github.com/google/go-cmp/cmp"
	"github.com/google/go-cmp/cmp/cmpopts"
	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	"knative.dev/pkg/apis"
	duckv1 "knative.dev/pkg/apis/duck/v1"
)

var (
	availableDeployment = &appsv1.Deployment{
		Status: appsv1.DeploymentStatus{
			Conditions: []appsv1.DeploymentCondition{
				{
					Type:   appsv1.DeploymentAvailable,
					Status: corev1.ConditionTrue,
				},
			},
		},
	}

	condReady = apis.Condition{
		Type:   MongoDbConditionReady,
		Status: corev1.ConditionTrue,
	}
)

func TestMongoDbSourceGetConditionSet(t *testing.T) {
	r := &MongoDbSource{}

	if got, want := r.GetConditionSet().GetTopLevelConditionType(), apis.ConditionReady; got != want {
		t.Errorf("GetTopLevelCondition=%v, want=%v", got, want)
	}
}

func TestMongoDbGetCondition(t *testing.T) {
	tests := []struct {
		name      string
		ms        *MongoDbSourceStatus
		condQuery apis.ConditionType
		want      *apis.Condition
	}{{
		name: "single condition",
		ms: &MongoDbSourceStatus{
			SourceStatus: duckv1.SourceStatus{
				Status: duckv1.Status{
					Conditions: []apis.Condition{
						condReady,
					},
				},
			},
		},
		condQuery: apis.ConditionReady,
		want:      &condReady,
	}, {
		name: "unknown condition",
		ms: &MongoDbSourceStatus{
			SourceStatus: duckv1.SourceStatus{
				Status: duckv1.Status{
					Conditions: []apis.Condition{
						condReady,
					},
				},
			},
		},
		condQuery: apis.ConditionType("foo"),
		want:      nil,
	}, {
		name: "mark deployed",
		ms: func() *MongoDbSourceStatus {
			m := &MongoDbSourceStatus{}
			m.InitializeConditions()
			m.PropagateDeploymentAvailability(availableDeployment)
			return m
		}(),
		condQuery: MongoDbConditionReady,
		want: &apis.Condition{
			Type:   MongoDbConditionReady,
			Status: corev1.ConditionUnknown,
		},
	}, {
		name: "mark sink and deployed",
		ms: func() *MongoDbSourceStatus {
			m := &MongoDbSourceStatus{}
			m.InitializeConditions()
			m.MarkSink(apis.HTTP("example"))
			m.MarkConnectionSuccess()
			m.PropagateDeploymentAvailability(availableDeployment)
			return m
		}(),
		condQuery: MongoDbConditionReady,
		want: &apis.Condition{
			Type:   MongoDbConditionReady,
			Status: corev1.ConditionTrue,
		},
	}}
	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			got := test.ms.GetCondition(test.condQuery)
			ignoreTime := cmpopts.IgnoreFields(apis.Condition{},
				"LastTransitionTime", "Severity")
			if diff := cmp.Diff(test.want, got, ignoreTime); diff != "" {
				t.Errorf("unexpected condition (-want, +got) = %v", diff)
			}
		})
	}
}

func TestMongoDbInitializeConditions(t *testing.T) {
	tests := []struct {
		name string
		ms   *MongoDbSourceStatus
		want *MongoDbSourceStatus
	}{{
		name: "empty",
		ms:   &MongoDbSourceStatus{},
		want: &MongoDbSourceStatus{
			SourceStatus: duckv1.SourceStatus{
				Status: duckv1.Status{
					Conditions: []apis.Condition{{
						Type:   MongoDbConditionConnectionEstablished,
						Status: corev1.ConditionUnknown,
					}, {
						Type:   MongoDbConditionDeployed,
						Status: corev1.ConditionUnknown,
					}, {
						Type:   MongoDbConditionReady,
						Status: corev1.ConditionUnknown,
					}, {
						Type:   MongoDbConditionSinkProvided,
						Status: corev1.ConditionUnknown,
					}},
				},
			},
		},
	}, {
		name: "one false",
		ms: &MongoDbSourceStatus{
			SourceStatus: duckv1.SourceStatus{
				Status: duckv1.Status{
					Conditions: []apis.Condition{{
						Type:   MongoDbConditionSinkProvided,
						Status: corev1.ConditionFalse,
					}},
				},
			},
		},
		want: &MongoDbSourceStatus{
			SourceStatus: duckv1.SourceStatus{
				Status: duckv1.Status{
					Conditions: []apis.Condition{{
						Type:   MongoDbConditionConnectionEstablished,
						Status: corev1.ConditionUnknown,
					}, {
						Type:   MongoDbConditionDeployed,
						Status: corev1.ConditionUnknown,
					}, {
						Type:   MongoDbConditionReady,
						Status: corev1.ConditionUnknown,
					}, {
						Type:   MongoDbConditionSinkProvided,
						Status: corev1.ConditionFalse,
					}},
				},
			},
		},
	}, {
		name: "one true",
		ms: &MongoDbSourceStatus{
			SourceStatus: duckv1.SourceStatus{
				Status: duckv1.Status{
					Conditions: []apis.Condition{{
						Type:   MongoDbConditionSinkProvided,
						Status: corev1.ConditionTrue,
					}},
				},
			},
		},
		want: &MongoDbSourceStatus{
			SourceStatus: duckv1.SourceStatus{
				Status: duckv1.Status{
					Conditions: []apis.Condition{{
						Type:   MongoDbConditionConnectionEstablished,
						Status: corev1.ConditionUnknown,
					}, {
						Type:   MongoDbConditionDeployed,
						Status: corev1.ConditionUnknown,
					}, {
						Type:   MongoDbConditionReady,
						Status: corev1.ConditionUnknown,
					}, {
						Type:   MongoDbConditionSinkProvided,
						Status: corev1.ConditionTrue,
					}},
				},
			},
		},
	}, {
		name: "marksink",
		ms: func() *MongoDbSourceStatus {
			status := MongoDbSourceStatus{}
			status.MarkSink(apis.HTTP("sink"))
			return &status
		}(),
		want: &MongoDbSourceStatus{
			SourceStatus: duckv1.SourceStatus{
				Status: duckv1.Status{
					Conditions: []apis.Condition{{
						Type:   MongoDbConditionConnectionEstablished,
						Status: corev1.ConditionUnknown,
					},{
						Type:   MongoDbConditionDeployed,
						Status: corev1.ConditionUnknown,
					}, {
						Type:   MongoDbConditionReady,
						Status: corev1.ConditionUnknown,
					}, {
						Type:   MongoDbConditionSinkProvided,
						Status: corev1.ConditionTrue,
					}},
				},
				SinkURI: apis.HTTP("sink"),
			},
		},
	}, {
		name: "marknosink",
		ms: func() *MongoDbSourceStatus {
			status := MongoDbSourceStatus{}
			status.MarkNoSink("nothere", "")
			return &status
		}(),
		want: &MongoDbSourceStatus{
			SourceStatus: duckv1.SourceStatus{
				Status: duckv1.Status{
					Conditions: []apis.Condition{{
						Type:   MongoDbConditionConnectionEstablished,
						Status: corev1.ConditionUnknown,
					},{
						Type:   MongoDbConditionDeployed,
						Status: corev1.ConditionUnknown,
					}, {
						Type:   MongoDbConditionReady,
						Status: corev1.ConditionFalse,
					}, {
						Type:   MongoDbConditionSinkProvided,
						Status: corev1.ConditionFalse,
					}},
				},
			},
		},
	}}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			test.ms.InitializeConditions()
			ignore := cmpopts.IgnoreFields(
				apis.Condition{},
				"LastTransitionTime", "Message", "Reason", "Severity")
			if diff := cmp.Diff(test.want, test.ms, ignore); diff != "" {
				t.Errorf("unexpected conditions (-want, +got) = %v", diff)
			}
		})
	}
}
<commit_msg>Reconciler: receive adapter maker + extra UT for lifecycle (#38)<commit_after>/*
Copyright 2020 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
	"errors"
	"testing"

	"github.com/google/go-cmp/cmp"
	"github.com/google/go-cmp/cmp/cmpopts"
	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	"knative.dev/pkg/apis"
	duckv1 "knative.dev/pkg/apis/duck/v1"
)

var (
	availableDeployment = &appsv1.Deployment{
		Status: appsv1.DeploymentStatus{
			Conditions: []appsv1.DeploymentCondition{
				{
					Type:   appsv1.DeploymentAvailable,
					Status: corev1.ConditionTrue,
				},
			},
		},
	}

	condReady = apis.Condition{
		Type:   MongoDbConditionReady,
		Status: corev1.ConditionTrue,
	}
)

func TestMongoDbSourceGetConditionSet(t *testing.T) {
	r := &MongoDbSource{}

	if got, want := r.GetConditionSet().GetTopLevelConditionType(), apis.ConditionReady; got != want {
		t.Errorf("GetTopLevelCondition=%v, want=%v", got, want)
	}
}

func TestMongoDbGetCondition(t *testing.T) {
	tests := []struct {
		name      string
		ms        *MongoDbSourceStatus
		condQuery apis.ConditionType
		want      *apis.Condition
	}{{
		name: "single condition",
		ms: &MongoDbSourceStatus{
			SourceStatus: duckv1.SourceStatus{
				Status: duckv1.Status{
					Conditions: []apis.Condition{
						condReady,
					},
				},
			},
		},
		condQuery: apis.ConditionReady,
		want:      &condReady,
	}, {
		name: "unknown condition",
		ms: &MongoDbSourceStatus{
			SourceStatus: duckv1.SourceStatus{
				Status: duckv1.Status{
					Conditions: []apis.Condition{
						condReady,
					},
				},
			},
		},
		condQuery: apis.ConditionType("foo"),
		want:      nil,
	}, {
		name: "mark deployed",
		ms: func() *MongoDbSourceStatus {
			m := &MongoDbSourceStatus{}
			m.InitializeConditions()
			m.PropagateDeploymentAvailability(availableDeployment)
			return m
		}(),
		condQuery: MongoDbConditionReady,
		want: &apis.Condition{
			Type:   MongoDbConditionReady,
			Status: corev1.ConditionUnknown,
		},
	}, {
		name: "mark sink, deployed and connection failed",
		ms: func() *MongoDbSourceStatus {
			m := &MongoDbSourceStatus{}
			m.InitializeConditions()
			m.MarkSink(apis.HTTP("example"))
			m.MarkConnectionFailed(errors.New(""))
			m.PropagateDeploymentAvailability(availableDeployment)
			return m
		}(),
		condQuery: MongoDbConditionReady,
		want: &apis.Condition{
			Type:   MongoDbConditionReady,
			Status: corev1.ConditionFalse,
			Reason: "Connection failed: incorrect credentials or database or collection not found",
		},
	}, {
		name: "mark sink, deployed and connection established",
		ms: func() *MongoDbSourceStatus {
			m := &MongoDbSourceStatus{}
			m.InitializeConditions()
			m.MarkSink(apis.HTTP("example"))
			m.MarkConnectionSuccess()
			m.PropagateDeploymentAvailability(availableDeployment)
			return m
		}(),
		condQuery: MongoDbConditionReady,
		want: &apis.Condition{
			Type:   MongoDbConditionReady,
			Status: corev1.ConditionTrue,
		},
	}}
	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			got := test.ms.GetCondition(test.condQuery)
			ignoreTime := cmpopts.IgnoreFields(apis.Condition{},
				"LastTransitionTime", "Severity")
			if diff := cmp.Diff(test.want, got, ignoreTime); diff != "" {
				t.Errorf("unexpected condition (-want, +got) = %v", diff)
			}
		})
	}
}

func TestMongoDbInitializeConditions(t *testing.T) {
	tests := []struct {
		name string
		ms   *MongoDbSourceStatus
		want *MongoDbSourceStatus
	}{{
		name: "empty",
		ms:   &MongoDbSourceStatus{},
		want: &MongoDbSourceStatus{
			SourceStatus: duckv1.SourceStatus{
				Status: duckv1.Status{
					Conditions: []apis.Condition{{
						Type:   MongoDbConditionConnectionEstablished,
						Status: corev1.ConditionUnknown,
					}, {
						Type:   MongoDbConditionDeployed,
						Status: corev1.ConditionUnknown,
					}, {
						Type:   MongoDbConditionReady,
						Status: corev1.ConditionUnknown,
					}, {
						Type:   MongoDbConditionSinkProvided,
						Status: corev1.ConditionUnknown,
					}},
				},
			},
		},
	}, {
		name: "one false",
		ms: &MongoDbSourceStatus{
			SourceStatus: duckv1.SourceStatus{
				Status: duckv1.Status{
					Conditions: []apis.Condition{{
						Type:   MongoDbConditionSinkProvided,
						Status: corev1.ConditionFalse,
					}},
				},
			},
		},
		want: &MongoDbSourceStatus{
			SourceStatus: duckv1.SourceStatus{
				Status: duckv1.Status{
					Conditions: []apis.Condition{{
						Type:   MongoDbConditionConnectionEstablished,
						Status: corev1.ConditionUnknown,
					}, {
						Type:   MongoDbConditionDeployed,
						Status: corev1.ConditionUnknown,
					}, {
						Type:   MongoDbConditionReady,
						Status: corev1.ConditionUnknown,
					}, {
						Type:   MongoDbConditionSinkProvided,
						Status: corev1.ConditionFalse,
					}},
				},
			},
		},
	}, {
		name: "one true",
		ms: &MongoDbSourceStatus{
			SourceStatus: duckv1.SourceStatus{
				Status: duckv1.Status{
					Conditions: []apis.Condition{{
						Type:   MongoDbConditionSinkProvided,
						Status: corev1.ConditionTrue,
					}},
				},
			},
		},
		want: &MongoDbSourceStatus{
			SourceStatus: duckv1.SourceStatus{
				Status: duckv1.Status{
					Conditions: []apis.Condition{{
						Type:   MongoDbConditionConnectionEstablished,
						Status: corev1.ConditionUnknown,
					}, {
						Type:   MongoDbConditionDeployed,
						Status: corev1.ConditionUnknown,
					}, {
						Type:   MongoDbConditionReady,
						Status: corev1.ConditionUnknown,
					}, {
						Type:   MongoDbConditionSinkProvided,
						Status: corev1.ConditionTrue,
					}},
				},
			},
		},
	}, {
		name: "marksink",
		ms: func() *MongoDbSourceStatus {
			status := MongoDbSourceStatus{}
			status.MarkSink(apis.HTTP("sink"))
			return &status
		}(),
		want: &MongoDbSourceStatus{
			SourceStatus: duckv1.SourceStatus{
				Status: duckv1.Status{
					Conditions: []apis.Condition{{
						Type:   MongoDbConditionConnectionEstablished,
						Status: corev1.ConditionUnknown,
					}, {
						Type:   MongoDbConditionDeployed,
						Status: corev1.ConditionUnknown,
					}, {
						Type:   MongoDbConditionReady,
						Status: corev1.ConditionUnknown,
					}, {
						Type:   MongoDbConditionSinkProvided,
						Status: corev1.ConditionTrue,
					}},
				},
				SinkURI: apis.HTTP("sink"),
			},
		},
	}, {
		name: "marknosink",
		ms: func() *MongoDbSourceStatus {
			status := MongoDbSourceStatus{}
			status.MarkNoSink("nothere", "")
			return &status
		}(),
		want: &MongoDbSourceStatus{
			SourceStatus: duckv1.SourceStatus{
				Status: duckv1.Status{
					Conditions: []apis.Condition{{
						Type:   MongoDbConditionConnectionEstablished,
						Status: corev1.ConditionUnknown,
					}, {
						Type:   MongoDbConditionDeployed,
						Status: corev1.ConditionUnknown,
					}, {
						Type:   MongoDbConditionReady,
						Status: corev1.ConditionFalse,
					}, {
						Type:   MongoDbConditionSinkProvided,
						Status: corev1.ConditionFalse,
					}},
				},
			},
		},
	}}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			test.ms.InitializeConditions()
			ignore := cmpopts.IgnoreFields(
				apis.Condition{},
				"LastTransitionTime", "Message", "Reason", "Severity")
			if diff := cmp.Diff(test.want, test.ms, ignore); diff != "" {
				t.Errorf("unexpected conditions (-want, +got) = %v", diff)
			}
		})
	}
}
<|endoftext|>


<commit_before>package grpcutil

import (
	"errors"
	"fmt"
	"net/url"
	"strconv"
	"strings"
)

const (
	// DefaultPachdNodePort is the pachd kubernetes service's default
	// NodePort.Port setting
	DefaultPachdNodePort = 30650

	// DefaultPachdPort is the pachd kubernetes service's default
	// Port (often used with Pachyderm ELBs)
	DefaultPachdPort = 650
)

var (
	// ErrNoPachdAddress is returned by ParsePachdAddress when the input is an
	// empty string
	ErrNoPachdAddress = errors.New("no pachd address specified")
	// DefaultPachdAddress is the default PachdAddress that should be used
	// if none is otherwise specified. It's a loopback that should rely on
	// port forwarding.
	DefaultPachdAddress = PachdAddress{
		Secured: false,
		Host:    "0.0.0.0",
		Port:    DefaultPachdNodePort,
	}
)

// PachdAddress represents a parsed pachd address value
type PachdAddress struct {
	// Secured specifies whether grpcs should be used
	Secured bool
	// Host specifies the pachd address host without the port
	Host string
	// Port specifies the pachd port
	Port uint16
}

// ParsePachdAddress parses a string into a pachd address, or returns an error
// if it's invalid
func ParsePachdAddress(value string) (*PachdAddress, error) {
	if value == "" {
		return nil, ErrNoPachdAddress
	}

	secured := false

	if strings.Contains(value, "://") {
		// only parse the url if it contains a scheme, as net/url doesn't
		// appropriately handle values without one

		u, err := url.Parse(value)
		if err != nil {
			return nil, fmt.Errorf("could not parse pachd address: %v", err)
		}

		if u.Scheme != "grpc" && u.Scheme != "grpcs" && u.Scheme != "http" && u.Scheme != "https" {
			return nil, fmt.Errorf("unrecognized scheme in pachd address: %s", u.Scheme)
		}
		if u.User != nil {
			return nil, errors.New("pachd address should not include login credentials")
		}
		if u.RawQuery != "" {
			return nil, errors.New("pachd address should not include a query string")
		}
		if u.Fragment != "" {
			return nil, errors.New("pachd address should not include a fragment")
		}
		if u.Path != "" {
			return nil, errors.New("pachd address should not include a path")
		}

		value = u.Host
		secured = u.Scheme == "grpcs" || u.Scheme == "https"
	}

	// port always starts after last colon, but net.SplitHostPort returns an
	// error on a hostport without a colon, which this might be
	colonIdx := strings.LastIndexByte(value, ':')
	host := value
	port := uint16(DefaultPachdNodePort)
	if colonIdx >= 0 {
		maybePort, err := strconv.ParseUint(value[colonIdx+1:], 10, 16)
		if err == nil {
			host = value[:colonIdx]
			port = uint16(maybePort)
		}
	}

	return &PachdAddress{
		Secured: secured,
		Host:    host,
		Port:    port,
	}, nil
}

// Qualified returns the "fully qualified" address, including the scheme
func (p *PachdAddress) Qualified() string {
	if p.Secured {
		return fmt.Sprintf("grpcs://%s:%d", p.Host, p.Port)
	}
	return fmt.Sprintf("grpc://%s:%d", p.Host, p.Port)
}

// Hostname returns the host:port combination of the pachd address, without
// the scheme
func (p *PachdAddress) Hostname() string {
	return fmt.Sprintf("%s:%d", p.Host, p.Port)
}

// IsUnusualPort returns true if the pachd address port is not one of the
// usual values
func (p *PachdAddress) IsUnusualPort() bool {
	return p.Port != DefaultPachdNodePort && p.Port != DefaultPachdPort
}

// IsLoopback returns whether the pachd address is referencing the loopback
// hostname
func (p *PachdAddress) IsLoopback() bool {
	return p.Host == "0.0.0.0" || p.Host == "127.0.0.1" || p.Host == "[::1]" || p.Host == "localhost"
}
<commit_msg>Use switch statements to check validity of values<commit_after>package grpcutil

import (
	"errors"
	"fmt"
	"net/url"
	"strconv"
	"strings"
)

const (
	// DefaultPachdNodePort is the pachd kubernetes service's default
	// NodePort.Port setting
	DefaultPachdNodePort = 30650

	// DefaultPachdPort is the pachd kubernetes service's default
	// Port (often used with Pachyderm ELBs)
	DefaultPachdPort = 650
)

var (
	// ErrNoPachdAddress is returned by ParsePachdAddress when the input is an
	// empty string
	ErrNoPachdAddress = errors.New("no pachd address specified")
	// DefaultPachdAddress is the default PachdAddress that should be used
	// if none is otherwise specified. It's a loopback that should rely on
	// port forwarding.
	DefaultPachdAddress = PachdAddress{
		Secured: false,
		Host:    "0.0.0.0",
		Port:    DefaultPachdNodePort,
	}
)

// PachdAddress represents a parsed pachd address value
type PachdAddress struct {
	// Secured specifies whether grpcs should be used
	Secured bool
	// Host specifies the pachd address host without the port
	Host string
	// Port specifies the pachd port
	Port uint16
}

// ParsePachdAddress parses a string into a pachd address, or returns an error
// if it's invalid
func ParsePachdAddress(value string) (*PachdAddress, error) {
	if value == "" {
		return nil, ErrNoPachdAddress
	}

	secured := false

	if strings.Contains(value, "://") {
		// only parse the url if it contains a scheme, as net/url doesn't
		// appropriately handle values without one

		u, err := url.Parse(value)
		if err != nil {
			return nil, fmt.Errorf("could not parse pachd address: %v", err)
		}

		switch u.Scheme {
		case "grpc", "grpcs", "http", "https":
		default:
			return nil, fmt.Errorf("unrecognized scheme in pachd address: %s", u.Scheme)
		}

		switch {
		case u.User != nil:
			return nil, errors.New("pachd address should not include login credentials")
		case u.RawQuery != "":
			return nil, errors.New("pachd address should not include a query string")
		case u.Fragment != "":
			return nil, errors.New("pachd address should not include a fragment")
		case u.Path != "":
			return nil, errors.New("pachd address should not include a path")
		}

		value = u.Host
		secured = u.Scheme == "grpcs" || u.Scheme == "https"
	}

	// port always starts after last colon, but net.SplitHostPort returns an
	// error on a hostport without a colon, which this might be
	colonIdx := strings.LastIndexByte(value, ':')
	host := value
	port := uint16(DefaultPachdNodePort)
	if colonIdx >= 0 {
		maybePort, err := strconv.ParseUint(value[colonIdx+1:], 10, 16)
		if err == nil {
			host = value[:colonIdx]
			port = uint16(maybePort)
		}
	}

	return &PachdAddress{
		Secured: secured,
		Host:    host,
		Port:    port,
	}, nil
}

// Qualified returns the "fully qualified" address, including the scheme
func (p *PachdAddress) Qualified() string {
	if p.Secured {
		return fmt.Sprintf("grpcs://%s:%d", p.Host, p.Port)
	}
	return fmt.Sprintf("grpc://%s:%d", p.Host, p.Port)
}

// Hostname returns the host:port combination of the pachd address, without
// the scheme
func (p *PachdAddress) Hostname() string {
	return fmt.Sprintf("%s:%d", p.Host, p.Port)
}

// IsUnusualPort returns true if the pachd address port is not one of the
// usual values
func (p *PachdAddress) IsUnusualPort() bool {
	return p.Port != DefaultPachdNodePort && p.Port != DefaultPachdPort
}

// IsLoopback returns whether the pachd address is referencing the loopback
// hostname
func (p *PachdAddress) IsLoopback() bool {
	return p.Host == "0.0.0.0" || p.Host == "127.0.0.1" || p.Host == "[::1]" || p.Host == "localhost"
}
<|endoftext|>