<commit_before>package integration_test

import (
	"net/http"
	"net/url"
	"os"
	"strings"
	"testing"

	. "launchpad.net/gocheck"
)

type R struct{}

var _ = Suite(&R{})

func (*R) SetUpTest(c *C) {
	setupTest(c)

	server := os.Getenv("GONUTS_IO_SERVER")
	u, err := url.Parse("http://" + server + "/debug/prepare_test")
	c.Assert(err, IsNil)
	res, err := http.Get(u.String())
	c.Assert(err, IsNil)
	res.Body.Close()
	c.Assert(res.StatusCode, Equals, 200)
}

func (r *R) TearDownTest(c *C) {
	r.SetUpTest(c)
}

func (*R) TestPublishGet(c *C) {
	if testing.Short() {
		c.Skip("-short passed")
		return
	}

	_, stderr := runNut(c, TestNut1, "pack -v")
	c.Check(strings.HasSuffix(stderr, `test_nut1-0.0.1.nut created.`), Equals, true)
	gitNoDiff(c, TestNut1)

	_, stderr = runNut(c, TestNut2, "pack -v")
	c.Check(strings.HasSuffix(stderr, `test_nut2-0.0.2.nut created.`), Equals, true)
	gitNoDiff(c, TestNut2)

	_, stderr = runNut(c, TestNut1, "publish -v test_nut1-0.0.1.nut")
	c.Check(strings.HasSuffix(stderr, `Nut debug/test_nut1 version 0.0.1 published.`), Equals, true)
	gitNoDiff(c, TestNut1)

	_, stderr = runNut(c, TestNut1, "publish -v test_nut1-0.0.1.nut", 1)
	c.Check(strings.HasSuffix(stderr, `Nut debug/test_nut1 version 0.0.1 already exists.`), Equals, true)
	gitNoDiff(c, TestNut1)

	_, stderr = runNut(c, TestNut2, "publish -v test_nut2-0.0.2.nut")
	c.Check(strings.HasSuffix(stderr, `Nut debug/test_nut2 version 0.0.2 published.`), Equals, true)
	gitNoDiff(c, TestNut2)

	// _, stderr = runNut(c, "", "get -v test_nut2/0.0.2")
	// c.Check(strings.HasSuffix(stderr, `Nut "test_nut2" version "0.0.2" published.`), Equals, true)
}
<commit_msg>Fix test skipping.<commit_after>package integration_test

import (
	"net/http"
	"net/url"
	"os"
	"strings"
	"testing"

	. "launchpad.net/gocheck"
)

type R struct{}

var _ = Suite(&R{})

func (*R) SetUpTest(c *C) {
	if testing.Short() {
		c.Skip("-short passed")
		return
	}

	setupTest(c)

	server := os.Getenv("GONUTS_IO_SERVER")
	u, err := url.Parse("http://" + server + "/debug/prepare_test")
	c.Assert(err, IsNil)
	res, err := http.Get(u.String())
	c.Assert(err, IsNil)
	res.Body.Close()
	c.Assert(res.StatusCode, Equals, 200)
}

func (r *R) TearDownTest(c *C) {
	r.SetUpTest(c)
}

func (*R) TestPublishGet(c *C) {
	_, stderr := runNut(c, TestNut1, "pack -v")
	c.Check(strings.HasSuffix(stderr, `test_nut1-0.0.1.nut created.`), Equals, true)
	gitNoDiff(c, TestNut1)

	_, stderr = runNut(c, TestNut2, "pack -v")
	c.Check(strings.HasSuffix(stderr, `test_nut2-0.0.2.nut created.`), Equals, true)
	gitNoDiff(c, TestNut2)

	_, stderr = runNut(c, TestNut1, "publish -v test_nut1-0.0.1.nut")
	c.Check(strings.HasSuffix(stderr, `Nut debug/test_nut1 version 0.0.1 published.`), Equals, true)
	gitNoDiff(c, TestNut1)

	_, stderr = runNut(c, TestNut1, "publish -v test_nut1-0.0.1.nut", 1)
	c.Check(strings.HasSuffix(stderr, `Nut debug/test_nut1 version 0.0.1 already exists.`), Equals, true)
	gitNoDiff(c, TestNut1)

	_, stderr = runNut(c, TestNut2, "publish -v test_nut2-0.0.2.nut")
	c.Check(strings.HasSuffix(stderr, `Nut debug/test_nut2 version 0.0.2 published.`), Equals, true)
	gitNoDiff(c, TestNut2)

	// _, stderr = runNut(c, "", "get -v test_nut2/0.0.2")
	// c.Check(strings.HasSuffix(stderr, `Nut "test_nut2" version "0.0.2" published.`), Equals, true)
}
<|endoftext|>


<commit_before>package dh_test

import (
	"github.com/devicehive/devicehive-go/dh"
	"github.com/matryer/is"
	"testing"
)

func TestUserCreationAndObtaining(t *testing.T) {
	is := is.New(t)

	user, err := client.CreateUser("go-test", "go-test", 1, nil, false)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err = user.Remove()
		if err != nil {
			t.Fatal(err)
		}
	}()

	is.True(user != nil)
	is.True(user.Id != 0)

	sameUser, err := client.GetUser(user.Id)
	if err != nil {
		t.Fatal(err)
	}

	is.True(sameUser != nil)

	currentUser, err := client.GetCurrentUser()
	if err != nil {
		t.Fatal(err)
	}

	is.True(currentUser != nil)
	is.Equal(currentUser.Id, int64(*userId))

	list, err := client.ListUsers(&dh.ListParams{
		UserStatus: 0,
	})
	if err != nil {
		t.Fatal(err)
	}

	is.True(len(list) > 0)
}

func TestUser(t *testing.T) {
	is := is.New(t)

	user, err := client.CreateUser("go-test", "go-test", 1, nil, false)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err = user.Remove()
		if err != nil {
			t.Fatal(err)
		}
	}()

	user.Data = map[string]interface{}{
		"test": "test",
	}

	err = user.Save()
	if err != nil {
		t.Fatal(err)
	}

	err = user.UpdatePassword("brand_new_password")
	if err != nil {
		t.Fatal(err)
	}

	network, err := client.CreateNetwork("go-test-user-network", "test")
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		network.Remove()
		if err != nil {
			t.Fatal(err)
		}
	}()

	err = user.AssignNetwork(network.Id)
	if err != nil {
		t.Fatal(err)
	}

	networkList, err := user.ListNetworks()
	if err != nil {
		t.Fatal(err)
	}

	is.Equal(len(networkList), 1)
	is.Equal(networkList[0].Name, "go-test-user-network")

	err = user.UnassignNetwork(network.Id)
	if err != nil {
		t.Fatal(err)
	}

	networkList, err = user.ListNetworks()
	if err != nil {
		t.Fatal(err)
	}

	is.Equal(len(networkList), 0)

	devType, err := client.CreateDeviceType("go-test-user-device-type", "test")
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err = devType.Remove()
		if err != nil {
			t.Fatal(err)
		}
	}()

	err = user.AssignDeviceType(devType.Id)
	if err != nil {
		t.Fatal(err)
	}

	devTypeList, err := user.ListDeviceTypes()
	if err != nil {
		t.Fatal(err)
	}

	is.Equal(len(devTypeList), 1)
	is.Equal(devTypeList[0].Name, "go-test-user-device-type")

	err = user.UnassignDeviceType(devType.Id)
	if err != nil {
		t.Fatal(err)
	}

	devTypeList, err = user.ListDeviceTypes()
	if err != nil {
		t.Fatal(err)
	}

	is.Equal(len(devTypeList), 0)

	err = user.AllowAllDeviceTypes()
	if err != nil {
		t.Fatal(err)
	}

	is.True(user.AllDeviceTypesAvailable)

	err = user.DisallowAllDeviceTypes()
	if err != nil {
		t.Fatal(err)
	}

	is.Equal(user.AllDeviceTypesAvailable, false)
}
<commit_msg>Refactor test for User model<commit_after>package dh_test

import (
	"github.com/devicehive/devicehive-go/dh"
	"github.com/matryer/is"
	"testing"
)

func TestUserCreationAndObtaining(t *testing.T) {
	is := is.New(t)

	user, err := client.CreateUser("go-test", "go-test", 1, nil, false)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err = user.Remove()
		if err != nil {
			t.Fatal(err)
		}
	}()

	is.True(user != nil)
	is.True(user.Id != 0)

	sameUser, err := client.GetUser(user.Id)
	if err != nil {
		t.Fatal(err)
	}

	is.True(sameUser != nil)

	currentUser, err := client.GetCurrentUser()
	if err != nil {
		t.Fatal(err)
	}

	is.True(currentUser != nil)
	is.Equal(currentUser.Id, int64(*userId))

	list, err := client.ListUsers(&dh.ListParams{
		UserStatus: 0,
	})
	if err != nil {
		t.Fatal(err)
	}

	is.True(len(list) > 0)
}

func TestUser(t *testing.T) {
	user, err := client.CreateUser("go-test", "go-test", 1, nil, false)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err = user.Remove()
		if err != nil {
			t.Fatal(err)
		}
	}()

	user.Data = map[string]interface{}{
		"test": "test",
	}

	err = user.Save()
	if err != nil {
		t.Fatal(err)
	}

	err = user.UpdatePassword("brand_new_password")
	if err != nil {
		t.Fatal(err)
	}
}

func TestUserNetworks(t *testing.T) {
	is := is.New(t)

	user, err := client.CreateUser("go-test", "go-test", 1, nil, false)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err = user.Remove()
		if err != nil {
			t.Fatal(err)
		}
	}()

	network, err := client.CreateNetwork("go-test-user-network", "test")
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		network.Remove()
		if err != nil {
			t.Fatal(err)
		}
	}()

	err = user.AssignNetwork(network.Id)
	if err != nil {
		t.Fatal(err)
	}

	networkList, err := user.ListNetworks()
	if err != nil {
		t.Fatal(err)
	}

	is.Equal(len(networkList), 1)
	is.Equal(networkList[0].Name, "go-test-user-network")

	err = user.UnassignNetwork(network.Id)
	if err != nil {
		t.Fatal(err)
	}

	networkList, err = user.ListNetworks()
	if err != nil {
		t.Fatal(err)
	}

	is.Equal(len(networkList), 0)
}

func TestUserDeviceTypes(t *testing.T) {
	is := is.New(t)

	user, err := client.CreateUser("go-test", "go-test", 1, nil, false)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err = user.Remove()
		if err != nil {
			t.Fatal(err)
		}
	}()

	devType, err := client.CreateDeviceType("go-test-user-device-type", "test")
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		err = devType.Remove()
		if err != nil {
			t.Fatal(err)
		}
	}()

	err = user.AssignDeviceType(devType.Id)
	if err != nil {
		t.Fatal(err)
	}

	devTypeList, err := user.ListDeviceTypes()
	if err != nil {
		t.Fatal(err)
	}

	is.Equal(len(devTypeList), 1)
	is.Equal(devTypeList[0].Name, "go-test-user-device-type")

	err = user.UnassignDeviceType(devType.Id)
	if err != nil {
		t.Fatal(err)
	}

	devTypeList, err = user.ListDeviceTypes()
	if err != nil {
		t.Fatal(err)
	}

	is.Equal(len(devTypeList), 0)

	err = user.AllowAllDeviceTypes()
	if err != nil {
		t.Fatal(err)
	}

	is.True(user.AllDeviceTypesAvailable)

	err = user.DisallowAllDeviceTypes()
	if err != nil {
		t.Fatal(err)
	}

	is.Equal(user.AllDeviceTypesAvailable, false)
}
<|endoftext|>


<commit_before>package api

import (
	"bytes"
	"strings"
	"text/template"

	"github.com/awslabs/aws-sdk-go/internal/util"
)

type Operation struct {
	API           *API `json: "-"`
	ExportedName  string
	Name          string
	Documentation string
	HTTP          HTTPInfo
	InputRef      ShapeRef `json:"input"`
	OutputRef     ShapeRef `json:"output"`
}

type HTTPInfo struct {
	Method       string
	RequestURI   string
	ResponseCode uint
}

func (o *Operation) HasInput() bool {
	return o.InputRef.ShapeName != ""
}

func (o *Operation) HasOutput() bool {
	return o.OutputRef.ShapeName != ""
}

var tplOperation = template.Must(template.New("operation").Parse(`
// {{ .ExportedName }}Request generates a request for the {{ .ExportedName }} operation.
func (c *{{ .API.StructName }}) {{ .ExportedName }}Request(` +
	`{{ if .HasInput }}input {{ .InputRef.GoType }}{{ end }}) ` +
	`(req *aws.Request{{ if .HasOutput }}, output {{ .OutputRef.GoType }}{{ end }}) {
	if op{{ .ExportedName }} == nil {
		op{{ .ExportedName }} = &aws.Operation{
			Name:       "{{ .Name }}",
			HTTPMethod: "{{ .HTTP.Method }}",
			HTTPPath:   "{{ .HTTP.RequestURI }}",
		}
	}

	req = aws.NewRequest(c.Service, op{{ .ExportedName }}, ` +
	`{{ if .HasInput }}input{{ else }}nil{{ end }}, {{ if .HasOutput }}output{{ else }}nil{{ end }})
	{{ if .HasOutput }}output = &{{ .OutputRef.GoTypeElem }}{}
	req.Data = output{{ end }}
	return
}

func (c *{{ .API.StructName }}) {{ .ExportedName }}(` +
	`{{ if .HasInput }}input {{ .InputRef.GoType }}{{ end }}) ` +
	`({{ if .HasOutput }}output {{ .OutputRef.GoType }},{{ end }} err error) {
	req{{ if .HasOutput }}, out{{ end }} := c.{{ .ExportedName }}Request({{ if .HasInput }}input{{ end }})
	{{ if .HasOutput }}output = out
	{{ end }}err = req.Send()
	return
}

var op{{ .ExportedName }} *aws.Operation
`))

func (o *Operation) GoCode() string {
	var buf bytes.Buffer
	err := tplOperation.Execute(&buf, o)
	if err != nil {
		panic(err)
	}

	return strings.TrimSpace(util.GoFmt(buf.String()))
}
<commit_msg>Don't generate HTTP bindings if not present in model<commit_after>package api

import (
	"bytes"
	"strings"
	"text/template"

	"github.com/awslabs/aws-sdk-go/internal/util"
)

type Operation struct {
	API           *API `json: "-"`
	ExportedName  string
	Name          string
	Documentation string
	HTTP          HTTPInfo
	InputRef      ShapeRef `json:"input"`
	OutputRef     ShapeRef `json:"output"`
}

type HTTPInfo struct {
	Method       string
	RequestURI   string
	ResponseCode uint
}

func (o *Operation) HasInput() bool {
	return o.InputRef.ShapeName != ""
}

func (o *Operation) HasOutput() bool {
	return o.OutputRef.ShapeName != ""
}

var tplOperation = template.Must(template.New("operation").Parse(`
// {{ .ExportedName }}Request generates a request for the {{ .ExportedName }} operation.
func (c *{{ .API.StructName }}) {{ .ExportedName }}Request(` +
	`{{ if .HasInput }}input {{ .InputRef.GoType }}{{ end }}) ` +
	`(req *aws.Request{{ if .HasOutput }}, output {{ .OutputRef.GoType }}{{ end }}) {
	if op{{ .ExportedName }} == nil {
		op{{ .ExportedName }} = &aws.Operation{
			Name:       "{{ .Name }}",
			{{ if ne .HTTP.Method "" }}HTTPMethod: "{{ .HTTP.Method }}",{{ end }}
			{{ if ne .HTTP.RequestURI "" }}HTTPPath:   "{{ .HTTP.RequestURI }}",{{ end }}
		}
	}

	req = aws.NewRequest(c.Service, op{{ .ExportedName }}, ` +
	`{{ if .HasInput }}input{{ else }}nil{{ end }}, {{ if .HasOutput }}output{{ else }}nil{{ end }})
	{{ if .HasOutput }}output = &{{ .OutputRef.GoTypeElem }}{}
	req.Data = output{{ end }}
	return
}

func (c *{{ .API.StructName }}) {{ .ExportedName }}(` +
	`{{ if .HasInput }}input {{ .InputRef.GoType }}{{ end }}) ` +
	`({{ if .HasOutput }}output {{ .OutputRef.GoType }},{{ end }} err error) {
	req{{ if .HasOutput }}, out{{ end }} := c.{{ .ExportedName }}Request({{ if .HasInput }}input{{ end }})
	{{ if .HasOutput }}output = out
	{{ end }}err = req.Send()
	return
}

var op{{ .ExportedName }} *aws.Operation
`))

func (o *Operation) GoCode() string {
	var buf bytes.Buffer
	err := tplOperation.Execute(&buf, o)
	if err != nil {
		panic(err)
	}

	return strings.TrimSpace(util.GoFmt(buf.String()))
}
<|endoftext|>


<commit_before>package object

import (
	"errors"
	"github.com/qiniu/go-sdk/v7/auth"
	"github.com/qiniu/go-sdk/v7/storage"
	"github.com/qiniu/qshell/v2/iqshell/common/alert"
	"github.com/qiniu/qshell/v2/iqshell/common/log"
	"github.com/qiniu/qshell/v2/iqshell/common/utils"
	"github.com/qiniu/qshell/v2/iqshell/common/workspace"
	"github.com/qiniu/qshell/v2/iqshell/storage/bucket"
)

type FetchApiInfo struct {
	Bucket  string
	Key     string
	FromUrl string
}

type FetchResult storage.FetchRet

func Fetch(info FetchApiInfo) (FetchResult, error) {
	if len(info.Bucket) == 0 {
		return FetchResult{}, errors.New(alert.CannotEmpty("bucket", ""))
	}

	if len(info.Key) == 0 {
		key, err := utils.KeyFromUrl(info.FromUrl)
		if err != nil || len(key) == 0 {
			return FetchResult{}, errors.New("get key from url failed:" + err.Error())
		}
		info.Key = key
	}

	if len(info.FromUrl) == 0 {
		return FetchResult{}, errors.New(alert.CannotEmpty("from url", ""))
	}

	log.DebugF("fetch start: %s => [%s|%s]", info.FromUrl, info.Bucket, info.Key)
	bucketManager, err := bucket.GetBucketManager()
	if err != nil {
		return FetchResult{}, err
	}
	fetchResult, err := bucketManager.Fetch(info.FromUrl, info.Bucket, info.Key)
	log.DebugF("fetch   end: %s => [%s|%s]", info.FromUrl, info.Bucket, info.Key)
	return FetchResult(fetchResult), err
}

type AsyncFetchApiInfo storage.AsyncFetchParam
type AsyncFetchApiResult storage.AsyncFetchRet

func AsyncFetch(info AsyncFetchApiInfo) (AsyncFetchApiResult, error) {
	bm, err := bucket.GetBucketManager()
	if err != nil {
		return AsyncFetchApiResult{}, err
	}
	ret, err := bm.AsyncFetch(storage.AsyncFetchParam(info))
	return AsyncFetchApiResult(ret), err
}

func CheckAsyncFetchStatus(toBucket, id string) (ret AsyncFetchApiResult, err error) {
	bm, gErr := bucket.GetBucketManager()
	if gErr != nil {
		err = gErr
		return
	}

	reqUrl, aErr := bm.ApiReqHost(toBucket)
	if aErr != nil {
		err = aErr
		return
	}

	mac, gErr := workspace.GetMac()
	if gErr != nil {
		err = gErr
		return
	}

	reqUrl += ("/sisyphus/fetch?id=" + id)
	ctx := auth.WithCredentialsType(workspace.GetContext(), mac, auth.TokenQiniu)
	err = bm.Client.Call(ctx, &ret, "GET", reqUrl, nil)
	return
}
<commit_msg>fetch & batch fetch add test case<commit_after>package object

import (
	"errors"
	"fmt"
	"github.com/qiniu/go-sdk/v7/auth"
	"github.com/qiniu/go-sdk/v7/storage"
	"github.com/qiniu/qshell/v2/iqshell/common/alert"
	"github.com/qiniu/qshell/v2/iqshell/common/log"
	"github.com/qiniu/qshell/v2/iqshell/common/utils"
	"github.com/qiniu/qshell/v2/iqshell/common/workspace"
	"github.com/qiniu/qshell/v2/iqshell/storage/bucket"
)

type FetchApiInfo struct {
	Bucket  string
	Key     string
	FromUrl string
}

type FetchResult storage.FetchRet

func Fetch(info FetchApiInfo) (FetchResult, error) {
	if len(info.Bucket) == 0 {
		return FetchResult{}, errors.New(alert.CannotEmpty("bucket", ""))
	}

	if len(info.Key) == 0 {
		key, err := utils.KeyFromUrl(info.FromUrl)
		if err != nil || len(key) == 0 {
			return FetchResult{}, fmt.Errorf("get key from url failed:%s error:%v", info.FromUrl, err)
		}
		info.Key = key
	}

	if len(info.FromUrl) == 0 {
		return FetchResult{}, errors.New(alert.CannotEmpty("from url", ""))
	}

	log.DebugF("fetch start: %s => [%s|%s]", info.FromUrl, info.Bucket, info.Key)
	bucketManager, err := bucket.GetBucketManager()
	if err != nil {
		return FetchResult{}, err
	}
	fetchResult, err := bucketManager.Fetch(info.FromUrl, info.Bucket, info.Key)
	log.DebugF("fetch   end: %s => [%s|%s]", info.FromUrl, info.Bucket, info.Key)
	return FetchResult(fetchResult), err
}

type AsyncFetchApiInfo storage.AsyncFetchParam
type AsyncFetchApiResult storage.AsyncFetchRet

func AsyncFetch(info AsyncFetchApiInfo) (AsyncFetchApiResult, error) {
	bm, err := bucket.GetBucketManager()
	if err != nil {
		return AsyncFetchApiResult{}, err
	}
	ret, err := bm.AsyncFetch(storage.AsyncFetchParam(info))
	return AsyncFetchApiResult(ret), err
}

func CheckAsyncFetchStatus(toBucket, id string) (ret AsyncFetchApiResult, err error) {
	bm, gErr := bucket.GetBucketManager()
	if gErr != nil {
		err = gErr
		return
	}

	reqUrl, aErr := bm.ApiReqHost(toBucket)
	if aErr != nil {
		err = aErr
		return
	}

	mac, gErr := workspace.GetMac()
	if gErr != nil {
		err = gErr
		return
	}

	reqUrl += ("/sisyphus/fetch?id=" + id)
	ctx := auth.WithCredentialsType(workspace.GetContext(), mac, auth.TokenQiniu)
	err = bm.Client.Call(ctx, &ret, "GET", reqUrl, nil)
	return
}
<|endoftext|>


<commit_before>package shp

import (
	"fmt"
	"math"
	"reflect"

	"github.com/ctessum/geom"
	"github.com/ctessum/geom/op"
	"github.com/jonas-p/go-shp"
)

// Shp2Geom converts a shapefile shape to a geometry
// object that can be used with other packages.
// This function can be used to wrap the go-shp "Shape()" method.
func shp2Geom(n int, s shp.Shape) (int, geom.T, error) {
	switch t := reflect.TypeOf(s); {
	case t == reflect.TypeOf(&shp.Point{}):
		return n, point2geom(*s.(*shp.Point)), nil
	case t == reflect.TypeOf(&shp.PointM{}):
		return n, pointM2geom(*s.(*shp.PointM)), nil
	case t == reflect.TypeOf(&shp.PointZ{}):
		return n, pointZ2geom(*s.(*shp.PointZ)), nil
	case t == reflect.TypeOf(&shp.Polygon{}):
		return n, polygon2geom(*s.(*shp.Polygon)), nil
	case t == reflect.TypeOf(&shp.PolygonM{}):
		return n, polygonM2geom(*s.(*shp.PolygonM)), nil
	case t == reflect.TypeOf(&shp.PolygonZ{}):
		return n, polygonZ2geom(*s.(*shp.PolygonZ)), nil
	case t == reflect.TypeOf(&shp.PolyLine{}):
		return n, polyLine2geom(*s.(*shp.PolyLine)), nil
	case t == reflect.TypeOf(&shp.PolyLineM{}):
		return n, polyLineM2geom(*s.(*shp.PolyLineM)), nil
	case t == reflect.TypeOf(&shp.PolyLineZ{}):
		return n, polyLineZ2geom(*s.(*shp.PolyLineZ)), nil
	//case t == "MultiPatch": // not yet supported
	case t == reflect.TypeOf(&shp.MultiPoint{}):
		return n, multiPoint2geom(*s.(*shp.MultiPoint)), nil
	case t == reflect.TypeOf(&shp.MultiPointM{}):
		return n, multiPointM2geom(*s.(*shp.MultiPointM)), nil
	case t == reflect.TypeOf(&shp.MultiPointZ{}):
		return n, multiPointZ2geom(*s.(*shp.MultiPointZ)), nil
	case t == reflect.TypeOf(&shp.Null{}):
		return n, nil, nil
	default:
		return n, nil, fmt.Errorf("Unsupported shape type: %v", t)
	}
}

// Functions for converting shp to geom

func point2geom(s shp.Point) geom.T {
	return geom.Point(s)
}
func pointM2geom(s shp.PointM) geom.T {
	return geom.Point{s.X, s.Y}
}
func pointZ2geom(s shp.PointZ) geom.T {
	return geom.Point{s.X, s.Y}
}
func getStartEnd(parts []int32, points []shp.Point, i int) (start, end int) {
	start = int(parts[i])
	if i == len(parts)-1 {
		end = len(points)
	} else {
		end = int(parts[i+1])
	}
	return
}
func polygon2geom(s shp.Polygon) geom.T {
	var pg geom.Polygon = make([][]geom.Point, len(s.Parts))
	for i := 0; i < len(s.Parts); i++ {
		start, end := getStartEnd(s.Parts, s.Points, i)
		pg[i] = make([]geom.Point, end-start)
		// Go backwards around the rings to switch to OGC format
		for j := end - 1; j >= start; j-- {
			pg[i][j-start] = geom.Point(s.Points[j])
		}
	}
	// Make sure the winding direction is correct
	op.FixOrientation(pg)
	return pg
}
func polygonM2geom(s shp.PolygonM) geom.T {
	var pg geom.Polygon = make([][]geom.Point, len(s.Parts))
	jj := 0
	for i := 0; i < len(s.Parts); i++ {
		start, end := getStartEnd(s.Parts, s.Points, i)
		jj += end - start
		pg[i] = make([]geom.Point, end-start)
		// Go backwards around the rings to switch to OGC format
		for j := end - 1; j >= start; j-- {
			ss := s.Points[j]
			pg[i][j-start] = geom.Point{ss.X, ss.Y} //, s.MArray[jj]}
			jj--
		}
	}
	// Make sure the winding direction is correct
	op.FixOrientation(pg)
	return pg
}

func polygonZ2geom(s shp.PolygonZ) geom.T {
	var pg geom.Polygon = make([][]geom.Point, len(s.Parts))
	jj := -1
	for i := 0; i < len(s.Parts); i++ {
		start, end := getStartEnd(s.Parts, s.Points, i)
		jj += end - start
		pg[i] = make([]geom.Point, end-start)
		// Go backwards around the rings to switch to OGC format
		for j := end - 1; j >= start; j-- {
			ss := s.Points[j]
			pg[i][j-start] = geom.Point{ss.X, ss.Y} //, s.ZArray[jj], s.MArray[jj]}
			jj--
		}
	}
	// Make sure the winding direction is correct
	op.FixOrientation(pg)
	return pg
}
func polyLine2geom(s shp.PolyLine) geom.T {
	var pl geom.MultiLineString = make([]geom.LineString, len(s.Parts))
	for i := 0; i < len(s.Parts); i++ {
		start, end := getStartEnd(s.Parts, s.Points, i)
		pl[i] = make([]geom.Point, end-start)
		for j := start; j < end; j++ {
			pl[i][j-start] = geom.Point(s.Points[j])
		}
	}
	return pl
}
func polyLineM2geom(s shp.PolyLineM) geom.T {
	var pl geom.MultiLineString = make([]geom.LineString, len(s.Parts))
	jj := 0
	for i := 0; i < len(s.Parts); i++ {
		start, end := getStartEnd(s.Parts, s.Points, i)
		pl[i] = make([]geom.Point, end-start)
		for j := start; j < end; j++ {
			ss := s.Points[j]
			pl[i][j-start] =
				geom.Point{ss.X, ss.Y} //, s.MArray[jj]}
			jj++
		}
	}
	return pl
}
func polyLineZ2geom(s shp.PolyLineZ) geom.T {
	var pl geom.MultiLineString = make([]geom.LineString, len(s.Parts))
	jj := 0
	for i := 0; i < len(s.Parts); i++ {
		start, end := getStartEnd(s.Parts, s.Points, i)
		pl[i] = make([]geom.Point, end-start)
		for j := start; j < end; j++ {
			ss := s.Points[j]
			pl[i][j-start] =
				geom.Point{ss.X, ss.Y} //, s.ZArray[jj], s.MArray[jj]}
			jj++
		}
	}
	return pl
}
func multiPoint2geom(s shp.MultiPoint) geom.T {
	var mp geom.MultiPoint = make([]geom.Point, len(s.Points))
	for i, p := range s.Points {
		mp[i] = geom.Point(p)
	}
	return mp
}
func multiPointM2geom(s shp.MultiPointM) geom.T {
	var mp geom.MultiPoint = make([]geom.Point, len(s.Points))
	for i, p := range s.Points {
		mp[i] = geom.Point{p.X, p.Y} //, s.MArray[i]}
	}
	return mp
}
func multiPointZ2geom(s shp.MultiPointZ) geom.T {
	var mp geom.MultiPoint = make([]geom.Point, len(s.Points))
	for i, p := range s.Points {
		mp[i] = geom.Point{p.X, p.Y} //, s.ZArray[i], s.MArray[i]}
	}
	return mp
}

// Geom2Shp converts a geometry object to a shapefile shape.
func geom2Shp(g geom.T) (shp.Shape, error) {
	if g == nil {
		return &shp.Null{}, nil
	}
	switch t := g.(type) {
	case geom.Point:
		return geom2point(g.(geom.Point)), nil
	case geom.Polygon:
		return geom2polygon(g.(geom.Polygon)), nil
	case geom.MultiLineString:
		return geom2polyLine(g.(geom.MultiLineString)), nil
	//case t == "MultiPatch": // not yet supported
	case geom.MultiPoint:
		return geom2multiPoint(g.(geom.MultiPoint)), nil
	default:
		return nil, fmt.Errorf("Unsupported geom type: %v", t)
	}
}

// Functions for converting geom to shp

func geom2point(g geom.Point) shp.Shape {
	p := shp.Point(g)
	return &p
}
func geom2polygon(g geom.Polygon) shp.Shape {
	parts := make([][]shp.Point, len(g))
	for i, r := range g {
		parts[i] = make([]shp.Point, len(r))
		// switch the winding direction
		for j := len(r) - 1; j >= 0; j-- {
			parts[i][j] = shp.Point(r[j])
		}
	}
	p := shp.Polygon(*shp.NewPolyLine(parts))
	return &p
}
func valrange(a []float64) [2]float64 {
	out := [2]float64{math.Inf(1), math.Inf(-1)}
	for _, val := range a {
		if val < out[0] {
			out[0] = val
		}
		if val > out[1] {
			out[1] = val
		}
	}
	return out
}
func geom2polyLine(g geom.MultiLineString) shp.Shape {
	parts := make([][]shp.Point, len(g))
	for i, r := range g {
		parts[i] = make([]shp.Point, len(r))
		for j, l := range r {
			parts[i][j] = shp.Point(l)
		}
	}
	return shp.NewPolyLine(parts)
}
func geom2multiPoint(g geom.MultiPoint) shp.Shape {
	mp := new(shp.MultiPoint)
	mp.Box = bounds2box(g)
	mp.NumPoints = int32(len(g))
	mp.Points = make([]shp.Point, len(g))
	for i, p := range g {
		mp.Points[i] = shp.Point(p)
	}
	return mp
}
func bounds2box(g geom.T) shp.Box {
	b := g.Bounds(nil)
	return shp.Box{b.Min.X, b.Min.Y, b.Max.X, b.Max.Y}
}
<commit_msg>made fixing shp geom optional<commit_after>package shp

import (
	"fmt"
	"math"
	"reflect"

	"github.com/ctessum/geom"
	"github.com/ctessum/geom/op"
	"github.com/jonas-p/go-shp"
)

// FixOrientation specifies whether to automatically check and fix the
// orientation of polygons imported from shapefiles.
var FixOrientation = false

// Shp2Geom converts a shapefile shape to a geometry
// object that can be used with other packages.
// This function can be used to wrap the go-shp "Shape()" method.
func shp2Geom(n int, s shp.Shape) (int, geom.T, error) {
	switch t := reflect.TypeOf(s); {
	case t == reflect.TypeOf(&shp.Point{}):
		return n, point2geom(*s.(*shp.Point)), nil
	case t == reflect.TypeOf(&shp.PointM{}):
		return n, pointM2geom(*s.(*shp.PointM)), nil
	case t == reflect.TypeOf(&shp.PointZ{}):
		return n, pointZ2geom(*s.(*shp.PointZ)), nil
	case t == reflect.TypeOf(&shp.Polygon{}):
		return n, polygon2geom(*s.(*shp.Polygon)), nil
	case t == reflect.TypeOf(&shp.PolygonM{}):
		return n, polygonM2geom(*s.(*shp.PolygonM)), nil
	case t == reflect.TypeOf(&shp.PolygonZ{}):
		return n, polygonZ2geom(*s.(*shp.PolygonZ)), nil
	case t == reflect.TypeOf(&shp.PolyLine{}):
		return n, polyLine2geom(*s.(*shp.PolyLine)), nil
	case t == reflect.TypeOf(&shp.PolyLineM{}):
		return n, polyLineM2geom(*s.(*shp.PolyLineM)), nil
	case t == reflect.TypeOf(&shp.PolyLineZ{}):
		return n, polyLineZ2geom(*s.(*shp.PolyLineZ)), nil
	//case t == "MultiPatch": // not yet supported
	case t == reflect.TypeOf(&shp.MultiPoint{}):
		return n, multiPoint2geom(*s.(*shp.MultiPoint)), nil
	case t == reflect.TypeOf(&shp.MultiPointM{}):
		return n, multiPointM2geom(*s.(*shp.MultiPointM)), nil
	case t == reflect.TypeOf(&shp.MultiPointZ{}):
		return n, multiPointZ2geom(*s.(*shp.MultiPointZ)), nil
	case t == reflect.TypeOf(&shp.Null{}):
		return n, nil, nil
	default:
		return n, nil, fmt.Errorf("Unsupported shape type: %v", t)
	}
}

// Functions for converting shp to geom

func point2geom(s shp.Point) geom.T {
	return geom.Point(s)
}
func pointM2geom(s shp.PointM) geom.T {
	return geom.Point{s.X, s.Y}
}
func pointZ2geom(s shp.PointZ) geom.T {
	return geom.Point{s.X, s.Y}
}
func getStartEnd(parts []int32, points []shp.Point, i int) (start, end int) {
	start = int(parts[i])
	if i == len(parts)-1 {
		end = len(points)
	} else {
		end = int(parts[i+1])
	}
	return
}
func polygon2geom(s shp.Polygon) geom.T {
	var pg geom.Polygon = make([][]geom.Point, len(s.Parts))
	for i := 0; i < len(s.Parts); i++ {
		start, end := getStartEnd(s.Parts, s.Points, i)
		pg[i] = make([]geom.Point, end-start)
		// Go backwards around the rings to switch to OGC format
		for j := end - 1; j >= start; j-- {
			pg[i][j-start] = geom.Point(s.Points[j])
		}
	}
	// Make sure the winding direction is correct
	if FixOrientation {
		op.FixOrientation(pg)
	}
	return pg
}
func polygonM2geom(s shp.PolygonM) geom.T {
	var pg geom.Polygon = make([][]geom.Point, len(s.Parts))
	jj := 0
	for i := 0; i < len(s.Parts); i++ {
		start, end := getStartEnd(s.Parts, s.Points, i)
		jj += end - start
		pg[i] = make([]geom.Point, end-start)
		// Go backwards around the rings to switch to OGC format
		for j := end - 1; j >= start; j-- {
			ss := s.Points[j]
			pg[i][j-start] = geom.Point{ss.X, ss.Y} //, s.MArray[jj]}
			jj--
		}
	}
	// Make sure the winding direction is correct
	op.FixOrientation(pg)
	return pg
}

func polygonZ2geom(s shp.PolygonZ) geom.T {
	var pg geom.Polygon = make([][]geom.Point, len(s.Parts))
	jj := -1
	for i := 0; i < len(s.Parts); i++ {
		start, end := getStartEnd(s.Parts, s.Points, i)
		jj += end - start
		pg[i] = make([]geom.Point, end-start)
		// Go backwards around the rings to switch to OGC format
		for j := end - 1; j >= start; j-- {
			ss := s.Points[j]
			pg[i][j-start] = geom.Point{ss.X, ss.Y} //, s.ZArray[jj], s.MArray[jj]}
			jj--
		}
	}
	// Make sure the winding direction is correct
	op.FixOrientation(pg)
	return pg
}
func polyLine2geom(s shp.PolyLine) geom.T {
	var pl geom.MultiLineString = make([]geom.LineString, len(s.Parts))
	for i := 0; i < len(s.Parts); i++ {
		start, end := getStartEnd(s.Parts, s.Points, i)
		pl[i] = make([]geom.Point, end-start)
		for j := start; j < end; j++ {
			pl[i][j-start] = geom.Point(s.Points[j])
		}
	}
	return pl
}
func polyLineM2geom(s shp.PolyLineM) geom.T {
	var pl geom.MultiLineString = make([]geom.LineString, len(s.Parts))
	jj := 0
	for i := 0; i < len(s.Parts); i++ {
		start, end := getStartEnd(s.Parts, s.Points, i)
		pl[i] = make([]geom.Point, end-start)
		for j := start; j < end; j++ {
			ss := s.Points[j]
			pl[i][j-start] =
				geom.Point{ss.X, ss.Y} //, s.MArray[jj]}
			jj++
		}
	}
	return pl
}
func polyLineZ2geom(s shp.PolyLineZ) geom.T {
	var pl geom.MultiLineString = make([]geom.LineString, len(s.Parts))
	jj := 0
	for i := 0; i < len(s.Parts); i++ {
		start, end := getStartEnd(s.Parts, s.Points, i)
		pl[i] = make([]geom.Point, end-start)
		for j := start; j < end; j++ {
			ss := s.Points[j]
			pl[i][j-start] =
				geom.Point{ss.X, ss.Y} //, s.ZArray[jj], s.MArray[jj]}
			jj++
		}
	}
	return pl
}
func multiPoint2geom(s shp.MultiPoint) geom.T {
	var mp geom.MultiPoint = make([]geom.Point, len(s.Points))
	for i, p := range s.Points {
		mp[i] = geom.Point(p)
	}
	return mp
}
func multiPointM2geom(s shp.MultiPointM) geom.T {
	var mp geom.MultiPoint = make([]geom.Point, len(s.Points))
	for i, p := range s.Points {
		mp[i] = geom.Point{p.X, p.Y} //, s.MArray[i]}
	}
	return mp
}
func multiPointZ2geom(s shp.MultiPointZ) geom.T {
	var mp geom.MultiPoint = make([]geom.Point, len(s.Points))
	for i, p := range s.Points {
		mp[i] = geom.Point{p.X, p.Y} //, s.ZArray[i], s.MArray[i]}
	}
	return mp
}

// Geom2Shp converts a geometry object to a shapefile shape.
func geom2Shp(g geom.T) (shp.Shape, error) {
	if g == nil {
		return &shp.Null{}, nil
	}
	switch t := g.(type) {
	case geom.Point:
		return geom2point(g.(geom.Point)), nil
	case geom.Polygon:
		return geom2polygon(g.(geom.Polygon)), nil
	case geom.MultiLineString:
		return geom2polyLine(g.(geom.MultiLineString)), nil
	//case t == "MultiPatch": // not yet supported
	case geom.MultiPoint:
		return geom2multiPoint(g.(geom.MultiPoint)), nil
	default:
		return nil, fmt.Errorf("Unsupported geom type: %v", t)
	}
}

// Functions for converting geom to shp

func geom2point(g geom.Point) shp.Shape {
	p := shp.Point(g)
	return &p
}
func geom2polygon(g geom.Polygon) shp.Shape {
	parts := make([][]shp.Point, len(g))
	for i, r := range g {
		parts[i] = make([]shp.Point, len(r))
		// switch the winding direction
		for j := len(r) - 1; j >= 0; j-- {
			parts[i][j] = shp.Point(r[j])
		}
	}
	p := shp.Polygon(*shp.NewPolyLine(parts))
	return &p
}
func valrange(a []float64) [2]float64 {
	out := [2]float64{math.Inf(1), math.Inf(-1)}
	for _, val := range a {
		if val < out[0] {
			out[0] = val
		}
		if val > out[1] {
			out[1] = val
		}
	}
	return out
}
func geom2polyLine(g geom.MultiLineString) shp.Shape {
	parts := make([][]shp.Point, len(g))
	for i, r := range g {
		parts[i] = make([]shp.Point, len(r))
		for j, l := range r {
			parts[i][j] = shp.Point(l)
		}
	}
	return shp.NewPolyLine(parts)
}
func geom2multiPoint(g geom.MultiPoint) shp.Shape {
	mp := new(shp.MultiPoint)
	mp.Box = bounds2box(g)
	mp.NumPoints = int32(len(g))
	mp.Points = make([]shp.Point, len(g))
	for i, p := range g {
		mp.Points[i] = shp.Point(p)
	}
	return mp
}
func bounds2box(g geom.T) shp.Box {
	b := g.Bounds(nil)
	return shp.Box{b.Min.X, b.Min.Y, b.Max.X, b.Max.Y}
}
<|endoftext|>


<commit_before>package scriptengine

import (
	schematypes "github.com/taskcluster/go-schematypes"
	"github.com/taskcluster/taskcluster-worker/runtime/util"
)

type configType struct {
	Command []string `json:"command"`
	Schema  struct {
		Type       string                 `json:"type"`
		Properties map[string]interface{} `json:"properties"`
		Required   []string               `json:"required"`
	} `json:"schema"`
}

var configSchema = schematypes.Object{
	Title:       "Script Engine Configuration",
	Description: `Configuration properties for the 'scriptengine'.`,
	Properties: schematypes.Properties{
		"command": schematypes.Array{
			Title: "Command to Execute",
			Description: util.Markdown(`
				Script and arguments to execute. This script will be fed
				a JSON string that matches the schema configured over 'stdin'.

				Output from the script over 'stdout' will be uploaded as task log.
				Output from the script over 'stderr' will be prefixed "[worker:error]"
				and merged with task log.

				The script will be executed with a temporary folder as
				_working directory_, this folder can be used for temporary storage and
				will be cleared between tasks.

				Files and folder stored in './artifacts/' relative to the
				_working directory_ will be uploaded as artifacts from
				the script. Hence, to make a public tar-ball artifact you create
				'./artifact/public/my-build.tar.gz' which will be uploaded as an
				artifact named 'public/my-build.tar.gz'.

				Exit codes from the script will be intepreted as follows:
				 * '0', task was executed successfully,
				 * '1', task was executed but failed,
				 * '2', task payload was not permitted, errors should be printed to stderr,
				 * '3', script had a non-fatal error, task is resolved exception
				 * '4', script had a fatal error, task is resolved exception, and
				   the worker crashes.
			`),
			Items: schematypes.String{},
		},
		"schema": schematypes.Object{
			Title: "Payload Schema",
			Description: util.Markdown(`
				JSON schema for 'task.payload'. A JSON string matching this
				schema will be piped to the script command over stdin.
			`),
			Properties: schematypes.Properties{
				"type":       schematypes.StringEnum{Options: []string{"object"}},
				"properties": schematypes.Object{AdditionalProperties: true},
				"required":   schematypes.Array{Items: schematypes.String{}},
			},
			Required: []string{
				"type",
				"properties",
				"required",
			},
		},
	},
	Required: []string{
		"command",
		"schema",
	},
}
<commit_msg>Fixed typos<commit_after>package scriptengine

import (
	schematypes "github.com/taskcluster/go-schematypes"
	"github.com/taskcluster/taskcluster-worker/runtime/util"
)

type configType struct {
	Command []string `json:"command"`
	Schema  struct {
		Type       string                 `json:"type"`
		Properties map[string]interface{} `json:"properties"`
		Required   []string               `json:"required"`
	} `json:"schema"`
}

var configSchema = schematypes.Object{
	Title:       "Script Engine Configuration",
	Description: `Configuration properties for the 'scriptengine'.`,
	Properties: schematypes.Properties{
		"command": schematypes.Array{
			Title: "Command to Execute",
			Description: util.Markdown(`
				Script and arguments to execute. This script will be fed
				a JSON string that matches the schema configured over 'stdin'.

				Output from the script over 'stdout' will be uploaded as task log.
				Output from the script over 'stderr' will be prefixed "[worker:error]"
				and merged with task log.

				The script will be executed with a temporary folder as
				_working directory_, this folder can be used for temporary storage and
				will be cleared between tasks.

				Files and folders stored in './artifacts/' relative to the
				_working directory_ will be uploaded as artifacts from
				the script. Hence, to make a public tar-ball artifact you create
				'./artifact/public/my-build.tar.gz' which will be uploaded as an
				artifact named 'public/my-build.tar.gz'.

				Exit codes from the script will be intepreted as follows:
				 * '0', task was executed successfully,
				 * '1', task was executed but failed,
				 * '2', task payload was not permitted, errors should be printed to stderr,
				 * '3', script had a non-fatal error, task is resolved exception
				 * '4', script had a fatal error, task is resolved exception, and
				   the worker crashes.
			`),
			Items: schematypes.String{},
		},
		"schema": schematypes.Object{
			Title: "Payload Schema",
			Description: util.Markdown(`
				JSON schema for 'task.payload'. A JSON string matching this
				schema will be piped to the script command over stdin.
			`),
			Properties: schematypes.Properties{
				"type":       schematypes.StringEnum{Options: []string{"object"}},
				"properties": schematypes.Object{AdditionalProperties: true},
				"required":   schematypes.Array{Items: schematypes.String{}},
			},
			Required: []string{
				"type",
				"properties",
				"required",
			},
		},
	},
	Required: []string{
		"command",
		"schema",
	},
}
<|endoftext|>


<commit_before>package text

import (
	"os"

	"github.com/segmentio/events"
	"golang.org/x/crypto/ssh/terminal"
)

func init() {
	if terminal.IsTerminal(1) {
		events.DefaultHandler = NewHandler("", os.Stdout)
	}
}
<commit_msg>add DefaultPrefix<commit_after>package text

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/segmentio/events"
	"golang.org/x/crypto/ssh/terminal"
)

// DefaultPrefix is used by the default handler configured when the program's
// standard output is a terminal.
//
// The value is "program-name[pid]: "
var DefaultPrefix string

func init() {
	DefaultPrefix = fmt.Sprintf("%s[%d]: ", filepath.Base(os.Args[0]), os.Getpid())

	if terminal.IsTerminal(1) {
		events.DefaultHandler = NewHandler(DefaultPrefix, os.Stdout)
	}
}
<|endoftext|>


<commit_before>package spiffeid

import (
	"net/url"
)

// ID is a SPIFFE ID
type ID struct {
}

// New creates a new ID using the trust domain (e.g. example.org) and path
// segments. An error is returned if the trust domain is not valid (see
// ParseTrustDomain).
func New(trustDomain string, segments ...string) (ID, error) {
	panic("not implemented")
}

// Must creates a new ID using the trust domain (e.g. example.org) and path
// segments. The function panics if the trust domain is not valid (see
// ParseTrustDomain).
func Must(trustDomain string, segments ...string) ID {
	panic("not implemented")
}

// Join returns the string representation of an ID inside the given trust
// domain (e.g. example.org) with the given path segments. An error is returned
// if the trust domain is not valid (see ParseTrustDomain).
func Join(trustDomain string, segments ...string) (string, error) {
	panic("not implemented")
}

// MustJoin returns the string representation of an ID inside the given trust
// domain (e.g. example.org) with the given path segments. The function panics
// if the trust domain is not valid (see ParseTrustDomain).
func MustJoin(trustDomain string, segments ...string) string {
	panic("not implemented")
}

// FromString parses a SPIFFE ID from a string.
func FromString(s string) (ID, error) {
	panic("not implemented")
}

// FromURI parses a SPIFFE ID from a URI.
func FromURI(u *url.URL) (ID, error) {
	panic("not implemented")
}

// TrustDomain returns the trust domain of the SPIFFE ID.
func (id ID) TrustDomain() TrustDomain {
	panic("not implemented")
}

// MemberOf returns true if the SPIFFE ID is a member of the given trust domain.
func (id ID) MemberOf(td TrustDomain) bool {
	panic("not implemented")
}

// Path returns the path of the SPIFFE ID inside the trust domain.
func (id ID) Path() string {
	panic("not implemented")
}

// String returns the string representation of the SPIFFE ID, e.g.,
// "spiffe://example.org/foo/bar".
func (id ID) String() string {
	panic("not implemented")
}

// URL returns a URL for SPIFFE ID.
func (id ID) URL() *url.URL {
	panic("not implemented")
}

// Empty returns true if the SPIFFE ID is empty.
func (id ID) Empty() bool {
	panic("not implemented")
}
<commit_msg>fix more comments<commit_after>package spiffeid

import (
	"net/url"
)

// ID is a SPIFFE ID
type ID struct {
}

// New creates a new ID using the trust domain (e.g. example.org) and path
// segments. An error is returned if the trust domain is not valid (see
// TrustDomainFromString).
func New(trustDomain string, segments ...string) (ID, error) {
	panic("not implemented")
}

// Must creates a new ID using the trust domain (e.g. example.org) and path
// segments. The function panics if the trust domain is not valid (see
// TrustDomainFromString).
func Must(trustDomain string, segments ...string) ID {
	panic("not implemented")
}

// Join returns the string representation of an ID inside the given trust
// domain (e.g. example.org) with the given path segments. An error is returned
// if the trust domain is not valid (see TrustDomainFromString).
func Join(trustDomain string, segments ...string) (string, error) {
	panic("not implemented")
}

// MustJoin returns the string representation of an ID inside the given trust
// domain (e.g. example.org) with the given path segments. The function panics
// if the trust domain is not valid (see TrustDomainFromString).
func MustJoin(trustDomain string, segments ...string) string {
	panic("not implemented")
}

// FromString parses a SPIFFE ID from a string.
func FromString(s string) (ID, error) {
	panic("not implemented")
}

// FromURI parses a SPIFFE ID from a URI.
func FromURI(u *url.URL) (ID, error) {
	panic("not implemented")
}

// TrustDomain returns the trust domain of the SPIFFE ID.
func (id ID) TrustDomain() TrustDomain {
	panic("not implemented")
}

// MemberOf returns true if the SPIFFE ID is a member of the given trust domain.
func (id ID) MemberOf(td TrustDomain) bool {
	panic("not implemented")
}

// Path returns the path of the SPIFFE ID inside the trust domain.
func (id ID) Path() string {
	panic("not implemented")
}

// String returns the string representation of the SPIFFE ID, e.g.,
// "spiffe://example.org/foo/bar".
func (id ID) String() string {
	panic("not implemented")
}

// URL returns a URL for SPIFFE ID.
func (id ID) URL() *url.URL {
	panic("not implemented")
}

// Empty returns true if the SPIFFE ID is empty.
func (id ID) Empty() bool {
	panic("not implemented")
}
<|endoftext|>


<commit_before>// proxiedsites manages a list of proxied sites, including a default set of
// sites (cloud) and user-applied customizations to that list. It provides an
// implementation of the http.Handler interface that serves up a PAC file based
// on the currently active proxied sites (cloud + additions - deletions).
package proxiedsites

import (
	"bytes"
	"fmt"
	"net/http"
	"sort"
	"sync"
	"text/template"

	"github.com/getlantern/golog"

	"gopkg.in/fatih/set.v0"
)

var (
	log = golog.LoggerFor("proxiedsites")

	parsedPacTmpl *template.Template
	cs            *configsets
	pacFile       string
	cfgMutex      sync.RWMutex
)

func init() {
	// Parse PACFile template on startup
	var err error
	parsedPacTmpl, err = template.New("pacfile").Parse(pactmpl)
	if err != nil {
		panic(fmt.Errorf("Could not parse PAC file template: %v", err))
	}
}

// Delta represents modifications to the proxied sites list.
type Delta struct {
	Additions []string `json:"Additions, omitempty"`
	Deletions []string `json:"Deletions, omitempty"`
}

// Merge merges the given delta into the existing one.
func (d *Delta) Merge(n *Delta) {
	oadd := toSet(d.Additions)
	odel := toSet(d.Deletions)
	nadd := toSet(n.Additions)
	ndel := toSet(n.Deletions)

	// First remove new deletions from old adds and vice versa
	fadd := set.Difference(oadd, ndel)
	fdel := set.Difference(odel, nadd)

	// Now add new adds and deletions
	fadd = set.Union(fadd, nadd)
	fdel = set.Union(fdel, ndel)

	d.Additions = toStrings(fadd)
	d.Deletions = toStrings(fdel)
}

// Config is the whole configuration for proxiedsites.
type Config struct {
	// User customizations
	*Delta

	// Global list of white-listed sites
	Cloud []string
}

// toCS converts this Config into a configsets
func (cfg *Config) toCS() *configsets {
	cs := &configsets{
		cloud: toSet(cfg.Cloud),
		add:   toSet(cfg.Delta.Additions),
		del:   toSet(cfg.Delta.Deletions),
	}
	cs.calculateActive()
	return cs
}

// toSet converts a slice of strings into a set
func toSet(s []string) set.Interface {
	if s == nil {
		return set.NewNonTS()
	}
	is := make([]interface{}, len(s))
	for i, s := range s {
		is[i] = s
	}
	return set.NewNonTS(is...)
}

// toStrings converts a set into a slice of strings
func toStrings(s set.Interface) []string {
	sl := s.List()
	l := make([]string, len(sl))
	for i, s := range sl {
		l[i] = s.(string)
	}
	sort.Strings(l)
	return l
}

// configsets is a version of a Config that uses sets instead of slices
type configsets struct {
	cloud      set.Interface
	add        set.Interface
	del        set.Interface
	active     set.Interface
	activeList []string
}

// calculateActive calculates the active sites for the given configsets and
// stores them in the active property.
func (cs *configsets) calculateActive() {
	cs.active = set.Difference(set.Union(cs.cloud, cs.add), cs.del)
	cs.activeList = toStrings(cs.active)
}

// equals checks whether this configsets is identical to some other configsets
func (cs *configsets) equals(other *configsets) bool {
	return cs.cloud.IsEqual(other.cloud) &&
		cs.add.IsEqual(other.add) &&
		cs.del.IsEqual(other.del)
}

// Configure applies the given configuration. If there were changes, a Delta is
// returned that includes the additions and deletions from the active list. If
// there were no changes, or the changes couldn't be applied, this method
// returns a nil Delta.
func Configure(cfg *Config) *Delta {
	newCS := cfg.toCS()
	if cs != nil && cs.equals(newCS) {
		log.Debug("Configuration unchanged")
		return nil
	}

	newPacFile, err := generatePACFile(newCS.activeList)
	if err != nil {
		log.Errorf("Error generating pac file, leaving configuration unchanged: %v", err)
		return nil
	}

	var delta *Delta
	if cs == nil {
		delta = &Delta{
			Additions: newCS.activeList,
		}
	} else {
		delta = &Delta{
			Additions: toStrings(set.Difference(newCS.active, cs.active)),
			Deletions: toStrings(set.Difference(cs.active, newCS.active)),
		}
	}
	cs = newCS
	pacFile = newPacFile
	log.Debug("Applied updated configuration")
	return delta
}

// ActiveDelta returns the active sites as a Delta of additions.
func ActiveDelta() *Delta {
	cfgMutex.RLock()
	d := &Delta{
		Additions: cs.activeList,
	}
	cfgMutex.RUnlock()
	return d
}

// ServePAC serves up the PAC file and can be used as an http.HandlerFunc
func ServePAC(resp http.ResponseWriter, req *http.Request) {
	resp.Header().Set("Content-Type", "application/x-ns-proxy-autoconfig")
	resp.WriteHeader(http.StatusOK)
	cfgMutex.RLock()
	resp.Write([]byte(pacFile))
	cfgMutex.RUnlock()
}

// generatePACFile generates a PAC File from the given active sites.
func generatePACFile(activeSites []string) (string, error) {
	data := make(map[string]interface{}, 0)
	data["Entries"] = activeSites
	buf := bytes.NewBuffer(nil)
	err := parsedPacTmpl.Execute(buf, data)
	if err != nil {
		return "", fmt.Errorf("Error generating updated PAC file: %s", err)
	}
	return string(buf.Bytes()), nil
}

const pactmpl = `var proxyDomains = new Array();
var i=0;

{{ range $key := .Entries }}
proxyDomains[i++] = "{{ $key }}";{{ end }}

for(i in proxyDomains) {
    proxyDomains[i] = proxyDomains[i].split(/\./).join("\\.");
}

var proxyDomainsRegx = new RegExp("(" + proxyDomains.join("|") + ")$", "i");

function FindProxyForURL(url, host) {
    if( host == "localhost" ||
        host == "127.0.0.1") {
        return "DIRECT";
    }

    if (proxyDomainsRegx.exec(host)) {
        return "PROXY 127.0.0.1:8787; DIRECT";
    }

    return "DIRECT";
}
`
<commit_msg>Code review fixes<commit_after>// proxiedsites manages a list of proxied sites, including a default set of
// sites (cloud) and user-applied customizations to that list. It provides an
// implementation of the http.Handler interface that serves up a PAC file based
// on the currently active proxied sites (cloud + additions - deletions).
package proxiedsites

import (
	"bytes"
	"fmt"
	"net/http"
	"sort"
	"sync"
	"text/template"

	"github.com/getlantern/golog"

	"gopkg.in/fatih/set.v0"
)

var (
	log = golog.LoggerFor("proxiedsites")

	parsedPacTmpl *template.Template
	cs            *configsets
	pacFile       string
	cfgMutex      sync.RWMutex
)

func init() {
	// Parse PACFile template on startup
	var err error
	parsedPacTmpl, err = template.New("pacfile").Parse(pactmpl)
	if err != nil {
		panic(fmt.Errorf("Could not parse PAC file template: %v", err))
	}
}

// Delta represents modifications to the proxied sites list.
type Delta struct {
	Additions []string `json:"Additions, omitempty"`
	Deletions []string `json:"Deletions, omitempty"`
}

// Merge merges the given delta into the existing one.
func (d *Delta) Merge(n *Delta) {
	oadd := toSet(d.Additions)
	odel := toSet(d.Deletions)
	nadd := toSet(n.Additions)
	ndel := toSet(n.Deletions)

	// First remove new deletions from old adds and vice versa
	fadd := set.Difference(oadd, ndel)
	fdel := set.Difference(odel, nadd)

	// Now add new adds and deletions
	fadd = set.Union(fadd, nadd)
	fdel = set.Union(fdel, ndel)

	d.Additions = toStrings(fadd)
	d.Deletions = toStrings(fdel)
}

// Config is the whole configuration for proxiedsites.
type Config struct {
	// User customizations
	*Delta

	// Global list of white-listed sites
	Cloud []string
}

// toCS converts this Config into a configsets
func (cfg *Config) toCS() *configsets {
	cs := &configsets{
		cloud: toSet(cfg.Cloud),
		add:   toSet(cfg.Delta.Additions),
		del:   toSet(cfg.Delta.Deletions),
	}
	cs.calculateActive()
	return cs
}

// toSet converts a slice of strings into a set
func toSet(s []string) set.Interface {
	if s == nil {
		return set.NewNonTS()
	}
	is := make([]interface{}, len(s))
	for i, s := range s {
		is[i] = s
	}
	return set.NewNonTS(is...)
}

// toStrings converts a set into a slice of strings
func toStrings(s set.Interface) []string {
	l := set.StringSlice(s)
	sort.Strings(l)
	return l
}

// configsets is a version of a Config that uses sets instead of slices
type configsets struct {
	cloud      set.Interface
	add        set.Interface
	del        set.Interface
	active     set.Interface
	activeList []string
}

// calculateActive calculates the active sites for the given configsets and
// stores them in the active property.
func (cs *configsets) calculateActive() {
	cs.active = set.Difference(set.Union(cs.cloud, cs.add), cs.del)
	cs.activeList = toStrings(cs.active)
}

// equals checks whether this configsets is identical to some other configsets
func (cs *configsets) equals(other *configsets) bool {
	return cs.cloud.IsEqual(other.cloud) &&
		cs.add.IsEqual(other.add) &&
		cs.del.IsEqual(other.del)
}

// Configure applies the given configuration. If there were changes, a Delta is
// returned that includes the additions and deletions from the active list. If
// there were no changes, or the changes couldn't be applied, this method
// returns a nil Delta.
func Configure(cfg *Config) *Delta {
	newCS := cfg.toCS()
	if cs != nil && cs.equals(newCS) {
		log.Debug("Configuration unchanged")
		return nil
	}

	newPacFile, err := generatePACFile(newCS.activeList)
	if err != nil {
		log.Errorf("Error generating pac file, leaving configuration unchanged: %v", err)
		return nil
	}

	var delta *Delta
	if cs == nil {
		delta = &Delta{
			Additions: newCS.activeList,
		}
	} else {
		delta = &Delta{
			Additions: toStrings(set.Difference(newCS.active, cs.active)),
			Deletions: toStrings(set.Difference(cs.active, newCS.active)),
		}
	}
	cs = newCS
	pacFile = newPacFile
	log.Debug("Applied updated configuration")
	return delta
}

// ActiveDelta returns the active sites as a Delta of additions.
func ActiveDelta() *Delta {
	cfgMutex.RLock()
	d := &Delta{
		Additions: cs.activeList,
	}
	cfgMutex.RUnlock()
	return d
}

// ServePAC serves up the PAC file and can be used as an http.HandlerFunc
func ServePAC(resp http.ResponseWriter, req *http.Request) {
	resp.Header().Set("Content-Type", "application/x-ns-proxy-autoconfig")
	resp.WriteHeader(http.StatusOK)
	cfgMutex.RLock()
	resp.Write([]byte(pacFile))
	cfgMutex.RUnlock()
}

// generatePACFile generates a PAC File from the given active sites.
func generatePACFile(activeSites []string) (string, error) {
	data := make(map[string]interface{}, 0)
	data["Entries"] = activeSites
	buf := bytes.NewBuffer(nil)
	err := parsedPacTmpl.Execute(buf, data)
	if err != nil {
		return "", fmt.Errorf("Error generating updated PAC file: %s", err)
	}
	return string(buf.Bytes()), nil
}

const pactmpl = `var proxyDomains = new Array();
var i=0;

{{ range $key := .Entries }}
proxyDomains[i++] = "{{ $key }}";{{ end }}

for(i in proxyDomains) {
    proxyDomains[i] = proxyDomains[i].split(/\./).join("\\.");
}

var proxyDomainsRegx = new RegExp("(" + proxyDomains.join("|") + ")$", "i");

function FindProxyForURL(url, host) {
    if( host == "localhost" ||
        host == "127.0.0.1") {
        return "DIRECT";
    }

    if (proxyDomainsRegx.exec(host)) {
        return "PROXY 127.0.0.1:8787; DIRECT";
    }

    return "DIRECT";
}
`
<|endoftext|>


<commit_before>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package topo

import (
	"context"
	"fmt"
	"io/ioutil"
	"sync"

	"github.com/golang/protobuf/proto"
	log "github.com/sirupsen/logrus"

	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"

	topologyclientv1 "github.com/google/kne/api/clientset/v1beta1"
	topologyv1 "github.com/google/kne/api/types/v1beta1"
	topopb "github.com/google/kne/proto/topo"
	"github.com/google/kne/topo/node"

	_ "github.com/google/kne/topo/node/ceos"
	_ "github.com/google/kne/topo/node/csr"
	_ "github.com/google/kne/topo/node/cxr"
	_ "github.com/google/kne/topo/node/frr"
	_ "github.com/google/kne/topo/node/host"
	_ "github.com/google/kne/topo/node/quagga"
	_ "github.com/google/kne/topo/node/srl"
)

var (
	meshNetCRD = map[string]string{
		"group":   "networkop.co.uk",
		"version": "v1beta1",
		"plural":  "topologies",
	}
)

// Manager is a topology instance manager for k8s cluster instance.
type Manager struct {
	kClient kubernetes.Interface
	tClient topologyclientv1.Interface
	rCfg    *rest.Config
	tpb     *topopb.Topology
	nodes   map[string]*node.Node
	links   map[string]*node.Link
}

// New creates a new topology manager based on the provided kubecfg and topology.
func New(kubecfg string, tpb *topopb.Topology) (*Manager, error) {
	log.Infof("Creating manager for: %s", tpb.Name)
	// use the current context in kubeconfig try in-cluster first if not fallback to kubeconfig
	log.Infof("Trying in-cluster configuration")
	rCfg, err := rest.InClusterConfig()
	if err != nil {
		log.Infof("Falling back to kubeconfig: %q", kubecfg)
		rCfg, err = clientcmd.BuildConfigFromFlags("", kubecfg)
		if err != nil {
			return nil, err
		}
	}
	// create the clientset
	kClient, err := kubernetes.NewForConfig(rCfg)
	if err != nil {
		return nil, err
	}
	tClient, err := topologyclientv1.NewForConfig(rCfg)
	if err != nil {
		return nil, err
	}

	return &Manager{
		kClient: kClient,
		tClient: tClient,
		rCfg:    rCfg,
		tpb:     tpb,
		nodes:   map[string]*node.Node{},
		links:   map[string]*node.Link{},
	}, nil
}

// Load creates an instance of the managed topology.
func (m *Manager) Load(ctx context.Context) error {
	for _, n := range m.tpb.Nodes {
		log.Infof("Adding Node: %s:%s", n.Name, n.Type)
		nn, err := node.New(m.tpb.Name, n, m.kClient, m.rCfg)
		if err != nil {
			return fmt.Errorf("failed to load topology: %w", err)
		}
		m.nodes[n.Name] = nn
	}
	uid := 0
	for _, l := range m.tpb.Links {
		log.Infof("Adding Link: %s:%s %s:%s", l.ANode, l.AInt, l.ZNode, l.ZInt)
		sNode, ok := m.nodes[l.ANode]
		if !ok {
			return fmt.Errorf("invalid topology: missing node %q", l.ANode)
		}
		dNode, ok := m.nodes[l.ZNode]
		if !ok {
			return fmt.Errorf("invalid topology: missing node %q", l.ZNode)
		}
		if _, ok := sNode.Interfaces[l.AInt]; ok {
			return fmt.Errorf("interface %s:%s already connected", l.ANode, l.AInt)
		}
		if _, ok := dNode.Interfaces[l.ZInt]; ok {
			return fmt.Errorf("interface %s:%s already connected", l.ZNode, l.ZInt)
		}
		link := &node.Link{
			UID:   uid,
			Proto: l,
		}
		sNode.Interfaces[l.AInt] = link
		dl := proto.Clone(l).(*topopb.Link)
		dl.AInt, dl.ZInt = dl.ZInt, dl.AInt
		dl.ANode, dl.ZNode = dl.ZNode, dl.ANode
		dLink := &node.Link{
			Proto: dl,
		}
		dNode.Interfaces[l.ZInt] = dLink
		uid++
	}
	return nil
}

// Pods gets all pods in the managed k8s cluster.
func (m *Manager) Pods(ctx context.Context) error {
	pods, err := m.kClient.CoreV1().Pods(m.tpb.Name).List(ctx, metav1.ListOptions{})
	if err != nil {
		return err
	}
	for _, p := range pods.Items {
		fmt.Println(p.Namespace, p.Name)
	}
	return nil
}

// Topology gets the topology CRDs for the cluster.
func (m *Manager) Topology(ctx context.Context) error {
	topology, err := m.tClient.Topology(m.tpb.Name).List(ctx, metav1.ListOptions{})
	if err != nil {
		return fmt.Errorf("failed to get topology CRDs: %v", err)
	}
	for _, t := range topology.Items {
		fmt.Printf("%+v\n", t)
	}
	return nil
}

// Push pushes the current topology to k8s.
func (m *Manager) Push(ctx context.Context) error {
	if _, err := m.kClient.CoreV1().Namespaces().Get(ctx, m.tpb.Name, metav1.GetOptions{}); err != nil {
		log.Infof("Creating namespace for topology: %q", m.tpb.Name)
		ns := &corev1.Namespace{
			ObjectMeta: metav1.ObjectMeta{
				Name: m.tpb.Name,
			},
		}
		sNs, err := m.kClient.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{})
		if err != nil {
			return err
		}
		log.Infof("Server Namespace: %+v", sNs)
	}

	log.Infof("Pushing Topology to k8s: %q", m.tpb.Name)
	for _, n := range m.nodes {
		t := &topologyv1.Topology{
			ObjectMeta: metav1.ObjectMeta{
				Name: n.Name(),
			},
			Spec: topologyv1.TopologySpec{},
		}
		var links []topologyv1.Link
		for _, intf := range n.Interfaces {
			link := topologyv1.Link{
				LocalIntf: intf.Proto.AInt,
				LocalIP:   "",
				PeerIntf:  intf.Proto.ZInt,
				PeerIP:    "",
				PeerPod:   intf.Proto.ZNode,
				UID:       intf.UID,
			}
			links = append(links, link)
		}
		t.Spec.Links = links
		sT, err := m.tClient.Topology(m.tpb.Name).Create(ctx, t)
		if err != nil {
			return err
		}
		log.Infof("Topology:\n%+v\n", sT)
	}
	log.Infof("Creating Node Pods")
	for k, n := range m.nodes {
		if err := n.Configure(ctx); err != nil {
			return err
		}
		if err := n.CreateService(ctx); err != nil {
			return err
		}
		if err := n.CreatePod(ctx); err != nil {
			return err
		}
		log.Infof("Node %q created", k)
	}
	return nil
}

// Delete deletes the topology from k8s.
func (m *Manager) Delete(ctx context.Context) error {
	if _, err := m.kClient.CoreV1().Namespaces().Get(ctx, m.tpb.Name, metav1.GetOptions{}); err != nil {
		return fmt.Errorf("topology %q does not exist in cluster", m.tpb.Name)
	}
	// Delete topology pods
	for _, n := range m.nodes {
		// Delete Service for node
		n.DeleteService(ctx)
		// Delete config maps for node
		n.Delete(ctx)
		// Delete Pod
		if err := m.kClient.CoreV1().Pods(m.tpb.Name).Delete(ctx, n.Name(), metav1.DeleteOptions{}); err != nil {
			log.Warnf("Error deleting pod %q: %v", n.Name(), err)
		}
		// Delete Topology for node
		if err := m.tClient.Topology(m.tpb.Name).Delete(ctx, n.Name(), metav1.DeleteOptions{}); err != nil {
			log.Warnf("Error deleting topology %q: %v", n.Name(), err)
		}
	}
	// Delete namespace
	prop := metav1.DeletePropagationForeground
	if err := m.kClient.CoreV1().Namespaces().Delete(ctx, m.tpb.Name, metav1.DeleteOptions{
		PropagationPolicy: &prop,
	}); err != nil {
		return err
	}
	return nil
}

// Load loads a Topology from fName.
func Load(fName string) (*topopb.Topology, error) {
	b, err := ioutil.ReadFile(fName)
	if err != nil {
		return nil, err
	}
	t := &topopb.Topology{}
	if err := proto.UnmarshalText(string(b), t); err != nil {
		return nil, err
	}
	return t, nil
}

type Resources struct {
	Services   map[string]*corev1.Service
	Pods       map[string]*corev1.Pod
	ConfigMaps map[string]*corev1.ConfigMap
	Topologies map[string]*topologyv1.Topology
}

// Resources gets the currently configured resources from the topology.
func (m *Manager) Resources(ctx context.Context) (*Resources, error) {
	r := Resources{
		Services:   map[string]*corev1.Service{},
		Pods:       map[string]*corev1.Pod{},
		ConfigMaps: map[string]*corev1.ConfigMap{},
		Topologies: map[string]*topologyv1.Topology{},
	}
	for _, n := range m.nodes {
		p, err := n.Pod(ctx)
		if err != nil {
			return nil, err
		}
		r.Pods[p.Name] = p
	}
	return &r, nil
}

var (
	muPort   sync.Mutex
	nextPort uint32 = 30001
)

func GetNextPort() uint32 {
	muPort.Lock()
	p := nextPort
	nextPort++
	muPort.Unlock()
	return p
}
<commit_msg>Update topo.go<commit_after>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package topo

import (
	"context"
	"fmt"
	"io/ioutil"
	"sync"

	"github.com/golang/protobuf/proto"
	log "github.com/sirupsen/logrus"

	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"

	topologyclientv1 "github.com/google/kne/api/clientset/v1beta1"
	topologyv1 "github.com/google/kne/api/types/v1beta1"
	topopb "github.com/google/kne/proto/topo"
	"github.com/google/kne/topo/node"

	_ "github.com/google/kne/topo/node/ceos"
	_ "github.com/google/kne/topo/node/csr"
	_ "github.com/google/kne/topo/node/cxr"
	_ "github.com/google/kne/topo/node/frr"
	_ "github.com/google/kne/topo/node/host"
	_ "github.com/google/kne/topo/node/quagga"
)

var (
	meshNetCRD = map[string]string{
		"group":   "networkop.co.uk",
		"version": "v1beta1",
		"plural":  "topologies",
	}
)

// Manager is a topology instance manager for k8s cluster instance.
type Manager struct {
	kClient kubernetes.Interface
	tClient topologyclientv1.Interface
	rCfg    *rest.Config
	tpb     *topopb.Topology
	nodes   map[string]*node.Node
	links   map[string]*node.Link
}

// New creates a new topology manager based on the provided kubecfg and topology.
func New(kubecfg string, tpb *topopb.Topology) (*Manager, error) {
	log.Infof("Creating manager for: %s", tpb.Name)
	// use the current context in kubeconfig try in-cluster first if not fallback to kubeconfig
	log.Infof("Trying in-cluster configuration")
	rCfg, err := rest.InClusterConfig()
	if err != nil {
		log.Infof("Falling back to kubeconfig: %q", kubecfg)
		rCfg, err = clientcmd.BuildConfigFromFlags("", kubecfg)
		if err != nil {
			return nil, err
		}
	}
	// create the clientset
	kClient, err := kubernetes.NewForConfig(rCfg)
	if err != nil {
		return nil, err
	}
	tClient, err := topologyclientv1.NewForConfig(rCfg)
	if err != nil {
		return nil, err
	}

	return &Manager{
		kClient: kClient,
		tClient: tClient,
		rCfg:    rCfg,
		tpb:     tpb,
		nodes:   map[string]*node.Node{},
		links:   map[string]*node.Link{},
	}, nil
}

// Load creates an instance of the managed topology.
func (m *Manager) Load(ctx context.Context) error {
	for _, n := range m.tpb.Nodes {
		log.Infof("Adding Node: %s:%s", n.Name, n.Type)
		nn, err := node.New(m.tpb.Name, n, m.kClient, m.rCfg)
		if err != nil {
			return fmt.Errorf("failed to load topology: %w", err)
		}
		m.nodes[n.Name] = nn
	}
	uid := 0
	for _, l := range m.tpb.Links {
		log.Infof("Adding Link: %s:%s %s:%s", l.ANode, l.AInt, l.ZNode, l.ZInt)
		sNode, ok := m.nodes[l.ANode]
		if !ok {
			return fmt.Errorf("invalid topology: missing node %q", l.ANode)
		}
		dNode, ok := m.nodes[l.ZNode]
		if !ok {
			return fmt.Errorf("invalid topology: missing node %q", l.ZNode)
		}
		if _, ok := sNode.Interfaces[l.AInt]; ok {
			return fmt.Errorf("interface %s:%s already connected", l.ANode, l.AInt)
		}
		if _, ok := dNode.Interfaces[l.ZInt]; ok {
			return fmt.Errorf("interface %s:%s already connected", l.ZNode, l.ZInt)
		}
		link := &node.Link{
			UID:   uid,
			Proto: l,
		}
		sNode.Interfaces[l.AInt] = link
		dl := proto.Clone(l).(*topopb.Link)
		dl.AInt, dl.ZInt = dl.ZInt, dl.AInt
		dl.ANode, dl.ZNode = dl.ZNode, dl.ANode
		dLink := &node.Link{
			Proto: dl,
		}
		dNode.Interfaces[l.ZInt] = dLink
		uid++
	}
	return nil
}

// Pods gets all pods in the managed k8s cluster.
func (m *Manager) Pods(ctx context.Context) error {
	pods, err := m.kClient.CoreV1().Pods(m.tpb.Name).List(ctx, metav1.ListOptions{})
	if err != nil {
		return err
	}
	for _, p := range pods.Items {
		fmt.Println(p.Namespace, p.Name)
	}
	return nil
}

// Topology gets the topology CRDs for the cluster.
func (m *Manager) Topology(ctx context.Context) error {
	topology, err := m.tClient.Topology(m.tpb.Name).List(ctx, metav1.ListOptions{})
	if err != nil {
		return fmt.Errorf("failed to get topology CRDs: %v", err)
	}
	for _, t := range topology.Items {
		fmt.Printf("%+v\n", t)
	}
	return nil
}

// Push pushes the current topology to k8s.
func (m *Manager) Push(ctx context.Context) error {
	if _, err := m.kClient.CoreV1().Namespaces().Get(ctx, m.tpb.Name, metav1.GetOptions{}); err != nil {
		log.Infof("Creating namespace for topology: %q", m.tpb.Name)
		ns := &corev1.Namespace{
			ObjectMeta: metav1.ObjectMeta{
				Name: m.tpb.Name,
			},
		}
		sNs, err := m.kClient.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{})
		if err != nil {
			return err
		}
		log.Infof("Server Namespace: %+v", sNs)
	}

	log.Infof("Pushing Topology to k8s: %q", m.tpb.Name)
	for _, n := range m.nodes {
		t := &topologyv1.Topology{
			ObjectMeta: metav1.ObjectMeta{
				Name: n.Name(),
			},
			Spec: topologyv1.TopologySpec{},
		}
		var links []topologyv1.Link
		for _, intf := range n.Interfaces {
			link := topologyv1.Link{
				LocalIntf: intf.Proto.AInt,
				LocalIP:   "",
				PeerIntf:  intf.Proto.ZInt,
				PeerIP:    "",
				PeerPod:   intf.Proto.ZNode,
				UID:       intf.UID,
			}
			links = append(links, link)
		}
		t.Spec.Links = links
		sT, err := m.tClient.Topology(m.tpb.Name).Create(ctx, t)
		if err != nil {
			return err
		}
		log.Infof("Topology:\n%+v\n", sT)
	}
	log.Infof("Creating Node Pods")
	for k, n := range m.nodes {
		if err := n.Configure(ctx); err != nil {
			return err
		}
		if err := n.CreateService(ctx); err != nil {
			return err
		}
		if err := n.CreatePod(ctx); err != nil {
			return err
		}
		log.Infof("Node %q created", k)
	}
	return nil
}

// Delete deletes the topology from k8s.
func (m *Manager) Delete(ctx context.Context) error {
	if _, err := m.kClient.CoreV1().Namespaces().Get(ctx, m.tpb.Name, metav1.GetOptions{}); err != nil {
		return fmt.Errorf("topology %q does not exist in cluster", m.tpb.Name)
	}
	// Delete topology pods
	for _, n := range m.nodes {
		// Delete Service for node
		n.DeleteService(ctx)
		// Delete config maps for node
		n.Delete(ctx)
		// Delete Pod
		if err := m.kClient.CoreV1().Pods(m.tpb.Name).Delete(ctx, n.Name(), metav1.DeleteOptions{}); err != nil {
			log.Warnf("Error deleting pod %q: %v", n.Name(), err)
		}
		// Delete Topology for node
		if err := m.tClient.Topology(m.tpb.Name).Delete(ctx, n.Name(), metav1.DeleteOptions{}); err != nil {
			log.Warnf("Error deleting topology %q: %v", n.Name(), err)
		}
	}
	// Delete namespace
	prop := metav1.DeletePropagationForeground
	if err := m.kClient.CoreV1().Namespaces().Delete(ctx, m.tpb.Name, metav1.DeleteOptions{
		PropagationPolicy: &prop,
	}); err != nil {
		return err
	}
	return nil
}

// Load loads a Topology from fName.
func Load(fName string) (*topopb.Topology, error) {
	b, err := ioutil.ReadFile(fName)
	if err != nil {
		return nil, err
	}
	t := &topopb.Topology{}
	if err := proto.UnmarshalText(string(b), t); err != nil {
		return nil, err
	}
	return t, nil
}

type Resources struct {
	Services   map[string]*corev1.Service
	Pods       map[string]*corev1.Pod
	ConfigMaps map[string]*corev1.ConfigMap
	Topologies map[string]*topologyv1.Topology
}

// Resources gets the currently configured resources from the topology.
func (m *Manager) Resources(ctx context.Context) (*Resources, error) {
	r := Resources{
		Services:   map[string]*corev1.Service{},
		Pods:       map[string]*corev1.Pod{},
		ConfigMaps: map[string]*corev1.ConfigMap{},
		Topologies: map[string]*topologyv1.Topology{},
	}
	for _, n := range m.nodes {
		p, err := n.Pod(ctx)
		if err != nil {
			return nil, err
		}
		r.Pods[p.Name] = p
	}
	return &r, nil
}

var (
	muPort   sync.Mutex
	nextPort uint32 = 30001
)

func GetNextPort() uint32 {
	muPort.Lock()
	p := nextPort
	nextPort++
	muPort.Unlock()
	return p
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"testing"
)

// Similar to the code under test
var DataDir = "data"
var MockRequestResponseFile = "requestResponseMap.json"

// global due to lazyness
var queryStr string
var dataFile string
var checkUp bool

// To process Json input file
type ReqRes struct {
	Qry string           `json:"query,omitempty"`
	Req *json.RawMessage `json:"req"`
	Res *json.RawMessage `json:"res"`
}

// read extra commandline arguments
func init() {
	flag.StringVar(&queryStr, "queryStr", "http://0.0.0.0/testingEnd?", "Testing End address, including 'debug' parameter if needed")
	mockRequestResponseFile := filepath.Dir(os.Args[0]) + filepath.FromSlash("/") + DataDir + filepath.FromSlash("/") + MockRequestResponseFile
	flag.StringVar(&dataFile, "dataFile", mockRequestResponseFile, "Data File with Request/Response map. No validation will be carried out.")
	flag.BoolVar(&checkUp, "checkUp", true, "Check it out that FastCGI is up and running through a HEAD request.")
	flag.Parse()
}

func TestRequests(t *testing.T) {

	// depends on your NGINX fastcgi configuration
	t.Log("-queryStr=" + queryStr)
	// depends on your test configuration
	t.Log("-dataFile=" + dataFile)
	// depends if the server under test supports HEAD queries
	t.Logf("-checkUp=%t\n", checkUp)

	// call that fastcgi to checkout whether it's up or not
	// TODO: Check it out if GSN supports HEAD method
	if checkUp {
		ping, err := http.Head(queryStr)
		if err != nil {
			t.Error("Unable to request for HEAD info to the server.")
			t.Fatal(err)
			t.FailNow()
		}
		if ping.StatusCode != http.StatusOK {
			t.Error("Probably FastCGI down.")
			t.Fatal(ping.Status)
			t.FailNow()
		}
	}

	// grab the real queries to launch
	dataMap, err := ioutil.ReadFile(dataFile)
	if err != nil {
		t.Error("Unable to read Mock Request Response File.")
		t.Fatal(err)
		t.FailNow()
	}

	// process json input
	dec := json.NewDecoder(strings.NewReader(string(dataMap)))
	err = ignoreFirstBracket(dec)
	if err != nil {
		t.Error("Unable to process Mock Request Response File.")
		t.Fatal(err)
		t.FailNow()
	}

	// resquests stats
	failedRequests := 0
	successRequests := 0

	// read object {"req": string, "res": string}
	for dec.More() {

		var rr ReqRes

		err = dec.Decode(&rr)
		if err != nil {
			t.Error("Unable to process Request Response object.")
			continue
		}

		if checkRequest(t, &rr) {
			successRequests++
		} else {
			failedRequests++
		}
	}

	err = ignoreLastBracket(dec)
	if err != nil {
		t.Error("Unable to process Mock Request Response File.")
		t.Fatal(err)
		t.FailNow()
	}

	t.Logf("Failed Requests: %d\n", failedRequests)
	t.Logf("Success Requests: %d\n", successRequests)
	t.Logf("Total requests sent: %d\n", failedRequests+successRequests)

	if failedRequests > 0 {
		t.Errorf("Failed Requests: %d\n", failedRequests)
		t.Fatalf("Failed Requests: %d\n", failedRequests)
		t.FailNow()
	}

}

// process specif request
func checkRequest(t *testing.T, rr *ReqRes) bool {

	query := queryStr
	if len(rr.Qry) > 0 {
		query += rr.Qry
	}

	// create the request
	req, err := toString(rr.Req)
	if err != nil {
		t.Error(err)
		return false
	}
	request, err := http.NewRequest("POST", query, strings.NewReader(req))
	if err != nil {
		t.Error("[" + query + "]" + req + ": " + err.Error())
		return false
	}
	//request.Header.Add("Accept-Encoding", "gzip")
	request.Header.Add("Content-Type", "application/json")
	request.Header.Add("Content-Length", strconv.Itoa(len(req)))

	// making the call
	client := &http.Client{}
	response, err := client.Do(request)
	if err != nil {
		t.Error("[" + query + "]" + req + ": " + err.Error())
		return false
	}
	defer response.Body.Close()
	if response.StatusCode != http.StatusOK {
		t.Error("[" + query + "]" + req + ": " + response.Status)
		return false
	}

	// double check the response
	res, err := ioutil.ReadAll(response.Body)
	if err != nil {
		t.Error("[" + query + "]" + req + ": " + err.Error())
		return false
	}
	expected, err := toString(rr.Res)
	if err != nil {
		t.Error("[" + query + "]" + req + ": " + err.Error())
		return false
	}
	if strings.EqualFold(string(res), expected) {
		// success
		return true
	} else {
		t.Error("[" + query + "]" + req + ": received->" + string(res) + " expected->" + expected)
		return false
	}
}

// convert into an string
func toString(raw *json.RawMessage) (string, error) {
	if raw != nil {
		noSoRaw, err := json.Marshal(raw)
		if err != nil {
			log.Fatal(err)
			return "", err
		}
		return string(noSoRaw), nil
	} else {
		return "", nil
	}
}

// ignore first bracket when json mock Request Response file is decoded
func ignoreFirstBracket(dec *json.Decoder) error {
	_, err := dec.Token()
	if err != nil {
		log.Fatal(err)
		return errors.New("Unable to process first token at Mock Request Response File")
	}
	return nil
}

// ignore last bracket when json mock Request Response file is decoded
func ignoreLastBracket(dec *json.Decoder) error {
	_, err := dec.Token()
	if err != nil {
		log.Fatal(err)
		return errors.New("Unable to process last token at Mock Request Response File")
	}
	return nil
}

// compact json to make it easy to look into the map for equivalent keys
func compactJson(loose []byte) (string, error) {

	compactedBuffer := new(bytes.Buffer)
	err := json.Compact(compactedBuffer, loose)
	if err != nil {
		log.Fatal(err)
		return "", err
	}
	return compactedBuffer.String(), nil
}
<commit_msg>GZIP encoding<commit_after>package main

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"errors"
	"flag"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"testing"
)

// Similar to the code under test
var DataDir = "data"
var MockRequestResponseFile = "requestResponseMap.json"

// global due to lazyness
var queryStr string
var dataFile string
var checkUp bool
var gzipOn bool

// To process Json input file
type ReqRes struct {
	Qry string           `json:"query,omitempty"`
	Req *json.RawMessage `json:"req"`
	Res *json.RawMessage `json:"res"`
}

// read extra commandline arguments
func init() {
	flag.StringVar(&queryStr, "queryStr", "http://0.0.0.0/testingEnd?", "Testing End address, including 'debug' parameter if needed")
	mockRequestResponseFile := filepath.Dir(os.Args[0]) + filepath.FromSlash("/") + DataDir + filepath.FromSlash("/") + MockRequestResponseFile
	flag.StringVar(&dataFile, "dataFile", mockRequestResponseFile, "Data File with Request/Response map. No validation will be carried out.")
	flag.BoolVar(&checkUp, "checkUp", true, "Check it out that FastCGI is up and running through a HEAD request.")
	flag.BoolVar(&gzipOn, "gzipOn", false, "Activate GZIP by adding specific header to the request. That might make all tests fail")
	flag.Parse()
}

func TestRequests(t *testing.T) {

	// depends on your NGINX fastcgi configuration
	t.Log("-queryStr=" + queryStr)
	// depends on your test configuration
	t.Log("-dataFile=" + dataFile)
	// depends if the server under test supports HEAD queries
	t.Logf("-checkUp=%t\n", checkUp)

	// call that fastcgi to checkout whether it's up or not
	if checkUp {
		ping, err := http.Head(queryStr)
		if err != nil {
			t.Error("Unable to request for HEAD info to the server.")
			t.Fatal(err)
			t.FailNow()
		}
		if ping.StatusCode != http.StatusOK {
			t.Error("Probably FastCGI down.")
			t.Fatal(ping.Status)
			t.FailNow()
		}
	}

	// grab the real queries to launch
	dataMap, err := ioutil.ReadFile(dataFile)
	if err != nil {
		t.Error("Unable to read Mock Request Response File.")
		t.Fatal(err)
		t.FailNow()
	}

	// process json input
	dec := json.NewDecoder(strings.NewReader(string(dataMap)))
	err = ignoreFirstBracket(dec)
	if err != nil {
		t.Error("Unable to process Mock Request Response File.")
		t.Fatal(err)
		t.FailNow()
	}

	// resquests stats
	failedRequests := 0
	successRequests := 0

	// read object {"req": string, "res": string}
	for dec.More() {

		var rr ReqRes

		err = dec.Decode(&rr)
		if err != nil {
			t.Error("Unable to process Request Response object.")
			continue
		}

		if checkRequest(t, &rr) {
			successRequests++
		} else {
			failedRequests++
		}
	}

	err = ignoreLastBracket(dec)
	if err != nil {
		t.Error("Unable to process Mock Request Response File.")
		t.Fatal(err)
		t.FailNow()
	}

	t.Logf("Failed Requests: %d\n", failedRequests)
	t.Logf("Success Requests: %d\n", successRequests)
	t.Logf("Total requests sent: %d\n", failedRequests+successRequests)

	if failedRequests > 0 {
		t.Errorf("Failed Requests: %d\n", failedRequests)
		t.Fatalf("Failed Requests: %d\n", failedRequests)
		t.FailNow()
	}

}

// process specif request
func checkRequest(t *testing.T, rr *ReqRes) bool {

	query := queryStr
	if len(rr.Qry) > 0 {
		query += rr.Qry
	}

	// create the request
	req, err := toString(rr.Req)
	if err != nil {
		t.Error(err)
		return false
	}
	request, err := http.NewRequest("POST", query, strings.NewReader(req))
	if err != nil {
		t.Error("[" + query + "]" + req + ": " + err.Error())
		return false
	}
	if gzipOn {
		request.Header.Add("Accept-Encoding", "gzip")
	}
	request.Header.Add("Content-Type", "application/json")
	request.Header.Add("Content-Length", strconv.Itoa(len(req)))

	// making the call
	client := &http.Client{}
	response, err := client.Do(request)
	if err != nil {
		t.Error("[" + query + "]" + req + ": " + err.Error())
		return false
	}
	defer response.Body.Close()
	if response.StatusCode != http.StatusOK {
		t.Error("[" + query + "]" + req + ": " + response.Status)
		return false
	}

	// double check the response depending on GZIP usage
	var reader io.ReadCloser
	switch response.Header.Get("Content-Encoding") {
	case "gzip":
		reader, err = gzip.NewReader(response.Body)
		defer reader.Close()
	default:
		reader = response.Body
	}
	res, err := ioutil.ReadAll(reader)
	if err != nil {
		t.Error("[" + query + "]" + req + ": " + err.Error())
		return false
	}
	responseStr := string(res)

	// what it's read from the file
	expected, err := toString(rr.Res)
	if err != nil {
		t.Error("[" + query + "]" + req + ": " + err.Error())
		return false
	}
	if strings.EqualFold(responseStr, expected) {
		// success
		return true
	}

	// failure
	t.Error("[" + query + "]" + req + ": received->" + responseStr + " expected->" + expected)
	return false
}

// convert into an string
func toString(raw *json.RawMessage) (string, error) {
	if raw != nil {
		noSoRaw, err := json.Marshal(raw)
		if err != nil {
			log.Fatal(err)
			return "", err
		}
		return string(noSoRaw), nil
	} else {
		return "", nil
	}
}

// ignore first bracket when json mock Request Response file is decoded
func ignoreFirstBracket(dec *json.Decoder) error {
	_, err := dec.Token()
	if err != nil {
		log.Fatal(err)
		return errors.New("Unable to process first token at Mock Request Response File")
	}
	return nil
}

// ignore last bracket when json mock Request Response file is decoded
func ignoreLastBracket(dec *json.Decoder) error {
	_, err := dec.Token()
	if err != nil {
		log.Fatal(err)
		return errors.New("Unable to process last token at Mock Request Response File")
	}
	return nil
}

// compact json to make it easy to look into the map for equivalent keys
func compactJson(loose []byte) (string, error) {

	compactedBuffer := new(bytes.Buffer)
	err := json.Compact(compactedBuffer, loose)
	if err != nil {
		log.Fatal(err)
		return "", err
	}
	return compactedBuffer.String(), nil
}
<|endoftext|>


<commit_before>package catena

import (
	"errors"
	"fmt"
	"path/filepath"
	"sort"
	"sync/atomic"

	"github.com/PreetamJinka/catena/partition"
	"github.com/PreetamJinka/catena/partition/memory"
	"github.com/PreetamJinka/catena/wal"
)

// InsertRows inserts the given rows into the database.
func (db *DB) InsertRows(rows []Row) error {
	keyToRows := map[int][]Row{}

	for _, row := range rows {
		key := int(row.Timestamp / db.partitionSize)
		keyToRows[key] = append(keyToRows[key], row)
	}

	keys := []int{}
	for key := range keyToRows {
		keys = append(keys, key)
	}
	sort.Ints(keys)

	for _, key := range keys {
		rowsForKey := keyToRows[key]

		minTimestampInRows := int64(0)
		maxTimestampInRows := int64(0)

		for i, row := range rowsForKey {
			if i == 1 {
				minTimestampInRows = row.Timestamp
				maxTimestampInRows = row.Timestamp
			}

			if row.Timestamp > maxTimestampInRows {
				maxTimestampInRows = row.Timestamp
			}

			if row.Timestamp < minTimestampInRows {
				minTimestampInRows = row.Timestamp
			}
		}

		var p partition.Partition

	FIND_PARTITION:

		i := db.partitionList.NewIterator()
		for i.Next() {
			val, _ := i.Value()
			val.Hold()

			if val.MinTimestamp()/db.partitionSize == int64(key) {
				p = val
				goto VALID_PARTITION
			}

			if val.MinTimestamp()/db.partitionSize < int64(key) && val.MaxTimestamp()/db.partitionSize >= int64(key) {
				p = val
				goto VALID_PARTITION
			}

			val.Release()
		}

		db.partitionCreateLock.Lock()
		if p == nil {
			if int64(key) < atomic.LoadInt64(&db.minTimestamp)/db.partitionSize {
				db.partitionCreateLock.Unlock()
				return errors.New("catena: row(s) being inserted are too old")
			}

			if db.partitionList.Size() == 0 ||
				int64(key) > atomic.LoadInt64(&db.maxTimestamp)/db.partitionSize {

				// Need to make a new partition
				newPartitionID := atomic.LoadInt64(&db.lastPartitionID) + 1
				w, err := wal.NewFileWAL(filepath.Join(db.baseDir,
					fmt.Sprintf("%d.wal", newPartitionID)))
				if err != nil {
					// Couldn't create the WAL. Maybe another writer has created
					// the WAL file. Retry.
					db.partitionCreateLock.Unlock()
					goto FIND_PARTITION
				}

				p = memory.NewMemoryPartition(w)
				db.partitionList.Insert(p)
				p.Hold()

				//if db.partitionList.Size() == 1 {
				atomic.SwapInt64(&db.minTimestamp, minTimestampInRows)
				atomic.SwapInt64(&db.maxTimestamp, maxTimestampInRows)
				//}

				if !atomic.CompareAndSwapInt64(&db.lastPartitionID, newPartitionID-1, newPartitionID) {
					p.Release()
					p.Destroy()
					db.partitionCreateLock.Unlock()
					goto FIND_PARTITION
				}
			}

			if p == nil {
				db.partitionCreateLock.Unlock()
				goto FIND_PARTITION
			}
		}

		db.partitionCreateLock.Unlock()

	VALID_PARTITION:

		if p.ReadOnly() {
			p.Release()
			return errors.New("catena: insert into read-only partition")
		}

		partitionRows := make([]partition.Row, len(rowsForKey))
		for i, row := range rowsForKey {
			partitionRows[i] = partition.Row(row)
		}

		err := p.InsertRows(partitionRows)
		if err != nil {
			p.Release()
			return err
		}

		p.Release()

		for min := atomic.LoadInt64(&db.minTimestamp); min > minTimestampInRows; min = atomic.LoadInt64(&db.minTimestamp) {
			if atomic.CompareAndSwapInt64(&db.minTimestamp, min, minTimestampInRows) {
				break
			}
		}

		for max := atomic.LoadInt64(&db.maxTimestamp); max < maxTimestampInRows; max = atomic.LoadInt64(&db.maxTimestamp) {
			if atomic.CompareAndSwapInt64(&db.maxTimestamp, max, maxTimestampInRows) {
				break
			}
		}
	}

	return nil
}
<commit_msg>Bring back if block in insert.go<commit_after>package catena

import (
	"errors"
	"fmt"
	"path/filepath"
	"sort"
	"sync/atomic"

	"github.com/PreetamJinka/catena/partition"
	"github.com/PreetamJinka/catena/partition/memory"
	"github.com/PreetamJinka/catena/wal"
)

// InsertRows inserts the given rows into the database.
func (db *DB) InsertRows(rows []Row) error {
	keyToRows := map[int][]Row{}

	for _, row := range rows {
		key := int(row.Timestamp / db.partitionSize)
		keyToRows[key] = append(keyToRows[key], row)
	}

	keys := []int{}
	for key := range keyToRows {
		keys = append(keys, key)
	}
	sort.Ints(keys)

	for _, key := range keys {
		rowsForKey := keyToRows[key]

		minTimestampInRows := int64(0)
		maxTimestampInRows := int64(0)

		for i, row := range rowsForKey {
			if i == 1 {
				minTimestampInRows = row.Timestamp
				maxTimestampInRows = row.Timestamp
			}

			if row.Timestamp > maxTimestampInRows {
				maxTimestampInRows = row.Timestamp
			}

			if row.Timestamp < minTimestampInRows {
				minTimestampInRows = row.Timestamp
			}
		}

		var p partition.Partition

	FIND_PARTITION:

		i := db.partitionList.NewIterator()
		for i.Next() {
			val, _ := i.Value()
			val.Hold()

			if val.MinTimestamp()/db.partitionSize == int64(key) {
				p = val
				goto VALID_PARTITION
			}

			if val.MinTimestamp()/db.partitionSize < int64(key) && val.MaxTimestamp()/db.partitionSize >= int64(key) {
				p = val
				goto VALID_PARTITION
			}

			val.Release()
		}

		db.partitionCreateLock.Lock()
		if p == nil {
			if int64(key) < atomic.LoadInt64(&db.minTimestamp)/db.partitionSize {
				db.partitionCreateLock.Unlock()
				return errors.New("catena: row(s) being inserted are too old")
			}

			if db.partitionList.Size() == 0 ||
				int64(key) > atomic.LoadInt64(&db.maxTimestamp)/db.partitionSize {

				// Need to make a new partition
				newPartitionID := atomic.LoadInt64(&db.lastPartitionID) + 1
				w, err := wal.NewFileWAL(filepath.Join(db.baseDir,
					fmt.Sprintf("%d.wal", newPartitionID)))
				if err != nil {
					// Couldn't create the WAL. Maybe another writer has created
					// the WAL file. Retry.
					db.partitionCreateLock.Unlock()
					goto FIND_PARTITION
				}

				p = memory.NewMemoryPartition(w)
				db.partitionList.Insert(p)
				p.Hold()

				if db.partitionList.Size() == 1 {
					atomic.SwapInt64(&db.minTimestamp, minTimestampInRows)
					atomic.SwapInt64(&db.maxTimestamp, maxTimestampInRows)
				}

				if !atomic.CompareAndSwapInt64(&db.lastPartitionID, newPartitionID-1, newPartitionID) {
					p.Release()
					p.Destroy()
					db.partitionCreateLock.Unlock()
					goto FIND_PARTITION
				}
			}

			if p == nil {
				db.partitionCreateLock.Unlock()
				goto FIND_PARTITION
			}
		}

		db.partitionCreateLock.Unlock()

	VALID_PARTITION:

		if p.ReadOnly() {
			p.Release()
			return errors.New("catena: insert into read-only partition")
		}

		partitionRows := make([]partition.Row, len(rowsForKey))
		for i, row := range rowsForKey {
			partitionRows[i] = partition.Row(row)
		}

		err := p.InsertRows(partitionRows)
		if err != nil {
			p.Release()
			return err
		}

		p.Release()

		for min := atomic.LoadInt64(&db.minTimestamp); min > minTimestampInRows; min = atomic.LoadInt64(&db.minTimestamp) {
			if atomic.CompareAndSwapInt64(&db.minTimestamp, min, minTimestampInRows) {
				break
			}
		}

		for max := atomic.LoadInt64(&db.maxTimestamp); max < maxTimestampInRows; max = atomic.LoadInt64(&db.maxTimestamp) {
			if atomic.CompareAndSwapInt64(&db.maxTimestamp, max, maxTimestampInRows) {
				break
			}
		}
	}

	return nil
}
<|endoftext|>