<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/bkeroack/travel"
	"io/ioutil"
	"log"
	"net/http"
	"os"
)

const (
	root_tree_path = "root_tree.json"
)

func get_root_tree() (map[string]interface{}, error) {
	var v map[string]interface{}
	d, err := ioutil.ReadFile(root_tree_path)
	if err != nil {
		return map[string]interface{}{}, err
	}
	json.Unmarshal(d, v)
	return v, nil
}

func save_root_tree(rt map[string]interface{}) error {
	b, err := json.Marshal(rt)
	if err != nil {
		return err
	}
	f, err := os.Create(root_tree_path)
	if err != nil {
		return err
	}
	_, err = f.Write(b)
	return err
}

func json_output(w http.ResponseWriter, val interface{}) {
	w.Header().Set("Content-Type", "application/json")
	w.Write(val)
}

func PrimaryHandler(w http.ResponseWriter, r *http.Request, c *travel.Context) {
	log.Printf("PrimaryHandler: %v\n", c)

	val, err := json.Marshal(c.CurrentObj)
	if err != nil {
		http.Error(w, fmt.Sprintf("couldn't marshal context: %v", err), 500)
		return
	}

	switch r.Method {
	case "GET":
		json_output(w, val)
	case "PUT":
		d := json.NewDecoder(r.Body)
		var b interface{}
		err := d.Decode(&b)
		if err != nil {
			http.Error(w, fmt.Sprintf("could not serialize request body: %v", err), 500)
			return
		}
		k := c.Subpath[0]
		c.CurrentObj[k] = b
		err = save_root_tree(c.RootTree)
		if err != nil {
			http.Error(w, fmt.Sprintf("error saving root tree: %v", err), 500)
			return
		}
		json_output(w, map[string]string{
			"success": "value written",
		})
		return
	case "DELETE":
		po := c.WalkBack(1)
		delete(po, c.Path[len(c.Path)-1])

	default:
		w.Header().Set("Accepts", "GET,PUT,DELETE")
		http.Error(w, "Method Not Allowed", 405)
	}
	return
}

func ErrorHandler(w http.ResponseWriter, r *http.Request, err travel.TraversalError) {
	log.Printf("ErrorHandler: %v\n", err)
	http.Error(w, err.Error(), err.Code())
}

func main() {
	hm := map[string]travel.TravelHandler{
		"": PrimaryHandler,
	}
	options := travel.TravelOptions{
		SubpathMaxLength: map[string]int{
			"GET":    0,
			"PUT":    1,
			"DELETE": 0,
		},
	}
	r := travel.NewRouter(get_root_tree, hm, ErrorHandler, &options)
	http.Handle("/", r)
	http.ListenAndServe("127.0.0.1:8000", nil)
}
<commit_msg>PUT starting to work<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/bkeroack/travel"
	"io/ioutil"
	"log"
	"net/http"
	"os"
)

const (
	root_tree_path = "root_tree.json"
)

func get_root_tree() (map[string]interface{}, error) {
	var v map[string]interface{}
	d, err := ioutil.ReadFile(root_tree_path)
	if err != nil {
		return map[string]interface{}{}, err
	}
	err = json.Unmarshal(d, &v)
	if err != nil {
		return map[string]interface{}{}, err
	}
	return v, nil
}

func save_root_tree(rt map[string]interface{}) error {
	b, err := json.Marshal(rt)
	if err != nil {
		return err
	}
	f, err := os.Create(root_tree_path)
	if err != nil {
		return err
	}
	_, err = f.Write(b)
	return err
}

func json_output(w http.ResponseWriter, val interface{}) {
	b, err := json.Marshal(val)
	if err != nil {
		http.Error(w, fmt.Sprintf("Error serializing output: %v", err), 500)
		return
	}
	w.Header().Set("Content-Type", "application/json")
	w.Write(b)
}

func PrimaryHandler(w http.ResponseWriter, r *http.Request, c *travel.Context) {
	log.Printf("PrimaryHandler: %v\n", c)

	save_rt := func() bool {
		err := save_root_tree(c.RootTree)
		if err != nil {
			http.Error(w, fmt.Sprintf("error saving root tree: %v", err), 500)
		}
		return err == nil
	}

	switch r.Method {
	case "GET":
		json_output(w, c.CurrentObj)
	case "PUT":
		d := json.NewDecoder(r.Body)
		var b interface{}
		err := d.Decode(&b)
		if err != nil {
			http.Error(w, fmt.Sprintf("could not serialize request body: %v", err), 500)
			return
		}
		k := c.Subpath[0]
		c.CurrentObj.(map[string]interface{})[k] = b
		if save_rt() {
			json_output(w, map[string]string{
				"success": "value written",
			})
		}
		return
	case "DELETE":
		po := c.WalkBack(1)
		delete(po, c.Path[len(c.Path)-1])
		if save_rt() {
			json_output(w, map[string]string{
				"success": "value deleted",
			})
		}
		return
	default:
		w.Header().Set("Accepts", "GET,PUT,DELETE")
		http.Error(w, "Method Not Allowed", 405)
	}
}

func ErrorHandler(w http.ResponseWriter, r *http.Request, err travel.TraversalError) {
	log.Printf("ErrorHandler: %v\n", err)
	http.Error(w, err.Error(), err.Code())
}

func main() {
	hm := map[string]travel.TravelHandler{
		"": PrimaryHandler,
	}
	options := travel.TravelOptions{
		SubpathMaxLength: map[string]int{
			"GET":    0,
			"PUT":    1,
			"DELETE": 0,
		},
	}
	r := travel.NewRouter(get_root_tree, hm, ErrorHandler, &options)
	http.Handle("/", r)
	http.ListenAndServe("127.0.0.1:8000", nil)
}
<|endoftext|>


<commit_before>package sgd

import (
	//	"fmt"
	"log"
	"math"
)

type LossFunc func(x []float64, y float64, θ []float64) []float64
type StepFunc func(K float64, τ float64, κ float64) float64

type Params struct {
	τ  float64
	κ  float64
	λ1 float64
	λ2 float64
}

type Obs struct {
	X []float64
	Y float64
}

func SgdKernel(dataChan chan Obs, paramChan chan Params, stateChan chan chan []float64, quitChan chan bool,
	J LossFunc, getStepSize StepFunc, θ_0 []float64) {
	// initialise the state
	θ := θ_0
	// initialise the counter
	n := 0
	// initialise the params
	var τ, κ, λ1, λ2 float64
	// initialise the adagrad step size modifier
	s := make([]float64, len(θ))
	for {
		select {
		case o := <-dataChan:
			// calculate the local gradient
			x := o.X
			y := o.Y
			grad := J(x, y, θ)
			// update the step size
			n += 1
			K := float64(n) // K is just the number of observations seen so far
			η := getStepSize(K, τ, κ)
			// update the new state
			for i, _ := range θ {
				s[i] += math.Pow(grad[i], 2)
				θ[i] += η * (grad[i] - λ1 - 2*λ2*θ[i]) / (τ + math.Sqrt(s[i]))
				//fmt.Printf("\t η:%.2f Δ:%.2f θ:%.2f\n", η, grad[i], θ[i])
				//fmt.Printf("\t θ:%.2f\n", θ[i])
			}
		case params := <-paramChan:
			τ = params.τ
			κ = params.κ
		case resChan := <-stateChan:
			resChan <- θ
		case <-quitChan:
			return
		}
	}
}

// step size update rules
func EtaConstant(K, τ, κ float64) float64 {
	return 0.1
}
func EtaInverse(K float64, τ float64, κ float64) float64 {
	return 1 / K
}
func EtaBottou(K float64, τ float64, κ float64) float64 {
	return math.Pow((τ + K), -κ)
}

// Link Functions
func identity(x float64) float64 {
	return x
}

func logit(x float64) float64 {
	return 1 / (1 + math.Exp(-x))
}

// Loss Functions
func LinearLoss(x, θ []float64, y, σ float64) float64 {
	// log((1/(2*pi*sigma**2))**0.5*exp(-1/(2*σ**2)*(y-w*x)**2))
	σ2 := math.Pow(σ, 2)
	z := math.Pow(1.0/(2*math.Pi*σ2), 0.5)
	yest := 0.0
	for i, xi := range x {
		yest += θ[i] * xi
	}
	return -math.Log(z * math.Exp(-1/(2*σ2)*math.Pow((y-yest), 2)))
}

func GradLinearLoss(x []float64, y float64, θ []float64) []float64 {
	grad := make([]float64, len(θ))
	yEst := 0.0
	for i, θi := range θ {
		yEst += x[i] * θi
	}
	err := y - yEst
	for i, xi := range x {
		grad[i] = err * xi
	}
	//log.Println("x", x, "y", y, "yEst", yEst, "err", err, "Δ", grad)
	return grad
}

func LogisticLoss(x, θ []float64, y float64) float64 {
	yest := 0.0
	for i, θi := range θ {
		yest += θi * x[i]
	}
	μ := logit(yest)
	if μ < 0 {
		log.Println("μ cannot be less than zero")
	}
	if μ > 1 {
		log.Println("μ cannot be greater than one")
	}
	ll := -(y*math.Log(μ) + (1-y)*math.Log(1-μ))
	if math.IsNaN(ll) {
		log.Println("NaN log likelihood!", y, μ, θ, x)
	}
	return ll
}

func GradLogisticLoss(x []float64, y float64, θ []float64) []float64 {
	grad := make([]float64, len(θ))
	yest := 0.0
	for i, θi := range θ {
		yest += θi * x[i]
	}
	//err := (logit(yest) - y)
	err := y - logit(yest)
	for i, xi := range x {
		grad[i] = err * xi
		//fmt.Printf("x:%+.2f y:%.0f, yest:%+.2f, logit(yest): %.2f err:%+.2f, Δ: %v\n", xi, y, yest, logit(yest), err, grad)
	}
	/*
		// grad = ( y - 1 / math.Exp(-(x * theta)) ) * x
		grad := make([]float64, len(θ))
		yEst := 0.0
		for i, θi := range θ {
			yEst += x[i] * θi
		}
		err := (y - logit(yEst))
		for i, xi := range x {
			grad[i] = err * xi
		}
		//log.Println("x", x, "y", y, "yEst", logit(yEst), "err", err, "Δ", grad)
	*/
	return grad
}
<commit_msg>Bug fix for regularization<commit_after>package sgd

import (
	//	"fmt"
	"log"
	"math"
)

type LossFunc func(x []float64, y float64, θ []float64) []float64
type StepFunc func(K float64, τ float64, κ float64) float64

type Params struct {
	τ  float64
	κ  float64
	λ1 float64
	λ2 float64
}

type Obs struct {
	X []float64
	Y float64
}

func SgdKernel(dataChan chan Obs, paramChan chan Params, stateChan chan chan []float64, quitChan chan bool,
	J LossFunc, getStepSize StepFunc, θ_0 []float64) {
	// initialise the state
	θ := θ_0
	// initialise the counter
	n := 0
	// initialise the params
	var τ, κ, λ1, λ2 float64
	// initialise the adagrad step size modifier
	s := make([]float64, len(θ))
	for {
		select {
		case o := <-dataChan:
			// calculate the local gradient
			x := o.X
			y := o.Y
			grad := J(x, y, θ)
			// update the step size
			n += 1
			K := float64(n) // K is just the number of observations seen so far
			η := getStepSize(K, τ, κ)
			// update the new state
			for i, _ := range θ {
				s[i] += math.Pow(grad[i], 2)
				θ[i] += η * (grad[i] - λ1 - 2*λ2*θ[i]) / (τ + math.Sqrt(s[i]))
				//fmt.Printf("\t η:%.2f Δ:%.2f θ:%.2f\n", η, grad[i], θ[i])
				//fmt.Printf("\t θ:%.2f\n", θ[i])
			}
		case params := <-paramChan:
			τ = params.τ
			κ = params.κ
			λ1 = params.λ1
			λ2 = params.λ2
		case resChan := <-stateChan:
			resChan <- θ
		case <-quitChan:
			return
		}
	}
}

// step size update rules
func EtaConstant(K, τ, κ float64) float64 {
	return 0.1
}
func EtaInverse(K float64, τ float64, κ float64) float64 {
	return 1 / K
}
func EtaBottou(K float64, τ float64, κ float64) float64 {
	return math.Pow((τ + K), -κ)
}

// Link Functions
func identity(x float64) float64 {
	return x
}

func logit(x float64) float64 {
	return 1 / (1 + math.Exp(-x))
}

// Loss Functions
func LinearLoss(x, θ []float64, y, σ float64) float64 {
	// log((1/(2*pi*sigma**2))**0.5*exp(-1/(2*σ**2)*(y-w*x)**2))
	σ2 := math.Pow(σ, 2)
	z := math.Pow(1.0/(2*math.Pi*σ2), 0.5)
	yest := 0.0
	for i, xi := range x {
		yest += θ[i] * xi
	}
	return -math.Log(z * math.Exp(-1/(2*σ2)*math.Pow((y-yest), 2)))
}

func GradLinearLoss(x []float64, y float64, θ []float64) []float64 {
	grad := make([]float64, len(θ))
	yEst := 0.0
	for i, θi := range θ {
		yEst += x[i] * θi
	}
	err := y - yEst
	for i, xi := range x {
		grad[i] = err * xi
	}
	//log.Println("x", x, "y", y, "yEst", yEst, "err", err, "Δ", grad)
	return grad
}

func LogisticLoss(x, θ []float64, y float64) float64 {
	yest := 0.0
	for i, θi := range θ {
		yest += θi * x[i]
	}
	μ := logit(yest)
	if μ < 0 {
		log.Println("μ cannot be less than zero")
	}
	if μ > 1 {
		log.Println("μ cannot be greater than one")
	}
	ll := -(y*math.Log(μ) + (1-y)*math.Log(1-μ))
	if math.IsNaN(ll) {
		log.Println("NaN log likelihood!", y, μ, θ, x)
	}
	return ll
}

func GradLogisticLoss(x []float64, y float64, θ []float64) []float64 {
	grad := make([]float64, len(θ))
	yest := 0.0
	for i, θi := range θ {
		yest += θi * x[i]
	}
	//err := (logit(yest) - y)
	err := y - logit(yest)
	for i, xi := range x {
		grad[i] = err * xi
		//fmt.Printf("x:%+.2f y:%.0f, yest:%+.2f, logit(yest): %.2f err:%+.2f, Δ: %v\n", xi, y, yest, logit(yest), err, grad)
	}
	/*
		// grad = ( y - 1 / math.Exp(-(x * theta)) ) * x
		grad := make([]float64, len(θ))
		yEst := 0.0
		for i, θi := range θ {
			yEst += x[i] * θi
		}
		err := (y - logit(yEst))
		for i, xi := range x {
			grad[i] = err * xi
		}
		//log.Println("x", x, "y", y, "yEst", logit(yEst), "err", err, "Δ", grad)
	*/
	return grad
}
<|endoftext|>


<commit_before>package sdk_test

import (
	"context"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"net/url"
	"reflect"
	"strconv"
	"testing"

	"github.com/grafana-tools/sdk"
	"github.com/stretchr/testify/assert"
)

func TestClient_SearchDashboards(t *testing.T) {
	type in struct {
		Query   string
		Starred bool
		Tags    []string
	}
	type testCase struct {
		In  in
		Out url.Values
	}
	dashboard := string(sdk.SearchTypeDashboard)
	for i, tc := range []testCase{
		{
			In:  in{},
			Out: url.Values{"type": []string{dashboard}, "starred": []string{"false"}},
		},
		{
			In:  in{Starred: true},
			Out: url.Values{"type": []string{dashboard}, "starred": []string{"true"}},
		},
		{
			In:  in{Query: "Foo"},
			Out: url.Values{"type": []string{dashboard}, "query": []string{"Foo"}, "starred": []string{"false"}},
		},
		{
			In:  in{Tags: []string{"Foo", "Bar"}},
			Out: url.Values{"type": []string{dashboard}, "starred": []string{"false"}, "tag": []string{"Foo", "Bar"}},
		},
	} {
		ts := httptest.NewServer(http.HandlerFunc(testSearchQuery(t, i, tc.Out)))
		client, _ := sdk.NewClient(ts.URL, "", ts.Client())
		ctx := context.Background()
		_, err := client.SearchDashboards(ctx, tc.In.Query, tc.In.Starred, tc.In.Tags...)
		ts.Close()
		if err != nil {
			t.Fatalf("SearchDashboards test %d failed: %s", i, err)
		}
	}
}

func TestClient_Search(t *testing.T) {
	type testCase struct {
		In  []sdk.SearchParam
		Out url.Values
	}
	for i, tc := range []testCase{
		{
			In:  []sdk.SearchParam{},
			Out: url.Values{},
		},
		{
			// Test all options given their correct usage.
			In: []sdk.SearchParam{
				sdk.SearchDashboardID(234),
				sdk.SearchDashboardID(432),
				sdk.SearchFolderID(123),
				sdk.SearchFolderID(321),
				sdk.SearchLimit(10),
				sdk.SearchPage(99),
				sdk.SearchQuery("Q"),
				sdk.SearchStarred(true),
				sdk.SearchTag("Foo"),
				sdk.SearchTag("Bar"),
				sdk.SearchType(sdk.SearchTypeFolder),
			},
			Out: url.Values{
				"dashboardIds": []string{"234", "432"},
				"folderIds":    []string{"123", "321"},
				"limit":        []string{"10"},
				"page":         []string{"99"},
				"query":        []string{"Q"},
				"starred":      []string{"true"},
				"tag":          []string{"Foo", "Bar"},
				"type":         []string{string(sdk.SearchTypeFolder)},
			},
		},
		{
			// Test non-repeatable options.
			In: []sdk.SearchParam{
				sdk.SearchLimit(10),
				sdk.SearchLimit(100),
				sdk.SearchPage(88),
				sdk.SearchPage(99),
				sdk.SearchQuery("Q1"),
				sdk.SearchQuery("Q2"),
				sdk.SearchStarred(true),
				sdk.SearchStarred(false),
				sdk.SearchType(sdk.SearchTypeFolder),
				sdk.SearchType(sdk.SearchTypeDashboard),
			},
			Out: url.Values{
				"limit":   []string{"100"},
				"page":    []string{"99"},
				"query":   []string{"Q2"},
				"starred": []string{"false"},
				"type":    []string{string(sdk.SearchTypeDashboard)},
			},
		},
	} {
		ts := httptest.NewServer(http.HandlerFunc(testSearchQuery(t, i, tc.Out)))
		client, _ := sdk.NewClient(ts.URL, "", ts.Client())
		ctx := context.Background()
		_, err := client.Search(ctx, tc.In...)
		ts.Close()
		if err != nil {
			t.Fatalf("SearchDashboards test %d failed: %s", i, err)
		}

	}
}

func testSearchQuery(t *testing.T, testID int, exp url.Values) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		if m := r.Method; m != http.MethodGet {
			t.Fatalf("unexpected http method for case %d: expected %s, got %s", testID, http.MethodGet, m)
		}
		if e := "/api/search"; r.URL.Path != e {
			t.Fatalf("unexpected http handler called for case %d: expected %s, got %s", testID, r.URL.Path, e)
		}
		qv, err := url.ParseQuery(r.URL.RawQuery)
		if err != nil {
			t.Fatalf("failed to parse query for case %d: %s, got err %s", testID, r.URL.RawQuery, err)
		}
		if !reflect.DeepEqual(exp, qv) {
			t.Fatalf("unexpected query arguments for case %d: expected %v, got %v", testID, exp, qv)
		}
		if _, err := w.Write([]byte("[]")); err != nil {
			t.Fatalf("failed to write http answer for case %d: %s", testID, err)
		}
	}
}

func TestSearchQuery(t *testing.T) {
	testStringSearchParam(t, sdk.SearchQuery, "query", []string{"foo", "bar"})
}

func TestSearchTag(t *testing.T) {
	testRepeatableStringSearchParam(t, sdk.SearchTag, "tag", []string{"foo", "bar"})
}

func TestSearchDashboardID(t *testing.T) {
	testRepeatableIntSearchParam(t, sdk.SearchDashboardID, "dashboardIds", []int{100, 200})
}

func TestSearchFolderID(t *testing.T) {
	testRepeatableIntSearchParam(t, sdk.SearchFolderID, "folderIds", []int{100, 200})
}

func TestSearchPage(t *testing.T) {
	testNonZeroUIntSearchParam(t, sdk.SearchPage, "page", []uint{100, 200})
}

func TestSearchLimit(t *testing.T) {
	testNonZeroUIntSearchParam(t, sdk.SearchLimit, "limit", []uint{100, 200})
}

func TestSearchStarred(t *testing.T) {
	testBoolSearchParam(t, sdk.SearchStarred, "starred", []bool{true, false})
}

func TestSearchType(t *testing.T) {
	var (
		sp         = sdk.SearchType
		key        = "type"
		testValues = []sdk.SearchParamType{sdk.SearchTypeFolder, sdk.SearchTypeDashboard}
	)

	v := make(url.Values)
	for _, testValue := range testValues {
		sp(testValue)(&v)
		expectedLen := 1
		gotLen := len(v[key])
		if gotLen != expectedLen {
			t.Errorf("expected length of %s to be %d, but was %d", key, expectedLen, gotLen)
		}
		value := v.Get(key)
		if value != string(testValue) {
			t.Errorf("expected value of %s to be %s, but was %s", key, testValue, value)
		}
	}
}

func testRepeatableStringSearchParam(t *testing.T, sp func(string) sdk.SearchParam, key string, testValues []string) {
	v := make(url.Values)
	for i, testValue := range testValues {
		sp(testValue)(&v)
		expectedLen := i + 1
		gotLen := len(v[key])
		if gotLen != expectedLen {
			t.Errorf("expected length of %s to be %d, but was %d", key, expectedLen, gotLen)
		}
		last := v[key][i]
		if last != testValue {
			t.Errorf("expected last %s to be %s, but was %s", key, testValue, last)
		}
	}
}

func testStringSearchParam(t *testing.T, sp func(string) sdk.SearchParam, key string, testValues []string) {
	v := make(url.Values)
	for _, testValue := range testValues {
		sp(testValue)(&v)
		expectedLen := 1
		gotLen := len(v[key])
		if gotLen != expectedLen {
			t.Errorf("expected length of %s to be %d, but was %d", key, expectedLen, gotLen)
		}
		value := v.Get(key)
		if value != testValue {
			t.Errorf("expected value of %s to be %s, but was %s", key, testValue, value)
		}
	}
}

func testBoolSearchParam(t *testing.T, sp func(bool) sdk.SearchParam, key string, testValues []bool) {
	v := make(url.Values)
	for _, testValue := range testValues {
		sp(testValue)(&v)
		expectedLen := 1
		gotLen := len(v[key])
		if gotLen != expectedLen {
			t.Errorf("expected length of %s to be %d, but was %d", key, expectedLen, gotLen)
		}
		value := v.Get(key)
		if value != strconv.FormatBool(testValue) {
			t.Errorf("expected value of %s to be %t, but was %s", key, testValue, value)
		}
	}
}

func testRepeatableIntSearchParam(t *testing.T, sp func(int) sdk.SearchParam, key string, testValues []int) {
	v := make(url.Values)
	for i, testValue := range testValues {
		sp(testValue)(&v)
		expectedLen := i + 1
		gotLen := len(v[key])
		if gotLen != expectedLen {
			t.Errorf("expected length of %s to be %d, but was %d", key, expectedLen, gotLen)
		}
		last := v[key][i]
		if last != strconv.Itoa(testValue) {
			t.Errorf("expected last %s to be %d, but was %s", key, testValue, last)
		}
	}
}

func testNonZeroUIntSearchParam(t *testing.T, sp func(uint) sdk.SearchParam, key string, testValues []uint) {
	v := make(url.Values)
	sp(0)(&v)
	value := v.Get(key)
	if value != "" {
		t.Errorf("expected value of %s to be unset, but was %s", key, value)
	}
	testUIntSearchParam(t, sp, key, testValues)
}

func testUIntSearchParam(t *testing.T, sp func(uint) sdk.SearchParam, key string, testValues []uint) {
	v := make(url.Values)
	for _, testValue := range testValues {
		sp(testValue)(&v)
		expectedLen := 1
		gotLen := len(v[key])
		if gotLen != expectedLen {
			t.Errorf("expected length of %s to be %d, but was %d", key, expectedLen, gotLen)
		}
		value := v.Get(key)
		if value != strconv.FormatUint(uint64(testValue), 10) {
			t.Errorf("expected value of %s to be %d, but was %s", key, testValue, value)
		}
	}
}

func Test_DecodeRawBoardPreserveId(t *testing.T) {
	r := getRawBoardRequest(t, true)
	//Serialize object
	data, err := json.Marshal(r)
	assert.Nil(t, err)
	//Deserialize data
	var rawData map[string]interface{} = make(map[string]interface{})
	assert.Nil(t, json.Unmarshal(data, &rawData))
	folderId := rawData["FolderID"]

	assert.Equal(t, folderId.(float64), float64(27))
	assert.Equal(t, rawData["Overwrite"].(bool), true)
	rawData = rawData["dashboard"].(map[string]interface{})
	assert.Equal(t, rawData["id"].(float64), float64(25))
	assert.Equal(t, rawData["title"].(string), "woot")

}

func Test_DecodeRawBoardWipeId(t *testing.T) {
	r := getRawBoardRequest(t, false)
	//Serialize object
	data, err := json.Marshal(r)
	assert.Nil(t, err)
	//Deserialize data
	var rawData map[string]interface{} = make(map[string]interface{})
	err = json.Unmarshal(data, &rawData)
	assert.Nil(t, err)
	folderId := rawData["FolderID"]

	assert.Equal(t, folderId.(float64), float64(27))
	assert.Equal(t, rawData["Overwrite"].(bool), true)
	rawData = rawData["dashboard"].(map[string]interface{})
	assert.Equal(t, rawData["id"].(float64), float64(0))
	assert.Equal(t, rawData["title"].(string), "woot")
}

func getRawBoardRequest(t *testing.T, preserveId bool) sdk.RawBoardRequest {
	r := sdk.RawBoardRequest{}
	board := sdk.Board{}
	board.ID = 25
	board.Title = "woot"
	rawBoard, err := json.Marshal(board)
	assert.Nil(t, err)

	params := sdk.SetDashboardParams{
		FolderID:   27,
		Overwrite:  true,
		PreserveId: preserveId,
	}
	r.Dashboard = rawBoard
	r.Parameters = params

	return r
}
<commit_msg>rest-dashboard: add serialization test (#168)<commit_after>package sdk_test

import (
	"context"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"net/url"
	"reflect"
	"strconv"
	"strings"
	"testing"

	"github.com/grafana-tools/sdk"
	"github.com/stretchr/testify/assert"
)

func TestClient_SearchDashboards(t *testing.T) {
	type in struct {
		Query   string
		Starred bool
		Tags    []string
	}
	type testCase struct {
		In  in
		Out url.Values
	}
	dashboard := string(sdk.SearchTypeDashboard)
	for i, tc := range []testCase{
		{
			In:  in{},
			Out: url.Values{"type": []string{dashboard}, "starred": []string{"false"}},
		},
		{
			In:  in{Starred: true},
			Out: url.Values{"type": []string{dashboard}, "starred": []string{"true"}},
		},
		{
			In:  in{Query: "Foo"},
			Out: url.Values{"type": []string{dashboard}, "query": []string{"Foo"}, "starred": []string{"false"}},
		},
		{
			In:  in{Tags: []string{"Foo", "Bar"}},
			Out: url.Values{"type": []string{dashboard}, "starred": []string{"false"}, "tag": []string{"Foo", "Bar"}},
		},
	} {
		ts := httptest.NewServer(http.HandlerFunc(testSearchQuery(t, i, tc.Out)))
		client, _ := sdk.NewClient(ts.URL, "", ts.Client())
		ctx := context.Background()
		_, err := client.SearchDashboards(ctx, tc.In.Query, tc.In.Starred, tc.In.Tags...)
		ts.Close()
		if err != nil {
			t.Fatalf("SearchDashboards test %d failed: %s", i, err)
		}
	}
}

func TestClient_Search(t *testing.T) {
	type testCase struct {
		In  []sdk.SearchParam
		Out url.Values
	}
	for i, tc := range []testCase{
		{
			In:  []sdk.SearchParam{},
			Out: url.Values{},
		},
		{
			// Test all options given their correct usage.
			In: []sdk.SearchParam{
				sdk.SearchDashboardID(234),
				sdk.SearchDashboardID(432),
				sdk.SearchFolderID(123),
				sdk.SearchFolderID(321),
				sdk.SearchLimit(10),
				sdk.SearchPage(99),
				sdk.SearchQuery("Q"),
				sdk.SearchStarred(true),
				sdk.SearchTag("Foo"),
				sdk.SearchTag("Bar"),
				sdk.SearchType(sdk.SearchTypeFolder),
			},
			Out: url.Values{
				"dashboardIds": []string{"234", "432"},
				"folderIds":    []string{"123", "321"},
				"limit":        []string{"10"},
				"page":         []string{"99"},
				"query":        []string{"Q"},
				"starred":      []string{"true"},
				"tag":          []string{"Foo", "Bar"},
				"type":         []string{string(sdk.SearchTypeFolder)},
			},
		},
		{
			// Test non-repeatable options.
			In: []sdk.SearchParam{
				sdk.SearchLimit(10),
				sdk.SearchLimit(100),
				sdk.SearchPage(88),
				sdk.SearchPage(99),
				sdk.SearchQuery("Q1"),
				sdk.SearchQuery("Q2"),
				sdk.SearchStarred(true),
				sdk.SearchStarred(false),
				sdk.SearchType(sdk.SearchTypeFolder),
				sdk.SearchType(sdk.SearchTypeDashboard),
			},
			Out: url.Values{
				"limit":   []string{"100"},
				"page":    []string{"99"},
				"query":   []string{"Q2"},
				"starred": []string{"false"},
				"type":    []string{string(sdk.SearchTypeDashboard)},
			},
		},
	} {
		ts := httptest.NewServer(http.HandlerFunc(testSearchQuery(t, i, tc.Out)))
		client, _ := sdk.NewClient(ts.URL, "", ts.Client())
		ctx := context.Background()
		_, err := client.Search(ctx, tc.In...)
		ts.Close()
		if err != nil {
			t.Fatalf("SearchDashboards test %d failed: %s", i, err)
		}

	}
}

func testSearchQuery(t *testing.T, testID int, exp url.Values) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		if m := r.Method; m != http.MethodGet {
			t.Fatalf("unexpected http method for case %d: expected %s, got %s", testID, http.MethodGet, m)
		}
		if e := "/api/search"; r.URL.Path != e {
			t.Fatalf("unexpected http handler called for case %d: expected %s, got %s", testID, r.URL.Path, e)
		}
		qv, err := url.ParseQuery(r.URL.RawQuery)
		if err != nil {
			t.Fatalf("failed to parse query for case %d: %s, got err %s", testID, r.URL.RawQuery, err)
		}
		if !reflect.DeepEqual(exp, qv) {
			t.Fatalf("unexpected query arguments for case %d: expected %v, got %v", testID, exp, qv)
		}
		if _, err := w.Write([]byte("[]")); err != nil {
			t.Fatalf("failed to write http answer for case %d: %s", testID, err)
		}
	}
}

func TestSearchQuery(t *testing.T) {
	testStringSearchParam(t, sdk.SearchQuery, "query", []string{"foo", "bar"})
}

func TestSearchTag(t *testing.T) {
	testRepeatableStringSearchParam(t, sdk.SearchTag, "tag", []string{"foo", "bar"})
}

func TestSearchDashboardID(t *testing.T) {
	testRepeatableIntSearchParam(t, sdk.SearchDashboardID, "dashboardIds", []int{100, 200})
}

func TestSearchFolderID(t *testing.T) {
	testRepeatableIntSearchParam(t, sdk.SearchFolderID, "folderIds", []int{100, 200})
}

func TestSearchPage(t *testing.T) {
	testNonZeroUIntSearchParam(t, sdk.SearchPage, "page", []uint{100, 200})
}

func TestSearchLimit(t *testing.T) {
	testNonZeroUIntSearchParam(t, sdk.SearchLimit, "limit", []uint{100, 200})
}

func TestSearchStarred(t *testing.T) {
	testBoolSearchParam(t, sdk.SearchStarred, "starred", []bool{true, false})
}

func TestSearchType(t *testing.T) {
	var (
		sp         = sdk.SearchType
		key        = "type"
		testValues = []sdk.SearchParamType{sdk.SearchTypeFolder, sdk.SearchTypeDashboard}
	)

	v := make(url.Values)
	for _, testValue := range testValues {
		sp(testValue)(&v)
		expectedLen := 1
		gotLen := len(v[key])
		if gotLen != expectedLen {
			t.Errorf("expected length of %s to be %d, but was %d", key, expectedLen, gotLen)
		}
		value := v.Get(key)
		if value != string(testValue) {
			t.Errorf("expected value of %s to be %s, but was %s", key, testValue, value)
		}
	}
}

func testRepeatableStringSearchParam(t *testing.T, sp func(string) sdk.SearchParam, key string, testValues []string) {
	v := make(url.Values)
	for i, testValue := range testValues {
		sp(testValue)(&v)
		expectedLen := i + 1
		gotLen := len(v[key])
		if gotLen != expectedLen {
			t.Errorf("expected length of %s to be %d, but was %d", key, expectedLen, gotLen)
		}
		last := v[key][i]
		if last != testValue {
			t.Errorf("expected last %s to be %s, but was %s", key, testValue, last)
		}
	}
}

func testStringSearchParam(t *testing.T, sp func(string) sdk.SearchParam, key string, testValues []string) {
	v := make(url.Values)
	for _, testValue := range testValues {
		sp(testValue)(&v)
		expectedLen := 1
		gotLen := len(v[key])
		if gotLen != expectedLen {
			t.Errorf("expected length of %s to be %d, but was %d", key, expectedLen, gotLen)
		}
		value := v.Get(key)
		if value != testValue {
			t.Errorf("expected value of %s to be %s, but was %s", key, testValue, value)
		}
	}
}

func testBoolSearchParam(t *testing.T, sp func(bool) sdk.SearchParam, key string, testValues []bool) {
	v := make(url.Values)
	for _, testValue := range testValues {
		sp(testValue)(&v)
		expectedLen := 1
		gotLen := len(v[key])
		if gotLen != expectedLen {
			t.Errorf("expected length of %s to be %d, but was %d", key, expectedLen, gotLen)
		}
		value := v.Get(key)
		if value != strconv.FormatBool(testValue) {
			t.Errorf("expected value of %s to be %t, but was %s", key, testValue, value)
		}
	}
}

func testRepeatableIntSearchParam(t *testing.T, sp func(int) sdk.SearchParam, key string, testValues []int) {
	v := make(url.Values)
	for i, testValue := range testValues {
		sp(testValue)(&v)
		expectedLen := i + 1
		gotLen := len(v[key])
		if gotLen != expectedLen {
			t.Errorf("expected length of %s to be %d, but was %d", key, expectedLen, gotLen)
		}
		last := v[key][i]
		if last != strconv.Itoa(testValue) {
			t.Errorf("expected last %s to be %d, but was %s", key, testValue, last)
		}
	}
}

func testNonZeroUIntSearchParam(t *testing.T, sp func(uint) sdk.SearchParam, key string, testValues []uint) {
	v := make(url.Values)
	sp(0)(&v)
	value := v.Get(key)
	if value != "" {
		t.Errorf("expected value of %s to be unset, but was %s", key, value)
	}
	testUIntSearchParam(t, sp, key, testValues)
}

func testUIntSearchParam(t *testing.T, sp func(uint) sdk.SearchParam, key string, testValues []uint) {
	v := make(url.Values)
	for _, testValue := range testValues {
		sp(testValue)(&v)
		expectedLen := 1
		gotLen := len(v[key])
		if gotLen != expectedLen {
			t.Errorf("expected length of %s to be %d, but was %d", key, expectedLen, gotLen)
		}
		value := v.Get(key)
		if value != strconv.FormatUint(uint64(testValue), 10) {
			t.Errorf("expected value of %s to be %d, but was %s", key, testValue, value)
		}
	}
}

func Test_DecodeRawBoardPreserveId(t *testing.T) {
	r := getRawBoardRequest(t, true)
	//Serialize object
	data, err := json.Marshal(r)
	assert.Nil(t, err)
	//Deserialize data
	var rawData map[string]interface{} = make(map[string]interface{})
	assert.Nil(t, json.Unmarshal(data, &rawData))
	folderId := rawData["FolderID"]

	assert.Equal(t, folderId.(float64), float64(27))
	assert.Equal(t, rawData["Overwrite"].(bool), true)
	rawData = rawData["dashboard"].(map[string]interface{})
	assert.Equal(t, rawData["id"].(float64), float64(25))
	assert.Equal(t, rawData["title"].(string), "woot")

}

func Test_DecodeRawBoardWipeId(t *testing.T) {
	r := getRawBoardRequest(t, false)
	//Serialize object
	data, err := json.Marshal(r)
	assert.Nil(t, err)
	//Deserialize data
	var rawData map[string]interface{} = make(map[string]interface{})
	err = json.Unmarshal(data, &rawData)
	assert.Nil(t, err)
	folderId := rawData["FolderID"]

	assert.Equal(t, folderId.(float64), float64(27))
	assert.Equal(t, rawData["Overwrite"].(bool), true)
	rawData = rawData["dashboard"].(map[string]interface{})
	assert.Equal(t, rawData["id"].(float64), float64(0))
	assert.Equal(t, rawData["title"].(string), "woot")
}

func getRawBoardRequest(t *testing.T, preserveId bool) sdk.RawBoardRequest {
	r := sdk.RawBoardRequest{}
	board := sdk.Board{}
	board.ID = 25
	board.Title = "woot"
	rawBoard, err := json.Marshal(board)
	assert.Nil(t, err)

	params := sdk.SetDashboardParams{
		FolderID:   27,
		Overwrite:  true,
		PreserveId: preserveId,
	}
	r.Dashboard = rawBoard
	r.Parameters = params

	return r
}

// TestCustomPanelSerialization serializes and de-serializes a sample dashboard
// object a few times and prints out final serialized JSON data to highlight
// that CustomPanel was nested in a strange way.
func TestCustomPanelSerialization(t *testing.T) {
	board := &sdk.Board{
		ID:              0,
		UID:             "",
		Slug:            "",
		Title:           "",
		OriginalTitle:   "",
		Tags:            nil,
		Style:           "",
		Timezone:        "",
		Editable:        false,
		HideControls:    false,
		SharedCrosshair: false,
		Panels: []*sdk.Panel{
			{
				CommonPanel:     sdk.CommonPanel{},
				GraphPanel:      nil,
				TablePanel:      nil,
				TextPanel:       nil,
				SinglestatPanel: nil,
				DashlistPanel:   nil,
				PluginlistPanel: nil,
				RowPanel:        nil,
				AlertlistPanel:  nil,
				CustomPanel: &sdk.CustomPanel{
					"key": "value",
				},
			},
		},
		Rows:       nil,
		Templating: sdk.Templating{},
		Annotations: struct {
			List []sdk.Annotation `json:"list"`
		}{},
		Refresh:       nil,
		SchemaVersion: 0,
		Version:       0,
		Links:         nil,
		Time:          sdk.Time{},
		Timepicker:    sdk.Timepicker{},
		GraphTooltip:  0,
	}

	jb, err := json.Marshal(board)
	if err != nil {
		t.Fatal(err)
	}

	for i := 0; i < 10; i++ {
		newBoard := &sdk.Board{}
		if err := json.Unmarshal(jb, newBoard); err != nil {
			t.Fatal(err)
		}

		jb, err = json.MarshalIndent(newBoard, "", "  ")
		if err != nil {
			t.Fatal(err)
		}
	}

	cnt := strings.Count(string(jb), "CustomPanel")
	if cnt != 0 {
		t.Fatalf("expected to not have any CustomPanel keys, got: %v", cnt)
	}
}
<|endoftext|>


<commit_before>package router

import (
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/prest/prest/adapters/postgres"
	"github.com/prest/prest/config"
	"github.com/prest/prest/testutils"
)

func init() {
	// load configuration at the beginning of the tests
	config.Load()
	postgres.Load()
}

func TestInitRouter(t *testing.T) {
	initRouter()
	if router == nil {
		t.Errorf("Router should not be nil.")
	}
}

func TestRoutes(t *testing.T) {
	router = nil
	r := Routes()
	if r == nil {
		t.Errorf("Should return a router.")
	}
}

func TestDefaultRouters(t *testing.T) {
	server := httptest.NewServer(GetRouter())
	defer server.Close()

	var testCases = []struct {
		url    string
		method string
		status int
	}{
		{"/databases", "GET", http.StatusOK},
		{"/schemas", "GET", http.StatusOK},
		{"/_QUERIES/{queriesLocation}/{script}", "GET", http.StatusBadRequest},
		{"/{database}/{schema}", "GET", http.StatusBadRequest},
		{"/show/{database}/{schema}/{table}", "GET", http.StatusBadRequest},
		{"/{database}/{schema}/{table}", "GET", http.StatusUnauthorized},
		{"/{database}/{schema}/{table}", "POST", http.StatusUnauthorized},
		{"/batch/{database}/{schema}/{table}", "POST", http.StatusBadRequest},
		{"/{database}/{schema}/{table}", "DELETE", http.StatusUnauthorized},
		{"/{database}/{schema}/{table}", "PUT", http.StatusUnauthorized},
		{"/{database}/{schema}/{table}", "PATCH", http.StatusUnauthorized},
		{"/", "GET", http.StatusNotFound},
	}
	for _, tc := range testCases {
		t.Log(tc.method, "\t", tc.url)
		testutils.DoRequest(t, server.URL+tc.url, nil, tc.method, tc.status, tc.url)
	}
}
<commit_msg>test to auth router<commit_after>package router

import (
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/prest/prest/adapters/postgres"
	"github.com/prest/prest/config"
	"github.com/prest/prest/testutils"
)

func init() {
	// load configuration at the beginning of the tests
	config.Load()
	postgres.Load()
}

func TestInitRouter(t *testing.T) {
	initRouter()
	if router == nil {
		t.Errorf("Router should not be nil.")
	}
}

func TestRoutes(t *testing.T) {
	router = nil
	r := Routes()
	if r == nil {
		t.Errorf("Should return a router.")
	}
}

func TestDefaultRouters(t *testing.T) {
	server := httptest.NewServer(GetRouter())
	defer server.Close()

	var testCases = []struct {
		url    string
		method string
		status int
	}{
		{"/databases", "GET", http.StatusOK},
		{"/schemas", "GET", http.StatusOK},
		{"/_QUERIES/{queriesLocation}/{script}", "GET", http.StatusBadRequest},
		{"/{database}/{schema}", "GET", http.StatusBadRequest},
		{"/show/{database}/{schema}/{table}", "GET", http.StatusBadRequest},
		{"/{database}/{schema}/{table}", "GET", http.StatusUnauthorized},
		{"/{database}/{schema}/{table}", "POST", http.StatusUnauthorized},
		{"/batch/{database}/{schema}/{table}", "POST", http.StatusBadRequest},
		{"/{database}/{schema}/{table}", "DELETE", http.StatusUnauthorized},
		{"/{database}/{schema}/{table}", "PUT", http.StatusUnauthorized},
		{"/{database}/{schema}/{table}", "PATCH", http.StatusUnauthorized},
		{"/auth", "GET", http.StatusNotFound},
		{"/", "GET", http.StatusNotFound},
	}
	for _, tc := range testCases {
		t.Log(tc.method, "\t", tc.url)
		testutils.DoRequest(t, server.URL+tc.url, nil, tc.method, tc.status, tc.url)
	}
}

func TestAuthRouterActive(t *testing.T) {
	config.PrestConf.AuthEnabled = true
	initRouter()
	server := httptest.NewServer(GetRouter())
	testutils.DoRequest(t, server.URL+"/auth", nil, "GET", http.StatusNotFound, "AuthEnable")
	testutils.DoRequest(t, server.URL+"/auth", nil, "POST", http.StatusUnauthorized, "AuthEnable")
}
<|endoftext|>


<commit_before>// Helper for converting SafeArray to array of objects.

package ole

import (
	"unsafe"
)

type SafeArrayConversion struct {
	Array *SafeArray
}

func (sac *SafeArrayConversion) ToStringArray() (strings []string) {
	totalElements, _ := sac.TotalElements(0)
	strings = make([]string, totalElements)

	for i := int32(0); i < totalElements; i++ {
		strings[int32(i)], _ = safeArrayGetElementString(sac.Array, i)
	}

	return
}

func (sac *SafeArrayConversion) ToByteArray() (bytes []byte) {
	totalElements, _ := sac.TotalElements(0)
	bytes = make([]byte, totalElements)

	for i := int32(0); i < totalElements; i++ {
		safeArrayGetElement(sac.Array, i, unsafe.Pointer(&bytes[int32(i)]))
	}

	return
}

func (sac *SafeArrayConversion) ToValueArray() (values []interface{}) {
	totalElements, _ := sac.TotalElements(0)
	values = make([]interface{}, totalElements)
	vt, _ := safeArrayGetVartype(sac.Array)

	for i := int32(0); i < totalElements; i++ {
		switch VT(vt) {
		case VT_BOOL:
			var v bool
			safeArrayGetElement(sac.Array, i, unsafe.Pointer(&v))
			values[i] = v
		case VT_I1:
			var v int8
			safeArrayGetElement(sac.Array, i, unsafe.Pointer(&v))
			values[i] = v
		case VT_I2:
			var v int16
			safeArrayGetElement(sac.Array, i, unsafe.Pointer(&v))
			values[i] = v
		case VT_I4:
			var v int32
			safeArrayGetElement(sac.Array, i, unsafe.Pointer(&v))
			values[i] = v
		case VT_I8:
			var v int64
			safeArrayGetElement(sac.Array, i, unsafe.Pointer(&v))
			values[i] = v
		case VT_UI1:
			var v uint8
			safeArrayGetElement(sac.Array, i, unsafe.Pointer(&v))
			values[i] = v
		case VT_UI2:
			var v uint16
			safeArrayGetElement(sac.Array, i, unsafe.Pointer(&v))
			values[i] = v
		case VT_UI4:
			var v uint32
			safeArrayGetElement(sac.Array, i, unsafe.Pointer(&v))
			values[i] = v
		case VT_UI8:
			var v uint64
			safeArrayGetElement(sac.Array, i, unsafe.Pointer(&v))
			values[i] = v
		case VT_R4:
			var v float32
			safeArrayGetElement(sac.Array, i, unsafe.Pointer(&v))
			values[i] = v
		case VT_R8:
			var v float64
			safeArrayGetElement(sac.Array, i, unsafe.Pointer(&v))
			values[i] = v
		case VT_BSTR:
			var v string
			safeArrayGetElement(sac.Array, i, unsafe.Pointer(&v))
			values[i] = v
		case VT_VARIANT:
			var v VARIANT
			safeArrayGetElement(sac.Array, i, unsafe.Pointer(&v))
			values[i] = v.Value()
		default:
			// TODO
		}
	}

	return
}

func (sac *SafeArrayConversion) GetType() (varType uint16, err error) {
	return safeArrayGetVartype(sac.Array)
}

func (sac *SafeArrayConversion) GetDimensions() (dimensions *uint32, err error) {
	return safeArrayGetDim(sac.Array)
}

func (sac *SafeArrayConversion) GetSize() (length *uint32, err error) {
	return safeArrayGetElementSize(sac.Array)
}

func (sac *SafeArrayConversion) TotalElements(index uint32) (totalElements int32, err error) {
	if index < 1 {
		index = 1
	}

	// Get array bounds
	var LowerBounds int32
	var UpperBounds int32

	LowerBounds, err = safeArrayGetLBound(sac.Array, index)
	if err != nil {
		return
	}

	UpperBounds, err = safeArrayGetUBound(sac.Array, index)
	if err != nil {
		return
	}

	totalElements = UpperBounds - LowerBounds + 1
	return
}

// Release Safe Array memory
func (sac *SafeArrayConversion) Release() {
	safeArrayDestroy(sac.Array)
}
<commit_msg>safeArrayGetElement error when type is VT_BSTR<commit_after>// Helper for converting SafeArray to array of objects.

package ole

import (
	"unsafe"
)

type SafeArrayConversion struct {
	Array *SafeArray
}

func (sac *SafeArrayConversion) ToStringArray() (strings []string) {
	totalElements, _ := sac.TotalElements(0)
	strings = make([]string, totalElements)

	for i := int32(0); i < totalElements; i++ {
		strings[int32(i)], _ = safeArrayGetElementString(sac.Array, i)
	}

	return
}

func (sac *SafeArrayConversion) ToByteArray() (bytes []byte) {
	totalElements, _ := sac.TotalElements(0)
	bytes = make([]byte, totalElements)

	for i := int32(0); i < totalElements; i++ {
		safeArrayGetElement(sac.Array, i, unsafe.Pointer(&bytes[int32(i)]))
	}

	return
}

func (sac *SafeArrayConversion) ToValueArray() (values []interface{}) {
	totalElements, _ := sac.TotalElements(0)
	values = make([]interface{}, totalElements)
	vt, _ := safeArrayGetVartype(sac.Array)

	for i := int32(0); i < totalElements; i++ {
		switch VT(vt) {
		case VT_BOOL:
			var v bool
			safeArrayGetElement(sac.Array, i, unsafe.Pointer(&v))
			values[i] = v
		case VT_I1:
			var v int8
			safeArrayGetElement(sac.Array, i, unsafe.Pointer(&v))
			values[i] = v
		case VT_I2:
			var v int16
			safeArrayGetElement(sac.Array, i, unsafe.Pointer(&v))
			values[i] = v
		case VT_I4:
			var v int32
			safeArrayGetElement(sac.Array, i, unsafe.Pointer(&v))
			values[i] = v
		case VT_I8:
			var v int64
			safeArrayGetElement(sac.Array, i, unsafe.Pointer(&v))
			values[i] = v
		case VT_UI1:
			var v uint8
			safeArrayGetElement(sac.Array, i, unsafe.Pointer(&v))
			values[i] = v
		case VT_UI2:
			var v uint16
			safeArrayGetElement(sac.Array, i, unsafe.Pointer(&v))
			values[i] = v
		case VT_UI4:
			var v uint32
			safeArrayGetElement(sac.Array, i, unsafe.Pointer(&v))
			values[i] = v
		case VT_UI8:
			var v uint64
			safeArrayGetElement(sac.Array, i, unsafe.Pointer(&v))
			values[i] = v
		case VT_R4:
			var v float32
			safeArrayGetElement(sac.Array, i, unsafe.Pointer(&v))
			values[i] = v
		case VT_R8:
			var v float64
			safeArrayGetElement(sac.Array, i, unsafe.Pointer(&v))
			values[i] = v
		case VT_BSTR:
			v , _ := safeArrayGetElementString(sac.Array, i)
			values[i] = v
		case VT_VARIANT:
			var v VARIANT
			safeArrayGetElement(sac.Array, i, unsafe.Pointer(&v))
			values[i] = v.Value()
		default:
			// TODO
		}
	}

	return
}

func (sac *SafeArrayConversion) GetType() (varType uint16, err error) {
	return safeArrayGetVartype(sac.Array)
}

func (sac *SafeArrayConversion) GetDimensions() (dimensions *uint32, err error) {
	return safeArrayGetDim(sac.Array)
}

func (sac *SafeArrayConversion) GetSize() (length *uint32, err error) {
	return safeArrayGetElementSize(sac.Array)
}

func (sac *SafeArrayConversion) TotalElements(index uint32) (totalElements int32, err error) {
	if index < 1 {
		index = 1
	}

	// Get array bounds
	var LowerBounds int32
	var UpperBounds int32

	LowerBounds, err = safeArrayGetLBound(sac.Array, index)
	if err != nil {
		return
	}

	UpperBounds, err = safeArrayGetUBound(sac.Array, index)
	if err != nil {
		return
	}

	totalElements = UpperBounds - LowerBounds + 1
	return
}

// Release Safe Array memory
func (sac *SafeArrayConversion) Release() {
	safeArrayDestroy(sac.Array)
}
<|endoftext|>


<commit_before>package vat

import (
	"bytes"
	"encoding/xml"
	"errors"
	"html/template"
	"io/ioutil"
	"net/http"
	"regexp"
	"strings"
	"time"
)

type viesResponse struct {
	CountryCode string
	VATnumber   string
	RequestDate time.Time
	Valid       bool
	Name        string
	Address     string
}

const serviceURL = "http://ec.europa.eu/taxation_customs/vies/services/checkVatService"
const envelope = `
<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:v1="http://schemas.conversesolutions.com/xsd/dmticta/v1">
<soapenv:Header/>
<soapenv:Body>
  <checkVat xmlns="urn:ec.europa.eu:taxud:vies:services:checkVat:types">
    <countryCode>{{.countryCode}}</countryCode>
    <vatNumber>{{.vatNumber}}</vatNumber>
  </checkVat>
</soapenv:Body>
</soapenv:Envelope>
`

var Timeout = 10 // seconds

var (
	ErrInvalidVATNumber   = errors.New("VAT number is invalid.")
	ErrServiceUnreachable = errors.New("Validation service is offline.")
)

func Validate(n string) (bool, error) {
	format, err := ValidateFormat(n)
	existence := false

	if format {
		existence, err = ValidateExistence(n)
	}

	return (format && existence), err
}

func ValidateFormat(n string) (bool, error) {
	patterns := map[string]string{
		"AT": "U[A-Z\\d]{8}",
		"BE": "(0\\d{9}|\\d{10})",
		"BG": "\\d{9,10}",
		"CY": "\\d{8}[A-Z]",
		"CZ": "\\d{8,10}",
		"DE": "\\d{9}",
		"DK": "(\\d{2} ?){3}\\d{2}",
		"EE": "\\d{9}",
		"EL": "\\d{9}",
		"ES": "[A-Z]\\d{7}[A-Z]|\\d{8}[A-Z]|[A-Z]\\d{8}",
		"FI": "\\d{8}",
		"FR": "([A-Z]{2}|\\d{2})\\d{9}",
		"GB": "\\d{9}|\\d{12}|(GD|HA)\\d{3}",
		"HR": "\\d{11}",
		"HU": "\\d{8}",
		"IE": "[A-Z\\d]{8}|[A-Z\\d]{9}",
		"IT": "\\d{11}",
		"LT": "(\\d{9}|\\d{12})",
		"LU": "\\d{8}",
		"LV": "\\d{11}",
		"MT": "\\d{8}",
		"NL": "\\d{9}B\\d{2}",
		"PL": "\\d{10}",
		"PT": "\\d{9}",
		"RO": "\\d{2,10}",
		"SE": "\\d{12}",
		"SI": "\\d{8}",
		"SK": "\\d{10}",
	}

	if len(n) < 3 {
		return false, nil
	}

	n = strings.ToUpper(n)
	pattern, ok := patterns[n[0:2]]
	if !ok {
		return false, nil
	}

	matched, err := regexp.MatchString(pattern, n[2:])
	return matched, err
}

func ValidateExistence(n string) (bool, error) {
	r, err := checkVAT(n)
	return r.Valid, err
}

// Check returns *VATresponse for vat number
func checkVAT(vatNumber string) (*viesResponse, error) {
	if len(vatNumber) < 3 {
		return nil, ErrInvalidVATNumber
	}

	e, err := getEnvelope(vatNumber)
	if err != nil {
		return nil, err
	}
	eb := bytes.NewBufferString(e)
	client := http.Client{
		Timeout: time.Duration(Timeout) * time.Second,
	}
	res, err := client.Post(serviceURL, "text/xml;charset=UTF-8", eb)
	if err != nil {
		return nil, ErrServiceUnreachable
	}
	defer res.Body.Close()

	// TODO: Use reader XML decoder
	xmlRes, err := ioutil.ReadAll(res.Body)

	// check if response contains "INVALID_INPUT" string
	if bytes.Contains(xmlRes, []byte("INVALID_INPUT")) {
		return nil, ErrInvalidVATNumber
	}

	var rd struct {
		XMLName xml.Name `xml:"Envelope"`
		Soap    struct {
			XMLName xml.Name `xml:"Body"`
			Soap    struct {
				XMLName     xml.Name `xml:"checkVatResponse"`
				CountryCode string   `xml:"countryCode"`
				VATnumber   string   `xml:"vatNumber"`
				RequestDate string   `xml:"requestDate"` // 2015-03-06+01:00
				Valid       bool     `xml:"valid"`
				Name        string   `xml:"name"`
				Address     string   `xml:"address"`
			}
		}
	}
	if err := xml.Unmarshal(xmlRes, &rd); err != nil {
		return nil, err
	}

	pDate, err := time.Parse("2006-01-02-07:00", rd.Soap.Soap.RequestDate)
	if err != nil {
		return nil, err
	}

	r := &viesResponse{
		CountryCode: rd.Soap.Soap.CountryCode,
		VATnumber:   rd.Soap.Soap.VATnumber,
		RequestDate: pDate,
		Valid:       rd.Soap.Soap.Valid,
		Name:        rd.Soap.Soap.Name,
		Address:     rd.Soap.Soap.Address,
	}

	return r, nil
}

// getEnvelope parses envelope template
func getEnvelope(vatNumber string) (string, error) {
	t, err := template.New("envelope").Parse(envelope)
	if err != nil {
		return "", err
	}

	var result bytes.Buffer
	if err := t.Execute(&result, map[string]string{
		"countryCode": strings.ToUpper(vatNumber[0:2]),
		"vatNumber":   vatNumber[2:],
	}); err != nil {
		return "", err
	}
	return result.String(), nil
}
<commit_msg>Add godoc for funcs.<commit_after>// Package vat provides VAT number verification for Golang.
package vat

import (
	"bytes"
	"encoding/xml"
	"errors"
	"html/template"
	"io/ioutil"
	"net/http"
	"regexp"
	"strings"
	"time"
)

type viesResponse struct {
	CountryCode string
	VATnumber   string
	RequestDate time.Time
	Valid       bool
	Name        string
	Address     string
}

const serviceURL = "http://ec.europa.eu/taxation_customs/vies/services/checkVatService"
const envelope = `
<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:v1="http://schemas.conversesolutions.com/xsd/dmticta/v1">
<soapenv:Header/>
<soapenv:Body>
  <checkVat xmlns="urn:ec.europa.eu:taxud:vies:services:checkVat:types">
    <countryCode>{{.countryCode}}</countryCode>
    <vatNumber>{{.vatNumber}}</vatNumber>
  </checkVat>
</soapenv:Body>
</soapenv:Envelope>
`

var (
	ErrInvalidVATNumber   = errors.New("VAT number is invalid.")
	ErrServiceUnreachable = errors.New("Validation service is offline.")
)

// Validate validates a VAT number by format and existence.
//
// The existence check uses the VIES VAT validation SOAP API and will only run when format validation passes.
func Validate(n string) (bool, error) {
	format, err := ValidateFormat(n)
	existence := false

	if format {
		existence, err = ValidateExistence(n)
	}

	return (format && existence), err
}

// ValidateFormat validates a VAT number by its format.
func ValidateFormat(n string) (bool, error) {
	patterns := map[string]string{
		"AT": "U[A-Z\\d]{8}",
		"BE": "(0\\d{9}|\\d{10})",
		"BG": "\\d{9,10}",
		"CY": "\\d{8}[A-Z]",
		"CZ": "\\d{8,10}",
		"DE": "\\d{9}",
		"DK": "(\\d{2} ?){3}\\d{2}",
		"EE": "\\d{9}",
		"EL": "\\d{9}",
		"ES": "[A-Z]\\d{7}[A-Z]|\\d{8}[A-Z]|[A-Z]\\d{8}",
		"FI": "\\d{8}",
		"FR": "([A-Z]{2}|\\d{2})\\d{9}",
		"GB": "\\d{9}|\\d{12}|(GD|HA)\\d{3}",
		"HR": "\\d{11}",
		"HU": "\\d{8}",
		"IE": "[A-Z\\d]{8}|[A-Z\\d]{9}",
		"IT": "\\d{11}",
		"LT": "(\\d{9}|\\d{12})",
		"LU": "\\d{8}",
		"LV": "\\d{11}",
		"MT": "\\d{8}",
		"NL": "\\d{9}B\\d{2}",
		"PL": "\\d{10}",
		"PT": "\\d{9}",
		"RO": "\\d{2,10}",
		"SE": "\\d{12}",
		"SI": "\\d{8}",
		"SK": "\\d{10}",
	}

	if len(n) < 3 {
		return false, nil
	}

	n = strings.ToUpper(n)
	pattern, ok := patterns[n[0:2]]
	if !ok {
		return false, nil
	}

	matched, err := regexp.MatchString(pattern, n[2:])
	return matched, err
}

// ValidateExistence validates a VAT number by its existence using the VIES VAT API (using SOAP)
func ValidateExistence(n string) (bool, error) {
	r, err := checkVAT(n)
	return r.Valid, err
}

// Check returns *VATresponse for vat number
func checkVAT(vatNumber string) (*viesResponse, error) {
	if len(vatNumber) < 3 {
		return nil, ErrInvalidVATNumber
	}

	e, err := getEnvelope(vatNumber)
	if err != nil {
		return nil, err
	}
	eb := bytes.NewBufferString(e)
	client := http.Client{
		Timeout: 10 * time.Second,
	}
	res, err := client.Post(serviceURL, "text/xml;charset=UTF-8", eb)
	if err != nil {
		return nil, ErrServiceUnreachable
	}
	defer res.Body.Close()

	// TODO: Use reader XML decoder
	xmlRes, err := ioutil.ReadAll(res.Body)

	// check if response contains "INVALID_INPUT" string
	if bytes.Contains(xmlRes, []byte("INVALID_INPUT")) {
		return nil, ErrInvalidVATNumber
	}

	var rd struct {
		XMLName xml.Name `xml:"Envelope"`
		Soap    struct {
			XMLName xml.Name `xml:"Body"`
			Soap    struct {
				XMLName     xml.Name `xml:"checkVatResponse"`
				CountryCode string   `xml:"countryCode"`
				VATnumber   string   `xml:"vatNumber"`
				RequestDate string   `xml:"requestDate"` // 2015-03-06+01:00
				Valid       bool     `xml:"valid"`
				Name        string   `xml:"name"`
				Address     string   `xml:"address"`
			}
		}
	}
	if err := xml.Unmarshal(xmlRes, &rd); err != nil {
		return nil, err
	}

	pDate, err := time.Parse("2006-01-02-07:00", rd.Soap.Soap.RequestDate)
	if err != nil {
		return nil, err
	}

	r := &viesResponse{
		CountryCode: rd.Soap.Soap.CountryCode,
		VATnumber:   rd.Soap.Soap.VATnumber,
		RequestDate: pDate,
		Valid:       rd.Soap.Soap.Valid,
		Name:        rd.Soap.Soap.Name,
		Address:     rd.Soap.Soap.Address,
	}

	return r, nil
}

// getEnvelope parses envelope template
func getEnvelope(vatNumber string) (string, error) {
	t, err := template.New("envelope").Parse(envelope)
	if err != nil {
		return "", err
	}

	var result bytes.Buffer
	if err := t.Execute(&result, map[string]string{
		"countryCode": strings.ToUpper(vatNumber[0:2]),
		"vatNumber":   vatNumber[2:],
	}); err != nil {
		return "", err
	}
	return result.String(), nil
}
<|endoftext|>


<commit_before>package lispy

import (
	"strings"
)

func Wbr(li *Lispy) string {
	if li.Content == "" {
		return "<wbr>"
	}
	needle := li.Get("needle")
	switch needle {
	case "(", ")":
		return li.Render(li.Content)
	}
	return li.Render(strings.Replace(li.Content, needle, needle+"<wbr>", -1))
}
<commit_msg>Made wbr more robust!<commit_after>package lispy

import (
	"strings"
)

func Wbr(li *Lispy) string {
	if li.Content == "" {
		return "<wbr>"
	}
	needle := li.Get("needle")
	if len(needle) <= 0 || len(needle) > 1 {
		return li.Render(li.Content)
	}

	switch needle {
	case "(", ")", ":", "|":
		return li.Render(li.Content)
	}
	return li.Render(strings.Replace(li.Content, needle, needle+"<wbr>", -1))
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"html/template"
	"net/http"
	"net/url"
	"sort"

	"github.com/blevesearch/bleve"
)

type offerData struct {
	Account  string
	Title    string
	Date     string
	Salary   string
	URL      string
	Location string
}

type sortedOfferDate []*offerData

func (s sortedOfferDate) Len() int {
	return len(s)
}

func (s sortedOfferDate) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (s sortedOfferDate) Less(i, j int) bool {
	return s[i].Date > s[j].Date
}

func serveQuery(templ *template.Template, store *Store, index bleve.Index,
	w http.ResponseWriter, r *http.Request) error {

	values, err := url.ParseQuery(r.URL.RawQuery)
	if err != nil {
		return err
	}
	query := values.Get("q")
	q := bleve.NewQueryStringQuery(query)
	rq := bleve.NewSearchRequest(q)
	rq.Size = 100
	offers := []*offerData{}
	maxDisplayed := 1000
	total := 0
	for {
		if query == "" {
			break
		}
		res, err := index.Search(rq)
		if err != nil {
			return err
		}
		total = int(res.Total)
		for _, doc := range res.Hits {
			if len(offers) >= maxDisplayed {
				break
			}
			data, err := store.Get(doc.ID)
			if err != nil {
				return err
			}
			o := &jsonOffer{}
			err = json.Unmarshal(data, o)
			if err != nil {
				return err
			}
			offer, err := convertOffer(o)
			if err != nil {
				fmt.Printf("error: cannot convert offer: %s\n", err)
				continue
			}
			salary := ""
			if offer.MinSalary > 0 {
				if offer.MaxSalary != offer.MinSalary {
					salary = fmt.Sprintf("(%d - %d kEUR)",
						offer.MinSalary, offer.MaxSalary)
				} else {
					salary = fmt.Sprintf("(%d kEUR)", offer.MinSalary)
				}
			}
			offers = append(offers, &offerData{
				Account:  offer.Account,
				Title:    offer.Title,
				Date:     offer.Date.Format("2006-01-02"),
				URL:      offer.URL,
				Salary:   salary,
				Location: offer.Location,
			})
		}
		if len(res.Hits) < rq.Size || len(offers) >= maxDisplayed {
			break
		}
		rq.From += rq.Size
	}
	sort.Sort(sortedOfferDate(offers))
	data := struct {
		Offers    []*offerData
		Displayed int
		Total     int
		Query     string
	}{
		Offers:    offers,
		Displayed: len(offers),
		Total:     total,
		Query:     query,
	}
	h := w.Header()
	h.Set("Content-Type", "text/html")
	templ.Execute(w, &data)
	return nil
}

func handleQuery(templ *template.Template, store *Store, index bleve.Index,
	w http.ResponseWriter, r *http.Request) {
	err := serveQuery(templ, store, index, w, r)
	if err != nil {
		w.Header().Set("Content-Type", "text/plain")
		w.WriteHeader(400)
		fmt.Fprintf(w, "error: %s\n", err)
	}
}

var (
	webCmd  = app.Command("web", "APEC web frontend")
	webHttp = webCmd.Flag("http", "http server address").Default(":6000").String()
)

func web(cfg *Config) error {
	store, err := OpenStore(cfg.Store())
	if err != nil {
		return fmt.Errorf("cannot open data store: %s", err)
	}
	index, err := bleve.Open(cfg.Index())
	if err != nil {
		return fmt.Errorf("cannot open index: %s", err)
	}
	defer index.Close()
	templ, err := template.ParseGlob("web/*.tmpl")
	if err != nil {
		return err
	}
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		handleQuery(templ, store, index, w, r)
	})
	return http.ListenAndServe(*webHttp, nil)
}
<commit_msg>web: change default port to 8081<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"html/template"
	"net/http"
	"net/url"
	"sort"

	"github.com/blevesearch/bleve"
)

type offerData struct {
	Account  string
	Title    string
	Date     string
	Salary   string
	URL      string
	Location string
}

type sortedOfferDate []*offerData

func (s sortedOfferDate) Len() int {
	return len(s)
}

func (s sortedOfferDate) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (s sortedOfferDate) Less(i, j int) bool {
	return s[i].Date > s[j].Date
}

func serveQuery(templ *template.Template, store *Store, index bleve.Index,
	w http.ResponseWriter, r *http.Request) error {

	values, err := url.ParseQuery(r.URL.RawQuery)
	if err != nil {
		return err
	}
	query := values.Get("q")
	q := bleve.NewQueryStringQuery(query)
	rq := bleve.NewSearchRequest(q)
	rq.Size = 100
	offers := []*offerData{}
	maxDisplayed := 1000
	total := 0
	for {
		if query == "" {
			break
		}
		res, err := index.Search(rq)
		if err != nil {
			return err
		}
		total = int(res.Total)
		for _, doc := range res.Hits {
			if len(offers) >= maxDisplayed {
				break
			}
			data, err := store.Get(doc.ID)
			if err != nil {
				return err
			}
			o := &jsonOffer{}
			err = json.Unmarshal(data, o)
			if err != nil {
				return err
			}
			offer, err := convertOffer(o)
			if err != nil {
				fmt.Printf("error: cannot convert offer: %s\n", err)
				continue
			}
			salary := ""
			if offer.MinSalary > 0 {
				if offer.MaxSalary != offer.MinSalary {
					salary = fmt.Sprintf("(%d - %d kEUR)",
						offer.MinSalary, offer.MaxSalary)
				} else {
					salary = fmt.Sprintf("(%d kEUR)", offer.MinSalary)
				}
			}
			offers = append(offers, &offerData{
				Account:  offer.Account,
				Title:    offer.Title,
				Date:     offer.Date.Format("2006-01-02"),
				URL:      offer.URL,
				Salary:   salary,
				Location: offer.Location,
			})
		}
		if len(res.Hits) < rq.Size || len(offers) >= maxDisplayed {
			break
		}
		rq.From += rq.Size
	}
	sort.Sort(sortedOfferDate(offers))
	data := struct {
		Offers    []*offerData
		Displayed int
		Total     int
		Query     string
	}{
		Offers:    offers,
		Displayed: len(offers),
		Total:     total,
		Query:     query,
	}
	h := w.Header()
	h.Set("Content-Type", "text/html")
	templ.Execute(w, &data)
	return nil
}

func handleQuery(templ *template.Template, store *Store, index bleve.Index,
	w http.ResponseWriter, r *http.Request) {
	err := serveQuery(templ, store, index, w, r)
	if err != nil {
		w.Header().Set("Content-Type", "text/plain")
		w.WriteHeader(400)
		fmt.Fprintf(w, "error: %s\n", err)
	}
}

var (
	webCmd  = app.Command("web", "APEC web frontend")
	webHttp = webCmd.Flag("http", "http server address").Default(":8081").String()
)

func web(cfg *Config) error {
	store, err := OpenStore(cfg.Store())
	if err != nil {
		return fmt.Errorf("cannot open data store: %s", err)
	}
	index, err := bleve.Open(cfg.Index())
	if err != nil {
		return fmt.Errorf("cannot open index: %s", err)
	}
	defer index.Close()
	templ, err := template.ParseGlob("web/*.tmpl")
	if err != nil {
		return err
	}
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		handleQuery(templ, store, index, w, r)
	})
	return http.ListenAndServe(*webHttp, nil)
}
<|endoftext|>


<commit_before>package scheduler

import (
	"strings"

	"github.com/spesnova/go-fleet/fleet"

	"github.com/spesnova/iruka/schema"
)

// CreateContaier creates and starts a new container as systemd unit via fleet
func (s *Scheduler) CreateContainer(c schema.Container) error {
	opts, err := s.containerToUnitOptions(c)
	if err != nil {
		return err
	}
	return s.fleet.Submit(c.Name+".service", opts, "launched")
}

func (s *Scheduler) DestroyContainer(c schema.Container) error {
	return s.fleet.Destroy(c.Name + ".service")
}

func (s *Scheduler) Containers() ([]schema.Container, error) {
	var containers []schema.Container

	units, err := s.fleet.Units()
	if err != nil {
		return nil, err
	}

	for _, unit := range units {
		container := schema.Container{
			Name:  strings.Replace(unit.Name, ".service", "", -1),
			State: unit.CurrentState,
		}
		containers = append(containers, container)
	}

	return containers, nil
}

func (s *Scheduler) containerToUnitOptions(c schema.Container) ([]*fleet.UnitOption, error) {
	return []*fleet.UnitOption{
		// Unit section example:
		//
		//   [Unit]
		//   Description=hello.web.a888f12a-0806-11e5-b898-5cf93896cc38
		//   Requires=docker.service
		//   After=docker.service
		//
		&fleet.UnitOption{
			Section: "Unit",
			Name:    "Description",
			Value:   c.Name,
		},
		&fleet.UnitOption{
			Section: "Unit",
			Name:    "Requires",
			Value:   "docker.service",
		},
		&fleet.UnitOption{
			Section: "Unit",
			Name:    "After",
			Value:   "docker.service",
		},
		// Service section example:
		//
		//   [Service]
		//   ExecStartPre=/usr/bin/docker pull quay.io/spesnova/example:latest
		//   ExecStartPre=-/usr/bin/docker kill hello.web.a888f12a-0806-11e5-b898-5cf93896cc38
		//   ExecStartPre=-/usr/bin/docker rm hello.web.a888f12a-0806-11e5-b898-5cf93896cc38
		//   ExecStart=/usr/bin/docker run \
		//     --name hello.web.a888f12a-0806-11e5-b898-5cf93896cc38 \
		//     -p 80 \
		//     -e HELLO=world \
		//     quay.io/spesnova/example:latest \
		//     /bin/hello
		//   ExecStop=/usr/bin/docker rm -f hello.web.a888f12a-0806-11e5-b898-5cf93896cc38
		//   Restart=on-failure
		//
		&fleet.UnitOption{
			Section: "Service",
			Name:    "ExecStartPre",
			Value:   strings.Join([]string{"/usr/bin/docker", "pull", c.Image}, " "),
		},
		&fleet.UnitOption{
			Section: "Service",
			Name:    "ExecStartPre",
			Value:   strings.Join([]string{"-/usr/bin/docker", "kill", c.Name}, " "),
		},
		&fleet.UnitOption{
			Section: "Service",
			Name:    "ExecStartPre",
			Value:   strings.Join([]string{"-/usr/bin/docker", "rm", c.Name}, " "),
		},
		&fleet.UnitOption{
			Section: "Service",
			Name:    "ExecStart",
			Value: strings.Join([]string{
				"/usr/bin/docker",
				"run",
				"--name",
				c.Name,
				configVarsOptions(c),
				portOptions(c),
				sizeOption(c),
				c.Image,
				c.Command,
			}, " "),
		},
		&fleet.UnitOption{
			Section: "Service",
			Name:    "ExecStop",
			Value:   strings.Join([]string{"/usr/bin/docker", "rm -f", c.Name}, " "),
		},
		&fleet.UnitOption{
			Section: "Service",
			Name:    "Restart",
			Value:   "on-failure",
		},
	}, nil
}

func configVarsOptions(c schema.Container) string {
	return ""
}

func portOptions(c schema.Container) string {
	return ""
}

func sizeOption(c schema.Container) string {
	return ""
}
<commit_msg>scheduler: Add supports exposing port option<commit_after>package scheduler

import (
	"strconv"
	"strings"

	"github.com/spesnova/go-fleet/fleet"

	"github.com/spesnova/iruka/schema"
)

// CreateContaier creates and starts a new container as systemd unit via fleet
func (s *Scheduler) CreateContainer(c schema.Container) error {
	opts, err := s.containerToUnitOptions(c)
	if err != nil {
		return err
	}
	return s.fleet.Submit(c.Name+".service", opts, "launched")
}

func (s *Scheduler) DestroyContainer(c schema.Container) error {
	return s.fleet.Destroy(c.Name + ".service")
}

func (s *Scheduler) Containers() ([]schema.Container, error) {
	var containers []schema.Container

	units, err := s.fleet.Units()
	if err != nil {
		return nil, err
	}

	for _, unit := range units {
		container := schema.Container{
			Name:  strings.Replace(unit.Name, ".service", "", -1),
			State: unit.CurrentState,
		}
		containers = append(containers, container)
	}

	return containers, nil
}

func (s *Scheduler) containerToUnitOptions(c schema.Container) ([]*fleet.UnitOption, error) {
	return []*fleet.UnitOption{
		// Unit section example:
		//
		//   [Unit]
		//   Description=hello.web.a888f12a-0806-11e5-b898-5cf93896cc38
		//   Requires=docker.service
		//   After=docker.service
		//
		&fleet.UnitOption{
			Section: "Unit",
			Name:    "Description",
			Value:   c.Name,
		},
		&fleet.UnitOption{
			Section: "Unit",
			Name:    "Requires",
			Value:   "docker.service",
		},
		&fleet.UnitOption{
			Section: "Unit",
			Name:    "After",
			Value:   "docker.service",
		},
		// Service section example:
		//
		//   [Service]
		//   ExecStartPre=/usr/bin/docker pull quay.io/spesnova/example:latest
		//   ExecStartPre=-/usr/bin/docker kill hello.web.a888f12a-0806-11e5-b898-5cf93896cc38
		//   ExecStartPre=-/usr/bin/docker rm hello.web.a888f12a-0806-11e5-b898-5cf93896cc38
		//   ExecStart=/usr/bin/docker run \
		//     --name hello.web.a888f12a-0806-11e5-b898-5cf93896cc38 \
		//     -p 80 \
		//     -e HELLO=world \
		//     quay.io/spesnova/example:latest \
		//     /bin/hello
		//   ExecStop=/usr/bin/docker rm -f hello.web.a888f12a-0806-11e5-b898-5cf93896cc38
		//   Restart=on-failure
		//
		&fleet.UnitOption{
			Section: "Service",
			Name:    "ExecStartPre",
			Value:   strings.Join([]string{"/usr/bin/docker", "pull", c.Image}, " "),
		},
		&fleet.UnitOption{
			Section: "Service",
			Name:    "ExecStartPre",
			Value:   strings.Join([]string{"-/usr/bin/docker", "kill", c.Name}, " "),
		},
		&fleet.UnitOption{
			Section: "Service",
			Name:    "ExecStartPre",
			Value:   strings.Join([]string{"-/usr/bin/docker", "rm", c.Name}, " "),
		},
		&fleet.UnitOption{
			Section: "Service",
			Name:    "ExecStart",
			Value: strings.Join([]string{
				"/usr/bin/docker",
				"run",
				"--name",
				c.Name,
				configVarsOptions(c),
				portOptions(c),
				sizeOption(c),
				c.Image,
				c.Command,
			}, " "),
		},
		&fleet.UnitOption{
			Section: "Service",
			Name:    "ExecStop",
			Value:   strings.Join([]string{"/usr/bin/docker", "rm -f", c.Name}, " "),
		},
		&fleet.UnitOption{
			Section: "Service",
			Name:    "Restart",
			Value:   "on-failure",
		},
	}, nil
}

func configVarsOptions(c schema.Container) string {
	return ""
}

func portOptions(c schema.Container) string {
	var opts []string
	for _, port := range c.Ports {
		opts = append(opts, strings.Join([]string{"-p", strconv.Itoa(port)}, " "))
	}
	return strings.Join(opts, " ")
}

func sizeOption(c schema.Container) string {
	return ""
}
<|endoftext|>


<commit_before>// Copyright 2017 Intel Corporation.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package scheduler

import (
	"github.com/intel-go/yanff/common"
	"github.com/intel-go/yanff/low"
	"runtime"
	"time"
)

// TODO: 1 is a delta of speeds. This delta should be
// verified or should be tunable from outside.
const speedDelta = 1

// Clones of flow functions. They are determined
// by their core and their stopper channel
type clonePair struct {
	core    int
	channel chan int
}

// UserContext is used inside flow packet and is going for user via it
type UserContext interface {
	Copy() interface{}
}

// Function types which are used inside flow functions
type uncloneFlowFunction func(interface{}, uint8)
type cloneFlowFunction func(interface{}, chan int, chan uint64, UserContext)
type conditionFlowFunction func(interface{}, uint64, bool) bool

// FlowFunction is a structure for all functions, is used inside flow package
type FlowFunction struct {
	// All parameters which gets these flow function
	Parameters interface{}
	// Main body of unclonable flow function
	uncloneFunction uncloneFlowFunction
	// Main body of clonable flow function
	cloneFunction cloneFlowFunction
	// Function true or false
	// and is used by scheduler
	checkPrintFunction conditionFlowFunction
	// Number of clones of this function
	cloneNumber int
	// Clones of this function. They are determined
	// by their core and their stopper channel
	clone []*clonePair
	// Channel for reporting current speed.
	// It is one for all clones
	report chan uint64
	// Current saved speed when making a clone.
	// It will be compared with immediate current speed to stop a clone.
	previousSpeed []float64
	// Name of flow function
	name string
	// Identifier which could be used together with flow function name
	// to uniquely identify this flow function
	identifier int
	context    UserContext
}

// NewUnclonableFlowFunction is a function for adding unclonable flow functions. Is used inside flow package
func (scheduler *Scheduler) NewUnclonableFlowFunction(name string, id int, ucfn uncloneFlowFunction,
	par interface{}) *FlowFunction {
	ff := new(FlowFunction)
	ff.name = name
	ff.identifier = id
	ff.uncloneFunction = ucfn
	ff.Parameters = par
	return ff
}

// NewClonableFlowFunction is a function for adding clonable flow functions. Is used inside flow package
func (scheduler *Scheduler) NewClonableFlowFunction(name string, id int, cfn cloneFlowFunction,
	par interface{}, check conditionFlowFunction, report chan uint64, context UserContext) *FlowFunction {
	ff := new(FlowFunction)
	ff.name = name
	ff.identifier = id
	ff.cloneFunction = cfn
	ff.Parameters = par
	ff.checkPrintFunction = check
	ff.report = report
	ff.previousSpeed = make([]float64, len(scheduler.freeCores), len(scheduler.freeCores))
	ff.context = context
	return ff
}

// Scheduler is a main structure for scheduler. Is used inside flow package
type Scheduler struct {
	Clonable          []*FlowFunction
	UnClonable        []*FlowFunction
	freeCores         []bool
	off               bool
	offRemove         bool
	stopDedicatedCore bool
	StopRing          *low.Queue
	usedCores         uint8
	checkTime         uint
	debugTime         uint
	Dropped           uint
}

// NewScheduler is a function for creation new scheduler. Is used inside flow package
func NewScheduler(coresNumber uint, schedulerOff bool, schedulerOffRemove bool,
	stopDedicatedCore bool, stopRing *low.Queue, checkTime uint, debugTime uint) Scheduler {
	// Init scheduler
	scheduler := new(Scheduler)
	scheduler.freeCores = make([]bool, coresNumber, coresNumber)
	for i := uint(0); i < coresNumber; i++ {
		scheduler.freeCores[i] = true
	}
	scheduler.off = schedulerOff
	scheduler.offRemove = schedulerOff || schedulerOffRemove
	scheduler.StopRing = stopRing
	scheduler.usedCores = 0
	scheduler.stopDedicatedCore = stopDedicatedCore
	scheduler.checkTime = checkTime
	scheduler.debugTime = debugTime
	scheduler.Dropped = 0

	return *scheduler
}

// SystemStart starts whole system. Is used inside flow package
func (scheduler *Scheduler) SystemStart() {
	core := scheduler.getCore(true)
	common.LogDebug(common.Initialization, "Start SCHEDULER at", core, "core")
	low.SetAffinity(uint8(core))
	if scheduler.stopDedicatedCore {
		core = scheduler.getCore(true)
		common.LogDebug(common.Initialization, "Start STOP at", core, "core")
	} else {
		common.LogDebug(common.Initialization, "Start STOP at scheduler", core, "core")
	}
	go func() {
		low.SetAffinity(uint8(core))
		low.Stop(scheduler.StopRing)
	}()
	for i := range scheduler.UnClonable {
		ff := scheduler.UnClonable[i]
		core := scheduler.getCore(true)
		common.LogDebug(common.Initialization, "Start unclonable FlowFunction", ff.name, ff.identifier, "at", core, "core")
		go func() {
			ff.uncloneFunction(ff.Parameters, uint8(core))
		}()
	}
	for i := range scheduler.Clonable {
		ff := scheduler.Clonable[i]
		core := scheduler.getCore(true)
		common.LogDebug(common.Initialization, "Start clonable FlowFunction", ff.name, ff.identifier, "at", core, "core")
		go func() {
			low.SetAffinity(uint8(core))
			ff.cloneNumber = 0
			if ff.context != nil {
				ff.cloneFunction(ff.Parameters, nil, ff.report, (ff.context.Copy()).(UserContext))
			} else {
				ff.cloneFunction(ff.Parameters, nil, ff.report, nil)
			}
		}()
	}
	// We need this to get a chance to all started goroutines to log their warnings.
	time.Sleep(time.Millisecond)
}

// Schedule - main execution. Is used inside flow package
func (scheduler *Scheduler) Schedule(schedTime uint) {
	tick := time.Tick(time.Duration(scheduler.checkTime) * time.Millisecond)
	debugTick := time.Tick(time.Duration(scheduler.debugTime) * time.Millisecond)
	checkRequired := false
	for {
		time.Sleep(time.Millisecond * time.Duration(schedTime))
		select {
		case <-tick:
			checkRequired = true
		case <-debugTick:
			// Report current state of system
			common.LogDebug(common.Debug, "---------------")
			common.LogDebug(common.Debug, "System is using", scheduler.usedCores, "cores now.", uint8(len(scheduler.freeCores))-scheduler.usedCores, "cores are left available.")
			low.Statistics(float32(scheduler.debugTime) / 1000)
			for i := range scheduler.Clonable {
				scheduler.Clonable[i].checkPrintFunction(scheduler.Clonable[i].Parameters, 0, true)
			}
			if scheduler.Dropped != 0 {
				common.LogDrop(common.Debug, "Flow functions together dropped", scheduler.Dropped, "packets")
				// It is race condition here, however it is just statistics.
				// It can be not accurate, but on the other hand doesn't influence performance
				scheduler.Dropped = 0
			}
			low.ReportMempoolsState()
		default:
		}
		// Procced with each clonable flow function
		for i := range scheduler.Clonable {
			ff := scheduler.Clonable[i]

			// Gather and sum current speeds from all clones of current flow function
			// Flow function itself and all clones put their speeds in one channel
			currentSpeed := uint64(0)
			t := len(ff.report) - ff.cloneNumber - 1
			for k := 0; k < t; k++ {
				<-ff.report
			}
			for k := 0; k < ff.cloneNumber+1; k++ {
				currentSpeed += <-ff.report
			}
			// We should multiply by 1000 because schedTime is in milliseconds
			speedPKTS := currentSpeed * 1000 / uint64(schedTime)
			// We multiply by 84 because it is the minimal packet size (64) plus before/after packet gaps in 40GB ethernet
			// We multiply by 8 to translate bytes to bits
			// We divide by 1000 and 1000 because networking suppose that megabit has 1000 kilobits instead of 1024
			// TODO This Mbits measurement should be used only for 84 packet size for debug purposes
//			speedMbits := speedPKTS * 84 * 8 / 1000 / 1000
			common.LogDebug(common.Debug, "Current speed of", ff.name, ff.identifier, "is", speedPKTS, "PKT/S")

			// After we gather current speed we can add or remove clones.

			// Firstly we check removing clones. We can remove one clone if:
			// 1. flow function has clones
			// 2. scheduler removing is switched on
			// 3. current speed of flow function is lower, than saved speed with less number of clones
			if (ff.cloneNumber != 0) && (scheduler.offRemove == false) && (float64(currentSpeed) < speedDelta*ff.previousSpeed[ff.cloneNumber-1]) {
				// Save current speed as speed of flow function with this number of clones before removing
				ff.previousSpeed[ff.cloneNumber] = float64(currentSpeed)
				ff.clone[ff.cloneNumber-1].channel <- -1
				scheduler.setCore(ff.clone[ff.cloneNumber-1].core)
				ff.clone = ff.clone[:len(ff.clone)-1]
				ff.cloneNumber--
				for j := 0; j < ff.cloneNumber; j++ {
					ff.clone[j].channel <- ff.cloneNumber
				}
				// After removing a clone we don't want to try to add clone immediately
				continue
			}

			// Secondly we check adding clones. We can add one clone if:
			// 1. check function signals that we need to clone
			// 2. scheduler is switched on
			// 3. we don't know flow function speed with more clones, or we know it and it is bigger than current speed
			if (ff.checkPrintFunction(ff.Parameters, speedPKTS, false) == true) && (scheduler.off == false) &&
				(ff.previousSpeed[ff.cloneNumber+1] == 0 || ff.previousSpeed[ff.cloneNumber+1] > speedDelta*float64(currentSpeed)) {
				// Save current speed as speed of flow function with this number of clones before adding
				ff.previousSpeed[ff.cloneNumber] = float64(currentSpeed)
				go func() {
					core := scheduler.getCore(false)
					if core != -1 {
						low.SetAffinity(uint8(core))
						quit := make(chan int)
						cp := new(clonePair)
						cp.channel = quit
						cp.core = core
						ff.clone = append(ff.clone, cp)
						ff.cloneNumber++
						for j := 0; j < ff.cloneNumber-1; j++ {
							// Here we want to add a pause to all clones. This pause depends on the number of clones.
							// As we can't use a channel from its goroutine we iterates to cloneNumber-1. We are OK
							// if the last clone will have zero pause as well as original (not cloned) function.
							ff.clone[j].channel <- ff.cloneNumber
						}
						if ff.context != nil {
							ff.cloneFunction(ff.Parameters, quit, ff.report, (ff.context.Copy()).(UserContext))
						} else {
							ff.cloneFunction(ff.Parameters, quit, ff.report, nil)
						}
					} else {
						common.LogWarning(common.Debug, "Can't start new clone for", ff.name, ff.identifier)
					}
				}()
				continue
			}

			// Scheduler can't add new clones if saved flow function speed with more
			// clones is slower than current. However this speed can be changed due to
			// external reasons. So flow function should check and update this speed
			// regular time.
			if checkRequired == true {
				// Regular flow function speed check after each checkTime milliseconds,
				// it is done by erasing previously measured speed data. After this scheduler can
				// add new clone. If performance decreases with added clone, it will be stopped
				// with setting speed data.
				ff.previousSpeed[ff.cloneNumber+1] = 0
			}
		}
		checkRequired = false
		runtime.Gosched()
	}
}

func (scheduler *Scheduler) setCore(i int) {
	scheduler.freeCores[i] = true
	scheduler.usedCores--
}

func (scheduler *Scheduler) getCore(startStage bool) int {
	for i := range scheduler.freeCores {
		if scheduler.freeCores[i] == true {
			scheduler.freeCores[i] = false
			scheduler.usedCores++
			return i
		}
	}
	if startStage == true {
		common.LogError(common.Initialization, "Requested number of cores isn't enough. System needs at least one core per each Set function (except Merger and Stopper) plus one additional core.")
	}
	return -1
}
<commit_msg>Refactoring of scheduler code. All helper functions are now outside of main-loop function.<commit_after>// Copyright 2017 Intel Corporation.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package scheduler

import (
	"github.com/intel-go/yanff/common"
	"github.com/intel-go/yanff/low"
	"runtime"
	"time"
)

// TODO: 1 is a delta of speeds. This delta should be
// verified or should be tunable from outside.
const speedDelta = 1

// Clones of flow functions. They are determined
// by their core and their stopper channel
type clonePair struct {
	core    int
	channel chan int
}

// UserContext is used inside flow packet and is going for user via it
type UserContext interface {
	Copy() interface{}
}

// Function types which are used inside flow functions
type uncloneFlowFunction func(interface{}, uint8)
type cloneFlowFunction func(interface{}, chan int, chan uint64, UserContext)
type conditionFlowFunction func(interface{}, uint64, bool) bool

// FlowFunction is a structure for all functions, is used inside flow package
type FlowFunction struct {
	// All parameters which gets these flow function
	Parameters interface{}
	// Main body of unclonable flow function
	uncloneFunction uncloneFlowFunction
	// Main body of clonable flow function
	cloneFunction cloneFlowFunction
	// Function true or false
	// and is used by scheduler
	checkPrintFunction conditionFlowFunction
	// Number of clones of this function
	cloneNumber int
	// Clones of this function. They are determined
	// by their core and their stopper channel
	clone []*clonePair
	// Channel for reporting current speed.
	// It is one for all clones
	report chan uint64
	// Current saved speed when making a clone.
	// It will be compared with immediate current speed to stop a clone.
	previousSpeed []float64
	// For debug purposes
	currentSpeed float64
	// Name of flow function
	name string
	// Identifier which could be used together with flow function name
	// to uniquely identify this flow function
	identifier int
	context    UserContext
}

// NewUnclonableFlowFunction is a function for adding unclonable flow functions. Is used inside flow package
func (scheduler *Scheduler) NewUnclonableFlowFunction(name string, id int, ucfn uncloneFlowFunction,
	par interface{}) *FlowFunction {
	ff := new(FlowFunction)
	ff.name = name
	ff.identifier = id
	ff.uncloneFunction = ucfn
	ff.Parameters = par
	return ff
}

// NewClonableFlowFunction is a function for adding clonable flow functions. Is used inside flow package
func (scheduler *Scheduler) NewClonableFlowFunction(name string, id int, cfn cloneFlowFunction,
	par interface{}, check conditionFlowFunction, report chan uint64, context UserContext) *FlowFunction {
	ff := new(FlowFunction)
	ff.name = name
	ff.identifier = id
	ff.cloneFunction = cfn
	ff.Parameters = par
	ff.checkPrintFunction = check
	ff.report = report
	ff.previousSpeed = make([]float64, len(scheduler.freeCores), len(scheduler.freeCores))
	ff.context = context
	return ff
}

// Scheduler is a main structure for scheduler. Is used inside flow package
type Scheduler struct {
	Clonable          []*FlowFunction
	UnClonable        []*FlowFunction
	freeCores         []bool
	off               bool
	offRemove         bool
	stopDedicatedCore bool
	StopRing          *low.Queue
	usedCores         uint8
	checkTime         uint
	debugTime         uint
	Dropped           uint
}

// NewScheduler is a function for creation new scheduler. Is used inside flow package
func NewScheduler(coresNumber uint, schedulerOff bool, schedulerOffRemove bool,
	stopDedicatedCore bool, stopRing *low.Queue, checkTime uint, debugTime uint) Scheduler {
	// Init scheduler
	scheduler := new(Scheduler)
	scheduler.freeCores = make([]bool, coresNumber, coresNumber)
	for i := uint(0); i < coresNumber; i++ {
		scheduler.freeCores[i] = true
	}
	scheduler.off = schedulerOff
	scheduler.offRemove = schedulerOff || schedulerOffRemove
	scheduler.StopRing = stopRing
	scheduler.usedCores = 0
	scheduler.stopDedicatedCore = stopDedicatedCore
	scheduler.checkTime = checkTime
	scheduler.debugTime = debugTime
	scheduler.Dropped = 0

	return *scheduler
}

// SystemStart starts whole system. Is used inside flow package
func (scheduler *Scheduler) SystemStart() {
	core := scheduler.getCore(true)
	common.LogDebug(common.Initialization, "Start SCHEDULER at", core, "core")
	low.SetAffinity(uint8(core))
	if scheduler.stopDedicatedCore {
		core = scheduler.getCore(true)
		common.LogDebug(common.Initialization, "Start STOP at", core, "core")
	} else {
		common.LogDebug(common.Initialization, "Start STOP at scheduler", core, "core")
	}
	go func() {
		low.SetAffinity(uint8(core))
		low.Stop(scheduler.StopRing)
	}()
	for i := range scheduler.UnClonable {
		ff := scheduler.UnClonable[i]
		core := scheduler.getCore(true)
		common.LogDebug(common.Initialization, "Start unclonable FlowFunction", ff.name, ff.identifier, "at", core, "core")
		go func() {
			ff.uncloneFunction(ff.Parameters, uint8(core))
		}()
	}
	for i := range scheduler.Clonable {
		ff := scheduler.Clonable[i]
		core := scheduler.getCore(true)
		common.LogDebug(common.Initialization, "Start clonable FlowFunction", ff.name, ff.identifier, "at", core, "core")
		go func() {
			low.SetAffinity(uint8(core))
			ff.cloneNumber = 0
			if ff.context != nil {
				ff.cloneFunction(ff.Parameters, nil, ff.report, (ff.context.Copy()).(UserContext))
			} else {
				ff.cloneFunction(ff.Parameters, nil, ff.report, nil)
			}
		}()
	}
	// We need this to get a chance to all started goroutines to log their warnings.
	time.Sleep(time.Millisecond)
}

// Schedule - main execution. Is used inside flow package
func (scheduler *Scheduler) Schedule(schedTime uint) {
	tick := time.Tick(time.Duration(scheduler.checkTime) * time.Millisecond)
	debugTick := time.Tick(time.Duration(scheduler.debugTime) * time.Millisecond)
	checkRequired := false
	for {
		time.Sleep(time.Millisecond * time.Duration(schedTime))
		select {
		case <-tick:
			checkRequired = true
		case <-debugTick:
			// Report current state of system
			common.LogDebug(common.Debug, "---------------")
			common.LogDebug(common.Debug, "System is using", scheduler.usedCores, "cores now.", uint8(len(scheduler.freeCores))-scheduler.usedCores, "cores are left available.")
			low.Statistics(float32(scheduler.debugTime) / 1000)
			for i := range scheduler.Clonable {
				scheduler.Clonable[i].printDebug(schedTime)
			}
			if scheduler.Dropped != 0 {
				common.LogDrop(common.Debug, "Flow functions together dropped", scheduler.Dropped, "packets")
				// It is race condition here, however it is just statistics.
				// It can be not accurate, but on the other hand doesn't influence performance
				scheduler.Dropped = 0
			}
			low.ReportMempoolsState()
		default:
		}
		// Procced with each clonable flow function
		for i := range scheduler.Clonable {
			ff := scheduler.Clonable[i]
			ff.updateCurrentSpeed()

			// Firstly we check removing clones. We can remove one clone if:
			// 1. flow function has clones
			// 2. scheduler removing is switched on
			// 3. current speed of flow function is lower, than saved speed with less number of clones
			if (ff.cloneNumber != 0) && (scheduler.offRemove == false) && (ff.currentSpeed < speedDelta*ff.previousSpeed[ff.cloneNumber-1]) {
				// Save current speed as speed of flow function with this number of clones before removing
				ff.previousSpeed[ff.cloneNumber] = ff.currentSpeed
				scheduler.removeClone(ff)
				for j := 0; j < ff.cloneNumber; j++ {
					ff.clone[j].channel <- ff.cloneNumber
				}
				// After removing a clone we don't want to try to add clone immediately
				continue
			}

			// Secondly we check adding clones. We can add one clone if:
			// 1. check function signals that we need to clone
			// 2. scheduler is switched on
			// 3. we don't know flow function speed with more clones, or we know it and it is bigger than current speed
			if (ff.checkPrintFunction(ff.Parameters, convertPKTS(ff.currentSpeed, schedTime), false) == true) && (scheduler.off == false) &&
				(ff.previousSpeed[ff.cloneNumber+1] == 0 || ff.previousSpeed[ff.cloneNumber+1] > speedDelta*ff.currentSpeed) {
				// Save current speed as speed of flow function with this number of clones before adding
				ff.previousSpeed[ff.cloneNumber] = ff.currentSpeed
				if scheduler.startClone(ff) == true {
					for j := 0; j < ff.cloneNumber-1; j++ {
						// Here we want to add a pause to all clones. This pause depends on the number of clones.
						// As we can't use a channel from its goroutine we iterates to cloneNumber-1. We are OK
						// if the last clone will have zero pause as well as original (not cloned) function.
						ff.clone[j].channel <- ff.cloneNumber
					}
				}
				continue
			}

			// Scheduler can't add new clones if saved flow function speed with more
			// clones is slower than current. However this speed can be changed due to
			// external reasons. So flow function should check and update this speed
			// regular time.
			if checkRequired == true {
				// Regular flow function speed check after each checkTime milliseconds,
				// it is done by erasing previously measured speed data. After this scheduler can
				// add new clone. If performance decreases with added clone, it will be stopped
				// with setting speed data.
				ff.previousSpeed[ff.cloneNumber+1] = 0
			}
		}
		checkRequired = false
		runtime.Gosched()
	}
}

func (scheduler *Scheduler) startClone(ff *FlowFunction) bool {
	core := scheduler.getCore(false)
	if core != -1 {
		go func() {
			low.SetAffinity(uint8(core))
			quit := make(chan int)
			cp := new(clonePair)
			cp.channel = quit
			cp.core = core
			ff.clone = append(ff.clone, cp)
			ff.cloneNumber++
			if ff.context != nil {
				ff.cloneFunction(ff.Parameters, quit, ff.report, (ff.context.Copy()).(UserContext))
			} else {
				ff.cloneFunction(ff.Parameters, quit, ff.report, nil)
			}
		}()
		return true
	} else {
		common.LogWarning(common.Debug, "Can't start new clone for", ff.name, ff.identifier)
		return false
	}
}

func (scheduler *Scheduler) removeClone(ff *FlowFunction) {
	ff.clone[ff.cloneNumber-1].channel <- -1
	scheduler.setCore(ff.clone[ff.cloneNumber-1].core)
	ff.clone = ff.clone[:len(ff.clone)-1]
	ff.cloneNumber--
}

func (ff *FlowFunction) printDebug(schedTime uint) {
	ff.checkPrintFunction(ff.Parameters, 0, true)
	speedPKTS := convertPKTS(ff.currentSpeed, schedTime)
	// We multiply by 84 because it is the minimal packet size (64) plus before/after packet gaps in 40GB ethernet
	// We multiply by 8 to translate bytes to bits
	// We divide by 1000 and 1000 because networking suppose that megabit has 1000 kilobits instead of 1024
	// TODO This Mbits measurement should be used only for 84 packet size for debug purposes
	// speedMbits := speedPKTS * 84 * 8 / 1000 / 1000
	common.LogDebug(common.Debug, "Current speed of", ff.name, ff.identifier, "is", speedPKTS, "PKT/S")
}

func convertPKTS(currentSpeed float64, schedTime uint) uint64 {
	// We should multiply by 1000 because schedTime is in milliseconds
	return uint64(currentSpeed) * 1000 / uint64(schedTime)
}

func (ff *FlowFunction) updateCurrentSpeed() {
	// Gather and sum current speeds from all clones of current flow function
	// Flow function itself and all clones put their speeds in one channel
	currentSpeed := uint64(0)
	t := len(ff.report) - ff.cloneNumber - 1
	for k := 0; k < t; k++ {
		<-ff.report
	}
	for k := 0; k < ff.cloneNumber+1; k++ {
		currentSpeed += <-ff.report
	}
	ff.currentSpeed = float64(currentSpeed)
}

func (scheduler *Scheduler) setCore(i int) {
	scheduler.freeCores[i] = true
	scheduler.usedCores--
}

func (scheduler *Scheduler) getCore(startStage bool) int {
	for i := range scheduler.freeCores {
		if scheduler.freeCores[i] == true {
			scheduler.freeCores[i] = false
			scheduler.usedCores++
			return i
		}
	}
	if startStage == true {
		common.LogError(common.Initialization, "Requested number of cores isn't enough. System needs at least one core per each Set function (except Merger and Stopper) plus one additional core.")
	}
	return -1
}
<|endoftext|>