<commit_before>package sarama

import (
	"testing"
	"time"
)

var (
	broker, coordinator *mockBroker
	testClient          Client
)

func TestNewOffsetManager(t *testing.T) {
	seedBroker := newMockBroker(t, 1)
	seedBroker.Returns(new(MetadataResponse))

	testClient, err := NewClient([]string{seedBroker.Addr()}, nil)
	if err != nil {
		t.Fatal(err)
	}

	_, err = NewOffsetManagerFromClient("grouop", testClient)
	if err != nil {
		t.Error(err)
	}

	testClient.Close()

	_, err = NewOffsetManagerFromClient("group", testClient)
	if err != ErrClosedClient {
		t.Errorf("Error expected for closed client; actual value: %v", err)
	}

	seedBroker.Close()
}

func initPOM(t *testing.T) PartitionOffsetManager {
	config := NewConfig()
	config.Metadata.Retry.Max = 0
	config.Consumer.Offsets.CommitInterval = 1 * time.Millisecond

	broker = newMockBroker(t, 1)
	coordinator = newMockBroker(t, 2)

	seedMeta := new(MetadataResponse)
	seedMeta.AddBroker(coordinator.Addr(), coordinator.BrokerID())
	// seedMeta.AddTopicPartition("my_topic", 0, 1, []int32{}, []int32{}, ErrNoError)
	broker.Returns(seedMeta)

	var err error
	testClient, err = NewClient([]string{broker.Addr()}, config)
	if err != nil {
		t.Fatal(err)
	}

	broker.Returns(&ConsumerMetadataResponse{
		CoordinatorID:   coordinator.BrokerID(),
		CoordinatorHost: "127.0.0.1",
		CoordinatorPort: coordinator.Port(),
	})

	fetchResponse := new(OffsetFetchResponse)
	fetchResponse.AddBlock("my_topic", 0, &OffsetFetchResponseBlock{
		Err:      ErrNoError,
		Offset:   5,
		Metadata: "test_meta",
	})
	coordinator.Returns(fetchResponse)

	om, err := NewOffsetManagerFromClient("group", testClient)
	if err != nil {
		t.Fatal(err)
	}

	pom, err := om.ManagePartition("my_topic", 0)
	if err != nil {
		t.Fatal(err)
	}
	return pom
}

func TestOffset(t *testing.T) {
	pom := initPOM(t)
	offset, meta := pom.Offset()
	if offset != 5 {
		t.Errorf("Expected offset 5. Actual: %v", offset)
	}
	if meta != "test_meta" {
		t.Errorf("Expected metadata \"test_meta\". Actual: %q", meta)
	}

	pom.Close()
	testClient.Close()
	broker.Close()
	coordinator.Close()
}

func TestSetOffset(t *testing.T) {
	pom := initPOM(t)

	ocResponse := new(OffsetCommitResponse)
	ocResponse.AddError("my_topic", 0, ErrNoError)
	coordinator.Returns(ocResponse)

	pom.SetOffset(100, "modified_meta")
	offset, meta := pom.Offset()

	if offset != 100 {
		t.Errorf("Expected offset 100. Actual: %v", offset)
	}
	if meta != "modified_meta" {
		t.Errorf("Expected metadata \"modified_meta\". Actual: %q", meta)
	}

	pom.Close()
	testClient.Close()
	broker.Close()
	coordinator.Close()
}

func TestCloseErrors(t *testing.T) {
	pom := initPOM(t)
	// pom.errors = "foo"
	err := pom.Close()
	if err != nil {
		t.Error(err)
	}

	testClient.Close()
	broker.Close()
	coordinator.Close()
}
<commit_msg>Test handling of commit errors when an offset is set<commit_after>package sarama

import (
	"log"
	"testing"
	"time"
)

var (
	broker, coordinator *mockBroker
	testClient          Client
)

func TestNewOffsetManager(t *testing.T) {
	seedBroker := newMockBroker(t, 1)
	seedBroker.Returns(new(MetadataResponse))

	testClient, err := NewClient([]string{seedBroker.Addr()}, nil)
	if err != nil {
		t.Fatal(err)
	}

	_, err = NewOffsetManagerFromClient("grouop", testClient)
	if err != nil {
		t.Error(err)
	}

	testClient.Close()

	_, err = NewOffsetManagerFromClient("group", testClient)
	if err != ErrClosedClient {
		t.Errorf("Error expected for closed client; actual value: %v", err)
	}

	seedBroker.Close()
}

func initPOM(t *testing.T) PartitionOffsetManager {
	config := NewConfig()
	config.Metadata.Retry.Max = 1
	config.Consumer.Offsets.CommitInterval = 1 * time.Millisecond

	broker = newMockBroker(t, 1)
	coordinator = newMockBroker(t, 2)

	seedMeta := new(MetadataResponse)
	seedMeta.AddBroker(coordinator.Addr(), coordinator.BrokerID())
	seedMeta.AddTopicPartition("my_topic", 0, 1, []int32{}, []int32{}, ErrNoError)
	seedMeta.AddTopicPartition("my_topic", 1, 1, []int32{}, []int32{}, ErrNoError)
	broker.Returns(seedMeta)

	var err error
	testClient, err = NewClient([]string{broker.Addr()}, config)
	if err != nil {
		t.Fatal(err)
	}

	broker.Returns(&ConsumerMetadataResponse{
		CoordinatorID:   coordinator.BrokerID(),
		CoordinatorHost: "127.0.0.1",
		CoordinatorPort: coordinator.Port(),
	})

	fetchResponse := new(OffsetFetchResponse)
	fetchResponse.AddBlock("my_topic", 0, &OffsetFetchResponseBlock{
		Err:      ErrNoError,
		Offset:   5,
		Metadata: "test_meta",
	})
	coordinator.Returns(fetchResponse)

	om, err := NewOffsetManagerFromClient("group", testClient)
	if err != nil {
		t.Fatal(err)
	}

	pom, err := om.ManagePartition("my_topic", 0)
	if err != nil {
		t.Fatal(err)
	}
	return pom
}

func TestOffset(t *testing.T) {
	pom := initPOM(t)
	offset, meta := pom.Offset()
	if offset != 5 {
		t.Errorf("Expected offset 5. Actual: %v", offset)
	}
	if meta != "test_meta" {
		t.Errorf("Expected metadata \"test_meta\". Actual: %q", meta)
	}

	pom.Close()
	testClient.Close()
	broker.Close()
	coordinator.Close()
}

func TestSetOffset(t *testing.T) {
	pom := initPOM(t)

	ocResponse := new(OffsetCommitResponse)
	ocResponse.AddError("my_topic", 0, ErrNoError)
	coordinator.Returns(ocResponse)

	pom.SetOffset(100, "modified_meta")
	offset, meta := pom.Offset()

	if offset != 100 {
		t.Errorf("Expected offset 100. Actual: %v", offset)
	}
	if meta != "modified_meta" {
		t.Errorf("Expected metadata \"modified_meta\". Actual: %q", meta)
	}

	pom.Close()
	testClient.Close()
	broker.Close()
	coordinator.Close()
}

// This test is not passing
func TestCommitErr(t *testing.T) {
	log.Println("TestCommitErr")
	log.Println("=============")
	pom := initPOM(t)

	ocResponse := new(OffsetCommitResponse)
	// ocResponse.Errors
	ocResponse.AddError("my_topic", 0, ErrOffsetOutOfRange)
	ocResponse.AddError("my_topic", 1, ErrNoError)
	coordinator.Returns(ocResponse)

	// ocResponse2 := new(OffsetCommitResponse)
	// // ocResponse.Errors
	// ocResponse2.AddError("my_topic", 1, ErrNoError)
	// coordinator.Returns(ocResponse2)

	freshCoordinator := newMockBroker(t, 3)

	// For RefreshCoordinator()
	coordinatorResponse3 := new(ConsumerMetadataResponse)
	coordinatorResponse3.CoordinatorID = freshCoordinator.BrokerID()
	coordinatorResponse3.CoordinatorHost = "127.0.0.1"
	coordinatorResponse3.CoordinatorPort = freshCoordinator.Port()
	broker.Returns(coordinatorResponse3)

	ocResponse2 := new(OffsetCommitResponse)
	// ocResponse.Errors
	// ocResponse2.AddError("my_topic", 0, ErrOffsetOutOfRange)
	ocResponse2.AddError("my_topic", 0, ErrNoError)
	freshCoordinator.Returns(ocResponse2)

	pom.SetOffset(100, "modified_meta")

	err := pom.Close()
	if err != nil {
		t.Error(err)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"fmt"
	"html"
	"io"
	"net"
	"net/http"
	"strconv"
)

func taskList(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/html")
	p := writerf(w)
	p("<title>runsit</title>")
	p("
runsit Admin UI

running tasks

\n")
	for _, t := range GetTasks() {
		p("
%s: %s
\n", t.Name, t.Name,
			html.EscapeString(t.Status()))
	}
	p("
\n")
	p("
runsit log

%s
\n", html.EscapeString(logBuf.String()))
	p("\n")
}

func killTask(w http.ResponseWriter, r *http.Request, t *Task) {
	w.Header().Set("Content-Type", "text/html")
	in, ok := t.RunningInstance()
	if !ok {
		http.Error(w, "task not running", 500)
		return
	}
	pid, _ := strconv.Atoi(r.FormValue("pid"))
	if in.Pid() != pid || pid == 0 {
		http.Error(w, "active task pid doesn't match pid parameter", 500)
		return
	}
	in.cmd.Process.Kill()
	p := writerf(w)
	p("killed pid %d.
back to %s status", pid, t.Name, t.Name)
}

func taskView(w http.ResponseWriter, r *http.Request) {
	path := r.URL.Path
	taskName := path[len("/task/"):]
	t, ok := GetTask(taskName)
	if !ok {
		http.NotFound(w, r)
		return
	}
	mode := r.FormValue("mode")
	switch mode {
	case "kill":
		killTask(w, r, t)
		return
	default:
		http.Error(w, "unknown mode", 400)
		return
	case "":
	}

	// Buffer to memory so we never block writing to a slow client
	// while holding the TaskOutput mutex.
	var buf bytes.Buffer
	p := writerf(&buf)
	defer io.Copy(w, &buf)

	p("<title>runsit; task %q</title>", t.Name)
	p("\n", css)
	p("\n", js)
	p("")
	p("
[runsit status]

%v
\n", t.Name)
	p("
status: %v
", html.EscapeString(t.Status()))

	in, ok := t.RunningInstance()
	if ok {
		if pid := in.Pid(); pid != 0 {
			p("
running instance: pid=%d ", pid)
			p("[kill] ", taskName, pid)
			p("
")
		}
		out := &in.output
		out.mu.Lock()
		defer out.mu.Unlock()
		p("
")
		for e := out.lines.Front(); e != nil; e = e.Next() {
			ol := e.Value.(*outputLine)
			p("
%s
\n", ol.name, ol.t, html.EscapeString(ol.data))
		}
		p("
\n")
	}

	p("\n")
}

func writerf(w io.Writer) func(string, ...interface{}) {
	return func(format string, args ...interface{}) {
		fmt.Fprintf(w, format, args...)
	}
}

func runWebServer(ln net.Listener) {
	mux := http.NewServeMux()
	// TODO: wrap mux in auth handler, making it available only to
	// TCP connections from localhost and owned by the uid/gid of
	// the running process.
	mux.HandleFunc("/", taskList)
	mux.HandleFunc("/task/", taskView)
	s := &http.Server{
		Handler: mux,
	}
	err := s.Serve(ln)
	if err != nil {
		logger.Fatalf("webserver exiting: %v", err)
	}
}

var css = `
#output {
   font-family: monospace;
   font-size: 10pt;
   border: 2px solid gray;
   padding: 0.5em;
   overflow: scroll;
   max-height: 25em;
}

#output div.stderr {
   color: #c00;
}

`

var js = `
window.addEventListener("load", function() {
   var d = document.getElementById("output");
   if (d) {
     d.scrollTop = d.scrollHeight;
   }
});

`<commit_msg>show command line and args<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"fmt"
	"html"
	"io"
	"net"
	"net/http"
	"strconv"
	"strings"
)

func taskList(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/html")
	p := writerf(w)
	p("<title>runsit</title>")
	p("
runsit Admin UI

running tasks

\n")
	for _, t := range GetTasks() {
		p("
%s: %s
\n", t.Name, t.Name,
			html.EscapeString(t.Status()))
	}
	p("
\n")
	p("
runsit log

%s
\n", html.EscapeString(logBuf.String()))
	p("\n")
}

func killTask(w http.ResponseWriter, r *http.Request, t *Task) {
	w.Header().Set("Content-Type", "text/html")
	in, ok := t.RunningInstance()
	if !ok {
		http.Error(w, "task not running", 500)
		return
	}
	pid, _ := strconv.Atoi(r.FormValue("pid"))
	if in.Pid() != pid || pid == 0 {
		http.Error(w, "active task pid doesn't match pid parameter", 500)
		return
	}
	in.cmd.Process.Kill()
	p := writerf(w)
	p("killed pid %d.
back to %s status", pid, t.Name, t.Name)
}

func taskView(w http.ResponseWriter, r *http.Request) {
	path := r.URL.Path
	taskName := path[len("/task/"):]
	t, ok := GetTask(taskName)
	if !ok {
		http.NotFound(w, r)
		return
	}
	mode := r.FormValue("mode")
	switch mode {
	case "kill":
		killTask(w, r, t)
		return
	default:
		http.Error(w, "unknown mode", 400)
		return
	case "":
	}

	// Buffer to memory so we never block writing to a slow client
	// while holding the TaskOutput mutex.
	var buf bytes.Buffer
	p := writerf(&buf)
	defer io.Copy(w, &buf)

	p("<title>runsit; task %q</title>", t.Name)
	p("\n", css)
	p("\n", js)
	p("")
	p("
[runsit status]

%v
\n", t.Name)
	p("
status: %v
", html.EscapeString(t.Status()))

	in, ok := t.RunningInstance()
	if ok {
		if pid := in.Pid(); pid != 0 {
			p("
running instance: pid=%d ", pid)
			p("[kill] ", taskName, pid)
			p("
")
		}
		p("
command: %s ", in.cmd.Path)
		for _, arg := range in.cmd.Args[1:] {
			qarg := arg
			if strings.Contains(arg, " ") || strings.Contains(arg, "\"") {
				qarg = fmt.Sprintf("%q", qarg)
			}
			p("%s ", html.EscapeString(qarg))
		}

		out := &in.output
		out.mu.Lock()
		defer out.mu.Unlock()
		p("
")
		for e := out.lines.Front(); e != nil; e = e.Next() {
			ol := e.Value.(*outputLine)
			p("
%s
\n", ol.name, ol.t, html.EscapeString(ol.data))
		}
		p("
\n")
	}

	p("\n")
}

func writerf(w io.Writer) func(string, ...interface{}) {
	return func(format string, args ...interface{}) {
		fmt.Fprintf(w, format, args...)
	}
}

func runWebServer(ln net.Listener) {
	mux := http.NewServeMux()
	// TODO: wrap mux in auth handler, making it available only to
	// TCP connections from localhost and owned by the uid/gid of
	// the running process.
	mux.HandleFunc("/", taskList)
	mux.HandleFunc("/task/", taskView)
	s := &http.Server{
		Handler: mux,
	}
	err := s.Serve(ln)
	if err != nil {
		logger.Fatalf("webserver exiting: %v", err)
	}
}

var css = `
#output {
   font-family: monospace;
   font-size: 10pt;
   border: 2px solid gray;
   padding: 0.5em;
   overflow: scroll;
   max-height: 25em;
}

#output div.stderr {
   color: #c00;
}

`

var js = `
window.addEventListener("load", function() {
   var d = document.getElementById("output");
   if (d) {
     d.scrollTop = d.scrollHeight;
   }
});

`
<|endoftext|>


<commit_before>package td_client

import "testing"

func TestShowTable(t *testing.T) {
	client, err := NewTDClient(Settings{
		Transport: &DummyTransport{[]byte(showTableResponse)},
	})
	if err != nil {
		t.Fatalf("failed create client: %s", err.Error())
	}
	table, err := client.ShowTable("test_database", "test_table")
	if err != nil {
		t.Fatalf("bad request: %s", err.Error())
	}
	if table.Name != "test_table" {
		t.Fatal("unexpected table name")
	}
	t.Log(table)
}

func TestListTables(t *testing.T) {
	client, err := NewTDClient(Settings{
		Transport: &DummyTransport{[]byte(listTablesResponse)},
	})
	if err != nil {
		t.Fatalf("failed create client: %s", err.Error())
	}
	tableList, err := client.ListTables("test")
	if err != nil {
		t.Fatalf("bad request: %s", err.Error())
	}
	if len(*tableList) != 3 {
		t.Fatal("not expected database count")
	}
}

func TestCreateLogTable(t *testing.T) {
	client, err := NewTDClient(Settings{
		Transport: &DummyTransport{[]byte(createLogTableResponse)},
	})
	if err != nil {
		t.Fatalf("failed create client: %s", err.Error())
	}
	err = client.CreateLogTable("test_database", "test_table")
	if err != nil {
		t.Fatalf("bad request: %s", err.Error())
	}
}

func TestSwapTable(t *testing.T) {
	client, err := NewTDClient(Settings{
		Transport: &DummyTransport{[]byte(swapTableResponse)},
	})
	if err != nil {
		t.Fatalf("failed create client: %s", err.Error())
	}
	err = client.SwapTable("test_database", "test_table2", "test_table")
	if err != nil {
		t.Fatalf("bad request: %s", err.Error())
	}

}

func TestUpdateSchema(t *testing.T) {
	client, err := NewTDClient(Settings{
		Transport: &DummyTransport{[]byte(updateSchemaResponse)},
	})
	if err != nil {
		t.Fatalf("failed create client: %s", err.Error())
	}
	err = client.UpdateSchema("test_database", "test_table", updateSchemaRequestParams)
	if err != nil {
		t.Fatalf("bad request: %s", err.Error())
	}

}

func TestUpdateExpire(t *testing.T) {
	client, err := NewTDClient(Settings{
		Transport: &DummyTransport{[]byte(updateExpireResponse)},
	})
	if err != nil {
		t.Fatalf("failed create client: %s", err.Error())
	}
	err = client.UpdateExpire("test_database", "test_table", 10)
	if err != nil {
		t.Fatalf("bad request: %s", err.Error())
	}
}

func TestDeleteTable(t *testing.T) {
	client, err := NewTDClient(Settings{
		Transport: &DummyTransport{[]byte(deleteTableResponse)},
	})
	if err != nil {
		t.Fatalf("failed create client: %s", err.Error())
	}
	_type, err := client.DeleteTable("test_database", "test_table")
	if err != nil {
		t.Fatalf("bad request: %s", err.Error())
	}
	if _type != "log" {
		t.Fatalf("unexpected type: %s", _type)
	}
}

func TestTail(t *testing.T) {}

const showTableResponse = `
{
   "id":999999,
   "name":"test_table",
   "estimated_storage_size":0,
   "counter_updated_at":null,
   "last_log_timestamp":null,
   "delete_protected":false,
   "created_at":"2017-05-14 12:19:37 UTC",
   "updated_at":"2017-05-14 15:53:17 UTC",
   "type":"log",
   "count":0,
   "schema":"[[\"col1\",\"string\"]]",
   "expire_days":10,
   "include_v":true
}
`

const listTablesResponse = `
{
   "database":"nh4_test",
   "tables":[
      {
         "id":99999991,
         "name":"test_table_1",
         "estimated_storage_size":67158,
         "counter_updated_at":"2017-05-13T11:39:53Z",
         "last_log_timestamp":"2017-05-07T08:59:35Z",
         "delete_protected":false,
         "created_at":"2017-03-30 09:00:14 UTC",
         "updated_at":"2017-05-07 08:59:40 UTC",
         "type":"log",
         "count":1751,
         "schema":"[[\"id\",\"long\"],[\"fizz\",\"string\"],[\"buzz\",\"string\"],[\"fizzbuzz\",\"string\"],[\"created_at\",\"string\"],[\"created_by\",\"long\"],[\"updated_at\",\"string\"],[\"updated_by\",\"long\"],[\"deleted_at\",\"string\"],[\"deleted_by\",\"long\"]]",
         "expire_days":null,
	 "include_v":true
      },
      {
         "id":99999992,
         "name":"test_table_2",
         "estimated_storage_size":84251,
         "counter_updated_at":"2017-05-13T11:39:53Z",
         "last_log_timestamp":"2016-12-28T20:59:52Z",
         "delete_protected":false,
         "created_at":"2016-11-07 11:57:15 UTC",
         "updated_at":"2017-05-07 08:52:15 UTC",
         "type":"log",
         "count":2539,
         "schema":"[[\"id\",\"long\"],[\"hogehoge\",\"string\"],[\"fugafuga\",\"long\"],[\"created_at\",\"string\"],[\"updated_at\",\"string\"],[\"created_by\",\"long\"],[\"updated_by\",\"long\"]]",
         "expire_days":null,
	 "include_v":true
      },
      {
         "id":99999993,
         "name":"test_table_3",
         "estimated_storage_size":1609,
         "counter_updated_at":"2017-05-13T11:39:53Z",
         "last_log_timestamp":"2017-05-07T08:52:24Z",
         "delete_protected":false,
         "created_at":"2016-11-07 11:56:58 UTC",
         "updated_at":"2017-05-07 08:52:27 UTC",
         "type":"log",
         "count":76,
         "schema":"[[\"id\",\"long\"],[\"foo\",\"string\"],[\"var\",\"string\"],[\"created_at\",\"string\"],[\"created_by\",\"long\"],[\"updated_at\",\"string\"],[\"updated_by\",\"long\"],[\"deleted_at\",\"string\"],[\"deleted_by\",\"long\"]]",
         "expire_days":null,
	 "include_v":true
      }
   ]
}
`

const createLogTableResponse = `
{
	"table":"test_table",
	"type":"log",
	"database":"test_database"
}
`

const swapTableResponse = `
{
	"database":"test_database",
	"table1":"test_table2",
	"table2":"test_table"
}
`

var updateSchemaRequestParams = []interface{}{
	[]interface{}{"col1", "string"},
	[]interface{}{"col2", "long"},
	[]interface{}{"col3", "string"},
}

const updateSchemaResponse = `
{
	"table":"test_table",
	"type":"log",
	"database":"test_database"
}
`
const updateExpireResponse = `
{
	"table":"test_table",
	"type":"log",
	"database":"test_database"
}
`

const deleteTableResponse = `
{
	"table":"test_table",
	"type":"log",
	"database":"test_database"
}
`
<commit_msg>Fix indent<commit_after>package td_client

import "testing"

func TestShowTable(t *testing.T) {
	client, err := NewTDClient(Settings{
		Transport: &DummyTransport{[]byte(showTableResponse)},
	})
	if err != nil {
		t.Fatalf("failed create client: %s", err.Error())
	}
	table, err := client.ShowTable("test_database", "test_table")
	if err != nil {
		t.Fatalf("bad request: %s", err.Error())
	}
	if table.Name != "test_table" {
		t.Fatal("unexpected table name")
	}
	t.Log(table)
}

func TestListTables(t *testing.T) {
	client, err := NewTDClient(Settings{
		Transport: &DummyTransport{[]byte(listTablesResponse)},
	})
	if err != nil {
		t.Fatalf("failed create client: %s", err.Error())
	}
	tableList, err := client.ListTables("test")
	if err != nil {
		t.Fatalf("bad request: %s", err.Error())
	}
	if len(*tableList) != 3 {
		t.Fatal("not expected database count")
	}
}

func TestCreateLogTable(t *testing.T) {
	client, err := NewTDClient(Settings{
		Transport: &DummyTransport{[]byte(createLogTableResponse)},
	})
	if err != nil {
		t.Fatalf("failed create client: %s", err.Error())
	}
	err = client.CreateLogTable("test_database", "test_table")
	if err != nil {
		t.Fatalf("bad request: %s", err.Error())
	}
}

func TestSwapTable(t *testing.T) {
	client, err := NewTDClient(Settings{
		Transport: &DummyTransport{[]byte(swapTableResponse)},
	})
	if err != nil {
		t.Fatalf("failed create client: %s", err.Error())
	}
	err = client.SwapTable("test_database", "test_table2", "test_table")
	if err != nil {
		t.Fatalf("bad request: %s", err.Error())
	}

}

func TestUpdateSchema(t *testing.T) {
	client, err := NewTDClient(Settings{
		Transport: &DummyTransport{[]byte(updateSchemaResponse)},
	})
	if err != nil {
		t.Fatalf("failed create client: %s", err.Error())
	}
	err = client.UpdateSchema("test_database", "test_table", updateSchemaRequestParams)
	if err != nil {
		t.Fatalf("bad request: %s", err.Error())
	}

}

func TestUpdateExpire(t *testing.T) {
	client, err := NewTDClient(Settings{
		Transport: &DummyTransport{[]byte(updateExpireResponse)},
	})
	if err != nil {
		t.Fatalf("failed create client: %s", err.Error())
	}
	err = client.UpdateExpire("test_database", "test_table", 10)
	if err != nil {
		t.Fatalf("bad request: %s", err.Error())
	}
}

func TestDeleteTable(t *testing.T) {
	client, err := NewTDClient(Settings{
		Transport: &DummyTransport{[]byte(deleteTableResponse)},
	})
	if err != nil {
		t.Fatalf("failed create client: %s", err.Error())
	}
	_type, err := client.DeleteTable("test_database", "test_table")
	if err != nil {
		t.Fatalf("bad request: %s", err.Error())
	}
	if _type != "log" {
		t.Fatalf("unexpected type: %s", _type)
	}
}

func TestTail(t *testing.T) {}

const showTableResponse = `
{
   "id":999999,
   "name":"test_table",
   "estimated_storage_size":0,
   "counter_updated_at":null,
   "last_log_timestamp":null,
   "delete_protected":false,
   "created_at":"2017-05-14 12:19:37 UTC",
   "updated_at":"2017-05-14 15:53:17 UTC",
   "type":"log",
   "count":0,
   "schema":"[[\"col1\",\"string\"]]",
   "expire_days":10,
   "include_v":true
}
`

const listTablesResponse = `
{
   "database":"nh4_test",
   "tables":[
      {
         "id":99999991,
         "name":"test_table_1",
         "estimated_storage_size":67158,
         "counter_updated_at":"2017-05-13T11:39:53Z",
         "last_log_timestamp":"2017-05-07T08:59:35Z",
         "delete_protected":false,
         "created_at":"2017-03-30 09:00:14 UTC",
         "updated_at":"2017-05-07 08:59:40 UTC",
         "type":"log",
         "count":1751,
         "schema":"[[\"id\",\"long\"],[\"fizz\",\"string\"],[\"buzz\",\"string\"],[\"fizzbuzz\",\"string\"],[\"created_at\",\"string\"],[\"created_by\",\"long\"],[\"updated_at\",\"string\"],[\"updated_by\",\"long\"],[\"deleted_at\",\"string\"],[\"deleted_by\",\"long\"]]",
         "expire_days":null,
         "include_v":true
      },
      {
         "id":99999992,
         "name":"test_table_2",
         "estimated_storage_size":84251,
         "counter_updated_at":"2017-05-13T11:39:53Z",
         "last_log_timestamp":"2016-12-28T20:59:52Z",
         "delete_protected":false,
         "created_at":"2016-11-07 11:57:15 UTC",
         "updated_at":"2017-05-07 08:52:15 UTC",
         "type":"log",
         "count":2539,
         "schema":"[[\"id\",\"long\"],[\"hogehoge\",\"string\"],[\"fugafuga\",\"long\"],[\"created_at\",\"string\"],[\"updated_at\",\"string\"],[\"created_by\",\"long\"],[\"updated_by\",\"long\"]]",
         "expire_days":null,
         "include_v":true
      },
      {
         "id":99999993,
         "name":"test_table_3",
         "estimated_storage_size":1609,
         "counter_updated_at":"2017-05-13T11:39:53Z",
         "last_log_timestamp":"2017-05-07T08:52:24Z",
         "delete_protected":false,
         "created_at":"2016-11-07 11:56:58 UTC",
         "updated_at":"2017-05-07 08:52:27 UTC",
         "type":"log",
         "count":76,
         "schema":"[[\"id\",\"long\"],[\"foo\",\"string\"],[\"var\",\"string\"],[\"created_at\",\"string\"],[\"created_by\",\"long\"],[\"updated_at\",\"string\"],[\"updated_by\",\"long\"],[\"deleted_at\",\"string\"],[\"deleted_by\",\"long\"]]",
         "expire_days":null,
         "include_v":true
      }
   ]
}
`

const createLogTableResponse = `
{
	"table":"test_table",
	"type":"log",
	"database":"test_database"
}
`

const swapTableResponse = `
{
	"database":"test_database",
	"table1":"test_table2",
	"table2":"test_table"
}
`

var updateSchemaRequestParams = []interface{}{
	[]interface{}{"col1", "string"},
	[]interface{}{"col2", "long"},
	[]interface{}{"col3", "string"},
}

const updateSchemaResponse = `
{
	"table":"test_table",
	"type":"log",
	"database":"test_database"
}
`
const updateExpireResponse = `
{
	"table":"test_table",
	"type":"log",
	"database":"test_database"
}
`

const deleteTableResponse = `
{
	"table":"test_table",
	"type":"log",
	"database":"test_database"
}
`
<|endoftext|>


<commit_before>/*
Crunchy PostgreSQL Operator API

The Crunchy PostgreSQL Operator API defines HTTP(S) interactions with the Crunchy PostgreSQL Operator.

## Direct API Calls

The API can also be accessed by interacting directly with the API server. This
can be done by making HTTP requests with curl to get information from the
server. In order to make these calls you will need to provide certificates along
with your request using the `--cacert`, `--key`, and `--cert` flags. Next you
will need to provide the username and password for the RBAC along with a header
that includes the content type and the `--insecure` flag. These flags will be
the same for all of your interactions with the API server and can be seen in the
following examples.

###### Get API Server Version

The most basic example of this interaction is getting the version of the API
server. You can send a GET request to `$PGO_APISERVER_URL/version` and this will
send back a json response including the API server version. You must specify the
client version that matches the API server version as part of the request.

The API server is setup to work with the pgo command line interface so the
parameters that are passed to the server can be found by looking at the related
flags. For example, the series parameter used in the `create` example below is
the same as the `-e, --series` flag that is described in the [pgo cli docs](https://access.crunchydata.com/documentation/postgres-operator/4.1.0/operatorcli/cli/pgo_create_cluster/).
```
curl --cacert $PGO_CA_CERT --key $PGO_CLIENT_KEY --cert $PGO_CA_CERT -u \
pgoadmin:examplepassword -H "Content-Type:application/json" --insecure -X \
GET $PGO_APISERVER_URL/version
```

#### Body examples
In the following examples data is being passed to the apiserver using a json
structure. These json structures are defined in the following documentation.

```
curl --cacert $PGO_CA_CERT --key $PGO_CLIENT_KEY --cert $PGO_CA_CERT -u \
pgoadmin:examplepassword -H "Content-Type:application/json" --insecure -X GET \
"$PGO_APISERVER_URL/workflow/<id>?version=<client-version>&namespace=<namespace>"
```

###### Create Cluster
You can create a cluster by sending a POST request to
`$PGO_APISERVER_URL/clusters`. In this example `--data` is being sent to the
API URL that includes the client version that was returned from the version
call, the namespace where the cluster should be created, the name of the new
cluster and the series number. Series sets the number of clusters that will be
created in the namespace.

```
curl --cacert $PGO_CA_CERT --key $PGO_CLIENT_KEY --cert $PGO_CA_CERT -u \
pgoadmin:examplepassword -H "Content-Type:application/json" --insecure -X \
POST --data \
    '{"ClientVersion":"4.1.0",
    "Namespace":"pgouser1",
    "Name":"mycluster",
    "Series":1}' \
$PGO_APISERVER_URL/clusters
```

###### Show and Delete Cluster
The last two examples show you how to `show` and `delete` a cluster. Notice
how instead of passing `"Name":"mycluster"` you pass `"Clustername":"mycluster"
to reference a cluster that has already been created. For the show cluster
example you can replace `"Clustername":"mycluster"` with `"AllFlag":true` to
show all of the clusters that are in the given namespace.

```
curl --cacert $PGO_CA_CERT --key $PGO_CLIENT_KEY --cert $PGO_CA_CERT -u \
pgoadmin:examplepassword -H "Content-Type:application/json" --insecure -X \
POST --data \
  '{"ClientVersion":"4.1.0",
  "Namespace":"pgouser1",
  "Clustername":"mycluster"}' \
$PGO_APISERVER_URL/showclusters
```

```
curl --cacert $PGO_CA_CERT --key $PGO_CLIENT_KEY --cert $PGO_CA_CERT -u \
pgoadmin:examplepassword -H "Content-Type:application/json" --insecure -X \
POST --data \
  '{"ClientVersion":"4.1.0",
  "Namespace":"pgouser1",
  "Clustername":"mycluster"}' \
$PGO_APISERVER_URL/clustersdelete
```

  Schemes: http, https
  BasePath: /
  Version: 4.1.0
  License: Apache 2.0 http://www.apache.org/licenses/LICENSE-2.0
  Contact: Crunchy Data<info@crunchydata.com> https://www.crunchydata.com/

	Consumes:
	- application/json

	Produces:
	- application/json

swagger:meta
*/
package v1

// +k8s:deepcopy-gen=package,register

/*
 Copyright 2017 - 2020 Crunchy Data Solutions, Inc.
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/
<commit_msg>Updates for 4.3 release<commit_after>/*
Crunchy PostgreSQL Operator API

The Crunchy PostgreSQL Operator API defines HTTP(S) interactions with the Crunchy PostgreSQL Operator.

## Direct API Calls

The API can also be accessed by interacting directly with the API server. This
can be done by making HTTP requests with curl to get information from the
server. In order to make these calls you will need to provide certificates along
with your request using the `--cacert`, `--key`, and `--cert` flags. Next you
will need to provide the username and password for the RBAC along with a header
that includes the content type and the `--insecure` flag. These flags will be
the same for all of your interactions with the API server and can be seen in the
following examples.

###### Get API Server Version

The most basic example of this interaction is getting the version of the API
server. You can send a GET request to `$PGO_APISERVER_URL/version` and this will
send back a json response including the API server version. You must specify the
client version that matches the API server version as part of the request.

The API server is setup to work with the pgo command line interface so the
parameters that are passed to the server can be found by looking at the related
flags. For example, the series parameter used in the `create` example below is
the same as the `-e, --series` flag that is described in the [pgo cli docs](https://access.crunchydata.com/documentation/postgres-operator/4.3.0/advanced/direct-api-calls).
```
curl --cacert $PGO_CA_CERT --key $PGO_CLIENT_KEY --cert $PGO_CA_CERT -u \
pgoadmin:examplepassword -H "Content-Type:application/json" --insecure -X \
GET $PGO_APISERVER_URL/version
```

#### Body examples
In the following examples data is being passed to the apiserver using a json
structure. These json structures are defined in the following documentation.

```
curl --cacert $PGO_CA_CERT --key $PGO_CLIENT_KEY --cert $PGO_CA_CERT -u \
pgoadmin:examplepassword -H "Content-Type:application/json" --insecure -X GET \
"$PGO_APISERVER_URL/workflow/<id>?version=<client-version>&namespace=<namespace>"
```

###### Create Cluster
You can create a cluster by sending a POST request to
`$PGO_APISERVER_URL/clusters`. In this example `--data` is being sent to the
API URL that includes the client version that was returned from the version
call, the namespace where the cluster should be created, the name of the new
cluster and the series number. Series sets the number of clusters that will be
created in the namespace.

```
curl --cacert $PGO_CA_CERT --key $PGO_CLIENT_KEY --cert $PGO_CA_CERT -u \
pgoadmin:examplepassword -H "Content-Type:application/json" --insecure -X \
POST --data \
    '{"ClientVersion":"4.3.0",
    "Namespace":"pgouser1",
    "Name":"mycluster",
    "Series":1}' \
$PGO_APISERVER_URL/clusters
```

###### Show and Delete Cluster
The last two examples show you how to `show` and `delete` a cluster. Notice
how instead of passing `"Name":"mycluster"` you pass `"Clustername":"mycluster"
to reference a cluster that has already been created. For the show cluster
example you can replace `"Clustername":"mycluster"` with `"AllFlag":true` to
show all of the clusters that are in the given namespace.

```
curl --cacert $PGO_CA_CERT --key $PGO_CLIENT_KEY --cert $PGO_CA_CERT -u \
pgoadmin:examplepassword -H "Content-Type:application/json" --insecure -X \
POST --data \
  '{"ClientVersion":"4.3.0",
  "Namespace":"pgouser1",
  "Clustername":"mycluster"}' \
$PGO_APISERVER_URL/showclusters
```

```
curl --cacert $PGO_CA_CERT --key $PGO_CLIENT_KEY --cert $PGO_CA_CERT -u \
pgoadmin:examplepassword -H "Content-Type:application/json" --insecure -X \
POST --data \
  '{"ClientVersion":"4.3.0",
  "Namespace":"pgouser1",
  "Clustername":"mycluster"}' \
$PGO_APISERVER_URL/clustersdelete
```

  Schemes: http, https
  BasePath: /
  Version: 4.3.0
  License: Apache 2.0 http://www.apache.org/licenses/LICENSE-2.0
  Contact: Crunchy Data<info@crunchydata.com> https://www.crunchydata.com/

	Consumes:
	- application/json

	Produces:
	- application/json

swagger:meta
*/
package v1

// +k8s:deepcopy-gen=package,register

/*
 Copyright 2017 - 2020 Crunchy Data Solutions, Inc.
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/
<|endoftext|>


<commit_before>// Package api provides rest-like server
package api

import (
	"encoding/xml"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/didip/tollbooth"
	"github.com/didip/tollbooth_chi"
	"github.com/go-chi/chi/v5"
	"github.com/go-chi/chi/v5/middleware"
	"github.com/go-chi/render"
	"github.com/go-pkgz/lcw"
	log "github.com/go-pkgz/lgr"
	"github.com/go-pkgz/rest"
	"github.com/go-pkgz/rest/logger"
	"github.com/umputun/feed-master/app/youtube"

	"github.com/umputun/feed-master/app/feed"
	"github.com/umputun/feed-master/app/proc"
)

// Server provides HTTP API
type Server struct {
	Version    string
	Conf       proc.Conf
	Store      *proc.BoltDB
	YoutubeSvc YoutubeSvc
	httpServer *http.Server
	cache      lcw.LoadingCache
}

// YoutubeSvc provides access to youtube's audio rss
type YoutubeSvc interface {
	RSSFeed(cinfo youtube.ChannelInfo) (string, error)
}

// Run starts http server for API with all routes
func (s *Server) Run(port int) {
	var err error
	if s.cache, err = lcw.NewExpirableCache(lcw.TTL(time.Minute*5), lcw.MaxCacheSize(10*1024*1024)); err != nil {
		log.Printf("[PANIC] failed to make loading cache, %v", err)
		return
	}

	router := chi.NewRouter()
	router.Use(middleware.RealIP, rest.Recoverer(log.Default()))
	router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second))
	router.Use(rest.AppInfo("feed-master", "umputun", s.Version), rest.Ping)
	router.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)))

	s.httpServer = &http.Server{
		Addr:              fmt.Sprintf(":%d", port),
		Handler:           router,
		ReadHeaderTimeout: 5 * time.Second,
		WriteTimeout:      30 * time.Second,
		IdleTimeout:       30 * time.Second,
	}

	router.Group(func(rimg chi.Router) {
		l := logger.New(logger.Log(log.Default()), logger.Prefix("[DEBUG]"))
		rimg.Use(l.Handler)
		rimg.Get("/images/{name}", s.getImageCtrl)
		rimg.Get("/image/{name}", s.getImageCtrl)
		rimg.Head("/image/{name}", s.getImageHeadCtrl)
		rimg.Head("/images/{name}", s.getImageHeadCtrl)
	})

	router.Group(func(rrss chi.Router) {
		l := logger.New(logger.Log(log.Default()), logger.Prefix("[INFO]"))
		rrss.Use(l.Handler)
		rrss.Get("/rss/{name}", s.getFeedCtrl)
		rrss.Get("/list", s.getListCtrl)
		rrss.Get("/feed/{name}", s.getFeedPageCtrl)
	})

	router.Route("/yt", func(r chi.Router) {
		l := logger.New(logger.Log(log.Default()), logger.Prefix("[INFO]"))
		r.Use(l.Handler)
		r.Get("/rss/{channel}", s.getYoutubeFeedCtrl)
	})

	baseYtURL, err := url.Parse(s.Conf.YouTube.BaseURL)
	if err != nil {
		log.Printf("[ERROR] failed to parse base url %s, %v", s.Conf.YouTube.BaseURL, err)
	}
	ytfs, err := rest.FileServer(baseYtURL.Path, s.Conf.YouTube.FilesLocation)
	if err == nil {
		router.Mount(baseYtURL.Path, ytfs)
	} else {
		log.Printf("[WARN] can't start static file server for yt, %v", err)
	}

	fs, err := rest.FileServer("/static", filepath.Join("webapp", "static"))
	if err == nil {
		router.Mount("/static", fs)
	} else {
		log.Printf("[WARN] can't start static file server, %v", err)
	}

	err = s.httpServer.ListenAndServe()
	log.Printf("[WARN] http server terminated, %s", err)
}

// GET /rss/{name} - returns rss for given feeds set
func (s *Server) getFeedCtrl(w http.ResponseWriter, r *http.Request) {
	feedName := chi.URLParam(r, "name")
	items, err := s.Store.Load(feedName, s.Conf.System.MaxTotal, true)
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest, err, "failed to get feed")
		return
	}

	for i, itm := range items {
		// add ts suffix to titles
		switch s.Conf.Feeds[feedName].ExtendDateTitle {
		case "yyyyddmm":
			items[i].Title = fmt.Sprintf("%s (%s)", itm.Title, itm.DT.Format("2006-02-01"))
		case "yyyymmdd":
			items[i].Title = fmt.Sprintf("%s (%s)", itm.Title, itm.DT.Format("2006-01-02"))
		}
	}

	rss := feed.Rss2{
		Version:       "2.0",
		ItemList:      items,
		Title:         s.Conf.Feeds[feedName].Title,
		Description:   s.Conf.Feeds[feedName].Description,
		Language:      s.Conf.Feeds[feedName].Language,
		Link:          s.Conf.Feeds[feedName].Link,
		PubDate:       items[0].PubDate,
		LastBuildDate: time.Now().Format(time.RFC822Z),
	}

	// replace link to UI page
	if s.Conf.System.BaseURL != "" {
		rss.Link = s.Conf.System.BaseURL + "/feed/" + feedName
	}

	b, err := xml.MarshalIndent(&rss, "", "  ")
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to marshal rss")
		return
	}
	w.Header().Set("Content-Type", "application/xml; charset=UTF-8")
	_, _ = fmt.Fprintf(w, "%s", string(b))
}

// GET /image/{name}
func (s *Server) getImageCtrl(w http.ResponseWriter, r *http.Request) {
	fm := chi.URLParam(r, "name")
	fm = strings.TrimSuffix(fm, ".png")
	feedConf, found := s.Conf.Feeds[fm]
	if !found {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest,
			fmt.Errorf("image %s not found", fm), "failed to load image")
		return
	}

	b, err := os.ReadFile(feedConf.Image)
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest,
			errors.New("can't read  "+chi.URLParam(r, "name")), "failed to read image")
		return
	}
	w.Header().Set("Content-Type", "image/png")
	if _, err := w.Write(b); err != nil {
		log.Printf("[WARN] failed to send image, %s", err)
	}
}

// HEAD /image/{name}
func (s *Server) getImageHeadCtrl(w http.ResponseWriter, r *http.Request) {
	fm := chi.URLParam(r, "name")
	fm = strings.TrimSuffix(fm, ".png")
	feedConf, found := s.Conf.Feeds[fm]
	if !found {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	info, err := os.Stat(feedConf.Image)
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		return
	}
	w.Header().Set("Content-Type", "image/png")
	w.Header().Set("Content-Length", strconv.Itoa(int(info.Size())))
	w.WriteHeader(http.StatusOK)
}

// GET /list - returns feed's image
func (s *Server) getListCtrl(w http.ResponseWriter, r *http.Request) {
	buckets, err := s.Store.Buckets()
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to read list")
		return
	}
	render.JSON(w, r, buckets)
}

// GET /yt/rss/{channel} - returns rss for given youtube channel
func (s *Server) getYoutubeFeedCtrl(w http.ResponseWriter, r *http.Request) {
	channel := chi.URLParam(r, "channel")

	res, err := s.YoutubeSvc.RSSFeed(youtube.ChannelInfo{ID: channel})
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to read yt list")
		return
	}

	w.Header().Set("Content-Type", "application/xml; charset=UTF-8")
	_, _ = fmt.Fprintf(w, "%s", res)
}
<commit_msg>don't fail on empty base url<commit_after>// Package api provides rest-like server
package api

import (
	"encoding/xml"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/didip/tollbooth"
	"github.com/didip/tollbooth_chi"
	"github.com/go-chi/chi/v5"
	"github.com/go-chi/chi/v5/middleware"
	"github.com/go-chi/render"
	"github.com/go-pkgz/lcw"
	log "github.com/go-pkgz/lgr"
	"github.com/go-pkgz/rest"
	"github.com/go-pkgz/rest/logger"
	"github.com/umputun/feed-master/app/youtube"

	"github.com/umputun/feed-master/app/feed"
	"github.com/umputun/feed-master/app/proc"
)

// Server provides HTTP API
type Server struct {
	Version    string
	Conf       proc.Conf
	Store      *proc.BoltDB
	YoutubeSvc YoutubeSvc
	httpServer *http.Server
	cache      lcw.LoadingCache
}

// YoutubeSvc provides access to youtube's audio rss
type YoutubeSvc interface {
	RSSFeed(cinfo youtube.ChannelInfo) (string, error)
}

// Run starts http server for API with all routes
func (s *Server) Run(port int) {
	var err error
	if s.cache, err = lcw.NewExpirableCache(lcw.TTL(time.Minute*5), lcw.MaxCacheSize(10*1024*1024)); err != nil {
		log.Printf("[PANIC] failed to make loading cache, %v", err)
		return
	}

	router := chi.NewRouter()
	router.Use(middleware.RealIP, rest.Recoverer(log.Default()))
	router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second))
	router.Use(rest.AppInfo("feed-master", "umputun", s.Version), rest.Ping)
	router.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)))

	s.httpServer = &http.Server{
		Addr:              fmt.Sprintf(":%d", port),
		Handler:           router,
		ReadHeaderTimeout: 5 * time.Second,
		WriteTimeout:      30 * time.Second,
		IdleTimeout:       30 * time.Second,
	}

	router.Group(func(rimg chi.Router) {
		l := logger.New(logger.Log(log.Default()), logger.Prefix("[DEBUG]"))
		rimg.Use(l.Handler)
		rimg.Get("/images/{name}", s.getImageCtrl)
		rimg.Get("/image/{name}", s.getImageCtrl)
		rimg.Head("/image/{name}", s.getImageHeadCtrl)
		rimg.Head("/images/{name}", s.getImageHeadCtrl)
	})

	router.Group(func(rrss chi.Router) {
		l := logger.New(logger.Log(log.Default()), logger.Prefix("[INFO]"))
		rrss.Use(l.Handler)
		rrss.Get("/rss/{name}", s.getFeedCtrl)
		rrss.Get("/list", s.getListCtrl)
		rrss.Get("/feed/{name}", s.getFeedPageCtrl)
	})

	router.Route("/yt", func(r chi.Router) {
		l := logger.New(logger.Log(log.Default()), logger.Prefix("[INFO]"))
		r.Use(l.Handler)
		r.Get("/rss/{channel}", s.getYoutubeFeedCtrl)
	})

	if s.Conf.YouTube.BaseURL != "" {
		baseYtURL, err := url.Parse(s.Conf.YouTube.BaseURL)
		if err != nil {
			log.Printf("[ERROR] failed to parse base url %s, %v", s.Conf.YouTube.BaseURL, err)
		}
		ytfs, err := rest.FileServer(baseYtURL.Path, s.Conf.YouTube.FilesLocation)
		if err == nil {
			router.Mount(baseYtURL.Path, ytfs)
		} else {
			log.Printf("[WARN] can't start static file server for yt, %v", err)
		}
	}

	fs, err := rest.FileServer("/static", filepath.Join("webapp", "static"))
	if err == nil {
		router.Mount("/static", fs)
	} else {
		log.Printf("[WARN] can't start static file server, %v", err)
	}

	err = s.httpServer.ListenAndServe()
	log.Printf("[WARN] http server terminated, %s", err)
}

// GET /rss/{name} - returns rss for given feeds set
func (s *Server) getFeedCtrl(w http.ResponseWriter, r *http.Request) {
	feedName := chi.URLParam(r, "name")
	items, err := s.Store.Load(feedName, s.Conf.System.MaxTotal, true)
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest, err, "failed to get feed")
		return
	}

	for i, itm := range items {
		// add ts suffix to titles
		switch s.Conf.Feeds[feedName].ExtendDateTitle {
		case "yyyyddmm":
			items[i].Title = fmt.Sprintf("%s (%s)", itm.Title, itm.DT.Format("2006-02-01"))
		case "yyyymmdd":
			items[i].Title = fmt.Sprintf("%s (%s)", itm.Title, itm.DT.Format("2006-01-02"))
		}
	}

	rss := feed.Rss2{
		Version:       "2.0",
		ItemList:      items,
		Title:         s.Conf.Feeds[feedName].Title,
		Description:   s.Conf.Feeds[feedName].Description,
		Language:      s.Conf.Feeds[feedName].Language,
		Link:          s.Conf.Feeds[feedName].Link,
		PubDate:       items[0].PubDate,
		LastBuildDate: time.Now().Format(time.RFC822Z),
	}

	// replace link to UI page
	if s.Conf.System.BaseURL != "" {
		rss.Link = s.Conf.System.BaseURL + "/feed/" + feedName
	}

	b, err := xml.MarshalIndent(&rss, "", "  ")
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to marshal rss")
		return
	}
	w.Header().Set("Content-Type", "application/xml; charset=UTF-8")
	_, _ = fmt.Fprintf(w, "%s", string(b))
}

// GET /image/{name}
func (s *Server) getImageCtrl(w http.ResponseWriter, r *http.Request) {
	fm := chi.URLParam(r, "name")
	fm = strings.TrimSuffix(fm, ".png")
	feedConf, found := s.Conf.Feeds[fm]
	if !found {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest,
			fmt.Errorf("image %s not found", fm), "failed to load image")
		return
	}

	b, err := os.ReadFile(feedConf.Image)
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest,
			errors.New("can't read  "+chi.URLParam(r, "name")), "failed to read image")
		return
	}
	w.Header().Set("Content-Type", "image/png")
	if _, err := w.Write(b); err != nil {
		log.Printf("[WARN] failed to send image, %s", err)
	}
}

// HEAD /image/{name}
func (s *Server) getImageHeadCtrl(w http.ResponseWriter, r *http.Request) {
	fm := chi.URLParam(r, "name")
	fm = strings.TrimSuffix(fm, ".png")
	feedConf, found := s.Conf.Feeds[fm]
	if !found {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	info, err := os.Stat(feedConf.Image)
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		return
	}
	w.Header().Set("Content-Type", "image/png")
	w.Header().Set("Content-Length", strconv.Itoa(int(info.Size())))
	w.WriteHeader(http.StatusOK)
}

// GET /list - returns feed's image
func (s *Server) getListCtrl(w http.ResponseWriter, r *http.Request) {
	buckets, err := s.Store.Buckets()
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to read list")
		return
	}
	render.JSON(w, r, buckets)
}

// GET /yt/rss/{channel} - returns rss for given youtube channel
func (s *Server) getYoutubeFeedCtrl(w http.ResponseWriter, r *http.Request) {
	channel := chi.URLParam(r, "channel")

	res, err := s.YoutubeSvc.RSSFeed(youtube.ChannelInfo{ID: channel})
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to read yt list")
		return
	}

	w.Header().Set("Content-Type", "application/xml; charset=UTF-8")
	_, _ = fmt.Fprintf(w, "%s", res)
}
<|endoftext|>


<commit_before>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build appengine

package build

import (
	"fmt"
	"sort"
	"strconv"
	"strings"

	"appengine"
	"appengine/datastore"
)

var knownTags = map[string]string{
	"go1":   "0051c7442fed9c888de6617fa9239a913904d96e",
	"go1.1": "d29da2ced72ba2cf48ed6a8f1ec4abc01e4c5bf1",
	"go1.2": "b1edf8faa5d6cbc50c6515785df9df9c19296564",
	"go1.3": "f153208c0a0e306bfca14f71ef11f09859ccabc8",
}

var lastRelease = "go1.3"

func splitBench(benchProcs string) (string, int) {
	ss := strings.Split(benchProcs, "-")
	procs, _ := strconv.Atoi(ss[1])
	return ss[0], procs
}

func dashPerfCommits(c appengine.Context, page int) ([]*Commit, error) {
	q := datastore.NewQuery("Commit").
		Ancestor((&Package{}).Key(c)).
		Order("-Num").
		Filter("NeedsBenchmarking =", true).
		Limit(commitsPerPage).
		Offset(page * commitsPerPage)
	var commits []*Commit
	_, err := q.GetAll(c, &commits)
	if err == nil && len(commits) == 0 {
		err = fmt.Errorf("no commits")
	}
	return commits, err
}

func perfChangeStyle(pc *PerfConfig, v float64, builder, benchmark, metric string) string {
	noise := pc.NoiseLevel(builder, benchmark, metric)
	if isNoise(v, noise) {
		return "noise"
	}
	if v > 0 {
		return "bad"
	}
	return "good"
}

func isNoise(diff, noise float64) bool {
	rnoise := -100 * noise / (noise + 100)
	return diff < noise && diff > rnoise
}

func perfDiff(old, new uint64) float64 {
	return 100*float64(new)/float64(old) - 100
}

func isPerfFailed(res *PerfResult, builder string) bool {
	data := res.ParseData()[builder]
	return data != nil && data["meta-done"] != nil && !data["meta-done"].OK
}

// PerfResultCache caches a set of PerfResults so that it's easy to access them
// without lots of duplicate accesses to datastore.
// It allows to iterate over newer or older results for some base commit.
type PerfResultCache struct {
	c       appengine.Context
	newer   bool
	iter    *datastore.Iterator
	results map[int]*PerfResult
}

func MakePerfResultCache(c appengine.Context, com *Commit, newer bool) *PerfResultCache {
	p := &Package{}
	q := datastore.NewQuery("PerfResult").Ancestor(p.Key(c)).Limit(100)
	if newer {
		q = q.Filter("CommitNum >=", com.Num).Order("CommitNum")
	} else {
		q = q.Filter("CommitNum <=", com.Num).Order("-CommitNum")
	}
	rc := &PerfResultCache{c: c, newer: newer, iter: q.Run(c), results: make(map[int]*PerfResult)}
	return rc
}

func (rc *PerfResultCache) Get(commitNum int) *PerfResult {
	rc.Next(commitNum) // fetch the commit, if necessary
	return rc.results[commitNum]
}

// Next returns the next PerfResult for the commit commitNum.
// It does not care whether the result has any data, failed or whatever.
func (rc *PerfResultCache) Next(commitNum int) (*PerfResult, error) {
	// See if we have next result in the cache.
	next := -1
	for ci := range rc.results {
		if rc.newer {
			if ci > commitNum && (next == -1 || ci < next) {
				next = ci
			}
		} else {
			if ci < commitNum && (next == -1 || ci > next) {
				next = ci
			}
		}
	}
	if next != -1 {
		return rc.results[next], nil
	}
	// Fetch next result from datastore.
	res := new(PerfResult)
	_, err := rc.iter.Next(res)
	if err == datastore.Done {
		return nil, nil
	}
	if err != nil {
		return nil, fmt.Errorf("fetching perf results: %v", err)
	}
	if (rc.newer && res.CommitNum < commitNum) || (!rc.newer && res.CommitNum > commitNum) {
		rc.c.Errorf("PerfResultCache.Next: bad commit num")
	}
	rc.results[res.CommitNum] = res
	return res, nil
}

// NextForComparison returns PerfResult which we need to use for performance comprison.
// It skips failed results, but does not skip results with no data.
func (rc *PerfResultCache) NextForComparison(commitNum int, builder string) (*PerfResult, error) {
	for {
		res, err := rc.Next(commitNum)
		if err != nil {
			return nil, err
		}
		if res == nil {
			return nil, nil
		}
		if res.CommitNum == commitNum {
			continue
		}
		parsed := res.ParseData()
		if builder != "" {
			// Comparing for a particular builder.
			// This is used in perf_changes and in email notifications.
			b := parsed[builder]
			if b == nil || b["meta-done"] == nil {
				// No results yet, must not do the comparison.
				return nil, nil
			}
			if b["meta-done"].OK {
				// Have complete results, compare.
				return res, nil
			}
		} else {
			// Comparing for all builders, find a result with at least
			// one successful meta-done.
			// This is used in perf_detail.
			for _, benchs := range parsed {
				if data := benchs["meta-done"]; data != nil && data.OK {
					return res, nil
				}
			}
		}
		// Failed, try next result.
		commitNum = res.CommitNum
	}
}

type PerfChange struct {
	Builder string
	Bench   string
	Metric  string
	Old     uint64
	New     uint64
	Diff    float64
}

func significantPerfChanges(pc *PerfConfig, builder string, prevRes, res *PerfResult) (changes []*PerfChange) {
	// First, collect all significant changes.
	for builder1, benchmarks1 := range res.ParseData() {
		if builder != "" && builder != builder1 {
			// This is not the builder you're looking for, Luke.
			continue
		}
		benchmarks0 := prevRes.ParseData()[builder1]
		if benchmarks0 == nil {
			continue
		}
		for benchmark, data1 := range benchmarks1 {
			data0 := benchmarks0[benchmark]
			if data0 == nil {
				continue
			}
			for metric, val := range data1.Metrics {
				val0 := data0.Metrics[metric]
				if val0 == 0 {
					continue
				}
				diff := perfDiff(val0, val)
				noise := pc.NoiseLevel(builder, benchmark, metric)
				if isNoise(diff, noise) {
					continue
				}
				ch := &PerfChange{Builder: builder, Bench: benchmark, Metric: metric, Old: val0, New: val, Diff: diff}
				changes = append(changes, ch)
			}
		}
	}
	// Then, strip non-repeatable changes (flakes).
	// The hypothesis is that a real change must show up with the majority of GOMAXPROCS values.
	majority := len(pc.ProcList(builder))/2 + 1
	cnt := make(map[string]int)
	for _, ch := range changes {
		b, _ := splitBench(ch.Bench)
		name := b + "|" + ch.Metric
		if ch.Diff < 0 {
			name += "--"
		}
		cnt[name] = cnt[name] + 1
	}
	for i := 0; i < len(changes); i++ {
		ch := changes[i]
		b, _ := splitBench(ch.Bench)
		name := b + "|" + ch.Metric
		if cnt[name] >= majority {
			continue
		}
		if cnt[name+"--"] >= majority {
			continue
		}
		// Remove flake.
		last := len(changes) - 1
		changes[i] = changes[last]
		changes = changes[:last]
		i--
	}
	return changes
}

// orderPerfTodo reorders commit nums for benchmarking todo.
// The resulting order is somewhat tricky. We want 2 things:
// 1. benchmark sequentially backwards (this provides information about most
// recent changes, and allows to estimate noise levels)
// 2. benchmark old commits in "scatter" order (this allows to quickly gather
// brief information about thousands of old commits)
// So this function interleaves the two orders.
func orderPerfTodo(nums []int) []int {
	sort.Ints(nums)
	n := len(nums)
	pow2 := uint32(0) // next power-of-two that is >= n
	npow2 := 0
	for npow2 <= n {
		pow2++
		npow2 = 1 << pow2
	}
	res := make([]int, n)
	resPos := n - 1            // result array is filled backwards
	present := make([]bool, n) // denotes values that already present in result array
	for i0, i1 := n-1, 0; i0 >= 0 || i1 < npow2; {
		// i0 represents "benchmark sequentially backwards" sequence
		// find the next commit that is not yet present and add it
		for cnt := 0; cnt < 2; cnt++ {
			for ; i0 >= 0; i0-- {
				if !present[i0] {
					present[i0] = true
					res[resPos] = nums[i0]
					resPos--
					i0--
					break
				}
			}
		}
		// i1 represents "scatter order" sequence
		// find the next commit that is not yet present and add it
		for ; i1 < npow2; i1++ {
			// do the "recursive split-ordering" trick
			idx := 0 // bitwise reverse of i1
			for j := uint32(0); j <= pow2; j++ {
				if (i1 & (1 << j)) != 0 {
					idx = idx | (1 << (pow2 - j - 1))
				}
			}
			if idx < n && !present[idx] {
				present[idx] = true
				res[resPos] = nums[idx]
				resPos--
				i1++
				break
			}
		}
	}
	// The above can't possibly be correct. Do dump check.
	res2 := make([]int, n)
	copy(res2, res)
	sort.Ints(res2)
	for i := range res2 {
		if res2[i] != nums[i] {
			panic(fmt.Sprintf("diff at %v: expect %v, want %v\nwas: %v\n become: %v",
				i, nums[i], res2[i], nums, res2))
		}
	}
	return res
}
<commit_msg>dashboard: tell perf dashboard about release 1.4<commit_after>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build appengine

package build

import (
	"fmt"
	"sort"
	"strconv"
	"strings"

	"appengine"
	"appengine/datastore"
)

var knownTags = map[string]string{
	"go1":   "0051c7442fed9c888de6617fa9239a913904d96e",
	"go1.1": "d29da2ced72ba2cf48ed6a8f1ec4abc01e4c5bf1",
	"go1.2": "b1edf8faa5d6cbc50c6515785df9df9c19296564",
	"go1.3": "f153208c0a0e306bfca14f71ef11f09859ccabc8",
	"go1.4": "faa3ed1dc30e42771a68b6337dcf8be9518d5c07",
}

var lastRelease = "go1.4"

func splitBench(benchProcs string) (string, int) {
	ss := strings.Split(benchProcs, "-")
	procs, _ := strconv.Atoi(ss[1])
	return ss[0], procs
}

func dashPerfCommits(c appengine.Context, page int) ([]*Commit, error) {
	q := datastore.NewQuery("Commit").
		Ancestor((&Package{}).Key(c)).
		Order("-Num").
		Filter("NeedsBenchmarking =", true).
		Limit(commitsPerPage).
		Offset(page * commitsPerPage)
	var commits []*Commit
	_, err := q.GetAll(c, &commits)
	if err == nil && len(commits) == 0 {
		err = fmt.Errorf("no commits")
	}
	return commits, err
}

func perfChangeStyle(pc *PerfConfig, v float64, builder, benchmark, metric string) string {
	noise := pc.NoiseLevel(builder, benchmark, metric)
	if isNoise(v, noise) {
		return "noise"
	}
	if v > 0 {
		return "bad"
	}
	return "good"
}

func isNoise(diff, noise float64) bool {
	rnoise := -100 * noise / (noise + 100)
	return diff < noise && diff > rnoise
}

func perfDiff(old, new uint64) float64 {
	return 100*float64(new)/float64(old) - 100
}

func isPerfFailed(res *PerfResult, builder string) bool {
	data := res.ParseData()[builder]
	return data != nil && data["meta-done"] != nil && !data["meta-done"].OK
}

// PerfResultCache caches a set of PerfResults so that it's easy to access them
// without lots of duplicate accesses to datastore.
// It allows to iterate over newer or older results for some base commit.
type PerfResultCache struct {
	c       appengine.Context
	newer   bool
	iter    *datastore.Iterator
	results map[int]*PerfResult
}

func MakePerfResultCache(c appengine.Context, com *Commit, newer bool) *PerfResultCache {
	p := &Package{}
	q := datastore.NewQuery("PerfResult").Ancestor(p.Key(c)).Limit(100)
	if newer {
		q = q.Filter("CommitNum >=", com.Num).Order("CommitNum")
	} else {
		q = q.Filter("CommitNum <=", com.Num).Order("-CommitNum")
	}
	rc := &PerfResultCache{c: c, newer: newer, iter: q.Run(c), results: make(map[int]*PerfResult)}
	return rc
}

func (rc *PerfResultCache) Get(commitNum int) *PerfResult {
	rc.Next(commitNum) // fetch the commit, if necessary
	return rc.results[commitNum]
}

// Next returns the next PerfResult for the commit commitNum.
// It does not care whether the result has any data, failed or whatever.
func (rc *PerfResultCache) Next(commitNum int) (*PerfResult, error) {
	// See if we have next result in the cache.
	next := -1
	for ci := range rc.results {
		if rc.newer {
			if ci > commitNum && (next == -1 || ci < next) {
				next = ci
			}
		} else {
			if ci < commitNum && (next == -1 || ci > next) {
				next = ci
			}
		}
	}
	if next != -1 {
		return rc.results[next], nil
	}
	// Fetch next result from datastore.
	res := new(PerfResult)
	_, err := rc.iter.Next(res)
	if err == datastore.Done {
		return nil, nil
	}
	if err != nil {
		return nil, fmt.Errorf("fetching perf results: %v", err)
	}
	if (rc.newer && res.CommitNum < commitNum) || (!rc.newer && res.CommitNum > commitNum) {
		rc.c.Errorf("PerfResultCache.Next: bad commit num")
	}
	rc.results[res.CommitNum] = res
	return res, nil
}

// NextForComparison returns PerfResult which we need to use for performance comprison.
// It skips failed results, but does not skip results with no data.
func (rc *PerfResultCache) NextForComparison(commitNum int, builder string) (*PerfResult, error) {
	for {
		res, err := rc.Next(commitNum)
		if err != nil {
			return nil, err
		}
		if res == nil {
			return nil, nil
		}
		if res.CommitNum == commitNum {
			continue
		}
		parsed := res.ParseData()
		if builder != "" {
			// Comparing for a particular builder.
			// This is used in perf_changes and in email notifications.
			b := parsed[builder]
			if b == nil || b["meta-done"] == nil {
				// No results yet, must not do the comparison.
				return nil, nil
			}
			if b["meta-done"].OK {
				// Have complete results, compare.
				return res, nil
			}
		} else {
			// Comparing for all builders, find a result with at least
			// one successful meta-done.
			// This is used in perf_detail.
			for _, benchs := range parsed {
				if data := benchs["meta-done"]; data != nil && data.OK {
					return res, nil
				}
			}
		}
		// Failed, try next result.
		commitNum = res.CommitNum
	}
}

type PerfChange struct {
	Builder string
	Bench   string
	Metric  string
	Old     uint64
	New     uint64
	Diff    float64
}

func significantPerfChanges(pc *PerfConfig, builder string, prevRes, res *PerfResult) (changes []*PerfChange) {
	// First, collect all significant changes.
	for builder1, benchmarks1 := range res.ParseData() {
		if builder != "" && builder != builder1 {
			// This is not the builder you're looking for, Luke.
			continue
		}
		benchmarks0 := prevRes.ParseData()[builder1]
		if benchmarks0 == nil {
			continue
		}
		for benchmark, data1 := range benchmarks1 {
			data0 := benchmarks0[benchmark]
			if data0 == nil {
				continue
			}
			for metric, val := range data1.Metrics {
				val0 := data0.Metrics[metric]
				if val0 == 0 {
					continue
				}
				diff := perfDiff(val0, val)
				noise := pc.NoiseLevel(builder, benchmark, metric)
				if isNoise(diff, noise) {
					continue
				}
				ch := &PerfChange{Builder: builder, Bench: benchmark, Metric: metric, Old: val0, New: val, Diff: diff}
				changes = append(changes, ch)
			}
		}
	}
	// Then, strip non-repeatable changes (flakes).
	// The hypothesis is that a real change must show up with the majority of GOMAXPROCS values.
	majority := len(pc.ProcList(builder))/2 + 1
	cnt := make(map[string]int)
	for _, ch := range changes {
		b, _ := splitBench(ch.Bench)
		name := b + "|" + ch.Metric
		if ch.Diff < 0 {
			name += "--"
		}
		cnt[name] = cnt[name] + 1
	}
	for i := 0; i < len(changes); i++ {
		ch := changes[i]
		b, _ := splitBench(ch.Bench)
		name := b + "|" + ch.Metric
		if cnt[name] >= majority {
			continue
		}
		if cnt[name+"--"] >= majority {
			continue
		}
		// Remove flake.
		last := len(changes) - 1
		changes[i] = changes[last]
		changes = changes[:last]
		i--
	}
	return changes
}

// orderPerfTodo reorders commit nums for benchmarking todo.
// The resulting order is somewhat tricky. We want 2 things:
// 1. benchmark sequentially backwards (this provides information about most
// recent changes, and allows to estimate noise levels)
// 2. benchmark old commits in "scatter" order (this allows to quickly gather
// brief information about thousands of old commits)
// So this function interleaves the two orders.
func orderPerfTodo(nums []int) []int {
	sort.Ints(nums)
	n := len(nums)
	pow2 := uint32(0) // next power-of-two that is >= n
	npow2 := 0
	for npow2 <= n {
		pow2++
		npow2 = 1 << pow2
	}
	res := make([]int, n)
	resPos := n - 1            // result array is filled backwards
	present := make([]bool, n) // denotes values that already present in result array
	for i0, i1 := n-1, 0; i0 >= 0 || i1 < npow2; {
		// i0 represents "benchmark sequentially backwards" sequence
		// find the next commit that is not yet present and add it
		for cnt := 0; cnt < 2; cnt++ {
			for ; i0 >= 0; i0-- {
				if !present[i0] {
					present[i0] = true
					res[resPos] = nums[i0]
					resPos--
					i0--
					break
				}
			}
		}
		// i1 represents "scatter order" sequence
		// find the next commit that is not yet present and add it
		for ; i1 < npow2; i1++ {
			// do the "recursive split-ordering" trick
			idx := 0 // bitwise reverse of i1
			for j := uint32(0); j <= pow2; j++ {
				if (i1 & (1 << j)) != 0 {
					idx = idx | (1 << (pow2 - j - 1))
				}
			}
			if idx < n && !present[idx] {
				present[idx] = true
				res[resPos] = nums[idx]
				resPos--
				i1++
				break
			}
		}
	}
	// The above can't possibly be correct. Do dump check.
	res2 := make([]int, n)
	copy(res2, res)
	sort.Ints(res2)
	for i := range res2 {
		if res2[i] != nums[i] {
			panic(fmt.Sprintf("diff at %v: expect %v, want %v\nwas: %v\n become: %v",
				i, nums[i], res2[i], nums, res2))
		}
	}
	return res
}
<|endoftext|>


<commit_before>package dbo

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"

	"github.com/byuoitav/authmiddleware/bearertoken"
	"github.com/byuoitav/configuration-database-microservice/accessors"
)

// GetData will run a get on the url, and attempt to fill the interface provided from the returned JSON.
func GetData(url string, structToFill interface{}) error {
	log.Printf("Getting data from URL: %s...", url)
	// Make an HTTP client so we can add custom headers (currently used for adding in the Bearer token for inter-microservice communication)

	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return err
	}

	err = setToken(req)
	if err != nil {
		return err
	}

	if req == nil {
		fmt.Printf("Alert! req is nil!")
	}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	if resp.StatusCode != http.StatusOK {
		errorString, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		return errors.New(string(errorString))
	}

	err = json.Unmarshal(b, structToFill)
	if err != nil {
		return err
	}
	log.Printf("Done.")
	return nil
}

//PostData hits POST endpoints
func PostData(url string, structToAdd interface{}, structToFill interface{}) error {
	log.Printf("Posting data to URL: %s...", url)

	body, err := json.Marshal(structToAdd)
	if err != nil {
		return err
	}
	client := &http.Client{}
	req, _ := http.NewRequest("POST", url, bytes.NewBuffer(body))

	req.Header.Set("Content-Type", "application/json")

	err = setToken(req)
	if err != nil {
		return err
	}

	response, err := client.Do(req)
	if err != nil {
		return err
	}
	if response.StatusCode != http.StatusOK {
		errorString, err := ioutil.ReadAll(response.Body)
		if err != nil {
			return err
		}
		return errors.New(string(errorString))
	}

	jsonArray, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return err
	}

	err = json.Unmarshal(jsonArray, structToFill)
	if err != nil {
		return err
	}

	return nil
}

func setToken(request *http.Request) error {
	fmt.Printf("Calling setToken on %v", request)

	if len(os.Getenv("LOCAL_ENVIRONMENT")) == 0 {

		log.Printf("Adding the bearer token for inter-service communication")

		token, err := bearertoken.GetToken()
		if err != nil {
			return err
		}

		request.Header.Set("Authorization", "Bearer "+token.Token)

	}

	return nil
}

// GetAllRawCommands retrieves all the commands
func GetAllRawCommands() (commands []accessors.RawCommand, err error) {
	log.Printf("Getting all commands.")
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/devices/commands"
	err = GetData(url, &commands)

	if err != nil {
		log.Printf("Error: %s", err.Error())
		return
	}

	log.Printf("Done.")
	return
}

// GetRoomByInfo simply retrieves a device's information from the databse.
func GetRoomByInfo(buildingName string, roomName string) (toReturn accessors.Room, err error) {
	log.Printf("Getting room %s in building %s...", roomName, buildingName)
	err = GetData(os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS")+"/buildings/"+buildingName+"/rooms/"+roomName, &toReturn)
	return
}

// GetDeviceByName simply retrieves a device's information from the databse.
func GetDeviceByName(buildingName string, roomName string, deviceName string) (toReturn accessors.Device, err error) {
	err = GetData(os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS")+"/buildings/"+buildingName+"/rooms/"+roomName+"/devices/"+deviceName, &toReturn)
	return
}

// GetDevicesByRoom will jut get the devices based on the room.
func GetDevicesByRoom(buildingName string, roomName string) (toReturn []accessors.Device, err error) {
	err = GetData(os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS")+"/buildings/"+buildingName+"/rooms/"+roomName+"/devices", &toReturn)
	return
}

// GetDevicesByBuildingAndRoomAndRole will get the devices with the given role from the DB
func GetDevicesByBuildingAndRoomAndRole(building string, room string, roleName string) (toReturn []accessors.Device, err error) {
	err = GetData(os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS")+"/buildings/"+building+"/rooms/"+room+"/devices/roles/"+roleName, &toReturn)
	return
}

// SetAudioInDB will set the audio levels in the database
func SetAudioInDB(building string, room string, device accessors.Device) error {
	log.Printf("Updating audio levels in DB.")

	if device.Volume != nil {
		url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/buildings/" + building + "/rooms/" + room + "/devices/" + device.Name + "/attributes/volume/" + strconv.Itoa(*device.Volume)
		request, err := http.NewRequest("PUT", url, nil)
		client := &http.Client{}
		_, err = client.Do(request)

		if err != nil {
			return err
		}
	}

	if device.Muted != nil {
		url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/buildings/" + building + "/rooms/" + room + "/devices/" + device.Name + "/attributes/muted/" + strconv.FormatBool(*device.Muted)
		request, err := http.NewRequest("PUT", url, nil)
		client := &http.Client{}
		_, err = client.Do(request)

		if err != nil {
			return err
		}
	}

	return nil
}

// GetBuildings will return all buildings
func GetBuildings() ([]accessors.Building, error) {
	log.Printf("getting all buildings...")
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/buildings"
	log.Printf(url)
	var buildings []accessors.Building
	err := GetData(url, &buildings)

	return buildings, err
}

// GetRooms returns all the rooms in a given building
func GetRoomsByBuilding(building string) ([]accessors.Room, error) {

	log.Printf("dbo.GetRoomsByBuilding called")

	log.Printf("getting all rooms from %v ...", building)
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/buildings/" + building + "/rooms"
	var rooms []accessors.Room
	err := GetData(url, &rooms)
	return rooms, err
}

// GetBuildingByShortname returns a building with a given shortname
func GetBuildingByShortname(building string) (accessors.Building, error) {
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/buildings/shortname/" + building
	var output accessors.Building
	err := GetData(url, &output)
	if err != nil {
		return output, err
	}
	return output, nil
}

// AddBuilding
func AddBuilding(buildingToAdd accessors.Building) (accessors.Building, error) {
	log.Printf("adding building %v to database", buildingToAdd.Shortname)
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/buildings/" + buildingToAdd.Shortname

	var buildingToFill accessors.Building
	err := PostData(url, buildingToAdd, &buildingToFill)
	if err != nil {
		return accessors.Building{}, err
	}

	return buildingToFill, nil
}

func AddRoom(building string, roomToAdd accessors.Room) (accessors.Room, error) {
	log.Printf("adding room %v to building %v in database", roomToAdd.Name, building)
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/buildings/" + building + "/rooms/" + roomToAdd.Name

	var roomToFill accessors.Room
	err := PostData(url, roomToAdd, &roomToFill)
	if err != nil {
		return accessors.Room{}, err
	}

	return roomToFill, nil
}

func GetDeviceTypes() ([]accessors.DeviceType, error) {

	log.Printf("getting all device types")
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/devices/types/"

	var DeviceTypes []accessors.DeviceType
	err := GetData(url, &DeviceTypes)
	if err != nil {
		return []accessors.DeviceType{}, err
	}

	return DeviceTypes, nil
}

func AddDeviceType(toAdd accessors.DeviceType) (accessors.DeviceType, error) {

	log.Printf("adding device type: %v to database", toAdd.Name)
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/devices/types/" + toAdd.Name

	var toFill accessors.DeviceType
	err := PostData(url, toAdd, &toFill)
	if err != nil {
		return accessors.DeviceType{}, err
	}

	return toFill, nil
}
func GetPowerStates() ([]accessors.PowerState, error) {

	log.Printf("getting all power states")
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/devices/powerstates/"

	var powerStates []accessors.PowerState
	err := GetData(url, &powerStates)
	if err != nil {
		return []accessors.PowerState{}, err
	}

	return powerStates, nil
}

func AddPowerState(toAdd accessors.PowerState) (accessors.PowerState, error) {

	log.Printf("adding power state: %v to database", toAdd.Name)
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/devices/powerstates/" + toAdd.Name

	var toFill accessors.PowerState
	err := PostData(url, toAdd, &toFill)
	if err != nil {
		return accessors.PowerState{}, err
	}

	return toFill, nil
}

func GetMicroservices() ([]accessors.Microservice, error) {

	log.Printf("getting all microservices")
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/devices/microservices"

	var microservices []accessors.Microservice
	err := GetData(url, &microservices)
	if err != nil {
		return []accessors.Microservice{}, err
	}

	return microservices, nil
}

func AddMicroservice(toAdd accessors.Microservice) (accessors.Microservice, error) {

	log.Printf("adding microservice: %v to database", toAdd.Name)
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/devices/microservices/" + toAdd.Name

	var toFill accessors.Microservice
	err := PostData(url, toAdd, &toFill)
	if err != nil {
		return accessors.Microservice{}, err
	}

	return toFill, nil
}

func GetEndpoints() ([]accessors.Endpoint, error) {

	log.Printf("getting all endpoints")
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/devices/endpoints"

	var endpoints []accessors.Endpoint
	err := GetData(url, &endpoints)
	if err != nil {
		return []accessors.Endpoint{}, err
	}

	return endpoints, nil
}

func AddEndpoint(toAdd accessors.Endpoint) (accessors.Endpoint, error) {

	log.Printf("adding endpoint: %v to database", toAdd.Name)
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/devices/endpoints/" + toAdd.Name

	var toFill accessors.Endpoint
	err := PostData(url, toAdd, &toFill)
	if err != nil {
		return accessors.Endpoint{}, err
	}

	return toFill, nil
}

func GetPorts() ([]accessors.PortType, error) {

	log.Printf("getting all ports")
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/devices/ports"

	var ports []accessors.PortType
	err := GetData(url, &ports)
	if err != nil {
		return []accessors.PortType{}, err
	}

	return ports, nil
}

func AddPort(portToAdd accessors.PortType) (accessors.PortType, error) {

	log.Printf("adding Port: %v to database", portToAdd.Name)
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/devices/ports/" + portToAdd.Name

	var portToFill accessors.PortType
	err := PostData(url, portToAdd, &portToFill)
	if err != nil {
		return accessors.PortType{}, err
	}

	return portToFill, nil
}

func GetDeviceRoleDefinitions() ([]accessors.DeviceRoleDef, error) {

	log.Printf("getting device role definitions")
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/devices/roledefinitions"

	var definitions []accessors.DeviceRoleDef
	err := GetData(url, definitions)
	if err != nil {
		return []accessors.DeviceRoleDef{}, err
	}

	return definitions, nil
}

func AddRoleDefinition(toAdd accessors.DeviceRoleDef) (accessors.DeviceRoleDef, error) {

	log.Printf("adding role definition: %v to database", toAdd.Name)
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/devices/roledefinitions/" + toAdd.Name

	var toFill accessors.DeviceRoleDef
	err := PostData(url, toAdd, &toFill)
	if err != nil {
		return accessors.DeviceRoleDef{}, err
	}

	return toFill, nil
}
<commit_msg>adding commands<commit_after>package dbo

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"

	"github.com/byuoitav/authmiddleware/bearertoken"
	"github.com/byuoitav/configuration-database-microservice/accessors"
)

// GetData will run a get on the url, and attempt to fill the interface provided from the returned JSON.
func GetData(url string, structToFill interface{}) error {
	log.Printf("Getting data from URL: %s...", url)
	// Make an HTTP client so we can add custom headers (currently used for adding in the Bearer token for inter-microservice communication)

	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return err
	}

	err = setToken(req)
	if err != nil {
		return err
	}

	if req == nil {
		fmt.Printf("Alert! req is nil!")
	}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	if resp.StatusCode != http.StatusOK {
		errorString, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		return errors.New(string(errorString))
	}

	err = json.Unmarshal(b, structToFill)
	if err != nil {
		return err
	}
	log.Printf("Done.")
	return nil
}

//PostData hits POST endpoints
func PostData(url string, structToAdd interface{}, structToFill interface{}) error {
	log.Printf("Posting data to URL: %s...", url)

	body, err := json.Marshal(structToAdd)
	if err != nil {
		return err
	}
	client := &http.Client{}
	req, _ := http.NewRequest("POST", url, bytes.NewBuffer(body))

	req.Header.Set("Content-Type", "application/json")

	err = setToken(req)
	if err != nil {
		return err
	}

	response, err := client.Do(req)
	if err != nil {
		return err
	}
	if response.StatusCode != http.StatusOK {
		errorString, err := ioutil.ReadAll(response.Body)
		if err != nil {
			return err
		}
		return errors.New(string(errorString))
	}

	jsonArray, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return err
	}

	err = json.Unmarshal(jsonArray, structToFill)
	if err != nil {
		return err
	}

	return nil
}

func setToken(request *http.Request) error {
	fmt.Printf("Calling setToken on %v", request)

	if len(os.Getenv("LOCAL_ENVIRONMENT")) == 0 {

		log.Printf("Adding the bearer token for inter-service communication")

		token, err := bearertoken.GetToken()
		if err != nil {
			return err
		}

		request.Header.Set("Authorization", "Bearer "+token.Token)

	}

	return nil
}

// GetAllRawCommands retrieves all the commands
func GetAllRawCommands() (commands []accessors.RawCommand, err error) {
	log.Printf("Getting all commands.")
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/devices/commands"
	err = GetData(url, &commands)

	if err != nil {
		log.Printf("Error: %s", err.Error())
		return
	}

	log.Printf("Done.")
	return
}

func AddRawCommand(toAdd accessors.RawCommand) (accessors.RawCommand, error) {
	log.Printf("adding raw command: %v to database", toAdd.Name)
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/devices/commands/" + toAdd.Name

	var toFill accessors.RawCommand
	err := PostData(url, toAdd, &toFill)
	if err != nil {
		return accessors.RawCommand{}, err
	}

	return toFill, nil
}

// GetRoomByInfo simply retrieves a device's information from the databse.
func GetRoomByInfo(buildingName string, roomName string) (toReturn accessors.Room, err error) {
	log.Printf("Getting room %s in building %s...", roomName, buildingName)
	err = GetData(os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS")+"/buildings/"+buildingName+"/rooms/"+roomName, &toReturn)
	return
}

// GetDeviceByName simply retrieves a device's information from the databse.
func GetDeviceByName(buildingName string, roomName string, deviceName string) (toReturn accessors.Device, err error) {
	err = GetData(os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS")+"/buildings/"+buildingName+"/rooms/"+roomName+"/devices/"+deviceName, &toReturn)
	return
}

// GetDevicesByRoom will jut get the devices based on the room.
func GetDevicesByRoom(buildingName string, roomName string) (toReturn []accessors.Device, err error) {
	err = GetData(os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS")+"/buildings/"+buildingName+"/rooms/"+roomName+"/devices", &toReturn)
	return
}

// GetDevicesByBuildingAndRoomAndRole will get the devices with the given role from the DB
func GetDevicesByBuildingAndRoomAndRole(building string, room string, roleName string) (toReturn []accessors.Device, err error) {
	err = GetData(os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS")+"/buildings/"+building+"/rooms/"+room+"/devices/roles/"+roleName, &toReturn)
	return
}

// SetAudioInDB will set the audio levels in the database
func SetAudioInDB(building string, room string, device accessors.Device) error {
	log.Printf("Updating audio levels in DB.")

	if device.Volume != nil {
		url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/buildings/" + building + "/rooms/" + room + "/devices/" + device.Name + "/attributes/volume/" + strconv.Itoa(*device.Volume)
		request, err := http.NewRequest("PUT", url, nil)
		client := &http.Client{}
		_, err = client.Do(request)

		if err != nil {
			return err
		}
	}

	if device.Muted != nil {
		url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/buildings/" + building + "/rooms/" + room + "/devices/" + device.Name + "/attributes/muted/" + strconv.FormatBool(*device.Muted)
		request, err := http.NewRequest("PUT", url, nil)
		client := &http.Client{}
		_, err = client.Do(request)

		if err != nil {
			return err
		}
	}

	return nil
}

// GetBuildings will return all buildings
func GetBuildings() ([]accessors.Building, error) {
	log.Printf("getting all buildings...")
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/buildings"
	log.Printf(url)
	var buildings []accessors.Building
	err := GetData(url, &buildings)

	return buildings, err
}

// GetRooms returns all the rooms in a given building
func GetRoomsByBuilding(building string) ([]accessors.Room, error) {

	log.Printf("dbo.GetRoomsByBuilding called")

	log.Printf("getting all rooms from %v ...", building)
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/buildings/" + building + "/rooms"
	var rooms []accessors.Room
	err := GetData(url, &rooms)
	return rooms, err
}

// GetBuildingByShortname returns a building with a given shortname
func GetBuildingByShortname(building string) (accessors.Building, error) {
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/buildings/shortname/" + building
	var output accessors.Building
	err := GetData(url, &output)
	if err != nil {
		return output, err
	}
	return output, nil
}

// AddBuilding
func AddBuilding(buildingToAdd accessors.Building) (accessors.Building, error) {
	log.Printf("adding building %v to database", buildingToAdd.Shortname)
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/buildings/" + buildingToAdd.Shortname

	var buildingToFill accessors.Building
	err := PostData(url, buildingToAdd, &buildingToFill)
	if err != nil {
		return accessors.Building{}, err
	}

	return buildingToFill, nil
}

func AddRoom(building string, roomToAdd accessors.Room) (accessors.Room, error) {
	log.Printf("adding room %v to building %v in database", roomToAdd.Name, building)
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/buildings/" + building + "/rooms/" + roomToAdd.Name

	var roomToFill accessors.Room
	err := PostData(url, roomToAdd, &roomToFill)
	if err != nil {
		return accessors.Room{}, err
	}

	return roomToFill, nil
}

func GetDeviceTypes() ([]accessors.DeviceType, error) {
	log.Printf("getting all device types")
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/devices/types/"

	var DeviceTypes []accessors.DeviceType
	err := GetData(url, &DeviceTypes)
	if err != nil {
		return []accessors.DeviceType{}, err
	}

	return DeviceTypes, nil
}

func AddDeviceType(toAdd accessors.DeviceType) (accessors.DeviceType, error) {
	log.Printf("adding device type: %v to database", toAdd.Name)
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/devices/types/" + toAdd.Name

	var toFill accessors.DeviceType
	err := PostData(url, toAdd, &toFill)
	if err != nil {
		return accessors.DeviceType{}, err
	}

	return toFill, nil
}
func GetPowerStates() ([]accessors.PowerState, error) {
	log.Printf("getting all power states")
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/devices/powerstates/"

	var powerStates []accessors.PowerState
	err := GetData(url, &powerStates)
	if err != nil {
		return []accessors.PowerState{}, err
	}

	return powerStates, nil
}

func AddPowerState(toAdd accessors.PowerState) (accessors.PowerState, error) {
	log.Printf("adding power state: %v to database", toAdd.Name)
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/devices/powerstates/" + toAdd.Name

	var toFill accessors.PowerState
	err := PostData(url, toAdd, &toFill)
	if err != nil {
		return accessors.PowerState{}, err
	}

	return toFill, nil
}

func GetMicroservices() ([]accessors.Microservice, error) {
	log.Printf("getting all microservices")
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/devices/microservices"

	var microservices []accessors.Microservice
	err := GetData(url, &microservices)
	if err != nil {
		return []accessors.Microservice{}, err
	}

	return microservices, nil
}

func AddMicroservice(toAdd accessors.Microservice) (accessors.Microservice, error) {
	log.Printf("adding microservice: %v to database", toAdd.Name)
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/devices/microservices/" + toAdd.Name

	var toFill accessors.Microservice
	err := PostData(url, toAdd, &toFill)
	if err != nil {
		return accessors.Microservice{}, err
	}

	return toFill, nil
}

func GetEndpoints() ([]accessors.Endpoint, error) {
	log.Printf("getting all endpoints")
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/devices/endpoints"

	var endpoints []accessors.Endpoint
	err := GetData(url, &endpoints)
	if err != nil {
		return []accessors.Endpoint{}, err
	}

	return endpoints, nil
}

func AddEndpoint(toAdd accessors.Endpoint) (accessors.Endpoint, error) {
	log.Printf("adding endpoint: %v to database", toAdd.Name)
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/devices/endpoints/" + toAdd.Name

	var toFill accessors.Endpoint
	err := PostData(url, toAdd, &toFill)
	if err != nil {
		return accessors.Endpoint{}, err
	}

	return toFill, nil
}

func GetPorts() ([]accessors.PortType, error) {
	log.Printf("getting all ports")
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/devices/ports"

	var ports []accessors.PortType
	err := GetData(url, &ports)
	if err != nil {
		return []accessors.PortType{}, err
	}

	return ports, nil
}

func AddPort(portToAdd accessors.PortType) (accessors.PortType, error) {
	log.Printf("adding Port: %v to database", portToAdd.Name)
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/devices/ports/" + portToAdd.Name

	var portToFill accessors.PortType
	err := PostData(url, portToAdd, &portToFill)
	if err != nil {
		return accessors.PortType{}, err
	}

	return portToFill, nil
}

func GetDeviceRoleDefinitions() ([]accessors.DeviceRoleDef, error) {
	log.Printf("getting device role definitions")
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/devices/roledefinitions"

	var definitions []accessors.DeviceRoleDef
	err := GetData(url, &definitions)
	if err != nil {
		return []accessors.DeviceRoleDef{}, err
	}

	return definitions, nil
}

func AddRoleDefinition(toAdd accessors.DeviceRoleDef) (accessors.DeviceRoleDef, error) {
	log.Printf("adding role definition: %v to database", toAdd.Name)
	url := os.Getenv("CONFIGURATION_DATABASE_MICROSERVICE_ADDRESS") + "/devices/roledefinitions/" + toAdd.Name

	var toFill accessors.DeviceRoleDef
	err := PostData(url, toAdd, &toFill)
	if err != nil {
		return accessors.DeviceRoleDef{}, err
	}

	return toFill, nil
}
<|endoftext|>


<commit_before>package waveform

import (
	"bytes"
	"math/rand"
	"testing"
	"time"

	"azul3d.org/audio.v1"
)

// BenchmarkNewWAV checks the performance of the New() function with a WAV file
func BenchmarkNewWAV(b *testing.B) {
	benchmarkNew(b, wavFile, nil)
}

// BenchmarkNewFLAC checks the performance of the New() function with a FLAC file
func BenchmarkNewFLAC(b *testing.B) {
	benchmarkNew(b, flacFile, nil)
}

// BenchmarkRMSF64Samples22050 checks the performance of the RMSF64Samples() function
// with 22050 samples
func BenchmarkRMSF64Samples22050(b *testing.B) {
	benchmarkRMSF64Samples(b, 22050)
}

// BenchmarkRMSF64Samples44100 checks the performance of the RMSF64Samples() function
// with 44100 samples
func BenchmarkRMSF64Samples44100(b *testing.B) {
	benchmarkRMSF64Samples(b, 44100)
}

// BenchmarkRMSF64Samples88200 checks the performance of the RMSF64Samples() function
// with 88200 samples
func BenchmarkRMSF64Samples88200(b *testing.B) {
	benchmarkRMSF64Samples(b, 88200)
}

// BenchmarkRMSF64Samples176400 checks the performance of the RMSF64Samples() function
// with 176400 samples
func BenchmarkRMSF64Samples176400(b *testing.B) {
	benchmarkRMSF64Samples(b, 176400)
}

// benchmarkNew contains common logic for benchmarking New
func benchmarkNew(b *testing.B, data []byte, options *Options) {
	for i := 0; i < b.N; i++ {
		New(bytes.NewReader(data), options)
	}
}

// benchmarkRMSF64Samples contains common logic for benchmarking RMSF64Samples
func benchmarkRMSF64Samples(b *testing.B, count int) {
	// Generate slice of samples
	rand.Seed(time.Now().UnixNano())
	var samples audio.F64Samples
	for i := 0; i < count; i++ {
		samples = append(samples, audio.F64(rand.Float64()))
	}

	// Reset timer and start benchmark
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		RMSF64Samples(samples)
	}
}
<commit_msg>Add proper benchmarks for ComputeValues and ImageFromValues<commit_after>package waveform

import (
	"bytes"
	"math/rand"
	"testing"
	"time"

	"azul3d.org/audio.v1"
)

// BenchmarkNewWAV checks the performance of the New() function with a WAV file
func BenchmarkNewWAV(b *testing.B) {
	benchmarkNew(b, wavFile, nil)
}

// BenchmarkNewFLAC checks the performance of the New() function with a FLAC file
func BenchmarkNewFLAC(b *testing.B) {
	benchmarkNew(b, flacFile, nil)
}

// BenchmarkComputeValuesWAV checks the performance of the ComputeValues() function with a WAV file
func BenchmarkComputeValuesWAV(b *testing.B) {
	benchmarkComputeValues(b, wavFile, nil)
}

// BenchmarkComputeValuesFLAC checks the performance of the ComputeValues() function with a FLAC file
func BenchmarkComputeValuesFLAC(b *testing.B) {
	benchmarkComputeValues(b, flacFile, nil)
}

// BenchmarkImageFromValues60 checks the performance of the ImageFromValues() function
// with approximately 60 seconds of computed values
func BenchmarkImageFromValues60(b *testing.B) {
	benchmarkImageFromValues(b, 60, nil)
}

// BenchmarkImageFromValues120 checks the performance of the ImageFromValues() function
// with approximately 120 seconds of computed values
func BenchmarkImageFromValues120(b *testing.B) {
	benchmarkImageFromValues(b, 120, nil)
}

// BenchmarkImageFromValues240 checks the performance of the ImageFromValues() function
// with approximately 240 seconds of computed values
func BenchmarkImageFromValues240(b *testing.B) {
	benchmarkImageFromValues(b, 240, nil)
}

// BenchmarkImageFromValues480 checks the performance of the ImageFromValues() function
// with approximately 480 seconds of computed values
func BenchmarkImageFromValues480(b *testing.B) {
	benchmarkImageFromValues(b, 480, nil)
}

// BenchmarkImageFromValues960 checks the performance of the ImageFromValues() function
// with approximately 960 seconds of computed values
func BenchmarkImageFromValues960(b *testing.B) {
	benchmarkImageFromValues(b, 960, nil)
}

// BenchmarkRMSF64Samples22050 checks the performance of the RMSF64Samples() function
// with 22050 samples
func BenchmarkRMSF64Samples22050(b *testing.B) {
	benchmarkRMSF64Samples(b, 22050)
}

// BenchmarkRMSF64Samples44100 checks the performance of the RMSF64Samples() function
// with 44100 samples
func BenchmarkRMSF64Samples44100(b *testing.B) {
	benchmarkRMSF64Samples(b, 44100)
}

// BenchmarkRMSF64Samples88200 checks the performance of the RMSF64Samples() function
// with 88200 samples
func BenchmarkRMSF64Samples88200(b *testing.B) {
	benchmarkRMSF64Samples(b, 88200)
}

// BenchmarkRMSF64Samples176400 checks the performance of the RMSF64Samples() function
// with 176400 samples
func BenchmarkRMSF64Samples176400(b *testing.B) {
	benchmarkRMSF64Samples(b, 176400)
}

// benchmarkNew contains common logic for benchmarking New
func benchmarkNew(b *testing.B, data []byte, options *Options) {
	for i := 0; i < b.N; i++ {
		New(bytes.NewReader(data), options)
	}
}

// benchmarkComputeValues contains common logic for benchmarking ComputeValues
func benchmarkComputeValues(b *testing.B, data []byte, options *ComputeOptions) {
	for i := 0; i < b.N; i++ {
		ComputeValues(bytes.NewReader(data), options)
	}
}

// benchmarkImageFromValues contains common logic for benchmarking ImageFromValues
func benchmarkImageFromValues(b *testing.B, count int, options *ImageOptions) {
	values := make([]float64, count)
	for i := 0; i < b.N; i++ {
		ImageFromValues(values, options)
	}
}

// benchmarkRMSF64Samples contains common logic for benchmarking RMSF64Samples
func benchmarkRMSF64Samples(b *testing.B, count int) {
	// Generate slice of samples
	rand.Seed(time.Now().UnixNano())
	var samples audio.F64Samples
	for i := 0; i < count; i++ {
		samples = append(samples, audio.F64(rand.Float64()))
	}

	// Reset timer and start benchmark
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		RMSF64Samples(samples)
	}
}
<|endoftext|>


<commit_before>package engine_test

import (
	"strings"
	"testing"

	"github.com/woobleio/wooblizer/wbzr/engine"
)

func TestIncludeHtml(t *testing.T) {
	src := `var Woobly=function Woobly(){_classCallCheck(this,Woobly);this.document=document.body.shadowRoot};`

	s, errs := engine.NewJS("objForTest", src, nil)

	if len(errs) > 0 {
		t.Error("The JS class is invalid")
	}

	s.IncludeHTMLCSS("
test
", "div { color: red }")

	expected := `var _sr_ = _t_.attachShadow({mode:'open'});var __b = document.createElement('div');__b.setAttribute('class', 'heelo');__b.setAttribute('id', 'hello');_sr_.appendChild(__b);var __c = document.createTextNode('test');__b.appendChild(__c);this.document = _sr_;var __s = document.createElement('style');__s.innerHTML = 'div { color: red }';this.document.appendChild(__s);`

	if !strings.Contains(s.Src, expected) {
		t.Error("Includes good HTML and good CSS : Unexpected source")
	}

	s, errs = engine.NewJS("objForTest", src, nil)

	s.IncludeHTMLCSS("", "div { color: red; }")

	expected = `function Woobly(_t_){_classCallCheck(this,Woobly);var _sr_ = _t_.attachShadow({mode:'open'});this.document = _sr_;var __s = document.createElement('style');__s.innerHTML = 'div { color: red; }';this.document.appendChild(__s);}`
	if !strings.Contains(s.Src, expected) {
		t.Error("Includes only HTML : Unexpected source")
	}

	s.Src = ""

	err := s.IncludeHTMLCSS("

", "")

	if err == nil {
		t.Error("Includes when no doc init is present : It should returns an error")
	}
}
<commit_msg>Fixes tests<commit_after>package engine_test

import (
	"strings"
	"testing"

	"github.com/woobleio/wooblizer/wbzr/engine"
)

func TestIncludeHtml(t *testing.T) {
	src := `var Woobly=function Woobly(){_classCallCheck(this,Woobly);this.document=document.body.shadowRoot};`

	s, errs := engine.NewJS("objForTest", src, nil)

	if len(errs) > 0 {
		t.Error("The JS class is invalid")
	}

	s.IncludeHTMLCSS("
test
", "div { color: red }")

	expected := `var _sr_ = _t_.attachShadow({mode:'open'});var __b = document.createElement('div');__b.setAttribute('class', 'heelo');__b.setAttribute('id', 'hello');_sr_.appendChild(__b);var __c = document.createTextNode('test');__b.appendChild(__c);this.document = _sr_;var __s = document.createElement('style');__s.innerHTML = 'div { color: red }';this.document.appendChild(__s);`

	if !strings.Contains(s.Src, expected) {
		t.Error("Includes good HTML and good CSS : Unexpected source")
	}

	s, errs = engine.NewJS("objForTest", src, nil)

	if len(errs) > 0 {
		t.Error("The JS class (second try) is invalid")
	}

	s.IncludeHTMLCSS("", "div { color: red; }")

	expected = `function Woobly(_t_){_classCallCheck(this,Woobly);var _sr_ = _t_.attachShadow({mode:'open'});this.document = _sr_;var __s = document.createElement('style');__s.innerHTML = 'div { color: red; }';this.document.appendChild(__s);}`
	if !strings.Contains(s.Src, expected) {
		t.Error("Includes only HTML : Unexpected source")
	}

	s.Src = ""

	err := s.IncludeHTMLCSS("

", "")

	if err == nil {
		t.Error("Includes when no doc init is present : It should returns an error")
	}
}
<|endoftext|>