<commit_before>package main

import (
	"bytes"
	"os"
	"strings"
	"testing"

	"v.io/x/devtools/lib/testutil"
	"v.io/x/devtools/lib/util"
	"v.io/x/lib/cmdline"
)

func TestTestProject(t *testing.T) {
	ctx := util.DefaultContext()

	// Setup an instance of vanadium universe.
	rootDir, err := ctx.Run().TempDir("", "")
	if err != nil {
		t.Fatalf("TempDir() failed: %v", err)
	}
	defer ctx.Run().RemoveAll(rootDir)
	oldRoot := os.Getenv("VANADIUM_ROOT")
	if err := os.Setenv("VANADIUM_ROOT", rootDir); err != nil {
		t.Fatalf("%v", err)
	}
	defer os.Setenv("VANADIUM_ROOT", oldRoot)

	config := util.NewConfig(util.ProjectTestsOpt(map[string][]string{"https://test-project": []string{"ignore-this"}}))
	createConfig(t, ctx, config)

	// Check that running the tests for the test project generates
	// the expected output.
	var out bytes.Buffer
	command := cmdline.Command{}
	command.Init(nil, &out, &out)
	if err := runTestProject(&command, []string{"https://test-project"}); err != nil {
		t.Fatalf("%v", err)
	}
	got, want := out.String(), `##### Running test "ignore-this" #####
##### PASSED #####
SUMMARY:
ignore-this PASSED
`
	if got != want {
		t.Fatalf("unexpected output:\ngot\n%v\nwant\n%v", got, want)
	}
}

func TestTestRun(t *testing.T) {
	ctx := util.DefaultContext()

	// Setup an instance of vanadium universe.
	rootDir, err := ctx.Run().TempDir("", "")
	if err != nil {
		t.Fatalf("TempDir() failed: %v", err)
	}
	defer ctx.Run().RemoveAll(rootDir)
	oldRoot := os.Getenv("VANADIUM_ROOT")
	if err := os.Setenv("VANADIUM_ROOT", rootDir); err != nil {
		t.Fatalf("%v", err)
	}
	defer os.Setenv("VANADIUM_ROOT", oldRoot)

	// Check that running the test generates the expected output.
	var out bytes.Buffer
	command := cmdline.Command{}
	command.Init(nil, &out, &out)
	if err := runTestRun(&command, []string{"ignore-this"}); err != nil {
		t.Fatalf("%v", err)
	}
	got, want := out.String(), `##### Running test "ignore-this" #####
##### PASSED #####
SUMMARY:
ignore-this PASSED
`
	if got != want {
		t.Fatalf("unexpected output:\ngot\n%v\nwant\n%v", got, want)
	}
}

func TestTestList(t *testing.T) {
	ctx := util.DefaultContext()

	// Setup an instance of vanadium universe.
	rootDir, err := ctx.Run().TempDir("", "")
	if err != nil {
		t.Fatalf("TempDir() failed: %v", err)
	}
	defer ctx.Run().RemoveAll(rootDir)
	oldRoot := os.Getenv("VANADIUM_ROOT")
	if err := os.Setenv("VANADIUM_ROOT", rootDir); err != nil {
		t.Fatalf("%v", err)
	}
	defer os.Setenv("VANADIUM_ROOT", oldRoot)

	// Check that listing existing tests generates the expected
	// output.
	var out bytes.Buffer
	command := cmdline.Command{}
	command.Init(nil, &out, &out)
	if err := runTestList(&command, []string{}); err != nil {
		t.Fatalf("%v", err)
	}
	testList, err := testutil.TestList()
	if err != nil {
		t.Fatalf("%v", err)
	}
	if got, want := strings.TrimSpace(out.String()), strings.Join(testList, "\n"); got != want {
		t.Fatalf("unexpected output:\ngot\n%v\nwant\n%v", got, want)
	}
}
<commit_msg>devtools: move (and rename) generateDummyTestReportFile from presubmit.go to run.go<commit_after>package main

import (
	"bytes"
	"os"
	"strings"
	"testing"

	"v.io/x/devtools/lib/testutil"
	"v.io/x/devtools/lib/util"
	"v.io/x/lib/cmdline"
)

func TestTestProject(t *testing.T) {
	ctx := util.DefaultContext()

	// Setup an instance of vanadium universe.
	rootDir, err := ctx.Run().TempDir("", "")
	if err != nil {
		t.Fatalf("TempDir() failed: %v", err)
	}
	defer ctx.Run().RemoveAll(rootDir)
	oldRoot := os.Getenv("VANADIUM_ROOT")
	if err := os.Setenv("VANADIUM_ROOT", rootDir); err != nil {
		t.Fatalf("%v", err)
	}
	defer os.Setenv("VANADIUM_ROOT", oldRoot)
	oldWorkspace := os.Getenv("WORKSPACE")
	if err := os.Setenv("WORKSPACE", rootDir); err != nil {
		t.Fatalf("%v", err)
	}
	defer os.Setenv("WORKSPACE", oldWorkspace)

	config := util.NewConfig(util.ProjectTestsOpt(map[string][]string{"https://test-project": []string{"ignore-this"}}))
	createConfig(t, ctx, config)

	// Check that running the tests for the test project generates
	// the expected output.
	var out bytes.Buffer
	command := cmdline.Command{}
	command.Init(nil, &out, &out)
	if err := runTestProject(&command, []string{"https://test-project"}); err != nil {
		t.Fatalf("%v", err)
	}
	got, want := out.String(), `##### Running test "ignore-this" #####
##### PASSED #####
SUMMARY:
ignore-this PASSED
`
	if got != want {
		t.Fatalf("unexpected output:\ngot\n%v\nwant\n%v", got, want)
	}
}

func TestTestRun(t *testing.T) {
	ctx := util.DefaultContext()

	// Setup an instance of vanadium universe.
	rootDir, err := ctx.Run().TempDir("", "")
	if err != nil {
		t.Fatalf("TempDir() failed: %v", err)
	}
	defer ctx.Run().RemoveAll(rootDir)
	oldRoot := os.Getenv("VANADIUM_ROOT")
	if err := os.Setenv("VANADIUM_ROOT", rootDir); err != nil {
		t.Fatalf("%v", err)
	}
	defer os.Setenv("VANADIUM_ROOT", oldRoot)
	oldWorkspace := os.Getenv("WORKSPACE")
	if err := os.Setenv("WORKSPACE", rootDir); err != nil {
		t.Fatalf("%v", err)
	}
	defer os.Setenv("WORKSPACE", oldWorkspace)

	// Check that running the test generates the expected output.
	var out bytes.Buffer
	command := cmdline.Command{}
	command.Init(nil, &out, &out)
	if err := runTestRun(&command, []string{"ignore-this"}); err != nil {
		t.Fatalf("%v", err)
	}
	got, want := out.String(), `##### Running test "ignore-this" #####
##### PASSED #####
SUMMARY:
ignore-this PASSED
`
	if got != want {
		t.Fatalf("unexpected output:\ngot\n%v\nwant\n%v", got, want)
	}
}

func TestTestList(t *testing.T) {
	ctx := util.DefaultContext()

	// Setup an instance of vanadium universe.
	rootDir, err := ctx.Run().TempDir("", "")
	if err != nil {
		t.Fatalf("TempDir() failed: %v", err)
	}
	defer ctx.Run().RemoveAll(rootDir)
	oldRoot := os.Getenv("VANADIUM_ROOT")
	if err := os.Setenv("VANADIUM_ROOT", rootDir); err != nil {
		t.Fatalf("%v", err)
	}
	defer os.Setenv("VANADIUM_ROOT", oldRoot)

	// Check that listing existing tests generates the expected
	// output.
	var out bytes.Buffer
	command := cmdline.Command{}
	command.Init(nil, &out, &out)
	if err := runTestList(&command, []string{}); err != nil {
		t.Fatalf("%v", err)
	}
	testList, err := testutil.TestList()
	if err != nil {
		t.Fatalf("%v", err)
	}
	if got, want := strings.TrimSpace(out.String()), strings.Join(testList, "\n"); got != want {
		t.Fatalf("unexpected output:\ngot\n%v\nwant\n%v", got, want)
	}
}
<|endoftext|>


<commit_before>package allocrunner

import (
	"fmt"
	"time"

	multierror "github.com/hashicorp/go-multierror"
	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
	"github.com/hashicorp/nomad/nomad/structs"
)

// allocHealthSetter is a shim to allow the alloc health watcher hook to set
// and clear the alloc health without full access to the alloc runner state
type allocHealthSetter struct {
	ar *allocRunner
}

// HasHealth returns true if a deployment status is already set.
func (a *allocHealthSetter) HasHealth() bool {
	a.ar.stateLock.Lock()
	defer a.ar.stateLock.Unlock()
	return a.ar.state.DeploymentStatus.HasHealth()
}

// ClearHealth allows the health watcher hook to clear the alloc's deployment
// health if the deployment id changes. It does not update the server as the
// status is only cleared when already receiving an update from the server.
//
// Only for use by health hook.
func (a *allocHealthSetter) ClearHealth() {
	a.ar.stateLock.Lock()
	a.ar.state.ClearDeploymentStatus()
	a.ar.persistDeploymentStatus(nil)
	a.ar.stateLock.Unlock()
}

// SetHealth allows the health watcher hook to set the alloc's
// deployment/migration health and emit task events.
//
// Only for use by health hook.
func (a *allocHealthSetter) SetHealth(healthy, isDeploy bool, trackerTaskEvents map[string]*structs.TaskEvent) {
	// Updating alloc deployment state is tricky because it may be nil, but
	// if it's not then we need to maintain the values of Canary and
	// ModifyIndex as they're only mutated by the server.
	a.ar.stateLock.Lock()
	a.ar.state.SetDeploymentStatus(time.Now(), healthy)
	a.ar.persistDeploymentStatus(a.ar.state.DeploymentStatus)
	terminalDesiredState := a.ar.alloc.ServerTerminalStatus()
	a.ar.stateLock.Unlock()

	// If deployment is unhealthy emit task events explaining why
	if !healthy && isDeploy && !terminalDesiredState {
		for task, event := range trackerTaskEvents {
			if tr, ok := a.ar.tasks[task]; ok {
				// Append but don't emit event since the server
				// will be updated below
				tr.AppendEvent(event)
			}
		}
	}

	// Gather the state of the other tasks
	states := make(map[string]*structs.TaskState, len(a.ar.tasks))
	for name, tr := range a.ar.tasks {
		states[name] = tr.TaskState()
	}

	// Build the client allocation
	calloc := a.ar.clientAlloc(states)

	// Update the server
	a.ar.stateUpdater.AllocStateUpdated(calloc)

	// Broadcast client alloc to listeners
	a.ar.allocBroadcaster.Send(calloc)
}

// initRunnerHooks intializes the runners hooks.
func (ar *allocRunner) initRunnerHooks() {
	hookLogger := ar.logger.Named("runner_hook")

	// create health setting shim
	hs := &allocHealthSetter{ar}

	// Create the alloc directory hook. This is run first to ensure the
	// directory path exists for other hooks.
	ar.runnerHooks = []interfaces.RunnerHook{
		newAllocDirHook(hookLogger, ar.allocDir),
		newUpstreamAllocsHook(hookLogger, ar.prevAllocWatcher),
		newDiskMigrationHook(hookLogger, ar.prevAllocMigrator, ar.allocDir),
		newAllocHealthWatcherHook(hookLogger, ar.Alloc(), hs, ar.Listener(), ar.consulClient),
	}
}

// prerun is used to run the runners prerun hooks.
func (ar *allocRunner) prerun() error {
	if ar.logger.IsTrace() {
		start := time.Now()
		ar.logger.Trace("running pre-run hooks", "start", start)
		defer func() {
			end := time.Now()
			ar.logger.Trace("finished pre-run hooks", "end", end, "duration", end.Sub(start))
		}()
	}

	for _, hook := range ar.runnerHooks {
		pre, ok := hook.(interfaces.RunnerPrerunHook)
		if !ok {
			continue
		}

		name := pre.Name()
		var start time.Time
		if ar.logger.IsTrace() {
			start = time.Now()
			ar.logger.Trace("running pre-run hook", "name", name, "start", start)
		}

		if err := pre.Prerun(); err != nil {
			return fmt.Errorf("pre-run hook %q failed: %v", name, err)
		}

		if ar.logger.IsTrace() {
			end := time.Now()
			ar.logger.Trace("finished pre-run hook", "name", name, "end", end, "duration", end.Sub(start))
		}
	}

	return nil
}

// update runs the alloc runner update hooks. Update hooks are run
// asynchronously with all other alloc runner operations.
func (ar *allocRunner) update(update *structs.Allocation) error {
	if ar.logger.IsTrace() {
		start := time.Now()
		ar.logger.Trace("running update hooks", "start", start)
		defer func() {
			end := time.Now()
			ar.logger.Trace("finished update hooks", "end", end, "duration", end.Sub(start))
		}()
	}

	req := &interfaces.RunnerUpdateRequest{
		Alloc: update,
	}

	var merr multierror.Error
	for _, hook := range ar.runnerHooks {
		h, ok := hook.(interfaces.RunnerUpdateHook)
		if !ok {
			continue
		}

		name := h.Name()
		var start time.Time
		if ar.logger.IsTrace() {
			start = time.Now()
			ar.logger.Trace("running pre-run hook", "name", name, "start", start)
		}

		if err := h.Update(req); err != nil {
			merr.Errors = append(merr.Errors, fmt.Errorf("update hook %q failed: %v", name, err))
		}

		if ar.logger.IsTrace() {
			end := time.Now()
			ar.logger.Trace("finished update hooks", "name", name, "end", end, "duration", end.Sub(start))
		}
	}

	return merr.ErrorOrNil()
}

// postrun is used to run the runners postrun hooks.
func (ar *allocRunner) postrun() error {
	if ar.logger.IsTrace() {
		start := time.Now()
		ar.logger.Trace("running post-run hooks", "start", start)
		defer func() {
			end := time.Now()
			ar.logger.Trace("finished post-run hooks", "end", end, "duration", end.Sub(start))
		}()
	}

	for _, hook := range ar.runnerHooks {
		post, ok := hook.(interfaces.RunnerPostrunHook)
		if !ok {
			continue
		}

		name := post.Name()
		var start time.Time
		if ar.logger.IsTrace() {
			start = time.Now()
			ar.logger.Trace("running post-run hook", "name", name, "start", start)
		}

		if err := post.Postrun(); err != nil {
			return fmt.Errorf("hook %q failed: %v", name, err)
		}

		if ar.logger.IsTrace() {
			end := time.Now()
			ar.logger.Trace("finished post-run hooks", "name", name, "end", end, "duration", end.Sub(start))
		}
	}

	return nil
}

// destroy is used to run the runners destroy hooks. All hooks are run and
// errors are returned as a multierror.
func (ar *allocRunner) destroy() error {
	if ar.logger.IsTrace() {
		start := time.Now()
		ar.logger.Trace("running destroy hooks", "start", start)
		defer func() {
			end := time.Now()
			ar.logger.Trace("finished destroy hooks", "end", end, "duration", end.Sub(start))
		}()
	}

	var merr multierror.Error
	for _, hook := range ar.runnerHooks {
		h, ok := hook.(interfaces.RunnerDestroyHook)
		if !ok {
			continue
		}

		name := h.Name()
		var start time.Time
		if ar.logger.IsTrace() {
			start = time.Now()
			ar.logger.Trace("running destroy hook", "name", name, "start", start)
		}

		if err := h.Destroy(); err != nil {
			merr.Errors = append(merr.Errors, fmt.Errorf("destroy hook %q failed: %v", name, err))
		}

		if ar.logger.IsTrace() {
			end := time.Now()
			ar.logger.Trace("finished destroy hooks", "name", name, "end", end, "duration", end.Sub(start))
		}
	}

	return merr.ErrorOrNil()
}

// shutdownHooks calls graceful shutdown hooks for when the agent is exiting.
func (ar *allocRunner) shutdownHooks() {
	for _, hook := range ar.runnerHooks {
		sh, ok := hook.(interfaces.ShutdownHook)
		if !ok {
			continue
		}

		name := sh.Name()
		var start time.Time
		if ar.logger.IsTrace() {
			start = time.Now()
			ar.logger.Trace("running shutdown hook", "name", name, "start", start)
		}

		sh.Shutdown()

		if ar.logger.IsTrace() {
			end := time.Now()
			ar.logger.Trace("finished shutdown hooks", "name", name, "end", end, "duration", end.Sub(start))
		}
	}
}
<commit_msg>client: synchronize access to ar.alloc<commit_after>package allocrunner

import (
	"fmt"
	"time"

	multierror "github.com/hashicorp/go-multierror"
	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
	"github.com/hashicorp/nomad/nomad/structs"
)

// allocHealthSetter is a shim to allow the alloc health watcher hook to set
// and clear the alloc health without full access to the alloc runner state
type allocHealthSetter struct {
	ar *allocRunner
}

// HasHealth returns true if a deployment status is already set.
func (a *allocHealthSetter) HasHealth() bool {
	a.ar.stateLock.Lock()
	defer a.ar.stateLock.Unlock()
	return a.ar.state.DeploymentStatus.HasHealth()
}

// ClearHealth allows the health watcher hook to clear the alloc's deployment
// health if the deployment id changes. It does not update the server as the
// status is only cleared when already receiving an update from the server.
//
// Only for use by health hook.
func (a *allocHealthSetter) ClearHealth() {
	a.ar.stateLock.Lock()
	a.ar.state.ClearDeploymentStatus()
	a.ar.persistDeploymentStatus(nil)
	a.ar.stateLock.Unlock()
}

// SetHealth allows the health watcher hook to set the alloc's
// deployment/migration health and emit task events.
//
// Only for use by health hook.
func (a *allocHealthSetter) SetHealth(healthy, isDeploy bool, trackerTaskEvents map[string]*structs.TaskEvent) {
	// Updating alloc deployment state is tricky because it may be nil, but
	// if it's not then we need to maintain the values of Canary and
	// ModifyIndex as they're only mutated by the server.
	a.ar.stateLock.Lock()
	a.ar.state.SetDeploymentStatus(time.Now(), healthy)
	a.ar.persistDeploymentStatus(a.ar.state.DeploymentStatus)
	terminalDesiredState := a.ar.Alloc().ServerTerminalStatus()
	a.ar.stateLock.Unlock()

	// If deployment is unhealthy emit task events explaining why
	if !healthy && isDeploy && !terminalDesiredState {
		for task, event := range trackerTaskEvents {
			if tr, ok := a.ar.tasks[task]; ok {
				// Append but don't emit event since the server
				// will be updated below
				tr.AppendEvent(event)
			}
		}
	}

	// Gather the state of the other tasks
	states := make(map[string]*structs.TaskState, len(a.ar.tasks))
	for name, tr := range a.ar.tasks {
		states[name] = tr.TaskState()
	}

	// Build the client allocation
	calloc := a.ar.clientAlloc(states)

	// Update the server
	a.ar.stateUpdater.AllocStateUpdated(calloc)

	// Broadcast client alloc to listeners
	a.ar.allocBroadcaster.Send(calloc)
}

// initRunnerHooks intializes the runners hooks.
func (ar *allocRunner) initRunnerHooks() {
	hookLogger := ar.logger.Named("runner_hook")

	// create health setting shim
	hs := &allocHealthSetter{ar}

	// Create the alloc directory hook. This is run first to ensure the
	// directory path exists for other hooks.
	ar.runnerHooks = []interfaces.RunnerHook{
		newAllocDirHook(hookLogger, ar.allocDir),
		newUpstreamAllocsHook(hookLogger, ar.prevAllocWatcher),
		newDiskMigrationHook(hookLogger, ar.prevAllocMigrator, ar.allocDir),
		newAllocHealthWatcherHook(hookLogger, ar.Alloc(), hs, ar.Listener(), ar.consulClient),
	}
}

// prerun is used to run the runners prerun hooks.
func (ar *allocRunner) prerun() error {
	if ar.logger.IsTrace() {
		start := time.Now()
		ar.logger.Trace("running pre-run hooks", "start", start)
		defer func() {
			end := time.Now()
			ar.logger.Trace("finished pre-run hooks", "end", end, "duration", end.Sub(start))
		}()
	}

	for _, hook := range ar.runnerHooks {
		pre, ok := hook.(interfaces.RunnerPrerunHook)
		if !ok {
			continue
		}

		name := pre.Name()
		var start time.Time
		if ar.logger.IsTrace() {
			start = time.Now()
			ar.logger.Trace("running pre-run hook", "name", name, "start", start)
		}

		if err := pre.Prerun(); err != nil {
			return fmt.Errorf("pre-run hook %q failed: %v", name, err)
		}

		if ar.logger.IsTrace() {
			end := time.Now()
			ar.logger.Trace("finished pre-run hook", "name", name, "end", end, "duration", end.Sub(start))
		}
	}

	return nil
}

// update runs the alloc runner update hooks. Update hooks are run
// asynchronously with all other alloc runner operations.
func (ar *allocRunner) update(update *structs.Allocation) error {
	if ar.logger.IsTrace() {
		start := time.Now()
		ar.logger.Trace("running update hooks", "start", start)
		defer func() {
			end := time.Now()
			ar.logger.Trace("finished update hooks", "end", end, "duration", end.Sub(start))
		}()
	}

	req := &interfaces.RunnerUpdateRequest{
		Alloc: update,
	}

	var merr multierror.Error
	for _, hook := range ar.runnerHooks {
		h, ok := hook.(interfaces.RunnerUpdateHook)
		if !ok {
			continue
		}

		name := h.Name()
		var start time.Time
		if ar.logger.IsTrace() {
			start = time.Now()
			ar.logger.Trace("running pre-run hook", "name", name, "start", start)
		}

		if err := h.Update(req); err != nil {
			merr.Errors = append(merr.Errors, fmt.Errorf("update hook %q failed: %v", name, err))
		}

		if ar.logger.IsTrace() {
			end := time.Now()
			ar.logger.Trace("finished update hooks", "name", name, "end", end, "duration", end.Sub(start))
		}
	}

	return merr.ErrorOrNil()
}

// postrun is used to run the runners postrun hooks.
func (ar *allocRunner) postrun() error {
	if ar.logger.IsTrace() {
		start := time.Now()
		ar.logger.Trace("running post-run hooks", "start", start)
		defer func() {
			end := time.Now()
			ar.logger.Trace("finished post-run hooks", "end", end, "duration", end.Sub(start))
		}()
	}

	for _, hook := range ar.runnerHooks {
		post, ok := hook.(interfaces.RunnerPostrunHook)
		if !ok {
			continue
		}

		name := post.Name()
		var start time.Time
		if ar.logger.IsTrace() {
			start = time.Now()
			ar.logger.Trace("running post-run hook", "name", name, "start", start)
		}

		if err := post.Postrun(); err != nil {
			return fmt.Errorf("hook %q failed: %v", name, err)
		}

		if ar.logger.IsTrace() {
			end := time.Now()
			ar.logger.Trace("finished post-run hooks", "name", name, "end", end, "duration", end.Sub(start))
		}
	}

	return nil
}

// destroy is used to run the runners destroy hooks. All hooks are run and
// errors are returned as a multierror.
func (ar *allocRunner) destroy() error {
	if ar.logger.IsTrace() {
		start := time.Now()
		ar.logger.Trace("running destroy hooks", "start", start)
		defer func() {
			end := time.Now()
			ar.logger.Trace("finished destroy hooks", "end", end, "duration", end.Sub(start))
		}()
	}

	var merr multierror.Error
	for _, hook := range ar.runnerHooks {
		h, ok := hook.(interfaces.RunnerDestroyHook)
		if !ok {
			continue
		}

		name := h.Name()
		var start time.Time
		if ar.logger.IsTrace() {
			start = time.Now()
			ar.logger.Trace("running destroy hook", "name", name, "start", start)
		}

		if err := h.Destroy(); err != nil {
			merr.Errors = append(merr.Errors, fmt.Errorf("destroy hook %q failed: %v", name, err))
		}

		if ar.logger.IsTrace() {
			end := time.Now()
			ar.logger.Trace("finished destroy hooks", "name", name, "end", end, "duration", end.Sub(start))
		}
	}

	return merr.ErrorOrNil()
}

// shutdownHooks calls graceful shutdown hooks for when the agent is exiting.
func (ar *allocRunner) shutdownHooks() {
	for _, hook := range ar.runnerHooks {
		sh, ok := hook.(interfaces.ShutdownHook)
		if !ok {
			continue
		}

		name := sh.Name()
		var start time.Time
		if ar.logger.IsTrace() {
			start = time.Now()
			ar.logger.Trace("running shutdown hook", "name", name, "start", start)
		}

		sh.Shutdown()

		if ar.logger.IsTrace() {
			end := time.Now()
			ar.logger.Trace("finished shutdown hooks", "name", name, "end", end, "duration", end.Sub(start))
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package capabilities

import "testing"

func TestMerge(t *testing.T) {
	testCases := []struct {
		name   string
		input1 map[string]interface{}
		input2 map[string]interface{}
		result map[string]interface{}
	}{
		{
			name:   "int,int",
			input1: map[string]interface{}{"v": 1},
			input2: map[string]interface{}{"v": 2},
			result: map[string]interface{}{"v": 2},
		},
		{
			name:   "string,bool",
			input1: map[string]interface{}{"v": "a string"},
			input2: map[string]interface{}{"v": true},
			result: map[string]interface{}{"v": true},
		},
		{
			name:   "int,string",
			input1: map[string]interface{}{"v": 1},
			input2: map[string]interface{}{"v": "a string"},
			result: map[string]interface{}{"v": "a string"},
		},
		{
			name:   "int,slice",
			input1: map[string]interface{}{"v": 1},
			input2: map[string]interface{}{"v": []interface{}{"string 1", 2, true, nil}},
			result: map[string]interface{}{"v": []interface{}{"string 1", 2, true, nil}},
		},
		{
			name:   "slice,slice",
			input1: map[string]interface{}{"v": []interface{}{"string 1", "string 2"}},
			input2: map[string]interface{}{"v": []interface{}{1, 2}},
			result: map[string]interface{}{"v": []interface{}{"string 1", "string 2", 1, 2}},
		},
		{
			name:   "int,map",
			input1: map[string]interface{}{"v": 1},
			input2: map[string]interface{}{"v": map[string]interface{}{
				"1": 1,
				"2": 2,
			}},
			result: map[string]interface{}{"v": map[string]interface{}{
				"1": 1,
				"2": 2,
			}},
		},
		{
			name: "map,map",
			input1: map[string]interface{}{"v": map[string]interface{}{
				"1": 1,
				"2": 2,
			}},
			input2: map[string]interface{}{"v": map[string]interface{}{
				"2": 3,
				"3": 4,
			}},
			result: map[string]interface{}{"v": map[string]interface{}{"1": 1, "2": 3, "3": 4}},
		},
		{
			name: "mixed",
			input1: map[string]interface{}{
				"1": []interface{}{1, 2},
				"2": map[string]interface{}{"a": "an a", "b": "a b", "c": "a c"},
				"3": 3,
			},
			input2: map[string]interface{}{
				"1": []interface{}{"a"},
				"2": map[string]interface{}{"a": "a c", "b": "a d"},
			},
			result: map[string]interface{}{
				"1": []interface{}{1, 2, "a"},
				"2": map[string]interface{}{"a": "a c", "b": "a d", "c": "a c"},
				"3": 3,
			},
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			if result := Merge(tc.input1, tc.input2); !Equals(tc.result, result) {
				t.Errorf("Got Merge(%+v, %+v) == %+v, expected %+v", tc.input1, tc.input2, result, tc.result)
			}
		})
	}
}

func TestEquals(t *testing.T) {
	testCases := []struct {
		name   string
		input1 map[string]interface{}
		input2 map[string]interface{}
		result bool
	}{
		{
			"empty",
			map[string]interface{}{},
			map[string]interface{}{},
			true,
		},
		{
			"int,nil",
			map[string]interface{}{"v": 1},
			map[string]interface{}{"v": nil},
			false,
		},
		{
			"string,nil",
			map[string]interface{}{"v": "hello"},
			map[string]interface{}{"v": nil},
			false,
		},
		{
			"bool,nil",
			map[string]interface{}{"v": true},
			map[string]interface{}{"v": nil},
			false,
		},
		{
			"slice,nil",
			map[string]interface{}{"v": []interface{}{}},
			map[string]interface{}{"v": nil},
			false,
		},
		{
			"map,nil",
			map[string]interface{}{"v": map[string]interface{}{}},
			map[string]interface{}{"v": nil},
			false,
		},
		{
			"map,map equals",
			map[string]interface{}{"v": map[string]interface{}{"a": 1}},
			map[string]interface{}{"v": map[string]interface{}{"a": 1}},
			true,
		},
		{
			"map,map different values",
			map[string]interface{}{"v": map[string]interface{}{"a": 1}},
			map[string]interface{}{"v": map[string]interface{}{"a": "hello"}},
			false,
		},
		{
			"map,map different keys",
			map[string]interface{}{"v": map[string]interface{}{"a": 1}},
			map[string]interface{}{"v": map[string]interface{}{"b": 1}},
			false,
		},
		{
			"map,map different lengths",
			map[string]interface{}{"v": map[string]interface{}{"a": 1}},
			map[string]interface{}{"v": map[string]interface{}{"a": 1, "b": 1}},
			false,
		},
		{
			"slice,slice equals",
			map[string]interface{}{"v": []interface{}{1}},
			map[string]interface{}{"v": []interface{}{1}},
			true,
		},
		{
			"slice,slice different values",
			map[string]interface{}{"v": []interface{}{1}},
			map[string]interface{}{"v": []interface{}{"hello"}},
			false,
		},
		{
			"slice,slice different lengths",
			map[string]interface{}{"v": []interface{}{1}},
			map[string]interface{}{"v": []interface{}{1, 1}},
			false,
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			if result := Equals(tc.input1, tc.input2); result != tc.result {
				t.Errorf("Got Equals(%+v, %+v) == %v, expected %v", tc.input1, tc.input2, result, tc.result)
			}
		})
	}
}

type kv struct {
	k string
	v interface{}
}

func TestGoogleCap(t *testing.T) {
	testCases := []struct {
		caps  map[string]interface{}
		wants []kv
	}{
		{
			map[string]interface{}{},
			[]kv{{"capName", nil}, {"otherCapName", nil}},
		},
		{
			map[string]interface{}{"google.capName": "vvvvvv"},
			[]kv{{"capName", "vvvvvv"}},
		},
		{
			map[string]interface{}{"capName": "vvvvvv"},
			[]kv{{"capName", nil}},
		},
	}

	for _, tc := range testCases {
		for _, want := range tc.wants {
			if got := GoogleCap(tc.caps, want.k); got != want.v {
				t.Errorf("GoogleCap(%v %q) is %v, want %v", tc.caps, want.k, got, want.v)
			}
			has := want.v != nil
			if got := HasGoogleCap(tc.caps, want.k); got != has {
				t.Errorf("HasGoogleCap(%v, %q) is %v, want %v", tc.caps, want.k, got, has)
			}
		}
	}
}

func TestSetGoogleCap(t *testing.T) {
	testCases := []struct {
		desc string
		caps map[string]interface{}
		k    string
		v    interface{}
		want map[string]interface{}
	}{
		{
			"set cap",
			map[string]interface{}{},
			"capName", "vvvvvv",
			map[string]interface{}{"google.capName": "vvvvvv"},
		},
		{
			"overwrite cap",
			map[string]interface{}{"google.capName": "xyz"},
			"capName", "vvvvvv",
			map[string]interface{}{"google.capName": "vvvvvv"},
		},
		{
			"overwrite google.* cap only",
			map[string]interface{}{"capName": "xyz"},
			"capName", "vvvvvv",
			map[string]interface{}{"google.capName": "vvvvvv", "capName": "xyz"},
		},
	}

	for _, tc := range testCases {
		SetGoogleCap(tc.caps, tc.k, tc.v)
		if !Equals(tc.caps, tc.want) {
			t.Errorf("[%v] got %v, want %v", tc.desc, tc.caps, tc.want)
		}
	}
}
<commit_msg>Use subtests. (#141)<commit_after>// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package capabilities

import "testing"

func TestMerge(t *testing.T) {
	testCases := []struct {
		name   string
		input1 map[string]interface{}
		input2 map[string]interface{}
		result map[string]interface{}
	}{
		{
			name:   "int,int",
			input1: map[string]interface{}{"v": 1},
			input2: map[string]interface{}{"v": 2},
			result: map[string]interface{}{"v": 2},
		},
		{
			name:   "string,bool",
			input1: map[string]interface{}{"v": "a string"},
			input2: map[string]interface{}{"v": true},
			result: map[string]interface{}{"v": true},
		},
		{
			name:   "int,string",
			input1: map[string]interface{}{"v": 1},
			input2: map[string]interface{}{"v": "a string"},
			result: map[string]interface{}{"v": "a string"},
		},
		{
			name:   "int,slice",
			input1: map[string]interface{}{"v": 1},
			input2: map[string]interface{}{"v": []interface{}{"string 1", 2, true, nil}},
			result: map[string]interface{}{"v": []interface{}{"string 1", 2, true, nil}},
		},
		{
			name:   "slice,slice",
			input1: map[string]interface{}{"v": []interface{}{"string 1", "string 2"}},
			input2: map[string]interface{}{"v": []interface{}{1, 2}},
			result: map[string]interface{}{"v": []interface{}{"string 1", "string 2", 1, 2}},
		},
		{
			name:   "int,map",
			input1: map[string]interface{}{"v": 1},
			input2: map[string]interface{}{"v": map[string]interface{}{
				"1": 1,
				"2": 2,
			}},
			result: map[string]interface{}{"v": map[string]interface{}{
				"1": 1,
				"2": 2,
			}},
		},
		{
			name: "map,map",
			input1: map[string]interface{}{"v": map[string]interface{}{
				"1": 1,
				"2": 2,
			}},
			input2: map[string]interface{}{"v": map[string]interface{}{
				"2": 3,
				"3": 4,
			}},
			result: map[string]interface{}{"v": map[string]interface{}{"1": 1, "2": 3, "3": 4}},
		},
		{
			name: "mixed",
			input1: map[string]interface{}{
				"1": []interface{}{1, 2},
				"2": map[string]interface{}{"a": "an a", "b": "a b", "c": "a c"},
				"3": 3,
			},
			input2: map[string]interface{}{
				"1": []interface{}{"a"},
				"2": map[string]interface{}{"a": "a c", "b": "a d"},
			},
			result: map[string]interface{}{
				"1": []interface{}{1, 2, "a"},
				"2": map[string]interface{}{"a": "a c", "b": "a d", "c": "a c"},
				"3": 3,
			},
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			if result := Merge(tc.input1, tc.input2); !Equals(tc.result, result) {
				t.Errorf("Got Merge(%+v, %+v) == %+v, expected %+v", tc.input1, tc.input2, result, tc.result)
			}
		})
	}
}

func TestEquals(t *testing.T) {
	testCases := []struct {
		name   string
		input1 map[string]interface{}
		input2 map[string]interface{}
		result bool
	}{
		{
			"empty",
			map[string]interface{}{},
			map[string]interface{}{},
			true,
		},
		{
			"int,nil",
			map[string]interface{}{"v": 1},
			map[string]interface{}{"v": nil},
			false,
		},
		{
			"string,nil",
			map[string]interface{}{"v": "hello"},
			map[string]interface{}{"v": nil},
			false,
		},
		{
			"bool,nil",
			map[string]interface{}{"v": true},
			map[string]interface{}{"v": nil},
			false,
		},
		{
			"slice,nil",
			map[string]interface{}{"v": []interface{}{}},
			map[string]interface{}{"v": nil},
			false,
		},
		{
			"map,nil",
			map[string]interface{}{"v": map[string]interface{}{}},
			map[string]interface{}{"v": nil},
			false,
		},
		{
			"map,map equals",
			map[string]interface{}{"v": map[string]interface{}{"a": 1}},
			map[string]interface{}{"v": map[string]interface{}{"a": 1}},
			true,
		},
		{
			"map,map different values",
			map[string]interface{}{"v": map[string]interface{}{"a": 1}},
			map[string]interface{}{"v": map[string]interface{}{"a": "hello"}},
			false,
		},
		{
			"map,map different keys",
			map[string]interface{}{"v": map[string]interface{}{"a": 1}},
			map[string]interface{}{"v": map[string]interface{}{"b": 1}},
			false,
		},
		{
			"map,map different lengths",
			map[string]interface{}{"v": map[string]interface{}{"a": 1}},
			map[string]interface{}{"v": map[string]interface{}{"a": 1, "b": 1}},
			false,
		},
		{
			"slice,slice equals",
			map[string]interface{}{"v": []interface{}{1}},
			map[string]interface{}{"v": []interface{}{1}},
			true,
		},
		{
			"slice,slice different values",
			map[string]interface{}{"v": []interface{}{1}},
			map[string]interface{}{"v": []interface{}{"hello"}},
			false,
		},
		{
			"slice,slice different lengths",
			map[string]interface{}{"v": []interface{}{1}},
			map[string]interface{}{"v": []interface{}{1, 1}},
			false,
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			if result := Equals(tc.input1, tc.input2); result != tc.result {
				t.Errorf("Got Equals(%+v, %+v) == %v, expected %v", tc.input1, tc.input2, result, tc.result)
			}
		})
	}
}

type kv struct {
	k string
	v interface{}
}

func TestGoogleCap(t *testing.T) {
	testCases := []struct {
		name  string
		caps  map[string]interface{}
		wants []kv
	}{
		{
			"not found",
			map[string]interface{}{},
			[]kv{{"capName", nil}, {"otherCapName", nil}},
		},
		{
			"found",
			map[string]interface{}{"google.capName": "vvvvvv"},
			[]kv{{"capName", "vvvvvv"}},
		},
		{
			"requires google prefix",
			map[string]interface{}{"capName": "vvvvvv"},
			[]kv{{"capName", nil}},
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			for _, want := range tc.wants {
				if got := GoogleCap(tc.caps, want.k); got != want.v {
					t.Errorf("GoogleCap(%v %q) is %v, want %v", tc.caps, want.k, got, want.v)
				}
				has := want.v != nil
				if got := HasGoogleCap(tc.caps, want.k); got != has {
					t.Errorf("HasGoogleCap(%v, %q) is %v, want %v", tc.caps, want.k, got, has)
				}
			}
		})
	}
}

func TestSetGoogleCap(t *testing.T) {
	testCases := []struct {
		name string
		caps map[string]interface{}
		k    string
		v    interface{}
		want map[string]interface{}
	}{
		{
			"set cap",
			map[string]interface{}{},
			"capName", "vvvvvv",
			map[string]interface{}{"google.capName": "vvvvvv"},
		},
		{
			"overwrite cap",
			map[string]interface{}{"google.capName": "xyz"},
			"capName", "vvvvvv",
			map[string]interface{}{"google.capName": "vvvvvv"},
		},
		{
			"overwrite google.* cap only",
			map[string]interface{}{"capName": "xyz"},
			"capName", "vvvvvv",
			map[string]interface{}{"google.capName": "vvvvvv", "capName": "xyz"},
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			SetGoogleCap(tc.caps, tc.k, tc.v)
			if !Equals(tc.caps, tc.want) {
				t.Errorf("got %v, want %v", tc.caps, tc.want)
			}
		})
	}
}
<|endoftext|>


<commit_before>package models

import "github.com/koding/bongo"

func (m MessageReply) GetId() int64 {
	return m.Id
}

func (m MessageReply) BongoName() string {
	return "api.message_reply"
}

func NewMessageReply() *MessageReply {
	return &MessageReply{}
}

func (m *MessageReply) BeforeCreate() error {
	return m.MarkIfExempt()
}

func (m *MessageReply) BeforeUpdate() error {
	return m.MarkIfExempt()
}

func (m *MessageReply) AfterCreate() {
	bongo.B.AfterCreate(m)
}

func (m *MessageReply) AfterUpdate() {
	bongo.B.AfterUpdate(m)
}

func (m *MessageReply) AfterDelete() {
	bongo.B.AfterDelete(m)
}

func (m *MessageReply) ById(id int64) error {
	return bongo.B.ById(m, id)
}

func (m *MessageReply) Create() error {
	return bongo.B.Create(m)
}

func (m *MessageReply) Update() error {
	return bongo.B.Update(m)
}

func (m *MessageReply) Some(data interface{}, q *bongo.Query) error {
	return bongo.B.Some(m, data, q)
}

func (m *MessageReply) One(q *bongo.Query) error {
	return bongo.B.One(m, m, q)
}
<commit_msg>Socialapi: implement fetchbyids methods for message reply<commit_after>package models

import "github.com/koding/bongo"

func (m MessageReply) GetId() int64 {
	return m.Id
}

func (m MessageReply) BongoName() string {
	return "api.message_reply"
}

func NewMessageReply() *MessageReply {
	return &MessageReply{}
}

func (m *MessageReply) BeforeCreate() error {
	return m.MarkIfExempt()
}

func (m *MessageReply) BeforeUpdate() error {
	return m.MarkIfExempt()
}

func (m *MessageReply) AfterCreate() {
	bongo.B.AfterCreate(m)
}

func (m *MessageReply) AfterUpdate() {
	bongo.B.AfterUpdate(m)
}

func (m *MessageReply) AfterDelete() {
	bongo.B.AfterDelete(m)
}

func (m *MessageReply) ById(id int64) error {
	return bongo.B.ById(m, id)
}

func (a *MessageReply) FetchByIds(ids []int64) ([]MessageReply, error) {
	var mrs []MessageReply

	if len(ids) == 0 {
		return mrs, nil
	}

	if err := bongo.B.FetchByIds(a, &mrs, ids); err != nil {
		return nil, err
	}

	return mrs, nil
}

func (m *MessageReply) Create() error {
	return bongo.B.Create(m)
}

func (m *MessageReply) Update() error {
	return bongo.B.Update(m)
}

func (m *MessageReply) Some(data interface{}, q *bongo.Query) error {
	return bongo.B.Some(m, data, q)
}

func (m *MessageReply) One(q *bongo.Query) error {
	return bongo.B.One(m, m, q)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"koding/messaging/rabbitmq"
	"koding/tools/config"
	"koding/tools/logger"
	realtime "socialapi/workers/realtime/lib"
)

func main() {
	flag.Parse()
	if *flagProfile == "" {
		log.Fatal("Please define config file with -c")
	}

	conf = config.MustConfig(*flagProfile)
	setLogLevel()

	// blocking
	realtime.Listen(rabbitmq.New(conf), startHandler)
	defer realtime.Consumer.Shutdown()
}

func setLogLevel() {
	var logLevel logger.Level

	if *flagDebug {
		logLevel = logger.DEBUG
	} else {
		logLevel = logger.INFO
	}
	log.SetLevel(logLevel)
}
<commit_msg>Social: add handler for relatimeworker<commit_after>package main

import (
	"flag"
	"koding/messaging/rabbitmq"
	"koding/tools/config"
	"koding/tools/logger"
	realtime "socialapi/workers/realtime/lib"

	"github.com/jinzhu/gorm"
	"github.com/streadway/amqp"
)

var (
	log         = logger.New("RealTimeWorker")
	conf        *config.Config
	flagProfile = flag.String("c", "", "Configuration profile from file")
	flagDebug   = flag.Bool("d", false, "Debug mode")
	handler     = realtime.NewRealTimeController(log)
)

func main() {
	flag.Parse()
	if *flagProfile == "" {
		log.Fatal("Please define config file with -c")
	}

	conf = config.MustConfig(*flagProfile)
	setLogLevel()

	// blocking
	realtime.Listen(rabbitmq.New(conf), startHandler)
	defer realtime.Consumer.Shutdown()
}

func setLogLevel() {
	var logLevel logger.Level

	if *flagDebug {
		logLevel = logger.DEBUG
	} else {
		logLevel = logger.INFO
	}
	log.SetLevel(logLevel)
}

func startHandler() func(delivery amqp.Delivery) {
	log.Info("Worker Started to Consume")
	return func(delivery amqp.Delivery) {
		err := handler.HandleEvent(delivery.Type, delivery.Body)
		switch err {
		case nil:
			delivery.Ack(false)
		case realtime.HandlerNotFoundErr:
			log.Notice("unknown event type (%s) recieved, \n deleting message from RMQ", delivery.Type)
			delivery.Ack(false)
		case gorm.RecordNotFound:
			log.Warning("Record not found in our db (%s) recieved, \n deleting message from RMQ", string(delivery.Body))
			delivery.Ack(false)
		default:
			// add proper error handling
			// instead of puttting message back to same queue, it is better
			// to put it to another maintenance queue/exchange
			log.Error("an error occured %s, \n putting message back to queue", err)
			// multiple false
			// reque true
			delivery.Nack(false, true)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package xio

import (
	"io"

	"github.com/m3db/m3/src/dbnode/ts"
	"github.com/m3db/m3/src/x/pool"
)

type segmentReader struct {
	segment ts.Segment
	si      int
	pool    SegmentReaderPool
}

// NewSegmentReader creates a new segment reader along with a specified segment.
func NewSegmentReader(segment ts.Segment) SegmentReader {
	return &segmentReader{segment: segment}
}

func (sr *segmentReader) Clone(
	pool pool.CheckedBytesPool,
) (SegmentReader, error) {
	return NewSegmentReader(sr.segment.Clone(pool)), nil
}

func (sr *segmentReader) Read(b []byte) (int, error) {
	if len(b) == 0 {
		return 0, nil
	}
	var head, tail []byte
	if b := sr.segment.Head; b != nil {
		head = b.Bytes()
	}
	if b := sr.segment.Tail; b != nil {
		tail = b.Bytes()
	}
	nh, nt := len(head), len(tail)
	if sr.si >= nh+nt {
		return 0, io.EOF
	}
	n := 0
	if sr.si < nh {
		nRead := copy(b, head[sr.si:])
		sr.si += nRead
		n += nRead
		if n == len(b) {
			return n, nil
		}
	}
	if sr.si < nh+nt {
		nRead := copy(b[n:], tail[sr.si-nh:])
		sr.si += nRead
		n += nRead
	}
	if n == 0 {
		return 0, io.EOF
	}
	return n, nil
}

func (sr *segmentReader) Segment() (ts.Segment, error) {
	return sr.segment, nil
}

func (sr *segmentReader) Reset(segment ts.Segment) {
	sr.segment = segment
	sr.si = 0
}

func (sr *segmentReader) Finalize() {
	// Finalize the segment
	sr.segment.Finalize()

	if pool := sr.pool; pool != nil {
		pool.Put(sr)
	}
}
<commit_msg>[dbnode] Decode avoid checked bytes in segment reader (#2190)<commit_after>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package xio

import (
	"io"

	"github.com/m3db/m3/src/dbnode/ts"
	"github.com/m3db/m3/src/x/pool"
)

type segmentReader struct {
	segment  ts.Segment
	lazyHead []byte
	lazyTail []byte
	si       int
	pool     SegmentReaderPool
}

// NewSegmentReader creates a new segment reader along with a specified segment.
func NewSegmentReader(segment ts.Segment) SegmentReader {
	return &segmentReader{segment: segment}
}

func (sr *segmentReader) Clone(
	pool pool.CheckedBytesPool,
) (SegmentReader, error) {
	return NewSegmentReader(sr.segment.Clone(pool)), nil
}

func (sr *segmentReader) Read(b []byte) (int, error) {
	if len(b) == 0 {
		return 0, nil
	}

	if b := sr.segment.Head; b != nil && len(sr.lazyHead) == 0 {
		sr.lazyHead = b.Bytes()
	}
	if b := sr.segment.Tail; b != nil && len(sr.lazyTail) == 0 {
		sr.lazyTail = b.Bytes()
	}

	nh, nt := len(sr.lazyHead), len(sr.lazyTail)
	if sr.si >= nh+nt {
		return 0, io.EOF
	}
	n := 0
	if sr.si < nh {
		nRead := copy(b, sr.lazyHead[sr.si:])
		sr.si += nRead
		n += nRead
		if n == len(b) {
			return n, nil
		}
	}
	if sr.si < nh+nt {
		nRead := copy(b[n:], sr.lazyTail[sr.si-nh:])
		sr.si += nRead
		n += nRead
	}
	if n == 0 {
		return 0, io.EOF
	}
	return n, nil
}

func (sr *segmentReader) Segment() (ts.Segment, error) {
	return sr.segment, nil
}

func (sr *segmentReader) Reset(segment ts.Segment) {
	sr.segment = segment
	sr.si = 0
	sr.lazyHead = sr.lazyHead[:0]
	sr.lazyTail = sr.lazyTail[:0]
}

func (sr *segmentReader) Finalize() {
	sr.segment.Finalize()
	sr.lazyHead = nil
	sr.lazyTail = nil

	if pool := sr.pool; pool != nil {
		pool.Put(sr)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	"log"
	"time"

	_ "github.com/lib/pq"
)

func main() {
	connStr := fmt.Sprintf(
		"postgres://%s:%s@%s:%s/%s?sslmode=%s&application_name=%s",
		"geo",
		"p",
		"localhost",
		"5432",
		"devel",
		"disable",
		"go-test-postgresql")

	db, err := sql.Open("postgres", connStr)

	if err != nil {
		log.Fatal(err)
	}

	defer db.Close()

	fmt.Println("opened...")

	rows, err := db.Query("select current_timestamp")

	if err != nil {
		log.Fatal(err)
	}

	defer rows.Close()

	for rows.Next() {
		var date time.Time
		err = rows.Scan(&date)

		if err != nil {
			log.Fatal(err)
		}

		// the format is interpreted by passing date value 2006-01-02 15:04:05.000
		// in your desired format
		fmt.Printf("Date: %v\n", date.Format("2006-01-02 15:04:05.000"))
	}
}
<commit_msg>Query + QeuryRow + sqlparams example<commit_after>package main

import (
	"database/sql"
	"fmt"
	"log"
	"time"

	_ "github.com/lib/pq"
)

func main() {
	connStr := fmt.Sprintf(
		"postgres://%s:%s@%s:%s/%s?sslmode=%s&application_name=%s",
		"geo",
		"p",
		"localhost",
		"5432",
		"devel",
		"disable",
		"go-test-postgresql")

	db, err := sql.Open("postgres", connStr)

	if err != nil {
		log.Fatal(err)
	}

	defer db.Close()

	err = db.Ping()

	if err != nil {
		log.Fatal(err)
	}

	fmt.Println("opened...")

	now, err := currentTime(db)

	if err != nil {
		log.Fatal(err)
	}

	count, err := getStatCount(db)

	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("Date: %s - Stat count: %d\n", now, count)
}

func getStatCount(db *sql.DB) (int64, error) {
	var count int64

	query := "select count(*) from pg_stat_activity where usename = $1"

	err := db.QueryRow(query, "geo").Scan(&count)

	switch {
	case err == sql.ErrNoRows:
		return 0, err
	case err != nil:
		return 0, err
	default:
		return count, nil
	}
}

func currentTime(db *sql.DB) (string, error) {
	var currentTime string

	rows, err := db.Query("select current_timestamp")

	if err != nil {
		return "", err
	}

	defer rows.Close()

	for rows.Next() {
		var date time.Time
		err = rows.Scan(&date)

		if err != nil {
			return "", err
		}

		// the format is interpreted by passing date value 2006-01-02 15:04:05.000
		// in your desired format
		currentTime = date.Format("2006-01-02 15:04:05.000")
	}

	if err := rows.Err(); err != nil {
		return "", err
	}

	return currentTime, nil
}
<|endoftext|>


<commit_before>/*
	Executable designed for plug in to the integration/validation suite.

	Reads yamelot on stdin; emits json on stdout.
*/
package main

import (
	"fmt"
)

func main() {
	fmt.Printf("{}\n") // nope
}
<commit_msg>basic yaml->json bounce for validation.<commit_after>/*
	Executable designed for plug in to the integration/validation suite.

	Reads yamelot on stdin; emits json on stdout.
*/
package main

import (
	"encoding/json"
	"io/ioutil"
	"os"

	"github.com/dropbox/yamelot/go/yaml"
)

func main() {
	input, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		panic(err)
	}
	var value interface{}
	err = yaml.Unmarshal(input, &value)
	if err != nil {
		panic(err)
	}
	json.NewEncoder(os.Stdout).Encode(value)
}
<|endoftext|>


<commit_before>package sync2

// What's in a name? Channels have all you need to emulate a counting
// semaphore with a boatload of extra functionality. However, in some
// cases, you just want a familiar API.

import ()

type Semaphore interface {
	Acquire()
	Release()
}

type semaphore struct {
	slots chan bool
}

func NewSemaphore(max int) Semaphore {
	return &semaphore{slots: make(chan bool, max)}
}

func (sem *semaphore) Acquire() {
	sem.slots <- true
}

func (sem *semaphore) Release() {
	<-sem.slots
}
<commit_msg>change semaphore implementation based on hot debate<commit_after>package sync2

// What's in a name? Channels have all you need to emulate a counting
// semaphore with a boatload of extra functionality. However, in some
// cases, you just want a familiar API.
//
// Additionally there is great debate on how to do this correctly:
//
// https://groups.google.com/forum/#!msg/golang-dev/ShqsqvCzkWg/Kg30VPN4QmUJ
// https://groups.google.com/forum/?fromgroups=#!topic/golang-nuts/Ug1DhZGGqTk
//
// http://golang.org/doc/effective_go.html
//
// It is looking like there might be a bug in the Effective Go
// example.  So it is implemented as the discussions suggest to
// protect against scheduler optimizations.

import ()

type Semaphore interface {
	Acquire()
	Release()
}

type semaphore struct {
	slots chan bool
}

func NewSemaphore(max int) Semaphore {
	sem := &semaphore{slots: make(chan bool, max)}
	for i := 0; i < max; i++ {
		sem.slots <- true
	}
	return sem
}

func (sem *semaphore) Acquire() {
	<-sem.slots
}

func (sem *semaphore) Release() {
	sem.slots <- true
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"k8s.io/contrib/cluster-autoscaler/cloudprovider"
	"k8s.io/contrib/cluster-autoscaler/cloudprovider/aws"
	"k8s.io/contrib/cluster-autoscaler/cloudprovider/gce"
	"k8s.io/contrib/cluster-autoscaler/config"
	"k8s.io/contrib/cluster-autoscaler/simulator"
	kube_util "k8s.io/contrib/cluster-autoscaler/utils/kubernetes"
	kube_api "k8s.io/kubernetes/pkg/api"
	kube_leaderelection "k8s.io/kubernetes/pkg/client/leaderelection"
	kube_record "k8s.io/kubernetes/pkg/client/record"
	kube_client "k8s.io/kubernetes/pkg/client/unversioned"
	kube_flag "k8s.io/kubernetes/pkg/util/flag"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/spf13/pflag"
)

// MultiStringFlag is a flag for passing multiple parameters using same flag
type MultiStringFlag []string

// String returns string representation of the node groups.
func (flag *MultiStringFlag) String() string {
	return "[" + strings.Join(*flag, " ") + "]"
}

// Set adds a new configuration.
func (flag *MultiStringFlag) Set(value string) error {
	*flag = append(*flag, value)
	return nil
}

var (
	nodeGroupsFlag          MultiStringFlag
	address                 = flag.String("address", ":8085", "The address to expose prometheus metrics.")
	kubernetes              = flag.String("kubernetes", "", "Kuberentes master location. Leave blank for default")
	cloudConfig             = flag.String("cloud-config", "", "The path to the cloud provider configuration file.  Empty string for no configuration file.")
	verifyUnschedulablePods = flag.Bool("verify-unschedulable-pods", true,
		"If enabled CA will ensure that each pod marked by Scheduler as unschedulable actually can't be scheduled on any node."+
			"This prevents from adding unnecessary nodes in situation when CA and Scheduler have different configuration.")
	scaleDownEnabled = flag.Bool("scale-down-enabled", true, "Should CA scale down the cluster")
	scaleDownDelay   = flag.Duration("scale-down-delay", 10*time.Minute,
		"Duration from the last scale up to the time when CA starts to check scale down options")
	scaleDownUnneededTime = flag.Duration("scale-down-unneeded-time", 10*time.Minute,
		"How long the node should be unneeded before it is eligible for scale down")
	scaleDownUtilizationThreshold = flag.Float64("scale-down-utilization-threshold", 0.5,
		"Node utilization level, defined as sum of requested resources divided by capacity, below which a node can be considered for scale down")
	scaleDownTrialInterval = flag.Duration("scale-down-trial-interval", 1*time.Minute,
		"How often scale down possiblity is check")
	scanInterval  = flag.Duration("scan-interval", 10*time.Second, "How often cluster is reevaluated for scale up or down")
	maxNodesTotal = flag.Int("max-nodes-total", 0, "Maximum number of nodes in all node groups. Cluster autoscaler will not grow the cluster beyond this number.")

	cloudProviderFlag = flag.String("cloud-provider", "gce", "Cloud provider type. Allowed values: gce")
)

func createKubeClient() *kube_client.Client {
	url, err := url.Parse(*kubernetes)
	if err != nil {
		glog.Fatalf("Failed to parse Kuberentes url: %v", err)
	}

	kubeConfig, err := config.GetKubeClientConfig(url)
	if err != nil {
		glog.Fatalf("Failed to build Kuberentes client configuration: %v", err)
	}

	return kube_client.NewOrDie(kubeConfig)
}

func createEventRecorder(kubeClient *kube_client.Client) kube_record.EventRecorder {
	eventBroadcaster := kube_record.NewBroadcaster()
	eventBroadcaster.StartLogging(glog.Infof)
	eventBroadcaster.StartRecordingToSink(kubeClient.Events(""))
	return eventBroadcaster.NewRecorder(kube_api.EventSource{Component: "cluster-autoscaler"})
}

// In order to meet interface criteria for LeaderElectionConfig we need to
// take stop channell as an argument. However, since we are committing a suicide
// after loosing mastership we can safely ignore it.
func run(_ <-chan struct{}) {
	kubeClient := createKubeClient()

	predicateChecker, err := simulator.NewPredicateChecker(kubeClient)
	if err != nil {
		glog.Fatalf("Failed to create predicate checker: %v", err)
	}
	unschedulablePodLister := kube_util.NewUnschedulablePodLister(kubeClient, kube_api.NamespaceAll)
	scheduledPodLister := kube_util.NewScheduledPodLister(kubeClient)
	nodeLister := kube_util.NewNodeLister(kubeClient)

	lastScaleUpTime := time.Now()
	lastScaleDownFailedTrial := time.Now()
	unneededNodes := make(map[string]time.Time)
	podLocationHints := make(map[string]string)
	usageTracker := simulator.NewUsageTracker()

	recorder := createEventRecorder(kubeClient)

	var cloudProvider cloudprovider.CloudProvider

	if *cloudProviderFlag == "gce" {
		// GCE Manager
		var gceManager *gce.GceManager
		var gceError error
		if *cloudConfig != "" {
			config, fileErr := os.Open(*cloudConfig)
			if fileErr != nil {
				glog.Fatalf("Couldn't open cloud provider configuration %s: %#v", *cloudConfig, err)
			}
			defer config.Close()
			gceManager, gceError = gce.CreateGceManager(config)
		} else {
			gceManager, gceError = gce.CreateGceManager(nil)
		}
		if gceError != nil {
			glog.Fatalf("Failed to create GCE Manager: %v", err)
		}
		cloudProvider, err = gce.BuildGceCloudProvider(gceManager, nodeGroupsFlag)
		if err != nil {
			glog.Fatalf("Failed to create GCE cloud provider: %v", err)
		}
	}

	if *cloudProviderFlag == "aws" {
		var awsManager *aws.AwsManager
		var awsError error
		if *cloudConfig != "" {
			config, fileErr := os.Open(*cloudConfig)
			if fileErr != nil {
				glog.Fatalf("Couldn't open cloud provider configuration %s: %#v", *cloudConfig, err)
			}
			defer config.Close()
			awsManager, awsError = aws.CreateAwsManager(config)
		} else {
			awsManager, awsError = aws.CreateAwsManager(nil)
		}
		if awsError != nil {
			glog.Fatalf("Failed to create AWS Manager: %v", err)
		}
		cloudProvider, err = aws.BuildAwsCloudProvider(awsManager, nodeGroupsFlag)
		if err != nil {
			glog.Fatalf("Failed to create AWS cloud provider: %v", err)
		}
	}

	for {
		select {
		case <-time.After(*scanInterval):
			{
				loopStart := time.Now()
				updateLastTime("main")

				nodes, err := nodeLister.List()
				if err != nil {
					glog.Errorf("Failed to list nodes: %v", err)
					continue
				}
				if len(nodes) == 0 {
					glog.Errorf("No nodes in the cluster")
					continue
				}

				if err := CheckGroupsAndNodes(nodes, cloudProvider); err != nil {
					glog.Warningf("Cluster is not ready for autoscaling: %v", err)
					continue
				}

				allUnschedulablePods, err := unschedulablePodLister.List()
				if err != nil {
					glog.Errorf("Failed to list unscheduled pods: %v", err)
					continue
				}

				allScheduled, err := scheduledPodLister.List()
				if err != nil {
					glog.Errorf("Failed to list scheduled pods: %v", err)
					continue
				}

				// We need to reset all pods that have been marked as unschedulable not after
				// the newest node became available for the scheduler.
				allNodesAvailableTime := GetAllNodesAvailableTime(nodes)
				podsToReset, unschedulablePodsToHelp := SlicePodsByPodScheduledTime(allUnschedulablePods, allNodesAvailableTime)
				ResetPodScheduledCondition(kubeClient, podsToReset)

				// We need to check whether pods marked as unschedulable are actually unschedulable.
				// This should prevent from adding unnecessary nodes. Example of such situation:
				// - CA and Scheduler has slightly different configuration
				// - Scheduler can't schedule a pod and marks it as unschedulable
				// - CA added a node which should help the pod
				// - Scheduler doesn't schedule the pod on the new node
				//   because according to it logic it doesn't fit there
				// - CA see the pod is still unschedulable, so it adds another node to help it
				//
				// With the check enabled the last point won't happen because CA will ignore a pod
				// which is supposed to schedule on an existing node.
				//
				// Without below check cluster might be unnecessary scaled up to the max allowed size
				// in the describe situation.
				schedulablePodsPresent := false
				if *verifyUnschedulablePods {
					newUnschedulablePodsToHelp := FilterOutSchedulable(unschedulablePodsToHelp, nodes, allScheduled, predicateChecker)

					if len(newUnschedulablePodsToHelp) != len(unschedulablePodsToHelp) {
						glog.V(2).Info("Schedulable pods present")
						schedulablePodsPresent = true
					}
					unschedulablePodsToHelp = newUnschedulablePodsToHelp
				}

				if len(unschedulablePodsToHelp) == 0 {
					glog.V(1).Info("No unschedulable pods")
				} else if *maxNodesTotal > 0 && len(nodes) >= *maxNodesTotal {
					glog.V(1).Info("Max total nodes in cluster reached")
				} else {
					scaleUpStart := time.Now()
					updateLastTime("scaleup")
					scaledUp, err := ScaleUp(unschedulablePodsToHelp, nodes, cloudProvider, kubeClient, predicateChecker, recorder,
						*maxNodesTotal)

					updateDuration("scaleup", scaleUpStart)

					if err != nil {
						glog.Errorf("Failed to scale up: %v", err)
						continue
					} else {
						if scaledUp {
							lastScaleUpTime = time.Now()
							// No scale down in this iteration.
							continue
						}
					}
				}

				if *scaleDownEnabled {
					unneededStart := time.Now()

					// In dry run only utilization is updated
					calculateUnneededOnly := lastScaleUpTime.Add(*scaleDownDelay).After(time.Now()) ||
						lastScaleDownFailedTrial.Add(*scaleDownTrialInterval).After(time.Now()) ||
						schedulablePodsPresent

					glog.V(4).Infof("Scale down status: unneededOnly=%v lastScaleUpTime=%s "+
						"lastScaleDownFailedTrail=%s schedulablePodsPresent=%v", calculateUnneededOnly,
						lastScaleUpTime, lastScaleDownFailedTrial, schedulablePodsPresent)

					updateLastTime("findUnneeded")
					glog.V(4).Infof("Calculating unneded nodes")

					usageTracker.CleanUp(time.Now().Add(-(*scaleDownUnneededTime)))
					unneededNodes, podLocationHints = FindUnneededNodes(
						nodes,
						unneededNodes,
						*scaleDownUtilizationThreshold,
						allScheduled,
						predicateChecker,
						podLocationHints,
						usageTracker, time.Now())

					updateDuration("findUnneeded", unneededStart)

					for key, val := range unneededNodes {
						if glog.V(4) {
							glog.V(4).Infof("%s is unneeded since %s duration %s", key, val.String(), time.Now().Sub(val).String())
						}
					}

					if !calculateUnneededOnly {
						glog.V(4).Infof("Starting scale down")

						scaleDownStart := time.Now()
						updateLastTime("scaledown")

						result, err := ScaleDown(
							nodes,
							unneededNodes,
							*scaleDownUnneededTime,
							allScheduled,
							cloudProvider,
							kubeClient,
							predicateChecker,
							podLocationHints,
							usageTracker)

						updateDuration("scaledown", scaleDownStart)

						// TODO: revisit result handling
						if err != nil {
							glog.Errorf("Failed to scale down: %v", err)
						} else {
							if result == ScaleDownError || result == ScaleDownNoNodeDeleted {
								lastScaleDownFailedTrial = time.Now()
							}
						}
					}
				}
				updateDuration("main", loopStart)
			}
		}
	}
}

func main() {
	leaderElection := kube_leaderelection.DefaultLeaderElectionConfiguration()
	leaderElection.LeaderElect = true

	kube_leaderelection.BindFlags(&leaderElection, pflag.CommandLine)
	flag.Var(&nodeGroupsFlag, "nodes", "sets min,max size and other configuration data for a node group in a format accepted by cloud provider."+
		"Can be used multiple times. Format: <min>:<max>:<other...>")
	kube_flag.InitFlags()

	glog.Infof("Cluster Autoscaler %s", ClusterAutoscalerVersion)

	go func() {
		http.Handle("/metrics", prometheus.Handler())
		err := http.ListenAndServe(*address, nil)
		glog.Fatalf("Failed to start metrics: %v", err)
	}()

	if !leaderElection.LeaderElect {
		run(nil)
	} else {
		id, err := os.Hostname()
		if err != nil {
			glog.Fatalf("Unable to get hostname: %v", err)
		}

		kubeClient := createKubeClient()
		kube_leaderelection.RunOrDie(kube_leaderelection.LeaderElectionConfig{
			EndpointsMeta: kube_api.ObjectMeta{
				Namespace: "kube-system",
				Name:      "cluster-autoscaler",
			},
			Client:        kubeClient,
			Identity:      id,
			EventRecorder: createEventRecorder(kubeClient),
			LeaseDuration: leaderElection.LeaseDuration.Duration,
			RenewDeadline: leaderElection.RenewDeadline.Duration,
			RetryPeriod:   leaderElection.RetryPeriod.Duration,
			Callbacks: kube_leaderelection.LeaderCallbacks{
				OnStartedLeading: run,
				OnStoppedLeading: func() {
					glog.Fatalf("lost master")
				},
			},
		})
	}
}

func updateDuration(label string, start time.Time) {
	duration.WithLabelValues(label).Observe(durationToMicro(start))
	lastDuration.WithLabelValues(label).Set(durationToMicro(start))
}

func updateLastTime(label string) {
	lastTimestamp.WithLabelValues(label).Set(float64(time.Now().Unix()))
}
<commit_msg>Add aws to allowed values on cluster autoscaler cloud-provider flag<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"k8s.io/contrib/cluster-autoscaler/cloudprovider"
	"k8s.io/contrib/cluster-autoscaler/cloudprovider/aws"
	"k8s.io/contrib/cluster-autoscaler/cloudprovider/gce"
	"k8s.io/contrib/cluster-autoscaler/config"
	"k8s.io/contrib/cluster-autoscaler/simulator"
	kube_util "k8s.io/contrib/cluster-autoscaler/utils/kubernetes"
	kube_api "k8s.io/kubernetes/pkg/api"
	kube_leaderelection "k8s.io/kubernetes/pkg/client/leaderelection"
	kube_record "k8s.io/kubernetes/pkg/client/record"
	kube_client "k8s.io/kubernetes/pkg/client/unversioned"
	kube_flag "k8s.io/kubernetes/pkg/util/flag"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/spf13/pflag"
)

// MultiStringFlag is a flag for passing multiple parameters using same flag
type MultiStringFlag []string

// String returns string representation of the node groups.
func (flag *MultiStringFlag) String() string {
	return "[" + strings.Join(*flag, " ") + "]"
}

// Set adds a new configuration.
func (flag *MultiStringFlag) Set(value string) error {
	*flag = append(*flag, value)
	return nil
}

var (
	nodeGroupsFlag          MultiStringFlag
	address                 = flag.String("address", ":8085", "The address to expose prometheus metrics.")
	kubernetes              = flag.String("kubernetes", "", "Kuberentes master location. Leave blank for default")
	cloudConfig             = flag.String("cloud-config", "", "The path to the cloud provider configuration file.  Empty string for no configuration file.")
	verifyUnschedulablePods = flag.Bool("verify-unschedulable-pods", true,
		"If enabled CA will ensure that each pod marked by Scheduler as unschedulable actually can't be scheduled on any node."+
			"This prevents from adding unnecessary nodes in situation when CA and Scheduler have different configuration.")
	scaleDownEnabled = flag.Bool("scale-down-enabled", true, "Should CA scale down the cluster")
	scaleDownDelay   = flag.Duration("scale-down-delay", 10*time.Minute,
		"Duration from the last scale up to the time when CA starts to check scale down options")
	scaleDownUnneededTime = flag.Duration("scale-down-unneeded-time", 10*time.Minute,
		"How long the node should be unneeded before it is eligible for scale down")
	scaleDownUtilizationThreshold = flag.Float64("scale-down-utilization-threshold", 0.5,
		"Node utilization level, defined as sum of requested resources divided by capacity, below which a node can be considered for scale down")
	scaleDownTrialInterval = flag.Duration("scale-down-trial-interval", 1*time.Minute,
		"How often scale down possiblity is check")
	scanInterval  = flag.Duration("scan-interval", 10*time.Second, "How often cluster is reevaluated for scale up or down")
	maxNodesTotal = flag.Int("max-nodes-total", 0, "Maximum number of nodes in all node groups. Cluster autoscaler will not grow the cluster beyond this number.")

	cloudProviderFlag = flag.String("cloud-provider", "gce", "Cloud provider type. Allowed values: gce, aws")
)

func createKubeClient() *kube_client.Client {
	url, err := url.Parse(*kubernetes)
	if err != nil {
		glog.Fatalf("Failed to parse Kuberentes url: %v", err)
	}

	kubeConfig, err := config.GetKubeClientConfig(url)
	if err != nil {
		glog.Fatalf("Failed to build Kuberentes client configuration: %v", err)
	}

	return kube_client.NewOrDie(kubeConfig)
}

func createEventRecorder(kubeClient *kube_client.Client) kube_record.EventRecorder {
	eventBroadcaster := kube_record.NewBroadcaster()
	eventBroadcaster.StartLogging(glog.Infof)
	eventBroadcaster.StartRecordingToSink(kubeClient.Events(""))
	return eventBroadcaster.NewRecorder(kube_api.EventSource{Component: "cluster-autoscaler"})
}

// In order to meet interface criteria for LeaderElectionConfig we need to
// take stop channell as an argument. However, since we are committing a suicide
// after loosing mastership we can safely ignore it.
func run(_ <-chan struct{}) {
	kubeClient := createKubeClient()

	predicateChecker, err := simulator.NewPredicateChecker(kubeClient)
	if err != nil {
		glog.Fatalf("Failed to create predicate checker: %v", err)
	}
	unschedulablePodLister := kube_util.NewUnschedulablePodLister(kubeClient, kube_api.NamespaceAll)
	scheduledPodLister := kube_util.NewScheduledPodLister(kubeClient)
	nodeLister := kube_util.NewNodeLister(kubeClient)

	lastScaleUpTime := time.Now()
	lastScaleDownFailedTrial := time.Now()
	unneededNodes := make(map[string]time.Time)
	podLocationHints := make(map[string]string)
	usageTracker := simulator.NewUsageTracker()

	recorder := createEventRecorder(kubeClient)

	var cloudProvider cloudprovider.CloudProvider

	if *cloudProviderFlag == "gce" {
		// GCE Manager
		var gceManager *gce.GceManager
		var gceError error
		if *cloudConfig != "" {
			config, fileErr := os.Open(*cloudConfig)
			if fileErr != nil {
				glog.Fatalf("Couldn't open cloud provider configuration %s: %#v", *cloudConfig, err)
			}
			defer config.Close()
			gceManager, gceError = gce.CreateGceManager(config)
		} else {
			gceManager, gceError = gce.CreateGceManager(nil)
		}
		if gceError != nil {
			glog.Fatalf("Failed to create GCE Manager: %v", err)
		}
		cloudProvider, err = gce.BuildGceCloudProvider(gceManager, nodeGroupsFlag)
		if err != nil {
			glog.Fatalf("Failed to create GCE cloud provider: %v", err)
		}
	}

	if *cloudProviderFlag == "aws" {
		var awsManager *aws.AwsManager
		var awsError error
		if *cloudConfig != "" {
			config, fileErr := os.Open(*cloudConfig)
			if fileErr != nil {
				glog.Fatalf("Couldn't open cloud provider configuration %s: %#v", *cloudConfig, err)
			}
			defer config.Close()
			awsManager, awsError = aws.CreateAwsManager(config)
		} else {
			awsManager, awsError = aws.CreateAwsManager(nil)
		}
		if awsError != nil {
			glog.Fatalf("Failed to create AWS Manager: %v", err)
		}
		cloudProvider, err = aws.BuildAwsCloudProvider(awsManager, nodeGroupsFlag)
		if err != nil {
			glog.Fatalf("Failed to create AWS cloud provider: %v", err)
		}
	}

	for {
		select {
		case <-time.After(*scanInterval):
			{
				loopStart := time.Now()
				updateLastTime("main")

				nodes, err := nodeLister.List()
				if err != nil {
					glog.Errorf("Failed to list nodes: %v", err)
					continue
				}
				if len(nodes) == 0 {
					glog.Errorf("No nodes in the cluster")
					continue
				}

				if err := CheckGroupsAndNodes(nodes, cloudProvider); err != nil {
					glog.Warningf("Cluster is not ready for autoscaling: %v", err)
					continue
				}

				allUnschedulablePods, err := unschedulablePodLister.List()
				if err != nil {
					glog.Errorf("Failed to list unscheduled pods: %v", err)
					continue
				}

				allScheduled, err := scheduledPodLister.List()
				if err != nil {
					glog.Errorf("Failed to list scheduled pods: %v", err)
					continue
				}

				// We need to reset all pods that have been marked as unschedulable not after
				// the newest node became available for the scheduler.
				allNodesAvailableTime := GetAllNodesAvailableTime(nodes)
				podsToReset, unschedulablePodsToHelp := SlicePodsByPodScheduledTime(allUnschedulablePods, allNodesAvailableTime)
				ResetPodScheduledCondition(kubeClient, podsToReset)

				// We need to check whether pods marked as unschedulable are actually unschedulable.
				// This should prevent from adding unnecessary nodes. Example of such situation:
				// - CA and Scheduler has slightly different configuration
				// - Scheduler can't schedule a pod and marks it as unschedulable
				// - CA added a node which should help the pod
				// - Scheduler doesn't schedule the pod on the new node
				//   because according to it logic it doesn't fit there
				// - CA see the pod is still unschedulable, so it adds another node to help it
				//
				// With the check enabled the last point won't happen because CA will ignore a pod
				// which is supposed to schedule on an existing node.
				//
				// Without below check cluster might be unnecessary scaled up to the max allowed size
				// in the describe situation.
				schedulablePodsPresent := false
				if *verifyUnschedulablePods {
					newUnschedulablePodsToHelp := FilterOutSchedulable(unschedulablePodsToHelp, nodes, allScheduled, predicateChecker)

					if len(newUnschedulablePodsToHelp) != len(unschedulablePodsToHelp) {
						glog.V(2).Info("Schedulable pods present")
						schedulablePodsPresent = true
					}
					unschedulablePodsToHelp = newUnschedulablePodsToHelp
				}

				if len(unschedulablePodsToHelp) == 0 {
					glog.V(1).Info("No unschedulable pods")
				} else if *maxNodesTotal > 0 && len(nodes) >= *maxNodesTotal {
					glog.V(1).Info("Max total nodes in cluster reached")
				} else {
					scaleUpStart := time.Now()
					updateLastTime("scaleup")
					scaledUp, err := ScaleUp(unschedulablePodsToHelp, nodes, cloudProvider, kubeClient, predicateChecker, recorder,
						*maxNodesTotal)

					updateDuration("scaleup", scaleUpStart)

					if err != nil {
						glog.Errorf("Failed to scale up: %v", err)
						continue
					} else {
						if scaledUp {
							lastScaleUpTime = time.Now()
							// No scale down in this iteration.
							continue
						}
					}
				}

				if *scaleDownEnabled {
					unneededStart := time.Now()

					// In dry run only utilization is updated
					calculateUnneededOnly := lastScaleUpTime.Add(*scaleDownDelay).After(time.Now()) ||
						lastScaleDownFailedTrial.Add(*scaleDownTrialInterval).After(time.Now()) ||
						schedulablePodsPresent

					glog.V(4).Infof("Scale down status: unneededOnly=%v lastScaleUpTime=%s "+
						"lastScaleDownFailedTrail=%s schedulablePodsPresent=%v", calculateUnneededOnly,
						lastScaleUpTime, lastScaleDownFailedTrial, schedulablePodsPresent)

					updateLastTime("findUnneeded")
					glog.V(4).Infof("Calculating unneded nodes")

					usageTracker.CleanUp(time.Now().Add(-(*scaleDownUnneededTime)))
					unneededNodes, podLocationHints = FindUnneededNodes(
						nodes,
						unneededNodes,
						*scaleDownUtilizationThreshold,
						allScheduled,
						predicateChecker,
						podLocationHints,
						usageTracker, time.Now())

					updateDuration("findUnneeded", unneededStart)

					for key, val := range unneededNodes {
						if glog.V(4) {
							glog.V(4).Infof("%s is unneeded since %s duration %s", key, val.String(), time.Now().Sub(val).String())
						}
					}

					if !calculateUnneededOnly {
						glog.V(4).Infof("Starting scale down")

						scaleDownStart := time.Now()
						updateLastTime("scaledown")

						result, err := ScaleDown(
							nodes,
							unneededNodes,
							*scaleDownUnneededTime,
							allScheduled,
							cloudProvider,
							kubeClient,
							predicateChecker,
							podLocationHints,
							usageTracker)

						updateDuration("scaledown", scaleDownStart)

						// TODO: revisit result handling
						if err != nil {
							glog.Errorf("Failed to scale down: %v", err)
						} else {
							if result == ScaleDownError || result == ScaleDownNoNodeDeleted {
								lastScaleDownFailedTrial = time.Now()
							}
						}
					}
				}
				updateDuration("main", loopStart)
			}
		}
	}
}

func main() {
	leaderElection := kube_leaderelection.DefaultLeaderElectionConfiguration()
	leaderElection.LeaderElect = true

	kube_leaderelection.BindFlags(&leaderElection, pflag.CommandLine)
	flag.Var(&nodeGroupsFlag, "nodes", "sets min,max size and other configuration data for a node group in a format accepted by cloud provider."+
		"Can be used multiple times. Format: <min>:<max>:<other...>")
	kube_flag.InitFlags()

	glog.Infof("Cluster Autoscaler %s", ClusterAutoscalerVersion)

	go func() {
		http.Handle("/metrics", prometheus.Handler())
		err := http.ListenAndServe(*address, nil)
		glog.Fatalf("Failed to start metrics: %v", err)
	}()

	if !leaderElection.LeaderElect {
		run(nil)
	} else {
		id, err := os.Hostname()
		if err != nil {
			glog.Fatalf("Unable to get hostname: %v", err)
		}

		kubeClient := createKubeClient()
		kube_leaderelection.RunOrDie(kube_leaderelection.LeaderElectionConfig{
			EndpointsMeta: kube_api.ObjectMeta{
				Namespace: "kube-system",
				Name:      "cluster-autoscaler",
			},
			Client:        kubeClient,
			Identity:      id,
			EventRecorder: createEventRecorder(kubeClient),
			LeaseDuration: leaderElection.LeaseDuration.Duration,
			RenewDeadline: leaderElection.RenewDeadline.Duration,
			RetryPeriod:   leaderElection.RetryPeriod.Duration,
			Callbacks: kube_leaderelection.LeaderCallbacks{
				OnStartedLeading: run,
				OnStoppedLeading: func() {
					glog.Fatalf("lost master")
				},
			},
		})
	}
}

func updateDuration(label string, start time.Time) {
	duration.WithLabelValues(label).Observe(durationToMicro(start))
	lastDuration.WithLabelValues(label).Set(durationToMicro(start))
}

func updateLastTime(label string) {
	lastTimestamp.WithLabelValues(label).Set(float64(time.Now().Unix()))
}
<|endoftext|>


<commit_before>package profile

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"

	"github.com/tormoder/fit/internal/types"
)

var camelRegex = regexp.MustCompile("[0-9A-Za-z]+")

func toCamelCase(s string) string {
	chunks := camelRegex.FindAllString(s, -1)
	for i, val := range chunks {
		chunks[i] = strings.Title(val)
	}
	return strings.Join(chunks, "")
}

var typeQuirks = map[string]string{
	"activity": "activity_mode",
}

func isTimestamp(name string) (types.Kind, bool) {
	if name == "date_time" {
		return types.TimeUTC, true
	}
	if name == "local_date_time" {
		return types.TimeLocal, true
	}
	return 0, false
}

func isCoordinate(name string) (types.Kind, bool) {
	if strings.HasSuffix(name, "_lat") {
		return types.Lat, true
	}
	if strings.HasSuffix(name, "_long") {
		return types.Lng, true
	}
	return 0, false
}

func TransformTypes(ptypes []*PType) (map[string]*Type, error) {
	types := make(map[string]*Type)
	for _, pt := range ptypes {
		t := Type{data: pt}
		skip, err := t.transform()
		if err != nil {
			return nil, err
		}
		if skip {
			continue
		}
		types[t.Name] = &t
	}

	return types, nil
}

func (t *Type) transform() (skip bool, err error) {
	_, isTS := isTimestamp(t.data.Header[tNAME])
	if isTS {
		return true, nil
	}

	name := t.data.Header[tNAME]
	if name == "" {
		return false, fmt.Errorf(
			"found empty type name in header %q",
			t.data.Header)
	}
	t.OrigName = name
	t.Name = toCamelCase(name)

	t.BaseType, err = types.BaseFromString(t.data.Header[tBTYPE])
	if err != nil {
		return false, err
	}

	for _, f := range t.data.Fields {
		vt := ValueTriple{
			Name:    toCamelCase(f[tVALNAME]),
			Value:   f[tVAL],
			Comment: f[tCOMMENT],
		}
		t.Values = append(t.Values, vt)
	}

	if renamed, found := typeQuirks[name]; found {
		t.Name = toCamelCase(renamed)
	}

	return false, nil
}

func TransformMsgs(pmsgs []*PMsg, ftypes map[string]*Type) ([]*Msg, error) {
	var msgs []*Msg
	for _, pmsg := range pmsgs {
		msg := Msg{
			Name:        pmsg.Header[mMSGNAME],
			FieldByName: make(map[string]*Field),
		}

		if len(msg.Name) == 0 {
			return nil, fmt.Errorf(
				"found empty message name in header %q",
				pmsg.Header)
		}
		msg.CCName = toCamelCase(msg.Name)
		debugln("transforming message", msg.CCName)

		for _, pfield := range pmsg.Fields {
			f := &Field{data: pfield.Field}
			skip, err := f.transform(false, ftypes)
			if err != nil {
				return nil, err
			}
			if skip {
				continue
			}
			msg.Fields = append(msg.Fields, f)
			msg.FieldByName[f.CCName] = f
			if len(pfield.Subfields) == 0 {
				continue
			}

			for _, sfield := range pfield.Subfields {
				sf := &Field{data: sfield}
				skip, err := sf.transform(true, ftypes)
				if err != nil {
					return nil, fmt.Errorf("error parsing subfield: %v", err)
				}
				if skip {
					continue
				}
				f.Subfields = append(f.Subfields, sf)
			}
		}
		msgs = append(msgs, &msg)
	}

	return msgs, nil
}

func (f *Field) transform(subfield bool, ftypes map[string]*Type) (skip bool, err error) {
	if f.data[mEXAMPLE] == "" {
		return true, nil
	}

	f.DefNum = f.data[mFDEFN]
	f.Name = f.data[mFNAME]
	f.CCName = toCamelCase(f.Name)

	f.parseArray()

	err = f.parseType(ftypes)
	if err != nil {
		return false, err
	}

	f.Units = f.data[mUNITS]
	f.Comment = f.data[mCOMMENT]
	f.Example = f.data[mEXAMPLE]

	if subfield {
		f.parseRefFields()
	}

	if f.data[mCOMPS] == "" {
		f.parseScaleOffset()
		return false, nil
	}

	return false, f.parseComponents(ftypes)
}

func (f *Field) parseArray() {
	arrayStr := strings.TrimFunc(
		f.data[mARRAY], func(r rune) bool {
			if r == '[' || r == ']' {
				return true
			}
			return false
		})
	switch arrayStr {
	case "":
		f.Array = "0"
	case "N":
		f.Array = "255"
	default:
		f.Array = arrayStr
	}
}

func (f *Field) parseType(ftypes map[string]*Type) error {
	array := f.Array != "0"

	coordKind, isCoord := isCoordinate(f.Name)
	if isCoord {
		f.FType = types.Make(coordKind, array)
		f.TypeName = f.FType.GoType()
		return nil
	}

	originalTypeName := f.data[mFTYPE]
	if rewritten, tfound := typeQuirks[originalTypeName]; tfound {
		f.TypeName = toCamelCase(rewritten)
	} else {
		f.TypeName = toCamelCase(originalTypeName)
	}

	tsKind, isTimestamp := isTimestamp(originalTypeName)
	if isTimestamp {
		f.FType = types.Make(tsKind, array)
		f.TypeName = f.FType.GoType()
		return nil
	}

	if f.TypeName == "Bool" {
		f.FType = types.MakeNative(types.BaseEnum, array)
		return nil
	}

	typeDef, found := ftypes[f.TypeName]
	if found {
		f.FType = types.MakeNative(typeDef.BaseType, array)
		if array {
			f.TypeName = "[]" + f.TypeName
		}
		return nil
	}

	// Assume base type.
	baseType, err := types.BaseFromString(originalTypeName)
	if err != nil {
		return err
	}
	f.FType = types.MakeNative(baseType, array)
	f.TypeName = f.FType.GoType()

	return nil
}

func (f *Field) parseRefFields() {
	f.RefFieldName = strings.Split(f.data[mRFNAME], ",")
	if len(f.RefFieldName) > 0 {
		for i, rfn := range f.RefFieldName {
			tmp := strings.TrimSpace(rfn)
			f.RefFieldName[i] = toCamelCase(tmp)
		}
	}

	f.RefFieldValue = strings.Split(f.data[mRFVAL], ",")
	if len(f.RefFieldValue) > 0 {
		for i, rfv := range f.RefFieldValue {
			tmp := strings.TrimSpace(rfv)
			f.RefFieldValue[i] = toCamelCase(tmp)
		}
	}
}

func (f *Field) parseScaleOffset() {
	if f.data[mSCALE] == "" {
		return
	}
	f.Scale = f.data[mSCALE]
	if f.data[mOFFSET] != "" {
		f.Offset = f.data[mOFFSET]
	}
}

func (f *Field) parseComponents(ftypes map[string]*Type) error {
	if f.data[mCOMPS] == "" {
		return nil
	}

	debugln("parsing components for field", f.CCName)

	switch f.FType.BaseType() {
	case types.BaseUint8, types.BaseUint16, types.BaseUint32:
	case types.BaseByte:
		if !f.FType.Array() {
			return fmt.Errorf("parseComponents: base type was byte but not an array")
		}
	default:
		return fmt.Errorf(
			"parseComponents: unhandled base type (%s) for field %s",
			f.FType.BaseType(), f.CCName)
	}

	components := strings.Split(f.data[mCOMPS], ",")
	if len(components) == 0 {
		return fmt.Errorf("parseComponents: zero components after string split")
	}

	bitsFull := f.data[mBITS]
	if new, rewrite := bitsRewrite[bitsFull]; rewrite {
		bitsFull = new
	}
	bits := strings.Split(bitsFull, ",")

	if len(components) != len(bits) {
		return fmt.Errorf("parseComponents: number of components (%d) and bits (%d) differ", len(components), len(bits))
	}

	accumulate := strings.Split(f.data[mACCUMU], ",")
	if len(accumulate) == 1 && accumulate[0] == "" {
		accumulate = nil
	}

	if len(accumulate) > 0 && (len(accumulate) != len(components)) {
		return fmt.Errorf(
			"parseComponents: number of components (%d) and accumulate flags (%d) differ",
			len(components), len(accumulate))
	}

	f.Components = make([]Component, len(components))

	var (
		err       error
		bitsTotal int
	)

	for i, comp := range components {
		f.Components[i].Name = strings.TrimSpace(comp)
		f.Components[i].Name = toCamelCase(f.Components[i].Name)
		f.Components[i].Bits = strings.TrimSpace(bits[i])
		f.Components[i].BitsInt, err = strconv.Atoi(f.Components[i].Bits)
		if err != nil {
			return fmt.Errorf("parseComponents: error converting bit to integer: %v", err)
		}
		bitsTotal += f.Components[i].BitsInt
		if len(accumulate) == len(components) {
			tmp := strings.TrimSpace(accumulate[i])
			f.Components[i].Accumulate, err = strconv.ParseBool(tmp)
			if err != nil {
				return fmt.Errorf("parseComponents: %v", err)
			}
		}
	}

	if bitsTotal > 32 || bitsTotal < 0 {
		return fmt.Errorf("parseComponents: illegal size for total number of bits: %d", bitsTotal)
	}

	if len(components) == 1 {
		// Set any scale on the "main" field.
		// TODO(tormoder): Verify that this is correct.
		f.parseScaleOffset()
		return nil
	}

	cscale := strings.Split(f.data[mSCALE], ",")
	coffset := strings.Split(f.data[mOFFSET], ",")

	if len(coffset) == 1 && coffset[0] == "" {
		coffset = nil
	}
	if len(cscale) != len(components) {
		return fmt.Errorf(
			"parseComponents: number of components (%d) and scales (%d) differ",
			len(components), len(cscale))
	}
	if len(coffset) != 0 && len(coffset) != len(components) {
		return fmt.Errorf(
			"parseComponents: #offset != 0 and number of components (%d) and offsets (%d) differ",
			len(components), len(coffset))
	}

	for i := range f.Components {
		f.Components[i].Scale = strings.TrimSpace(cscale[i])
		if len(coffset) == 0 {
			continue
		}
		f.Components[i].Offset = strings.TrimSpace(coffset[i])
	}

	return nil
}

var bitsRewrite = map[string]string{
	"1616":     "16,16",
	"88888888": "8,8,8,8,8,8,8,8",
	"53":       "5,3",
	"44":       "4,4",
}
<commit_msg>internal/profile: handle scale (new sdk)<commit_after>package profile

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"

	"github.com/tormoder/fit/internal/types"
)

var camelRegex = regexp.MustCompile("[0-9A-Za-z]+")

func toCamelCase(s string) string {
	chunks := camelRegex.FindAllString(s, -1)
	for i, val := range chunks {
		chunks[i] = strings.Title(val)
	}
	return strings.Join(chunks, "")
}

var typeQuirks = map[string]string{
	"activity": "activity_mode",
}

func isTimestamp(name string) (types.Kind, bool) {
	if name == "date_time" {
		return types.TimeUTC, true
	}
	if name == "local_date_time" {
		return types.TimeLocal, true
	}
	return 0, false
}

func isCoordinate(name string) (types.Kind, bool) {
	if strings.HasSuffix(name, "_lat") {
		return types.Lat, true
	}
	if strings.HasSuffix(name, "_long") {
		return types.Lng, true
	}
	return 0, false
}

func TransformTypes(ptypes []*PType) (map[string]*Type, error) {
	types := make(map[string]*Type)
	for _, pt := range ptypes {
		t := Type{data: pt}
		skip, err := t.transform()
		if err != nil {
			return nil, err
		}
		if skip {
			continue
		}
		types[t.Name] = &t
	}

	return types, nil
}

func (t *Type) transform() (skip bool, err error) {
	_, isTS := isTimestamp(t.data.Header[tNAME])
	if isTS {
		return true, nil
	}

	name := t.data.Header[tNAME]
	if name == "" {
		return false, fmt.Errorf(
			"found empty type name in header %q",
			t.data.Header)
	}
	t.OrigName = name
	t.Name = toCamelCase(name)

	t.BaseType, err = types.BaseFromString(t.data.Header[tBTYPE])
	if err != nil {
		return false, err
	}

	for _, f := range t.data.Fields {
		vt := ValueTriple{
			Name:    toCamelCase(f[tVALNAME]),
			Value:   f[tVAL],
			Comment: f[tCOMMENT],
		}
		t.Values = append(t.Values, vt)
	}

	if renamed, found := typeQuirks[name]; found {
		t.Name = toCamelCase(renamed)
	}

	return false, nil
}

func TransformMsgs(pmsgs []*PMsg, ftypes map[string]*Type) ([]*Msg, error) {
	var msgs []*Msg
	for _, pmsg := range pmsgs {
		msg := Msg{
			Name:        pmsg.Header[mMSGNAME],
			FieldByName: make(map[string]*Field),
		}

		if len(msg.Name) == 0 {
			return nil, fmt.Errorf(
				"found empty message name in header %q",
				pmsg.Header)
		}
		msg.CCName = toCamelCase(msg.Name)
		debugln("transforming message", msg.CCName)

		for _, pfield := range pmsg.Fields {
			f := &Field{data: pfield.Field}
			skip, err := f.transform(false, ftypes)
			if err != nil {
				return nil, err
			}
			if skip {
				continue
			}
			msg.Fields = append(msg.Fields, f)
			msg.FieldByName[f.CCName] = f
			if len(pfield.Subfields) == 0 {
				continue
			}

			for _, sfield := range pfield.Subfields {
				sf := &Field{data: sfield}
				skip, err := sf.transform(true, ftypes)
				if err != nil {
					return nil, fmt.Errorf("error parsing subfield: %v", err)
				}
				if skip {
					continue
				}
				f.Subfields = append(f.Subfields, sf)
			}
		}
		msgs = append(msgs, &msg)
	}

	return msgs, nil
}

func (f *Field) transform(subfield bool, ftypes map[string]*Type) (skip bool, err error) {
	if f.data[mEXAMPLE] == "" {
		return true, nil
	}

	f.DefNum = f.data[mFDEFN]
	f.Name = f.data[mFNAME]
	f.CCName = toCamelCase(f.Name)

	f.parseArray()

	err = f.parseType(ftypes)
	if err != nil {
		return false, err
	}

	f.Units = f.data[mUNITS]
	f.Comment = f.data[mCOMMENT]
	f.Example = f.data[mEXAMPLE]

	if subfield {
		f.parseRefFields()
	}

	if f.data[mCOMPS] == "" {
		f.parseScaleOffset()
		return false, nil
	}

	return false, f.parseComponents(ftypes)
}

func (f *Field) parseArray() {
	arrayStr := strings.TrimFunc(
		f.data[mARRAY], func(r rune) bool {
			if r == '[' || r == ']' {
				return true
			}
			return false
		})
	switch arrayStr {
	case "":
		f.Array = "0"
	case "N":
		f.Array = "255"
	default:
		f.Array = arrayStr
	}
}

func (f *Field) parseType(ftypes map[string]*Type) error {
	array := f.Array != "0"

	coordKind, isCoord := isCoordinate(f.Name)
	if isCoord {
		f.FType = types.Make(coordKind, array)
		f.TypeName = f.FType.GoType()
		return nil
	}

	originalTypeName := f.data[mFTYPE]
	if rewritten, tfound := typeQuirks[originalTypeName]; tfound {
		f.TypeName = toCamelCase(rewritten)
	} else {
		f.TypeName = toCamelCase(originalTypeName)
	}

	tsKind, isTimestamp := isTimestamp(originalTypeName)
	if isTimestamp {
		f.FType = types.Make(tsKind, array)
		f.TypeName = f.FType.GoType()
		return nil
	}

	if f.TypeName == "Bool" {
		f.FType = types.MakeNative(types.BaseEnum, array)
		return nil
	}

	typeDef, found := ftypes[f.TypeName]
	if found {
		f.FType = types.MakeNative(typeDef.BaseType, array)
		if array {
			f.TypeName = "[]" + f.TypeName
		}
		return nil
	}

	// Assume base type.
	baseType, err := types.BaseFromString(originalTypeName)
	if err != nil {
		return err
	}
	f.FType = types.MakeNative(baseType, array)
	f.TypeName = f.FType.GoType()

	return nil
}

func (f *Field) parseRefFields() {
	f.RefFieldName = strings.Split(f.data[mRFNAME], ",")
	if len(f.RefFieldName) > 0 {
		for i, rfn := range f.RefFieldName {
			tmp := strings.TrimSpace(rfn)
			f.RefFieldName[i] = toCamelCase(tmp)
		}
	}

	f.RefFieldValue = strings.Split(f.data[mRFVAL], ",")
	if len(f.RefFieldValue) > 0 {
		for i, rfv := range f.RefFieldValue {
			tmp := strings.TrimSpace(rfv)
			f.RefFieldValue[i] = toCamelCase(tmp)
		}
	}
}

func (f *Field) parseScaleOffset() {
	if f.data[mSCALE] == "" {
		return
	}
	f.Scale = f.data[mSCALE]
	if f.data[mOFFSET] != "" {
		f.Offset = f.data[mOFFSET]
	}
}

func (f *Field) parseComponents(ftypes map[string]*Type) error {
	if f.data[mCOMPS] == "" {
		return nil
	}

	debugln("parsing components for field", f.CCName)

	switch f.FType.BaseType() {
	case types.BaseUint8, types.BaseUint16, types.BaseUint32:
	case types.BaseByte:
		if !f.FType.Array() {
			return fmt.Errorf("parseComponents: base type was byte but not an array")
		}
	default:
		return fmt.Errorf(
			"parseComponents: unhandled base type (%s) for field %s",
			f.FType.BaseType(), f.CCName)
	}

	components := strings.Split(f.data[mCOMPS], ",")
	if len(components) == 0 {
		return fmt.Errorf("parseComponents: zero components after string split")
	}

	bitsFull := f.data[mBITS]
	if new, rewrite := bitsRewrite[bitsFull]; rewrite {
		bitsFull = new
	}
	bits := strings.Split(bitsFull, ",")

	if len(components) != len(bits) {
		return fmt.Errorf("parseComponents: number of components (%d) and bits (%d) differ", len(components), len(bits))
	}

	accumulate := strings.Split(f.data[mACCUMU], ",")
	if len(accumulate) == 1 && accumulate[0] == "" {
		accumulate = nil
	}

	if len(accumulate) > 0 && (len(accumulate) != len(components)) {
		return fmt.Errorf(
			"parseComponents: number of components (%d) and accumulate flags (%d) differ",
			len(components), len(accumulate))
	}

	f.Components = make([]Component, len(components))

	var (
		err       error
		bitsTotal int
	)

	for i, comp := range components {
		f.Components[i].Name = strings.TrimSpace(comp)
		f.Components[i].Name = toCamelCase(f.Components[i].Name)
		f.Components[i].Bits = strings.TrimSpace(bits[i])
		f.Components[i].BitsInt, err = strconv.Atoi(f.Components[i].Bits)
		if err != nil {
			return fmt.Errorf("parseComponents: error converting bit to integer: %v", err)
		}
		bitsTotal += f.Components[i].BitsInt
		if len(accumulate) == len(components) {
			tmp := strings.TrimSpace(accumulate[i])
			f.Components[i].Accumulate, err = strconv.ParseBool(tmp)
			if err != nil {
				return fmt.Errorf("parseComponents: %v", err)
			}
		}
	}

	if bitsTotal > 32 || bitsTotal < 0 {
		return fmt.Errorf("parseComponents: illegal size for total number of bits: %d", bitsTotal)
	}

	if len(components) == 1 {
		// Set any scale on the "main" field.
		// TODO(tormoder): Verify that this is correct.
		f.parseScaleOffset()
		return nil
	}

	cscaleFull := f.data[mSCALE]
	if new, rewrite := scaleRewrite[cscaleFull]; rewrite {
		cscaleFull = new
	}
	cscale := strings.Split(cscaleFull, ",")
	coffset := strings.Split(f.data[mOFFSET], ",")

	if len(coffset) == 1 && coffset[0] == "" {
		coffset = nil
	}
	if len(cscale) != len(components) {
		return fmt.Errorf(
			"parseComponents: number of components (%d) and scales (%d) differ",
			len(components), len(cscale))
	}
	if len(coffset) != 0 && len(coffset) != len(components) {
		return fmt.Errorf(
			"parseComponents: #offset != 0 and number of components (%d) and offsets (%d) differ",
			len(components), len(coffset))
	}

	for i := range f.Components {
		f.Components[i].Scale = strings.TrimSpace(cscale[i])
		if len(coffset) == 0 {
			continue
		}
		f.Components[i].Offset = strings.TrimSpace(coffset[i])
	}

	return nil
}

var bitsRewrite = map[string]string{
	"1616":     "16,16",
	"88888888": "8,8,8,8,8,8,8,8",
	"53":       "5,3",
	"44":       "4,4",
}

var scaleRewrite = map[string]string{
	"11":   "1,1",
	"1111": "1,1,1,1",
}
<|endoftext|>