<commit_before>package nut

import (
	"testing"
	"time"
)

var stringTests = []struct {
	in  string
	out string
	ok  bool
}{
	// @todo: tests go here
}

var intTests = []struct {
	in  string
	out int64
	ok  bool
}{
	// decimal
	{"0", 0, true},
	{"10", 10, true},
	{"123456789", 123456789, true},

	// hexadecimal
	{"0x02", 2, true},
	{"0xff", 255, true},
	{"0xc", 12, true},

	// octal
	{"010", 8, true},
	{"01234567", 342391, true},
	{"012345678", 0, false},

	// signs
	{"+0", 0, true},
	{"-0", 0, true},
	{"+10", 10, true},
	{"-0x00", 0, true},
	{"-0x10", -16, true},
	{"+01", 1, true},
	{"-010", -8, true},

	// limits
	{"9223372036854775807", 1<<63 - 1, true},
	{"9223372036854775808", 0, false},
	{"9223372036854775809", 0, false},
	{"-9223372036854775807", -(1<<63 - 1), true},
	{"-9223372036854775808", -1 << 63, true},
	{"-9223372036854775809", 0, false},

	{"0x7FFFFFFFFFFFFFFF", 1<<63 - 1, true},
	{"0X8000000000000000", 0, false},
	{"0X8000000000000001", 0, false},
	{"-0x7FFFFFFFFFFFFFFF", -(1<<63 - 1), true},
	{"-0X8000000000000000", -1 << 63, true},
	{"-0X8000000000000001", 0, false},

	{"0777777777777777777777", 1<<63 - 1, true},
	{"01000000000000000000000", 0, false},
	{"01000000000000000000001", 0, false},
	{"-0777777777777777777777", -(1<<63 - 1), true},
	{"-01000000000000000000000", -1 << 63, true},
	{"-01000000000000000000001", 0, false},

	// invalid
	{"", 0, false},
	{"abc", 0, false},
	{"100 blue", 0, false},
	{"-0-", 0, false},
	{"++0", 0, false},
}

var boolTests = []struct {
	in  string
	out bool
	ok  bool
}{
	// truthy
	{"true", true, true},
	{"yes", true, true},
	{"on", true, true},

	// falsy
	{"false", false, true},
	{"no", false, true},
	{"off", false, true},

	// invalid
	{"", false, false},
	{"y", false, false},
	{"foo", false, false},
}

var durationTests = []struct {
	in  string
	out time.Duration
	ok  bool
}{
	// simple formats
	{"0", 0, true},
	{"0s", 0, true},
	{"5s", 5 * time.Second, true},
	{"37s", 37 * time.Second, true},
	{"010s", 10 * time.Second, true},
	{"3d", 3 * 24 * time.Hour, true},

	// all units
	{"10ns", 10 * time.Nanosecond, true},
	{"10µs", 10 * time.Microsecond, true},
	{"10μs", 10 * time.Microsecond, true},
	{"10us", 10 * time.Microsecond, true},
	{"10ms", 10 * time.Millisecond, true},
	{"10s", 10 * time.Second, true},
	{"10m", 10 * time.Minute, true},
	{"10h", 10 * time.Hour, true},
	{"10d", 10 * 24 * time.Hour, true},
	{"10w", 10 * 7 * 24 * time.Hour, true},

	// mixed units
	{"1h1m1s", time.Hour + time.Minute + time.Second, true},
	{"4h30m", 4*time.Hour + 30*time.Minute, true},
	{"1s500ms", time.Second + 500*time.Millisecond, true},
	{"1w1d24h1440m", 10 * 24 * time.Hour, true},

	// allow (ignore) spaces
	{"1h 1m1s", time.Hour + time.Minute + time.Second, true},
	{"4h 30m", 4*time.Hour + 30*time.Minute, true},
	{"1s    500ms", time.Second + 500*time.Millisecond, true},
	{"1w 1d 24h 1440m", 10 * 24 * time.Hour, true},

	// disallow signs and decimal values
	{"-3h", -1, false},
	{"+5m", -1, false},
	{"300.5h", -1, false},
	{"1h 1m 1.3s", -1, false},
	{"10w -3d", -1, false},
	{"1.2d20m", -1, false},

	// various invalid formats
	{"", -1, false},
	{"1sm", -1, false},
	{"100", -1, false},
	{"1d 200", -1, false},
	{"3 4 5ms", -1, false},
}

var timeTests = []struct {
	in  string
	out *time.Time
	ok  bool
}{
	// @todo: tests go here
}

func TestParseString(t *testing.T) {
	for _, test := range stringTests {
		t.Logf("> parseString(%#v)\n", test.in)
		out, ok := parseString(test.in)

		if ok != test.ok {
			if test.ok {
				t.Fatalf("parsing failed unexpectedly\n")
			} else {
				t.Fatalf("parsing should not succeed\n")
			}
		}

		if out != test.out {
			t.Fatalf("%#v != %#v\n", out, test.out)
		}
	}
}

func TestParseInt(t *testing.T) {
	for _, test := range intTests {
		t.Logf("> parseInt(%#v)\n", test.in)
		out, ok := parseInt(test.in)

		if ok != test.ok {
			if test.ok {
				t.Fatalf("parsing failed unexpectedly\n")
			} else {
				t.Fatalf("parsing should not succeed\n")
			}
		}

		if out != test.out {
			t.Fatalf("%#v != %#v\n", out, test.out)
		}
	}
}

func TestParseBool(t *testing.T) {
	for _, test := range boolTests {
		t.Logf("> parseBool(%#v)\n", test.in)
		out, ok := parseBool(test.in)

		if ok != test.ok {
			if test.ok {
				t.Fatalf("parsing failed unexpectedly\n")
			} else {
				t.Fatalf("parsing should not succeed\n")
			}
		}

		if out != test.out {
			t.Fatalf("%#v != %#v\n", out, test)
		}
	}
}

func TestParseDuration(t *testing.T) {
	for _, test := range durationTests {
		t.Logf("> parseDuration(%#v)\n", test.in)
		out, ok := parseDuration(test.in)

		if ok != test.ok {
			if test.ok {
				t.Fatalf("parsing failed unexpectedly\n")
			} else {
				t.Fatalf("parsing should not succeed\n")
			}
		}

		if out != test.out {
			t.Fatalf("%#v != %#v\n", out.String(), test.out.String())
		}
	}
}

func TestParseTime(t *testing.T) {
	for _, test := range timeTests {
		t.Logf("> parseTime(%#v)\n", test.in)
		out, ok := parseTime(test.in)

		if ok != test.ok {
			if test.ok {
				t.Fatalf("parsing failed unexpectedly\n")
			} else {
				t.Fatalf("parsing should not succeed\n")
			}
		}

		if out == nil {
			if test.out != nil {
				t.Fatalf("returned nil\n")
			}
			return
		}

		if !out.Equal(*test.out) {
			t.Fatalf("%#v != %#v\n", out.String(), test.out.String())
		}
	}
}

// Convenience function for creating a `time.Time` struct of a certain value,
// then returning a pointer to it.
func t(value string) *time.Time {
	t, err := time.Parse("2006-01-02 15:04:05 -07:00", value)
	if err != nil {
		panic(err)
	}

	return &t
}
<commit_msg>Simplify failure reporting in the test suite<commit_after>package nut

import (
	"testing"
	"time"
)

var stringTests = []struct {
	in  string
	out string
	ok  bool
}{
	// @todo: tests go here
}

var intTests = []struct {
	in  string
	out int64
	ok  bool
}{
	// decimal
	{"0", 0, true},
	{"10", 10, true},
	{"123456789", 123456789, true},

	// hexadecimal
	{"0x02", 2, true},
	{"0xff", 255, true},
	{"0xc", 12, true},

	// octal
	{"010", 8, true},
	{"01234567", 342391, true},
	{"012345678", 0, false},

	// signs
	{"+0", 0, true},
	{"-0", 0, true},
	{"+10", 10, true},
	{"-0x00", 0, true},
	{"-0x10", -16, true},
	{"+01", 1, true},
	{"-010", -8, true},

	// limits
	{"9223372036854775807", 1<<63 - 1, true},
	{"9223372036854775808", 0, false},
	{"9223372036854775809", 0, false},
	{"-9223372036854775807", -(1<<63 - 1), true},
	{"-9223372036854775808", -1 << 63, true},
	{"-9223372036854775809", 0, false},

	{"0x7FFFFFFFFFFFFFFF", 1<<63 - 1, true},
	{"0X8000000000000000", 0, false},
	{"0X8000000000000001", 0, false},
	{"-0x7FFFFFFFFFFFFFFF", -(1<<63 - 1), true},
	{"-0X8000000000000000", -1 << 63, true},
	{"-0X8000000000000001", 0, false},

	{"0777777777777777777777", 1<<63 - 1, true},
	{"01000000000000000000000", 0, false},
	{"01000000000000000000001", 0, false},
	{"-0777777777777777777777", -(1<<63 - 1), true},
	{"-01000000000000000000000", -1 << 63, true},
	{"-01000000000000000000001", 0, false},

	// invalid
	{"", 0, false},
	{"abc", 0, false},
	{"100 blue", 0, false},
	{"-0-", 0, false},
	{"++0", 0, false},
}

var boolTests = []struct {
	in  string
	out bool
	ok  bool
}{
	// truthy
	{"true", true, true},
	{"yes", true, true},
	{"on", true, true},

	// falsy
	{"false", false, true},
	{"no", false, true},
	{"off", false, true},

	// invalid
	{"", false, false},
	{"y", false, false},
	{"foo", false, false},
}

var durationTests = []struct {
	in  string
	out time.Duration
	ok  bool
}{
	// simple formats
	{"0", 0, true},
	{"0s", 0, true},
	{"5s", 5 * time.Second, true},
	{"37s", 37 * time.Second, true},
	{"010s", 10 * time.Second, true},
	{"3d", 3 * 24 * time.Hour, true},

	// all units
	{"10ns", 10 * time.Nanosecond, true},
	{"10µs", 10 * time.Microsecond, true},
	{"10μs", 10 * time.Microsecond, true},
	{"10us", 10 * time.Microsecond, true},
	{"10ms", 10 * time.Millisecond, true},
	{"10s", 10 * time.Second, true},
	{"10m", 10 * time.Minute, true},
	{"10h", 10 * time.Hour, true},
	{"10d", 10 * 24 * time.Hour, true},
	{"10w", 10 * 7 * 24 * time.Hour, true},

	// mixed units
	{"1h1m1s", time.Hour + time.Minute + time.Second, true},
	{"4h30m", 4*time.Hour + 30*time.Minute, true},
	{"1s500ms", time.Second + 500*time.Millisecond, true},
	{"1w1d24h1440m", 10 * 24 * time.Hour, true},

	// allow (ignore) spaces
	{"1h 1m1s", time.Hour + time.Minute + time.Second, true},
	{"4h 30m", 4*time.Hour + 30*time.Minute, true},
	{"1s    500ms", time.Second + 500*time.Millisecond, true},
	{"1w 1d 24h 1440m", 10 * 24 * time.Hour, true},

	// disallow signs and decimal values
	{"-3h", -1, false},
	{"+5m", -1, false},
	{"300.5h", -1, false},
	{"1h 1m 1.3s", -1, false},
	{"10w -3d", -1, false},
	{"1.2d20m", -1, false},

	// various invalid formats
	{"", -1, false},
	{"1sm", -1, false},
	{"100", -1, false},
	{"1d 200", -1, false},
	{"3 4 5ms", -1, false},
}

var timeTests = []struct {
	in  string
	out *time.Time
	ok  bool
}{
	// @todo: tests go here
}

func TestParseString(t *testing.T) {
	for _, test := range stringTests {
		fail := setup(t, "parseString", test.in)
		out, ok := parseString(test.in)

		if ok != test.ok {
			if test.ok {
				fail("parsing failed unexpectedly")
			} else {
				fail("parsing should not succeed")
			}
		}

		if out != test.out {
			fail("%#v != %#v", out, test.out)
		}
	}
}

func TestParseInt(t *testing.T) {
	for _, test := range intTests {
		fail := setup(t, "parseInt", test.in)
		out, ok := parseInt(test.in)

		if ok != test.ok {
			if test.ok {
				fail("parsing failed unexpectedly")
			} else {
				fail("parsing should not succeed")
			}
		}

		if out != test.out {
			fail("%#v != %#v", out, test.out)
		}
	}
}

func TestParseBool(t *testing.T) {
	for _, test := range boolTests {
		fail := setup(t, "parseBool", test.in)
		out, ok := parseBool(test.in)

		if ok != test.ok {
			if test.ok {
				fail("parsing failed unexpectedly")
			} else {
				fail("parsing should not succeed")
			}
		}

		if out != test.out {
			fail("%#v != %#v", out, test)
		}
	}
}

func TestParseDuration(t *testing.T) {
	for _, test := range durationTests {
		fail := setup(t, "parseDuration", test.in)
		out, ok := parseDuration(test.in)

		if ok != test.ok {
			if test.ok {
				fail("parsing failed unexpectedly")
			} else {
				fail("parsing should not succeed")
			}
		}

		if out != test.out {
			fail("%#v != %#v", out.String(), test.out.String())
		}
	}
}

func TestParseTime(t *testing.T) {
	for _, test := range timeTests {
		fail := setup(t, "parseTime", test.in)
		out, ok := parseTime(test.in)

		if ok != test.ok {
			if test.ok {
				fail("parsing failed unexpectedly")
			} else {
				fail("parsing should not succeed")
			}
		}

		if out == nil {
			if test.out != nil {
				fail("returned nil")
			}
			return
		}

		if !out.Equal(*test.out) {
			fail("%#v != %#v", out.String(), test.out.String())
		}
	}
}

// Simplify error reporting.
type failReporter func(format string, values ...interface{})

func setup(t *testing.T, fn string, input interface{}) failReporter {
	return func(format string, values ...interface{}) {
		t.Logf(fn+"(%#v)\n", input)
		t.Fatalf(format+"\n", values...)
	}
}

// Convenience function for creating a `time.Time` struct of a certain value,
// then returning a pointer to it.
func t(value string) *time.Time {
	t, err := time.Parse("2006-01-02 15:04:05 -07:00", value)
	if err != nil {
		panic(err)
	}

	return &t
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"os/exec"
	"runtime"

	"k8s.io/test-infra/kubetest/util"
)

const (
	buildDefault = "quick"
)

type buildStrategy string

// Support both --build and --build=foo
func (b *buildStrategy) IsBoolFlag() bool {
	return true
}

// Return b as a string
func (b *buildStrategy) String() string {
	return string(*b)
}

// Set to --build=B or buildDefault if just --build
func (b *buildStrategy) Set(value string) error {
	if value == "true" { // just --build, choose default
		if runtime.GOARCH == "amd64" {
			value = buildDefault
		} else {
			value = "host-go"
		}
	}
	switch value {
	case "bazel", "e2e", "host-go", "quick", "release":
		*b = buildStrategy(value)
		return nil
	}
	return fmt.Errorf("bad build strategy: %v (use: bazel, e2e, host-go, quick, release)", value)
}

func (b *buildStrategy) Type() string {
	return "buildStrategy"
}

// True when this kubetest invocation wants to build a release
func (b *buildStrategy) Enabled() bool {
	return *b != ""
}

// Build kubernetes according to specified strategy.
// This may be a bazel, host-go, quick or full release build depending on --build=B.
func (b *buildStrategy) Build() error {
	var target string
	switch *b {
	case "bazel":
		target = "bazel-release"
	case "e2e":
		//TODO(Q-Lee): we should have a better way of build just the e2e tests
		target = "bazel-release"
	// you really should use "bazel" or "quick" in most cases, but in CI
	// we are mimicking these in our job container without an extra level
	// of sandboxing in some cases
	case "host-go":
		target = "all"
	case "quick":
		target = "quick-release"
	case "release":
		target = "release"
	default:
		return fmt.Errorf("Unknown build strategy: %v", b)
	}

	// TODO(fejta): FIX ME
	// The build-release script needs stdin to ask the user whether
	// it's OK to download the docker image.
	return control.FinishRunning(exec.Command("make", "-C", util.K8s("kubernetes"), target))
}
<commit_msg>kubetest: build windows and linux artifcats with bazel<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"os/exec"
	"runtime"

	"k8s.io/test-infra/kubetest/util"
)

const (
	buildDefault = "quick"
)

type buildStrategy string

// Support both --build and --build=foo
func (b *buildStrategy) IsBoolFlag() bool {
	return true
}

// Return b as a string
func (b *buildStrategy) String() string {
	return string(*b)
}

// Set to --build=B or buildDefault if just --build
func (b *buildStrategy) Set(value string) error {
	if value == "true" { // just --build, choose default
		if runtime.GOARCH == "amd64" {
			value = buildDefault
		} else {
			value = "host-go"
		}
	}
	switch value {
	case "bazel", "e2e", "host-go", "quick", "release", "gce-windows-bazel":
		*b = buildStrategy(value)
		return nil
	}
	return fmt.Errorf("bad build strategy: %v (use: bazel, e2e, host-go, quick, release)", value)
}

func (b *buildStrategy) Type() string {
	return "buildStrategy"
}

// True when this kubetest invocation wants to build a release
func (b *buildStrategy) Enabled() bool {
	return *b != ""
}

// Build kubernetes according to specified strategy.
// This may be a bazel, host-go, quick or full release build depending on --build=B.
func (b *buildStrategy) Build() error {
	var target string
	switch *b {
	case "bazel":
		target = "bazel-release"
	case "e2e":
		//TODO(Q-Lee): we should have a better way of build just the e2e tests
		target = "bazel-release"
	// you really should use "bazel" or "quick" in most cases, but in CI
	// we are mimicking these in our job container without an extra level
	// of sandboxing in some cases
	case "host-go":
		target = "all"
	case "quick":
		target = "quick-release"
	case "release":
		target = "release"
	case "gce-windows-bazel":
		// bazel doesn't support building multiple platforms simultaneously
		// yet. We add custom logic here to build both Windows and Linux
		// release tars. https://github.com/kubernetes/kubernetes/issues/76470
		// TODO: remove this after bazel supports the feature.
	default:
		return fmt.Errorf("Unknown build strategy: %v", b)
	}

	if *b == "gce-windows-bazel" {
		// Build Linux aritifacts
		cmd := exec.Command("bazel", "build", "--config=cross:linux_amd64", "//build/release-tars")
		cmd.Dir = util.K8s("kubernetes")
		err := control.FinishRunning(cmd)
		if err != nil {
			return err
		}
		// Build windows aritifacts
		cmd = exec.Command("bazel", "build", "--config=cross:windows_amd64", "//build/release-tars")
		cmd.Dir = util.K8s("kubernetes")
		return control.FinishRunning(cmd)
	}

	// TODO(fejta): FIX ME
	// The build-release script needs stdin to ask the user whether
	// it's OK to download the docker image.
	return control.FinishRunning(exec.Command("make", "-C", util.K8s("kubernetes"), target))
}
<|endoftext|>


<commit_before>package desugar

import "github.com/tisp-lang/tisp/src/lib/ast"

func signatureToNames(s ast.Signature) names {
	ns := newNames(append(s.PosReqs(), s.KeyReqs()...)...)

	for _, o := range s.PosOpts() {
		ns.add(o.Name())
	}

	if r := s.PosRest(); r != "" {
		ns.add(r)
	}

	for _, o := range s.KeyOpts() {
		ns.add(o.Name())
	}

	if r := s.KeyRest(); r != "" {
		ns.add(r)
	}

	return ns
}

func prependPosReqsToSig(ns []string, s ast.Signature) ast.Signature {
	return ast.NewSignature(
		append(ns, s.PosReqs()...), s.PosOpts(), s.PosRest(),
		s.KeyReqs(), s.KeyOpts(), s.KeyRest())
}
<commit_msg>Refactor signatureToNames function<commit_after>package desugar

import "github.com/tisp-lang/tisp/src/lib/ast"

func signatureToNames(s ast.Signature) names {
	ns := newNames()

	for n := range s.NameToIndex() {
		ns.add(n)
	}

	return ns
}

func prependPosReqsToSig(ns []string, s ast.Signature) ast.Signature {
	return ast.NewSignature(
		append(ns, s.PosReqs()...), s.PosOpts(), s.PosRest(),
		s.KeyReqs(), s.KeyOpts(), s.KeyRest())
}
<|endoftext|>


<commit_before>// Copyright 2012-present Oliver Eilhard. All rights reserved.
// Use of this source code is governed by a MIT-license.
// See http://olivere.mit-license.org/license.txt for details.

package elastic

import (
	"encoding/json"
	"fmt"
	"strings"
)

// BulkUpdateRequest is a request to update a document in Elasticsearch.
//
// See https://www.elastic.co/guide/en/elasticsearch/reference/5.2/docs-bulk.html
// for details.
type BulkUpdateRequest struct {
	BulkableRequest
	index string
	typ   string
	id    string

	routing         string
	parent          string
	script          *Script
	scriptedUpsert  *bool
	version         int64  // default is MATCH_ANY
	versionType     string // default is "internal"
	retryOnConflict *int
	upsert          interface{}
	docAsUpsert     *bool
	detectNoop      *bool
	doc             interface{}

	source []string
}

// NewBulkUpdateRequest returns a new BulkUpdateRequest.
func NewBulkUpdateRequest() *BulkUpdateRequest {
	return &BulkUpdateRequest{}
}

// Index specifies the Elasticsearch index to use for this update request.
// If unspecified, the index set on the BulkService will be used.
func (r *BulkUpdateRequest) Index(index string) *BulkUpdateRequest {
	r.index = index
	r.source = nil
	return r
}

// Type specifies the Elasticsearch type to use for this update request.
// If unspecified, the type set on the BulkService will be used.
func (r *BulkUpdateRequest) Type(typ string) *BulkUpdateRequest {
	r.typ = typ
	r.source = nil
	return r
}

// Id specifies the identifier of the document to update.
func (r *BulkUpdateRequest) Id(id string) *BulkUpdateRequest {
	r.id = id
	r.source = nil
	return r
}

// Routing specifies a routing value for the request.
func (r *BulkUpdateRequest) Routing(routing string) *BulkUpdateRequest {
	r.routing = routing
	r.source = nil
	return r
}

// Parent specifies the identifier of the parent document (if available).
func (r *BulkUpdateRequest) Parent(parent string) *BulkUpdateRequest {
	r.parent = parent
	r.source = nil
	return r
}

// Script specifies an update script.
// See https://www.elastic.co/guide/en/elasticsearch/reference/5.2/docs-bulk.html#bulk-update
// and https://www.elastic.co/guide/en/elasticsearch/reference/5.2/modules-scripting.html
// for details.
func (r *BulkUpdateRequest) Script(script *Script) *BulkUpdateRequest {
	r.script = script
	r.source = nil
	return r
}

// ScripedUpsert specifies if your script will run regardless of
// whether the document exists or not.
//
// See https://www.elastic.co/guide/en/elasticsearch/reference/5.2/docs-update.html#_literal_scripted_upsert_literal
func (r *BulkUpdateRequest) ScriptedUpsert(upsert bool) *BulkUpdateRequest {
	r.scriptedUpsert = &upsert
	r.source = nil
	return r
}

// RetryOnConflict specifies how often to retry in case of a version conflict.
func (r *BulkUpdateRequest) RetryOnConflict(retryOnConflict int) *BulkUpdateRequest {
	r.retryOnConflict = &retryOnConflict
	r.source = nil
	return r
}

// Version indicates the version of the document as part of an optimistic
// concurrency model.
func (r *BulkUpdateRequest) Version(version int64) *BulkUpdateRequest {
	r.version = version
	r.source = nil
	return r
}

// VersionType can be "internal" (default), "external", "external_gte",
// "external_gt", or "force".
func (r *BulkUpdateRequest) VersionType(versionType string) *BulkUpdateRequest {
	r.versionType = versionType
	r.source = nil
	return r
}

// Doc specifies the updated document.
func (r *BulkUpdateRequest) Doc(doc interface{}) *BulkUpdateRequest {
	r.doc = doc
	r.source = nil
	return r
}

// DocAsUpsert indicates whether the contents of Doc should be used as
// the Upsert value.
//
// See https://www.elastic.co/guide/en/elasticsearch/reference/5.2/docs-update.html#_literal_doc_as_upsert_literal
// for details.
func (r *BulkUpdateRequest) DocAsUpsert(docAsUpsert bool) *BulkUpdateRequest {
	r.docAsUpsert = &docAsUpsert
	r.source = nil
	return r
}

// DetectNoop specifies whether changes that don't affect the document
// should be ignored (true) or unignored (false). This is enabled by default
// in Elasticsearch.
func (r *BulkUpdateRequest) DetectNoop(detectNoop bool) *BulkUpdateRequest {
	r.detectNoop = &detectNoop
	r.source = nil
	return r
}

// Upsert specifies the document to use for upserts. It will be used for
// create if the original document does not exist.
func (r *BulkUpdateRequest) Upsert(doc interface{}) *BulkUpdateRequest {
	r.upsert = doc
	r.source = nil
	return r
}

// String returns the on-wire representation of the update request,
// concatenated as a single string.
func (r *BulkUpdateRequest) String() string {
	lines, err := r.Source()
	if err != nil {
		return fmt.Sprintf("error: %v", err)
	}
	return strings.Join(lines, "\n")
}

func (r *BulkUpdateRequest) getSourceAsString(data interface{}) (string, error) {
	switch t := data.(type) {
	default:
		body, err := json.Marshal(data)
		if err != nil {
			return "", err
		}
		return string(body), nil
	case json.RawMessage:
		return string(t), nil
	case *json.RawMessage:
		return string(*t), nil
	case string:
		return t, nil
	case *string:
		return *t, nil
	}
}

// Source returns the on-wire representation of the update request,
// split into an action-and-meta-data line and an (optional) source line.
// See https://www.elastic.co/guide/en/elasticsearch/reference/5.2/docs-bulk.html
// for details.
func (r BulkUpdateRequest) Source() ([]string, error) {
	// { "update" : { "_index" : "test", "_type" : "type1", "_id" : "1", ... } }
	// { "doc" : { "field1" : "value1", ... } }
	// or
	// { "update" : { "_index" : "test", "_type" : "type1", "_id" : "1", ... } }
	// { "script" : { ... } }

	if r.source != nil {
		return r.source, nil
	}

	lines := make([]string, 2)

	// "update" ...
	command := make(map[string]interface{})
	updateCommand := make(map[string]interface{})
	if r.index != "" {
		updateCommand["_index"] = r.index
	}
	if r.typ != "" {
		updateCommand["_type"] = r.typ
	}
	if r.id != "" {
		updateCommand["_id"] = r.id
	}
	if r.routing != "" {
		updateCommand["_routing"] = r.routing
	}
	if r.parent != "" {
		updateCommand["_parent"] = r.parent
	}
	if r.version > 0 {
		updateCommand["_version"] = r.version
	}
	if r.versionType != "" {
		updateCommand["_version_type"] = r.versionType
	}
	if r.retryOnConflict != nil {
		updateCommand["_retry_on_conflict"] = *r.retryOnConflict
	}
	command["update"] = updateCommand
	line, err := json.Marshal(command)
	if err != nil {
		return nil, err
	}
	lines[0] = string(line)

	// 2nd line: {"doc" : { ... }} or {"script": {...}}
	source := make(map[string]interface{})
	if r.docAsUpsert != nil {
		source["doc_as_upsert"] = *r.docAsUpsert
	}
	if r.detectNoop != nil {
		source["detect_noop"] = *r.detectNoop
	}
	if r.upsert != nil {
		source["upsert"] = r.upsert
	}
	if r.scriptedUpsert != nil {
		source["scripted_upsert"] = *r.scriptedUpsert
	}
	if r.doc != nil {
		// {"doc":{...}}
		source["doc"] = r.doc
	} else if r.script != nil {
		// {"script":...}
		src, err := r.script.Source()
		if err != nil {
			return nil, err
		}
		source["script"] = src
	}
	lines[1], err = r.getSourceAsString(source)
	if err != nil {
		return nil, err
	}

	r.source = lines
	return lines, nil
}
<commit_msg>fix bulk update request bug<commit_after>// Copyright 2012-present Oliver Eilhard. All rights reserved.
// Use of this source code is governed by a MIT-license.
// See http://olivere.mit-license.org/license.txt for details.

package elastic

import (
	"encoding/json"
	"fmt"
	"strings"
)

// BulkUpdateRequest is a request to update a document in Elasticsearch.
//
// See https://www.elastic.co/guide/en/elasticsearch/reference/5.2/docs-bulk.html
// for details.
type BulkUpdateRequest struct {
	BulkableRequest
	index string
	typ   string
	id    string

	routing         string
	parent          string
	script          *Script
	scriptedUpsert  *bool
	version         int64  // default is MATCH_ANY
	versionType     string // default is "internal"
	retryOnConflict *int
	upsert          interface{}
	docAsUpsert     *bool
	detectNoop      *bool
	doc             interface{}

	source []string
}

// NewBulkUpdateRequest returns a new BulkUpdateRequest.
func NewBulkUpdateRequest() *BulkUpdateRequest {
	return &BulkUpdateRequest{}
}

// Index specifies the Elasticsearch index to use for this update request.
// If unspecified, the index set on the BulkService will be used.
func (r *BulkUpdateRequest) Index(index string) *BulkUpdateRequest {
	r.index = index
	r.source = nil
	return r
}

// Type specifies the Elasticsearch type to use for this update request.
// If unspecified, the type set on the BulkService will be used.
func (r *BulkUpdateRequest) Type(typ string) *BulkUpdateRequest {
	r.typ = typ
	r.source = nil
	return r
}

// Id specifies the identifier of the document to update.
func (r *BulkUpdateRequest) Id(id string) *BulkUpdateRequest {
	r.id = id
	r.source = nil
	return r
}

// Routing specifies a routing value for the request.
func (r *BulkUpdateRequest) Routing(routing string) *BulkUpdateRequest {
	r.routing = routing
	r.source = nil
	return r
}

// Parent specifies the identifier of the parent document (if available).
func (r *BulkUpdateRequest) Parent(parent string) *BulkUpdateRequest {
	r.parent = parent
	r.source = nil
	return r
}

// Script specifies an update script.
// See https://www.elastic.co/guide/en/elasticsearch/reference/5.2/docs-bulk.html#bulk-update
// and https://www.elastic.co/guide/en/elasticsearch/reference/5.2/modules-scripting.html
// for details.
func (r *BulkUpdateRequest) Script(script *Script) *BulkUpdateRequest {
	r.script = script
	r.source = nil
	return r
}

// ScripedUpsert specifies if your script will run regardless of
// whether the document exists or not.
//
// See https://www.elastic.co/guide/en/elasticsearch/reference/5.2/docs-update.html#_literal_scripted_upsert_literal
func (r *BulkUpdateRequest) ScriptedUpsert(upsert bool) *BulkUpdateRequest {
	r.scriptedUpsert = &upsert
	r.source = nil
	return r
}

// RetryOnConflict specifies how often to retry in case of a version conflict.
func (r *BulkUpdateRequest) RetryOnConflict(retryOnConflict int) *BulkUpdateRequest {
	r.retryOnConflict = &retryOnConflict
	r.source = nil
	return r
}

// Version indicates the version of the document as part of an optimistic
// concurrency model.
func (r *BulkUpdateRequest) Version(version int64) *BulkUpdateRequest {
	r.version = version
	r.source = nil
	return r
}

// VersionType can be "internal" (default), "external", "external_gte",
// "external_gt", or "force".
func (r *BulkUpdateRequest) VersionType(versionType string) *BulkUpdateRequest {
	r.versionType = versionType
	r.source = nil
	return r
}

// Doc specifies the updated document.
func (r *BulkUpdateRequest) Doc(doc interface{}) *BulkUpdateRequest {
	r.doc = doc
	r.source = nil
	return r
}

// DocAsUpsert indicates whether the contents of Doc should be used as
// the Upsert value.
//
// See https://www.elastic.co/guide/en/elasticsearch/reference/5.2/docs-update.html#_literal_doc_as_upsert_literal
// for details.
func (r *BulkUpdateRequest) DocAsUpsert(docAsUpsert bool) *BulkUpdateRequest {
	r.docAsUpsert = &docAsUpsert
	r.source = nil
	return r
}

// DetectNoop specifies whether changes that don't affect the document
// should be ignored (true) or unignored (false). This is enabled by default
// in Elasticsearch.
func (r *BulkUpdateRequest) DetectNoop(detectNoop bool) *BulkUpdateRequest {
	r.detectNoop = &detectNoop
	r.source = nil
	return r
}

// Upsert specifies the document to use for upserts. It will be used for
// create if the original document does not exist.
func (r *BulkUpdateRequest) Upsert(doc interface{}) *BulkUpdateRequest {
	r.upsert = doc
	r.source = nil
	return r
}

// String returns the on-wire representation of the update request,
// concatenated as a single string.
func (r *BulkUpdateRequest) String() string {
	lines, err := r.Source()
	if err != nil {
		return fmt.Sprintf("error: %v", err)
	}
	return strings.Join(lines, "\n")
}

func (r *BulkUpdateRequest) getSourceAsString(data interface{}) (string, error) {
	switch t := data.(type) {
	default:
		body, err := json.Marshal(data)
		if err != nil {
			return "", err
		}
		return string(body), nil
	case json.RawMessage:
		return string(t), nil
	case *json.RawMessage:
		return string(*t), nil
	case string:
		return t, nil
	case *string:
		return *t, nil
	}
}

// Source returns the on-wire representation of the update request,
// split into an action-and-meta-data line and an (optional) source line.
// See https://www.elastic.co/guide/en/elasticsearch/reference/5.2/docs-bulk.html
// for details.
func (r *BulkUpdateRequest) Source() ([]string, error) {
	// { "update" : { "_index" : "test", "_type" : "type1", "_id" : "1", ... } }
	// { "doc" : { "field1" : "value1", ... } }
	// or
	// { "update" : { "_index" : "test", "_type" : "type1", "_id" : "1", ... } }
	// { "script" : { ... } }

	if r.source != nil {
		return r.source, nil
	}

	lines := make([]string, 2)

	// "update" ...
	command := make(map[string]interface{})
	updateCommand := make(map[string]interface{})
	if r.index != "" {
		updateCommand["_index"] = r.index
	}
	if r.typ != "" {
		updateCommand["_type"] = r.typ
	}
	if r.id != "" {
		updateCommand["_id"] = r.id
	}
	if r.routing != "" {
		updateCommand["_routing"] = r.routing
	}
	if r.parent != "" {
		updateCommand["_parent"] = r.parent
	}
	if r.version > 0 {
		updateCommand["_version"] = r.version
	}
	if r.versionType != "" {
		updateCommand["_version_type"] = r.versionType
	}
	if r.retryOnConflict != nil {
		updateCommand["_retry_on_conflict"] = *r.retryOnConflict
	}
	command["update"] = updateCommand
	line, err := json.Marshal(command)
	if err != nil {
		return nil, err
	}
	lines[0] = string(line)

	// 2nd line: {"doc" : { ... }} or {"script": {...}}
	source := make(map[string]interface{})
	if r.docAsUpsert != nil {
		source["doc_as_upsert"] = *r.docAsUpsert
	}
	if r.detectNoop != nil {
		source["detect_noop"] = *r.detectNoop
	}
	if r.upsert != nil {
		source["upsert"] = r.upsert
	}
	if r.scriptedUpsert != nil {
		source["scripted_upsert"] = *r.scriptedUpsert
	}
	if r.doc != nil {
		// {"doc":{...}}
		source["doc"] = r.doc
	} else if r.script != nil {
		// {"script":...}
		src, err := r.script.Source()
		if err != nil {
			return nil, err
		}
		source["script"] = src
	}
	lines[1], err = r.getSourceAsString(source)
	if err != nil {
		return nil, err
	}

	r.source = lines
	return lines, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"net"
	"os"
	"runtime"
	"time"

	sf "bitbucket.org/krepa098/gosfml2"

	"github.com/gabriel-comeau/multiplayer-game-test/protocol"
	"github.com/gabriel-comeau/multiplayer-game-test/shared"
)

var (
	// This block of variables is shared global state throughout the client.  Obviously not great
	// but since our client program is pretty simple, this is quick and effective.

	// Our current movement velocity
	velocity sf.Vector2f

	// Current state of input - which buttons are being pressed
	inputState *shared.InputState

	// Socket connection to the server
	conn net.Conn

	// Unique identifying ID sent over by the server on connection
	myPlayerId int64

	// Keep track of the entities we need to draw.  The key is their UUID.  Our player entity
	// is just another in this list.
	entities map[int64]*Unit

	// Hold messages from the server in a queue
	messageQueue *protocol.MessageQueue

	// Channel to send outgoing messages on
	outgoing chan protocol.Message

	// Keep a list of inputs that have been processed locally through client-side prediction
	// but haven't yet been acknowledged on the server
	unacked []*protocol.SendInputMessage

	// Current sequence number for input messages we'll be sending
	currentSeq int64
)

func init() {
	runtime.LockOSThread()
	inputState = new(shared.InputState)
	entities = make(map[int64]*Unit)
	messageQueue = protocol.CreateMessageQueue()
	outgoing = make(chan protocol.Message)
	currentSeq = 0
	unacked = make([]*protocol.SendInputMessage, 0)
}

func main() {

	// Open the game window.
	renderWindow := sf.NewRenderWindow(sf.VideoMode{1024, 768, 32}, "Wow! Much client-side-interpretation", sf.StyleDefault, sf.DefaultContextSettings())

	// Because we send a message to the server every frame where input is present, we'll limit the frames so that
	// machines capable of rendering hundreds of frames per second don't try to send hundreds of network message
	// per second.
	//
	// This could be handled better of course, having artificial limiting on only the networking
	// portion, but for simplicity's sake, this will do the job.
	renderWindow.SetFramerateLimit(60)

	// establish connection to server
	conn = connectToServer()

	// Preset up the timestep stuff so there's a value for the first rendered frame
	lastTick := time.Now()
	var dt time.Duration = 0

	// This is the start of our main game loop.  As long as the window remains open, this will continue.
	for renderWindow.IsOpen() {

		// process user input, changing the value of the inputstate struct
		inputState = handleUserInput(renderWindow, inputState)
		if inputState.HasInput() {
			velocity = shared.GetVectorFromInputAndDt(inputState, dt)

			// client side prediction
			player, ok := entities[myPlayerId]
			if ok {
				player.Move(velocity)
			}

			// We need to send this input to the server, so build a message object
			// stick it in the unacked map and then transmit
			inputMsg := protocol.CreateSendInputMessage(inputState, currentSeq, dt, myPlayerId)
			unacked = append(unacked, inputMsg)
			currentSeq++
			outgoing <- inputMsg
		}

		// process incoming messages
		incoming := messageQueue.PopAll()
		for _, message := range incoming {
			switch message.GetMessageType() {

			case protocol.WORLD_STATE_MESSAGE:
				typed, ok := message.(*protocol.WorldStateMessage)
				if !ok {
					fmt.Println("Got a message with WORLD_STATE_MESSAGE id but couldn't be cast")
					continue
				}

				// Now we're going to iterate through the entities contained in the message.  On the
				// first pass we're going to do a couple of different things.  If the entity doesn't
				// exist, we're going to add it.  If the entity is ours, we'll apply interpolation based
				// on our past inputs.  If the entity is someone else's and exists we'll move it.  If the
				// entity doesn't exist, we'll create it.

				for _, msgEnt := range typed.Entities {
					// first check if this thing exists at all
					existingEnt, ok := entities[msgEnt.Id]
					if !ok {
						// not in the map, let's create it - we'll bail after this because even if
						// this is our own entity we'll start to worry about interpolation on the next
						// pass only
						addEntityToGameWorld(msgEnt.Id, msgEnt.Position)
						continue
					}

					// Is this ours or someone else's?
					if msgEnt.Id == myPlayerId {

						// First, set the position to wherever the server thinks it was
						existingEnt.SetPosition(msgEnt.Position)

						// Next, let's go through our pending inputs list and get rid of everything older
						// than this seq number
						newUnacked := make([]*protocol.SendInputMessage, 0)
						for _, oldMsg := range unacked {
							if oldMsg.Seq > msgEnt.LastSeq {
								// not processed yet, so reapply and keep it in the list
								newUnacked = append(newUnacked, oldMsg)
								existingEnt.Move(shared.GetVectorFromInputAndDt(oldMsg.Input, oldMsg.Dt))
							}
						}
						unacked = newUnacked

					} else {
						// This is someone else's entity, so just move it
						existingEnt.SetPosition(msgEnt.Position)
					}
				}

				// Now, we've also got to check to see if we still have any remaining entities
				// that belong to players who've left.  This means we have to another ugly iteration
				// but that's life.
				removeDisconnectedPlayers(typed.Entities)
			}
		}

		now := time.Now()
		dt = now.Sub(lastTick)
		lastTick = now

		renderWindow.Clear(sf.Color{0, 0, 0, 0})

		for _, unit := range entities {
			unit.Draw(renderWindow, sf.DefaultRenderStates())
		}

		renderWindow.Display()
	}
}

// Look over the events coming in, check them against the current keystates, and then update
// the keystates to match.  This is one of those bad functions which mutates the package-wide
// keystate struct but really this is the only function which writes to it so why bother copying
// things?
func handleUserInput(renderWindow *sf.RenderWindow, inputState *shared.InputState) *shared.InputState {

	// Handle user input
	for event := renderWindow.PollEvent(); event != nil; event = renderWindow.PollEvent() {
		switch ev := event.(type) {
		case sf.EventKeyReleased:
			switch ev.Code {

			case sf.KeyEscape:
				renderWindow.Close()

			case sf.KeyLeft:
				if inputState.KeyLeftDown {
					inputState.KeyLeftDown = false
				}

			case sf.KeyRight:
				if inputState.KeyRightDown {
					inputState.KeyRightDown = false
				}

			case sf.KeyDown:
				if inputState.KeyDownDown {
					inputState.KeyDownDown = false
				}

			case sf.KeyUp:
				if inputState.KeyUpDown {
					inputState.KeyUpDown = false
				}

			}

		case sf.EventKeyPressed:
			switch ev.Code {

			case sf.KeyLeft:
				if !inputState.KeyLeftDown {
					inputState.KeyLeftDown = true
				}

			case sf.KeyRight:
				if !inputState.KeyRightDown {
					inputState.KeyRightDown = true
				}

			case sf.KeyDown:
				if !inputState.KeyDownDown {
					inputState.KeyDownDown = true
				}

			case sf.KeyUp:
				if !inputState.KeyUpDown {
					inputState.KeyUpDown = true
				}
			}
		}
	}

	return inputState
}

// Add a new entity to the game world.  It will use the Player constructor (for the player texture)
// if the entity ID matches the player ID and the Other constructor otherwise.
func addEntityToGameWorld(id int64, pos sf.Vector2f) {
	if id == myPlayerId {
		_, ok := entities[myPlayerId]
		if ok {
			fmt.Println("ERROR - tried to add a new player with the same ID")
			return
		}

		player := NewPlayer(pos)
		entities[myPlayerId] = player
	} else {
		_, ok := entities[id]
		if ok {
			fmt.Println("ERROR - tried to add a new other entity with an ID that was already in the system")
			return
		}

		other := NewOther(pos)
		entities[id] = other
	}
}

// Go through the entities which the server is tracking and compare to the entities the client is
// tracking.  Remove any client-side entities that aren't in the server's list.
func removeDisconnectedPlayers(serverEntities []protocol.MessageEntity) {
	// We'll make a map of the IDs so we can do more convenient lookups
	// I used bool because it's small.
	serverEnts := make(map[int64]bool)
	for _, ent := range serverEntities {
		serverEnts[ent.Id] = true
	}

	// Now iterate through the map of stored client-side entities.  Remove anything
	// that doesn't appear in the server entities map
	for id, _ := range entities {
		_, ok := serverEnts[id]
		if !ok {
			if id == myPlayerId {
				panic("Whoops trying to remove myself - this is an error condition so we out")
			}

			delete(entities, id)
		}
	}
}

// Establish a connection to the game server and start the two goroutines which require it
func connectToServer() net.Conn {
	conn, err := net.Dial("tcp", shared.HOST+":"+shared.PORT)
	if err != nil {
		panic(err.Error())
	}

	// Now we're going to wait for the server to give us an entity ID
	b := bufio.NewReader(conn)
	for {
		line, err := b.ReadBytes('\n')

		if err != nil {
			conn.Close()
			fmt.Println("Error while trying to accept player id")
			os.Exit(1)
			break
		}

		if string(line) == "" || string(line) == "\n" {
			continue
		}

		message, err := protocol.DecodeMessage(line)
		if err != nil {
			fmt.Println("ERROR during decode: " + err.Error())
			continue
		}

		if message.GetMessageType() == protocol.PLAYER_UUID_MESSAGE {
			typed, ok := message.(*protocol.PlayerUUIDMessage)
			if !ok {
				fmt.Println("Message couldn't be asserted into PlayerUUIDMessage though that was message id")
				conn.Close()
				os.Exit(1)
			}
			myPlayerId = typed.UUID
			break
		} else {
			fmt.Println("Got the wrong type of message - expected PLAYER_UUID_MESSAGE")
			conn.Close()
			os.Exit(1)
		}
	}

	go listenForMessages(conn)
	go writeMessages(conn, outgoing)

	return conn
}

// Listens for incoming messages from the server, decodes the serialized versions into
// message objects and then pushes them into the message queue.
//
// This is a concurrent function - it runs simultaneously to the main game loop as a goroutine
func listenForMessages(conn net.Conn) {
	b := bufio.NewReader(conn)
	for {
		line, err := b.ReadBytes('\n')

		if err != nil {
			conn.Close()
			fmt.Println("ERROR, CLOSING CONN: " + err.Error())
			break
		}

		if string(line) == "" || string(line) == "\n" {
			continue
		}

		// Deal with incoming messages from the server
		message, err := protocol.DecodeMessage(line)
		if err != nil {
			fmt.Println("Error decoding message: " + err.Error())
			continue
		}
		messageQueue.PushMessage(message)
	}
}

// This function writes outgoing messages to the connection.
//
// This is a concurrent function - it runs simultaneously to the main game loop as a goroutine
func writeMessages(conn net.Conn, msgChan chan protocol.Message) {
	for {
		msg := <-msgChan
		conn.Write(msg.Encode())
	}
}
<commit_msg>Fix some comments.<commit_after>package main

import (
	"bufio"
	"fmt"
	"net"
	"os"
	"runtime"
	"time"

	sf "bitbucket.org/krepa098/gosfml2"

	"github.com/gabriel-comeau/multiplayer-game-test/protocol"
	"github.com/gabriel-comeau/multiplayer-game-test/shared"
)

var (
	// This block of variables is shared global state throughout the client.  Obviously not great
	// but since our client program is pretty simple, this is quick and effective.

	// Our current movement velocity
	velocity sf.Vector2f

	// Current state of input - which buttons are being pressed
	inputState *shared.InputState

	// Socket connection to the server
	conn net.Conn

	// Unique identifying ID sent over by the server on connection
	myPlayerId int64

	// Keep track of the entities we need to draw.  The key is their UUID.  Our player entity
	// is just another in this list.
	entities map[int64]*Unit

	// Hold messages from the server in a queue
	messageQueue *protocol.MessageQueue

	// Channel to send outgoing messages on
	outgoing chan protocol.Message

	// Keep a list of inputs that have been processed locally through client-side prediction
	// but haven't yet been acknowledged on the server
	unacked []*protocol.SendInputMessage

	// Current sequence number for input messages we'll be sending
	currentSeq int64
)

func init() {
	runtime.LockOSThread()
	inputState = new(shared.InputState)
	entities = make(map[int64]*Unit)
	messageQueue = protocol.CreateMessageQueue()
	outgoing = make(chan protocol.Message)
	currentSeq = 0
	unacked = make([]*protocol.SendInputMessage, 0)
}

func main() {

	// Open the game window.
	renderWindow := sf.NewRenderWindow(sf.VideoMode{1024, 768, 32}, "Wow! Much client-side-interpretation", sf.StyleDefault, sf.DefaultContextSettings())

	// Because we send a message to the server every frame where input is present, we'll limit the frames so that
	// machines capable of rendering hundreds of frames per second don't try to send hundreds of network message
	// per second.
	//
	// This could be handled better of course, having artificial limiting on only the networking
	// portion, but for simplicity's sake, this will do the job.
	renderWindow.SetFramerateLimit(60)

	// establish connection to server
	conn = connectToServer()

	// Preset up the timestep stuff so there's a value for the first rendered frame
	lastTick := time.Now()
	var dt time.Duration = 0

	// This is the start of our main game loop.  As long as the window remains open, this will continue.
	for renderWindow.IsOpen() {

		// process user input, changing the value of the inputstate struct
		inputState = handleUserInput(renderWindow, inputState)
		if inputState.HasInput() {
			velocity = shared.GetVectorFromInputAndDt(inputState, dt)

			// client side prediction
			player, ok := entities[myPlayerId]
			if ok {
				player.Move(velocity)
			}

			// We need to send this input to the server, so build a message object
			// stick it in the unacked map and then transmit
			inputMsg := protocol.CreateSendInputMessage(inputState, currentSeq, dt, myPlayerId)
			unacked = append(unacked, inputMsg)
			currentSeq++
			outgoing <- inputMsg
		}

		// process incoming messages
		incoming := messageQueue.PopAll()
		for _, message := range incoming {
			switch message.GetMessageType() {

			case protocol.WORLD_STATE_MESSAGE:
				typed, ok := message.(*protocol.WorldStateMessage)
				if !ok {
					fmt.Println("Got a message with WORLD_STATE_MESSAGE id but couldn't be cast")
					continue
				}

				// Now we're going to iterate through the entities contained in the message.  On the
				// first pass we're going to do a couple of different things.  If the entity doesn't
				// exist, we're going to add it.  If the entity is ours, we'll apply interpolation based
				// on our past inputs.  If the entity is someone else's and exists we'll move it.

				for _, msgEnt := range typed.Entities {
					// first check if this thing exists at all
					existingEnt, ok := entities[msgEnt.Id]
					if !ok {
						// not in the map, let's create it - we'll bail after this because even if
						// this is our own entity we'll start to worry about interpolation on the next
						// pass only
						addEntityToGameWorld(msgEnt.Id, msgEnt.Position)
						continue
					}

					// Is this ours or someone else's?
					if msgEnt.Id == myPlayerId {

						// First, set the position to wherever the server thinks it was
						existingEnt.SetPosition(msgEnt.Position)

						// Next, let's go through our pending inputs list and get rid of everything older
						// than this seq number
						newUnacked := make([]*protocol.SendInputMessage, 0)
						for _, oldMsg := range unacked {
							if oldMsg.Seq > msgEnt.LastSeq {
								// not processed yet, so reapply and keep it in the list
								newUnacked = append(newUnacked, oldMsg)
								existingEnt.Move(shared.GetVectorFromInputAndDt(oldMsg.Input, oldMsg.Dt))
							}
						}
						unacked = newUnacked

					} else {
						// This is someone else's entity, so just move it
						existingEnt.SetPosition(msgEnt.Position)
					}
				}

				// Now, we've also got to check to see if we still have any remaining entities
				// that belong to players who've left.  This means we have to do another ugly iteration
				// but that's life.
				removeDisconnectedPlayers(typed.Entities)
			}
		}

		now := time.Now()
		dt = now.Sub(lastTick)
		lastTick = now

		renderWindow.Clear(sf.Color{0, 0, 0, 0})

		for _, unit := range entities {
			unit.Draw(renderWindow, sf.DefaultRenderStates())
		}

		renderWindow.Display()
	}
}

// Look over the events coming in, check them against the current keystates, and then update
// the keystates to match.  This is one of those bad functions which mutates the package-wide
// keystate struct but really this is the only function which writes to it so why bother copying
// things?
func handleUserInput(renderWindow *sf.RenderWindow, inputState *shared.InputState) *shared.InputState {

	// Handle user input
	for event := renderWindow.PollEvent(); event != nil; event = renderWindow.PollEvent() {
		switch ev := event.(type) {
		case sf.EventKeyReleased:
			switch ev.Code {

			case sf.KeyEscape:
				renderWindow.Close()

			case sf.KeyLeft:
				if inputState.KeyLeftDown {
					inputState.KeyLeftDown = false
				}

			case sf.KeyRight:
				if inputState.KeyRightDown {
					inputState.KeyRightDown = false
				}

			case sf.KeyDown:
				if inputState.KeyDownDown {
					inputState.KeyDownDown = false
				}

			case sf.KeyUp:
				if inputState.KeyUpDown {
					inputState.KeyUpDown = false
				}

			}

		case sf.EventKeyPressed:
			switch ev.Code {

			case sf.KeyLeft:
				if !inputState.KeyLeftDown {
					inputState.KeyLeftDown = true
				}

			case sf.KeyRight:
				if !inputState.KeyRightDown {
					inputState.KeyRightDown = true
				}

			case sf.KeyDown:
				if !inputState.KeyDownDown {
					inputState.KeyDownDown = true
				}

			case sf.KeyUp:
				if !inputState.KeyUpDown {
					inputState.KeyUpDown = true
				}
			}
		}
	}

	return inputState
}

// Add a new entity to the game world.  It will use the Player constructor (for the player texture)
// if the entity ID matches the player ID and the Other constructor otherwise.
func addEntityToGameWorld(id int64, pos sf.Vector2f) {
	if id == myPlayerId {
		_, ok := entities[myPlayerId]
		if ok {
			fmt.Println("ERROR - tried to add a new player with the same ID")
			return
		}

		player := NewPlayer(pos)
		entities[myPlayerId] = player
	} else {
		_, ok := entities[id]
		if ok {
			fmt.Println("ERROR - tried to add a new other entity with an ID that was already in the system")
			return
		}

		other := NewOther(pos)
		entities[id] = other
	}
}

// Go through the entities which the server is tracking and compare to the entities the client is
// tracking.  Remove any client-side entities that aren't in the server's list.
func removeDisconnectedPlayers(serverEntities []protocol.MessageEntity) {
	// We'll make a map of the IDs so we can do more convenient lookups
	// I used bool because it's small.
	serverEnts := make(map[int64]bool)
	for _, ent := range serverEntities {
		serverEnts[ent.Id] = true
	}

	// Now iterate through the map of stored client-side entities.  Remove anything
	// that doesn't appear in the server entities map
	for id, _ := range entities {
		_, ok := serverEnts[id]
		if !ok {
			if id == myPlayerId {
				panic("Whoops trying to remove myself - this is an error condition so we out")
			}

			delete(entities, id)
		}
	}
}

// Establish a connection to the game server and start the two goroutines which require it
func connectToServer() net.Conn {
	conn, err := net.Dial("tcp", shared.HOST+":"+shared.PORT)
	if err != nil {
		panic(err.Error())
	}

	// Now we're going to wait for the server to give us an entity ID
	b := bufio.NewReader(conn)
	for {
		line, err := b.ReadBytes('\n')

		if err != nil {
			conn.Close()
			fmt.Println("Error while trying to accept player id")
			os.Exit(1)
			break
		}

		if string(line) == "" || string(line) == "\n" {
			continue
		}

		message, err := protocol.DecodeMessage(line)
		if err != nil {
			fmt.Println("ERROR during decode: " + err.Error())
			continue
		}

		if message.GetMessageType() == protocol.PLAYER_UUID_MESSAGE {
			typed, ok := message.(*protocol.PlayerUUIDMessage)
			if !ok {
				fmt.Println("Message couldn't be asserted into PlayerUUIDMessage though that was message id")
				conn.Close()
				os.Exit(1)
			}
			myPlayerId = typed.UUID
			break
		} else {
			fmt.Println("Got the wrong type of message - expected PLAYER_UUID_MESSAGE")
			conn.Close()
			os.Exit(1)
		}
	}

	go listenForMessages(conn)
	go writeMessages(conn, outgoing)

	return conn
}

// Listens for incoming messages from the server, decodes the serialized versions into
// message objects and then pushes them into the message queue.
//
// This is a concurrent function - it runs simultaneously to the main game loop as a goroutine
func listenForMessages(conn net.Conn) {
	b := bufio.NewReader(conn)
	for {
		line, err := b.ReadBytes('\n')

		if err != nil {
			conn.Close()
			fmt.Println("ERROR, CLOSING CONN: " + err.Error())
			break
		}

		if string(line) == "" || string(line) == "\n" {
			continue
		}

		// Deal with incoming messages from the server
		message, err := protocol.DecodeMessage(line)
		if err != nil {
			fmt.Println("Error decoding message: " + err.Error())
			continue
		}
		messageQueue.PushMessage(message)
	}
}

// This function writes outgoing messages to the connection.
//
// This is a concurrent function - it runs simultaneously to the main game loop as a goroutine
func writeMessages(conn net.Conn, msgChan chan protocol.Message) {
	for {
		msg := <-msgChan
		conn.Write(msg.Encode())
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"net/http"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/log"
)

const (
	namespace = "namenode"
)

var (
	listenAddress  = flag.String("web.listen-address", ":9070", "Address on which to expose metrics and web interface.")
	metricsPath    = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.")
	namenodeJmxUrl = flag.String("namenode.jmx.url", "http://localhost:50070/jmx", "Hadoop JMX URL.")
)

type Exporter struct {
	url                      string
	MissingBlocks            prometheus.Gauge
	CapacityTotal            prometheus.Gauge
	CapacityUsed             prometheus.Gauge
	CapacityRemaining        prometheus.Gauge
	CapacityUsedNonDFS       prometheus.Gauge
	BlocksTotal              prometheus.Gauge
	FilesTotal               prometheus.Gauge
	CorruptBlocks            prometheus.Gauge
	ExcessBlocks             prometheus.Gauge
	StaleDataNodes           prometheus.Gauge
	cmsGcCount               prometheus.Counter
	cmsGcTime                prometheus.Counter
	heapMemoryUsageCommitted prometheus.Gauge
	heapMemoryUsageInit      prometheus.Gauge
	heapMemoryUsageMax       prometheus.Gauge
	heapMemoryUsageUsed      prometheus.Gauge
}

func NewExporter(url string) *Exporter {
	return &Exporter{
		url: url,
		MissingBlocks: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "MissingBlocks",
			Help:      "MissingBlocks",
		}),
		CapacityTotal: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "CapacityTotal",
			Help:      "CapacityTotal",
		}),
		CapacityUsed: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "CapacityUsed",
			Help:      "CapacityUsed",
		}),
		CapacityRemaining: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "CapacityRemaining",
			Help:      "CapacityRemaining",
		}),
		CapacityUsedNonDFS: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "CapacityUsedNonDFS",
			Help:      "CapacityUsedNonDFS",
		}),
		BlocksTotal: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "BlocksTotal",
			Help:      "BlocksTotal",
		}),
		FilesTotal: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "FilesTotal",
			Help:      "FilesTotal",
		}),
		CorruptBlocks: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "CorruptBlocks",
			Help:      "CorruptBlocks",
		}),
		ExcessBlocks: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "ExcessBlocks",
			Help:      "ExcessBlocks",
		}),
		StaleDataNodes: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "StaleDataNodes",
			Help:      "StaleDataNodes",
		}),
		cmsGcCount: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: namespace,
			Name:      "ConcurrentMarkSweep_CollectionCount",
			Help:      "ConcurrentMarkSweep GC Count",
		}),
		cmsGcTime: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: namespace,
			Name:      "ConcurrentMarkSweep_CollectionTime",
			Help:      "ConcurrentMarkSweep GC Time",
		}),
		heapMemoryUsageCommitted: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "heapMemoryUsageCommitted",
			Help:      "heapMemoryUsageCommitted",
		}),
		heapMemoryUsageInit: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "heapMemoryUsageInit",
			Help:      "heapMemoryUsageInit",
		}),
		heapMemoryUsageMax: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "heapMemoryUsageMax",
			Help:      "heapMemoryUsageMax",
		}),
		heapMemoryUsageUsed: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "heapMemoryUsageUsed",
			Help:      "heapMemoryUsageUsed",
		}),
	}
}

// Describe implements the prometheus.Collector interface.
func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
	e.MissingBlocks.Describe(ch)
	e.CapacityTotal.Describe(ch)
	e.CapacityUsed.Describe(ch)
	e.CapacityRemaining.Describe(ch)
	e.CapacityUsedNonDFS.Describe(ch)
	e.BlocksTotal.Describe(ch)
	e.FilesTotal.Describe(ch)
	e.CorruptBlocks.Describe(ch)
	e.ExcessBlocks.Describe(ch)
	e.StaleDataNodes.Describe(ch)
	e.cmsGcCount.Describe(ch)
	e.heapMemoryUsageCommitted.Describe(ch)
	e.heapMemoryUsageInit.Describe(ch)
	e.heapMemoryUsageMax.Describe(ch)
	e.heapMemoryUsageUsed.Describe(ch)
}

// Collect implements the prometheus.Collector interface.
func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
	resp, err := http.Get(e.url)
	if err != nil {
		log.Error(err)
	}
	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Error(err)
	}
	var f interface{}
	err = json.Unmarshal(data, &f)
	if err != nil {
		log.Error(err)
	}
	// {"beans":[{"name":"Hadoop:service=NameNode,name=FSNamesystem", ...}, {"name":"java.lang:type=MemoryPool,name=Code Cache", ...}, ...]}
	m := f.(map[string]interface{})
	// [{"name":"Hadoop:service=NameNode,name=FSNamesystem", ...}, {"name":"java.lang:type=MemoryPool,name=Code Cache", ...}, ...]
	var nameList = m["beans"].([]interface{})
	for _, nameData := range nameList {
		nameDataMap := nameData.(map[string]interface{})
		/*
			{
				"name" : "Hadoop:service=NameNode,name=FSNamesystem",
				"modelerType" : "FSNamesystem",
				"tag.Context" : "dfs",
				"tag.HAState" : "active",
				"tag.TotalSyncTimes" : "23 6 ",
				"tag.Hostname" : "CNHORTO7502.line.ism",
				"MissingBlocks" : 0,
				"MissingReplOneBlocks" : 0,
				"ExpiredHeartbeats" : 0,
				"TransactionsSinceLastCheckpoint" : 2007,
				"TransactionsSinceLastLogRoll" : 7,
				"LastWrittenTransactionId" : 172706,
				"LastCheckpointTime" : 1456089173101,
				"CapacityTotal" : 307099828224,
				"CapacityTotalGB" : 286.0,
				"CapacityUsed" : 1471291392,
				"CapacityUsedGB" : 1.0,
				"CapacityRemaining" : 279994568704,
				"CapacityRemainingGB" : 261.0,
				"CapacityUsedNonDFS" : 25633968128,
				"TotalLoad" : 6,
				"SnapshottableDirectories" : 0,
				"Snapshots" : 0,
				"LockQueueLength" : 0,
				"BlocksTotal" : 67,
				"NumFilesUnderConstruction" : 0,
				"NumActiveClients" : 0,
				"FilesTotal" : 184,
				"PendingReplicationBlocks" : 0,
				"UnderReplicatedBlocks" : 0,
				"CorruptBlocks" : 0,
				"ScheduledReplicationBlocks" : 0,
				"PendingDeletionBlocks" : 0,
				"ExcessBlocks" : 0,
				"PostponedMisreplicatedBlocks" : 0,
				"PendingDataNodeMessageCount" : 0,
				"MillisSinceLastLoadedEdits" : 0,
				"BlockCapacity" : 2097152,
				"StaleDataNodes" : 0,
				"TotalFiles" : 184,
				"TotalSyncCount" : 7
			}
		*/
		if nameDataMap["name"] == "Hadoop:service=NameNode,name=FSNamesystem" {
			e.MissingBlocks.Set(nameDataMap["MissingBlocks"].(float64))
			e.CapacityTotal.Set(nameDataMap["CapacityTotal"].(float64))
			e.CapacityUsed.Set(nameDataMap["CapacityUsed"].(float64))
			e.CapacityRemaining.Set(nameDataMap["CapacityRemaining"].(float64))
			e.CapacityUsedNonDFS.Set(nameDataMap["CapacityUsedNonDFS"].(float64))
			e.BlocksTotal.Set(nameDataMap["BlocksTotal"].(float64))
			e.FilesTotal.Set(nameDataMap["FilesTotal"].(float64))
			e.CorruptBlocks.Set(nameDataMap["CorruptBlocks"].(float64))
			e.ExcessBlocks.Set(nameDataMap["ExcessBlocks"].(float64))
			e.StaleDataNodes.Set(nameDataMap["StaleDataNodes"].(float64))
		}
		if nameDataMap["name"] == "java.lang:type=GarbageCollector,name=ConcurrentMarkSweep" {
			e.cmsGcCount.Set(nameDataMap["CollectionCount"].(float64))
			e.cmsGcTime.Set(nameDataMap["CollectionTime"].(float64))
		}
		/*
			"name" : "java.lang:type=Memory",
			"modelerType" : "sun.management.MemoryImpl",
			"HeapMemoryUsage" : {
				"committed" : 1060372480,
				"init" : 1073741824,
				"max" : 1060372480,
				"used" : 124571464
			},
		*/
		if nameDataMap["name"] == "java.lang:type=Memory" {
			heapMemoryUsage := nameDataMap["HeapMemoryUsage"].(map[string]interface{})
			e.heapMemoryUsageCommitted.Set(heapMemoryUsage["committed"].(float64))
			e.heapMemoryUsageInit.Set(heapMemoryUsage["init"].(float64))
			e.heapMemoryUsageMax.Set(heapMemoryUsage["max"].(float64))
			e.heapMemoryUsageUsed.Set(heapMemoryUsage["used"].(float64))
		}

	}
	e.MissingBlocks.Collect(ch)
	e.CapacityTotal.Collect(ch)
	e.CapacityUsed.Collect(ch)
	e.CapacityRemaining.Collect(ch)
	e.CapacityUsedNonDFS.Collect(ch)
	e.BlocksTotal.Collect(ch)
	e.FilesTotal.Collect(ch)
	e.CorruptBlocks.Collect(ch)
	e.ExcessBlocks.Collect(ch)
	e.StaleDataNodes.Collect(ch)
	e.cmsGcCount.Collect(ch)
	e.cmsGcTime.Collect(ch)
	e.heapMemoryUsageCommitted.Collect(ch)
	e.heapMemoryUsageInit.Collect(ch)
	e.heapMemoryUsageMax.Collect(ch)
	e.heapMemoryUsageUsed.Collect(ch)
}

func main() {
	flag.Parse()

	exporter := NewExporter(*namenodeJmxUrl)
	prometheus.MustRegister(exporter)

	log.Printf("Starting Server: %s", *listenAddress)
	http.Handle(*metricsPath, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
		<title>NameNode Exporter</title>

NameNode Exporter

<a href="` + *metricsPath + `">Metrics

		`))
	})
	err := http.ListenAndServe(*listenAddress, nil)
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Add ParNew<commit_after>package main

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"net/http"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/log"
)

const (
	namespace = "namenode"
)

var (
	listenAddress  = flag.String("web.listen-address", ":9070", "Address on which to expose metrics and web interface.")
	metricsPath    = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.")
	namenodeJmxUrl = flag.String("namenode.jmx.url", "http://localhost:50070/jmx", "Hadoop JMX URL.")
)

type Exporter struct {
	url                      string
	MissingBlocks            prometheus.Gauge
	CapacityTotal            prometheus.Gauge
	CapacityUsed             prometheus.Gauge
	CapacityRemaining        prometheus.Gauge
	CapacityUsedNonDFS       prometheus.Gauge
	BlocksTotal              prometheus.Gauge
	FilesTotal               prometheus.Gauge
	CorruptBlocks            prometheus.Gauge
	ExcessBlocks             prometheus.Gauge
	StaleDataNodes           prometheus.Gauge
	pnGcCount               prometheus.Counter
	pnGcTime                prometheus.Counter
	cmsGcCount               prometheus.Counter
	cmsGcTime                prometheus.Counter
	heapMemoryUsageCommitted prometheus.Gauge
	heapMemoryUsageInit      prometheus.Gauge
	heapMemoryUsageMax       prometheus.Gauge
	heapMemoryUsageUsed      prometheus.Gauge
}

func NewExporter(url string) *Exporter {
	return &Exporter{
		url: url,
		MissingBlocks: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "MissingBlocks",
			Help:      "MissingBlocks",
		}),
		CapacityTotal: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "CapacityTotal",
			Help:      "CapacityTotal",
		}),
		CapacityUsed: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "CapacityUsed",
			Help:      "CapacityUsed",
		}),
		CapacityRemaining: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "CapacityRemaining",
			Help:      "CapacityRemaining",
		}),
		CapacityUsedNonDFS: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "CapacityUsedNonDFS",
			Help:      "CapacityUsedNonDFS",
		}),
		BlocksTotal: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "BlocksTotal",
			Help:      "BlocksTotal",
		}),
		FilesTotal: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "FilesTotal",
			Help:      "FilesTotal",
		}),
		CorruptBlocks: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "CorruptBlocks",
			Help:      "CorruptBlocks",
		}),
		ExcessBlocks: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "ExcessBlocks",
			Help:      "ExcessBlocks",
		}),
		StaleDataNodes: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "StaleDataNodes",
			Help:      "StaleDataNodes",
		}),
		pnGcCount: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: namespace,
			Name:      "ParNew_CollectionCount",
			Help:      "ParNew GC Count",
		}),
		pnGcTime: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: namespace,
			Name:      "ParNew_CollectionTime",
			Help:      "ParNew GC Time",
		}),
		cmsGcCount: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: namespace,
			Name:      "ConcurrentMarkSweep_CollectionCount",
			Help:      "ConcurrentMarkSweep GC Count",
		}),
		cmsGcTime: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: namespace,
			Name:      "ConcurrentMarkSweep_CollectionTime",
			Help:      "ConcurrentMarkSweep GC Time",
		}),
		heapMemoryUsageCommitted: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "heapMemoryUsageCommitted",
			Help:      "heapMemoryUsageCommitted",
		}),
		heapMemoryUsageInit: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "heapMemoryUsageInit",
			Help:      "heapMemoryUsageInit",
		}),
		heapMemoryUsageMax: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "heapMemoryUsageMax",
			Help:      "heapMemoryUsageMax",
		}),
		heapMemoryUsageUsed: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "heapMemoryUsageUsed",
			Help:      "heapMemoryUsageUsed",
		}),
	}
}

// Describe implements the prometheus.Collector interface.
func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
	e.MissingBlocks.Describe(ch)
	e.CapacityTotal.Describe(ch)
	e.CapacityUsed.Describe(ch)
	e.CapacityRemaining.Describe(ch)
	e.CapacityUsedNonDFS.Describe(ch)
	e.BlocksTotal.Describe(ch)
	e.FilesTotal.Describe(ch)
	e.CorruptBlocks.Describe(ch)
	e.ExcessBlocks.Describe(ch)
	e.StaleDataNodes.Describe(ch)
	e.pnGcCount.Describe(ch)
	e.pnGcTime.Describe(ch)
	e.cmsGcCount.Describe(ch)
	e.cmsGcTime.Describe(ch)
	e.heapMemoryUsageCommitted.Describe(ch)
	e.heapMemoryUsageInit.Describe(ch)
	e.heapMemoryUsageMax.Describe(ch)
	e.heapMemoryUsageUsed.Describe(ch)
}

// Collect implements the prometheus.Collector interface.
func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
	resp, err := http.Get(e.url)
	if err != nil {
		log.Error(err)
	}
	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Error(err)
	}
	var f interface{}
	err = json.Unmarshal(data, &f)
	if err != nil {
		log.Error(err)
	}
	// {"beans":[{"name":"Hadoop:service=NameNode,name=FSNamesystem", ...}, {"name":"java.lang:type=MemoryPool,name=Code Cache", ...}, ...]}
	m := f.(map[string]interface{})
	// [{"name":"Hadoop:service=NameNode,name=FSNamesystem", ...}, {"name":"java.lang:type=MemoryPool,name=Code Cache", ...}, ...]
	var nameList = m["beans"].([]interface{})
	for _, nameData := range nameList {
		nameDataMap := nameData.(map[string]interface{})
		/*
			{
				"name" : "Hadoop:service=NameNode,name=FSNamesystem",
				"modelerType" : "FSNamesystem",
				"tag.Context" : "dfs",
				"tag.HAState" : "active",
				"tag.TotalSyncTimes" : "23 6 ",
				"tag.Hostname" : "CNHORTO7502.line.ism",
				"MissingBlocks" : 0,
				"MissingReplOneBlocks" : 0,
				"ExpiredHeartbeats" : 0,
				"TransactionsSinceLastCheckpoint" : 2007,
				"TransactionsSinceLastLogRoll" : 7,
				"LastWrittenTransactionId" : 172706,
				"LastCheckpointTime" : 1456089173101,
				"CapacityTotal" : 307099828224,
				"CapacityTotalGB" : 286.0,
				"CapacityUsed" : 1471291392,
				"CapacityUsedGB" : 1.0,
				"CapacityRemaining" : 279994568704,
				"CapacityRemainingGB" : 261.0,
				"CapacityUsedNonDFS" : 25633968128,
				"TotalLoad" : 6,
				"SnapshottableDirectories" : 0,
				"Snapshots" : 0,
				"LockQueueLength" : 0,
				"BlocksTotal" : 67,
				"NumFilesUnderConstruction" : 0,
				"NumActiveClients" : 0,
				"FilesTotal" : 184,
				"PendingReplicationBlocks" : 0,
				"UnderReplicatedBlocks" : 0,
				"CorruptBlocks" : 0,
				"ScheduledReplicationBlocks" : 0,
				"PendingDeletionBlocks" : 0,
				"ExcessBlocks" : 0,
				"PostponedMisreplicatedBlocks" : 0,
				"PendingDataNodeMessageCount" : 0,
				"MillisSinceLastLoadedEdits" : 0,
				"BlockCapacity" : 2097152,
				"StaleDataNodes" : 0,
				"TotalFiles" : 184,
				"TotalSyncCount" : 7
			}
		*/
		if nameDataMap["name"] == "Hadoop:service=NameNode,name=FSNamesystem" {
			e.MissingBlocks.Set(nameDataMap["MissingBlocks"].(float64))
			e.CapacityTotal.Set(nameDataMap["CapacityTotal"].(float64))
			e.CapacityUsed.Set(nameDataMap["CapacityUsed"].(float64))
			e.CapacityRemaining.Set(nameDataMap["CapacityRemaining"].(float64))
			e.CapacityUsedNonDFS.Set(nameDataMap["CapacityUsedNonDFS"].(float64))
			e.BlocksTotal.Set(nameDataMap["BlocksTotal"].(float64))
			e.FilesTotal.Set(nameDataMap["FilesTotal"].(float64))
			e.CorruptBlocks.Set(nameDataMap["CorruptBlocks"].(float64))
			e.ExcessBlocks.Set(nameDataMap["ExcessBlocks"].(float64))
			e.StaleDataNodes.Set(nameDataMap["StaleDataNodes"].(float64))
		}
		if nameDataMap["name"] == "java.lang:type=GarbageCollector,name=ParNew" {
			e.pnGcCount.Set(nameDataMap["CollectionCount"].(float64))
			e.pnGcTime.Set(nameDataMap["CollectionTime"].(float64))
		}
		if nameDataMap["name"] == "java.lang:type=GarbageCollector,name=ConcurrentMarkSweep" {
			e.cmsGcCount.Set(nameDataMap["CollectionCount"].(float64))
			e.cmsGcTime.Set(nameDataMap["CollectionTime"].(float64))
		}
		/*
			"name" : "java.lang:type=Memory",
			"modelerType" : "sun.management.MemoryImpl",
			"HeapMemoryUsage" : {
				"committed" : 1060372480,
				"init" : 1073741824,
				"max" : 1060372480,
				"used" : 124571464
			},
		*/
		if nameDataMap["name"] == "java.lang:type=Memory" {
			heapMemoryUsage := nameDataMap["HeapMemoryUsage"].(map[string]interface{})
			e.heapMemoryUsageCommitted.Set(heapMemoryUsage["committed"].(float64))
			e.heapMemoryUsageInit.Set(heapMemoryUsage["init"].(float64))
			e.heapMemoryUsageMax.Set(heapMemoryUsage["max"].(float64))
			e.heapMemoryUsageUsed.Set(heapMemoryUsage["used"].(float64))
		}

	}
	e.MissingBlocks.Collect(ch)
	e.CapacityTotal.Collect(ch)
	e.CapacityUsed.Collect(ch)
	e.CapacityRemaining.Collect(ch)
	e.CapacityUsedNonDFS.Collect(ch)
	e.BlocksTotal.Collect(ch)
	e.FilesTotal.Collect(ch)
	e.CorruptBlocks.Collect(ch)
	e.ExcessBlocks.Collect(ch)
	e.StaleDataNodes.Collect(ch)
	e.pnGcCount.Collect(ch)
	e.pnGcTime.Collect(ch)
	e.cmsGcCount.Collect(ch)
	e.cmsGcTime.Collect(ch)
	e.heapMemoryUsageCommitted.Collect(ch)
	e.heapMemoryUsageInit.Collect(ch)
	e.heapMemoryUsageMax.Collect(ch)
	e.heapMemoryUsageUsed.Collect(ch)
}

func main() {
	flag.Parse()

	exporter := NewExporter(*namenodeJmxUrl)
	prometheus.MustRegister(exporter)

	log.Printf("Starting Server: %s", *listenAddress)
	http.Handle(*metricsPath, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
		<title>NameNode Exporter</title>

NameNode Exporter

<a href="` + *metricsPath + `">Metrics

		`))
	})
	err := http.ListenAndServe(*listenAddress, nil)
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 Masterminds

// This package provides the execution context for a Cookoo request.
package context

type Datasource struct {
}

type ExecutionContext struct {
	// Need the following:
  // Datasources -- probably a hashtable
  // Context vars -- hashtable
}

func (cxt *ExecutionContext) Init() *ExecutionContext {
  return cxt
}

func (cxt *ExecutionContext) Add(name string, value string) {
}

func (cxt *ExecutionContext) Get(name string) {
}

func (cxt *ExecutionContext) Datasource() *Datasource {
  return null
}

func (cxt *ExecutionContext) AddDatasource(*Datasource) {
}
<commit_msg>Added a constructor function for ExecutionContext. This is in the same format as Go uses itself.<commit_after>// Copyright 2013 Masterminds

// This package provides the execution context for a Cookoo request.
package context

type Datasource struct {
}

type ExecutionContext struct {
	// Need the following:
	// Datasources -- probably a hashtable
	// Context vars -- hashtable
}

func NewExecutionContext() *ExecutionContext {
	cxt := new(ExecutionContext).Init()
	return cxt
}

func (cxt *ExecutionContext) Init() *ExecutionContext {
	return cxt
}

func (cxt *ExecutionContext) Add(name string, value string) {
}

func (cxt *ExecutionContext) Get(name string) {
}

func (cxt *ExecutionContext) Datasource() *Datasource {
	return null
}

func (cxt *ExecutionContext) AddDatasource(*Datasource) {
}
<|endoftext|>


<commit_before>package croc

import (
	"crypto/rand"
	"io/ioutil"
	"os"
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func TestSendReceive(t *testing.T) {
	generateRandomFile(100)

	var wg sync.WaitGroup
	wg.Add(2)
	go func() {
		defer wg.Done()
		c := Init(true)
		assert.Nil(t, c.Send("100mb.file", "test"))
	}()
	go func() {
		defer wg.Done()
		time.Sleep(100 * time.Millisecond)
		os.MkdirAll("test", 0755)
		os.Chdir("test")
		c := Init(true)
		assert.Nil(t, c.Receive("test"))
	}()
	wg.Wait()
}

func generateRandomFile(megabytes int) {
	// generate a random file
	bigBuff := make([]byte, 1024*1024*megabytes)
	rand.Read(bigBuff)
	ioutil.WriteFile("100mb.file", bigBuff, 0666)
}
<commit_msg>test works<commit_after>package croc

import (
	"crypto/rand"
	"fmt"
	"io/ioutil"
	"os"
	"sync"
	"testing"
	"time"

	"github.com/schollz/croc/src/utils"
	"github.com/stretchr/testify/assert"
)

func TestSendReceive(t *testing.T) {
	var startTime time.Time
	var durationPerMegabyte float64
	generateRandomFile(100)
	var wg sync.WaitGroup
	wg.Add(2)
	go func() {
		defer wg.Done()
		c := Init(true)
		assert.Nil(t, c.Send("100mb.file", "test"))
	}()
	go func() {
		defer wg.Done()
		time.Sleep(2 * time.Second)
		os.MkdirAll("test", 0755)
		os.Chdir("test")
		c := Init(true)
		startTime = time.Now()
		assert.Nil(t, c.Receive("test"))
		durationPerMegabyte = 100.0 / time.Since(startTime).Seconds()
		assert.True(t, utils.Exists("100mb.file"))
	}()
	wg.Wait()
	os.Chdir("..")
	os.RemoveAll("test")
	os.Remove("100mb.file")
	fmt.Printf("\n-----\n%2.1f MB/s\n----\n", durationPerMegabyte)
}

func generateRandomFile(megabytes int) {
	// generate a random file
	bigBuff := make([]byte, 1024*1024*megabytes)
	rand.Read(bigBuff)
	ioutil.WriteFile("100mb.file", bigBuff, 0666)
}
<|endoftext|>


<commit_before>package build

import (
	"fmt"
	"github.com/fatih/color"
	"neon/util"
	"strings"
	"unicode/utf8"
)

const (
	defaultTheme = "bee"
)

type colorizer func(a ...interface{}) string

// Themes is a map of themes by name
var Themes = map[string]map[string][]color.Attribute{
	"bee": {
		"title": {color.FgYellow},
		"ok":    {color.FgGreen, color.Bold},
		"error": {color.FgRed, color.Bold},
	},
	"marine": {
		"title": {color.FgBlue},
		"ok":    {color.FgGreen, color.BgBlack, color.Bold},
		"error": {color.FgRed, color.BgBlack, color.Bold},
	},
	"bold": {
		"title": {color.FgYellow, color.Bold},
		"ok":    {color.FgGreen, color.Underline, color.Bold},
		"error": {color.FgRed, color.Underline, color.Bold},
	},
}

// Grey is a flag that tells if we print on console without color
var Grey = false

// Color definitions
var colorTitle colorizer
var colorOk colorizer
var colorError colorizer

// apply default theme
func init() {
	ApplyTheme(defaultTheme)
}

// ApplyTheme applies named theme
func ApplyTheme(theme string) error {
	if _, ok := Themes[theme]; !ok {
		return fmt.Errorf("unknown theme '%s'", theme)
	}
	colorTitle = color.New(Themes[theme]["title"]...).SprintFunc()
	colorOk = color.New(Themes[theme]["ok"]...).SprintFunc()
	colorError = color.New(Themes[theme]["error"]...).SprintFunc()
	return nil
}

// Message prints a message on console:
// - text: text to print (that might embed fields to print, such as "%s")
// - args: arguments for the text to print
func Message(text string, args ...interface{}) {
	printGrey(text, args...)
}

// Title prints a title on the console
// - text: text of the title to print
func Title(text string) {
	length := util.TerminalWidth() - (4 + utf8.RuneCountInString(text))
	if length < 2 {
		length = 2
	}
	message := fmt.Sprintf("%s %s --", strings.Repeat("-", length), text)
	if Grey {
		printGrey(message)
	} else {
		printColor(colorTitle(message))
	}
}

// PrintOk prints a green OK on the console
func PrintOk() {
	if Grey {
		printGrey("OK")
	} else {
		printColor(colorOk("OK"))
	}
}

// PrintError prints a red ERROR on the console followed with an explanatory
// text
// - text: the explanatory text to print
func PrintError(text string) {
	if Grey {
		printGrey("ERROR %s", text)
	} else {
		printColor("%s %s", colorError("ERROR"), text)
	}
}

// PrintColor prints a string with arguments in given color
// - text: the text to print
// - args: the arguments for the text to print
func printColor(text string, args ...interface{}) {
	if len(args) > 0 {
		fmt.Fprintf(color.Output, text, args...)
		fmt.Println()
	} else {
		fmt.Println(text)
	}
}

// PrintGrey prints a string with arguments in grey
// - text: the text to print
// - args: the arguments for the text to print
func printGrey(text string, fields ...interface{}) {
	if len(fields) > 0 {
		fmt.Printf(text, fields...)
		fmt.Println()
	} else {
		fmt.Println(text)
	}
}
<commit_msg>Added morethemes<commit_after>package build

import (
	"fmt"
	"github.com/fatih/color"
	"neon/util"
	"strings"
	"unicode/utf8"
)

const (
	defaultTheme = "bee"
)

type colorizer func(a ...interface{}) string

// Themes is a map of themes by name
var Themes = map[string]map[string][]color.Attribute{
	"bee": {
		"title": {color.FgYellow},
		"ok":    {color.FgGreen, color.Bold},
		"error": {color.FgRed, color.Bold},
	},
	"red": {
		"title": {color.FgRed},
		"ok":    {color.FgRed, color.Bold},
		"error": {color.FgRed, color.Bold, color.ReverseVideo},
	},
	"green": {
		"title": {color.FgGreen},
		"ok":    {color.FgGreen, color.Bold},
		"error": {color.FgGreen, color.Bold, color.ReverseVideo},
	},
	"blue": {
		"title": {color.FgBlue},
		"ok":    {color.FgBlue, color.Bold},
		"error": {color.FgBlue, color.Bold, color.ReverseVideo},
	},
	"fire": {
		"title": {color.FgRed},
		"ok":    {color.FgGreen, color.Bold, color.Underline},
		"error": {color.FgRed, color.Bold, color.Underline},
	},
	"marine": {
		"title": {color.FgBlue},
		"ok":    {color.FgGreen, color.Bold, color.Underline},
		"error": {color.FgRed, color.Bold, color.Underline},
	},
	"nature": {
		"title": {color.FgGreen},
		"ok":    {color.FgGreen, color.Bold, color.Underline},
		"error": {color.FgRed, color.Bold, color.Underline},
	},
	"bold": {
		"title": {color.FgYellow, color.Bold},
		"ok":    {color.FgGreen, color.Underline, color.Bold},
		"error": {color.FgRed, color.Underline, color.Bold},
	},
	"reverse": {
		"title": {color.ReverseVideo},
		"ok":    {color.ReverseVideo, color.Bold},
		"error": {color.ReverseVideo, color.Bold},
	},
}

// Grey is a flag that tells if we print on console without color
var Grey = false

// Color definitions
var colorTitle colorizer
var colorOk colorizer
var colorError colorizer

// apply default theme
func init() {
	ApplyTheme(defaultTheme)
}

// ApplyTheme applies named theme
func ApplyTheme(theme string) error {
	if _, ok := Themes[theme]; !ok {
		return fmt.Errorf("unknown theme '%s'", theme)
	}
	colorTitle = color.New(Themes[theme]["title"]...).SprintFunc()
	colorOk = color.New(Themes[theme]["ok"]...).SprintFunc()
	colorError = color.New(Themes[theme]["error"]...).SprintFunc()
	return nil
}

// Message prints a message on console:
// - text: text to print (that might embed fields to print, such as "%s")
// - args: arguments for the text to print
func Message(text string, args ...interface{}) {
	printGrey(text, args...)
}

// Title prints a title on the console
// - text: text of the title to print
func Title(text string) {
	length := util.TerminalWidth() - (4 + utf8.RuneCountInString(text))
	if length < 2 {
		length = 2
	}
	message := fmt.Sprintf("%s %s --", strings.Repeat("-", length), text)
	if Grey {
		printGrey(message)
	} else {
		printColor(colorTitle(message))
	}
}

// PrintOk prints a green OK on the console
func PrintOk() {
	if Grey {
		printGrey("OK")
	} else {
		printColor(colorOk("OK"))
	}
}

// PrintError prints a red ERROR on the console followed with an explanatory
// text
// - text: the explanatory text to print
func PrintError(text string) {
	if Grey {
		printGrey("ERROR %s", text)
	} else {
		printColor("%s %s", colorError("ERROR"), text)
	}
}

// PrintColor prints a string with arguments in given color
// - text: the text to print
// - args: the arguments for the text to print
func printColor(text string, args ...interface{}) {
	if len(args) > 0 {
		fmt.Fprintf(color.Output, text, args...)
		fmt.Println()
	} else {
		fmt.Println(text)
	}
}

// PrintGrey prints a string with arguments in grey
// - text: the text to print
// - args: the arguments for the text to print
func printGrey(text string, fields ...interface{}) {
	if len(fields) > 0 {
		fmt.Printf(text, fields...)
		fmt.Println()
	} else {
		fmt.Println(text)
	}
}
<|endoftext|>