<commit_before>// Copyright 2015 Square Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package transform

import (
	"fmt"
	"math"
	"strconv"

	"github.com/square/metrics/api"
	"github.com/square/metrics/function"
)

var Timeshift = function.MetricFunction{
	Name:         "transform.timeshift",
	MinArguments: 2,
	MaxArguments: 2,
	Compute: func(context function.EvaluationContext, arguments []function.Expression, groups function.Groups) (function.Value, error) {
		value, err := arguments[1].Evaluate(context)
		if err != nil {
			return nil, err
		}
		duration, err := value.ToDuration()
		if err != nil {
			return nil, err
		}
		newContext := context
		newContext.Timerange = newContext.Timerange.Shift(duration)

		result, err := arguments[0].Evaluate(newContext)
		if err != nil {
			return nil, err
		}

		if seriesValue, ok := result.(api.SeriesList); ok {
			seriesValue.Timerange = context.Timerange
			seriesValue.Query = fmt.Sprintf("transform.timeshift(%s,%s)", result.GetName(), value.GetName())
			seriesValue.Name = seriesValue.Query
			return seriesValue, nil
		}
		return result, nil
	},
}

var MovingAverage = function.MetricFunction{
	Name:         "transform.moving_average",
	MinArguments: 2,
	MaxArguments: 2,
	Compute: func(context function.EvaluationContext, arguments []function.Expression, groups function.Groups) (function.Value, error) {
		// Applying a similar trick as did TimeshiftFunction. It fetches data prior to the start of the timerange.

		sizeValue, err := arguments[1].Evaluate(context)
		if err != nil {
			return nil, err
		}
		size, err := sizeValue.ToDuration()
		if err != nil {
			return nil, err
		}
		limit := int(float64(size)/float64(context.Timerange.Resolution()) + 0.5) // Limit is the number of items to include in the average
		if limit < 1 {
			// At least one value must be included at all times
			limit = 1
		}

		newContext := context
		timerange := context.Timerange
		newContext.Timerange, err = api.NewSnappedTimerange(timerange.Start()-int64(limit-1)*timerange.ResolutionMillis(), timerange.End(), timerange.ResolutionMillis())
		if err != nil {
			return nil, err
		}
		// The new context has a timerange which is extended beyond the query's.
		listValue, err := arguments[0].Evaluate(newContext)
		if err != nil {
			return nil, err
		}

		// This value must be a SeriesList.
		list, err := listValue.ToSeriesList(newContext.Timerange)
		if err != nil {
			return nil, err
		}

		// The timerange must be reverted.
		list.Timerange = context.Timerange

		// Update each series in the list.
		for index, series := range list.Series {
			// The series will be given a (shorter) replaced list of values.
			results := make([]float64, context.Timerange.Slots())
			count := 0
			sum := 0.0
			for i := range series.Values {
				// Add the new element, if it isn't NaN.
				if !math.IsNaN(series.Values[i]) {
					sum += series.Values[i]
					count++
				}
				// Remove the oldest element, if it isn't NaN, and it's in range.
				// (e.g., if limit = 1, then this removes the previous element from the sum).
				if i >= limit && !math.IsNaN(series.Values[i-limit]) {
					sum -= series.Values[i-limit]
					count--
				}
				// Numerical error could (possibly) cause count == 0 but sum != 0.
				if i-limit+1 >= 0 {
					if count == 0 {
						results[i-limit+1] = math.NaN()
					} else {
						results[i-limit+1] = sum / float64(count)
					}
				}
			}
			list.Series[index].Values = results
		}
		list.Query = fmt.Sprintf("transform.moving_average(%s, %s)", listValue.GetName(), sizeValue.GetName())
		list.Name = list.Query
		return list, nil
	},
}

var Alias = function.MetricFunction{
	Name:         "transform.alias",
	MinArguments: 2,
	MaxArguments: 2,
	Compute: func(context function.EvaluationContext, arguments []function.Expression, groups function.Groups) (function.Value, error) {
		value, err := arguments[0].Evaluate(context)
		if err != nil {
			return nil, err
		}
		list, err := value.ToSeriesList(context.Timerange)
		if err != nil {
			return nil, err
		}
		nameValue, err := arguments[1].Evaluate(context)
		if err != nil {
			return nil, err
		}
		name, err := nameValue.ToString()
		if err != nil {
			return nil, err
		}
		list.Name = name
		list.Query = fmt.Sprintf("transform.alias(%s, %s)", value.GetName(), strconv.Quote(name))
		return list, nil
	},
}

// Derivative is special because it needs to get one extra data point to the left
// This transform estimates the "change per second" between the two samples (scaled consecutive difference)
var Derivative = newDerivativeBasedTransform("derivative", derivative)

func derivative(values []float64, parameters []function.Value, scale float64) ([]float64, error) {
	result := make([]float64, len(values)-1)
	for i := range values {
		if i == 0 {
			continue
		}
		// Scaled difference
		result[i-1] = (values[i] - values[i-1]) / scale
	}
	return result, nil
}

// Rate is special because it needs to get one extra data point to the left.
// This transform functions mostly like Derivative but bounds the result to be positive.
// Specifically this function is designed for strictly increasing counters that
// only decrease when reset to zero. That is, thie function returns consecutive
// differences which are at least 0, or math.Max of the newly reported value and 0
var Rate = newDerivativeBasedTransform("rate", rate)

func rate(values []float64, parameters []function.Value, scale float64) ([]float64, error) {
	result := make([]float64, len(values)-1)
	for i := range values {
		if i == 0 {
			continue
		}
		// Scaled difference
		result[i-1] = (values[i] - values[i-1]) / scale
		if result[i-1] < 0 {
			// values[i] is our best approximatation of the delta between i-1 and i
			// Why? This should only be used on counters, so if v[i] - v[i-1] < 0 then
			// the counter has reset, and we know *at least* v[i] increments have happened
			result[i-1] = math.Max(values[i], 0) / scale
		}
	}
	return result, nil
}

// newDerivativeBasedTransform returns a function.MetricFunction that performs
// a delta between two data points. The transform parameter is a function of type
// transform is expected to return an array of values whose length is 1 less
// than the given series
func newDerivativeBasedTransform(name string, transformer transform) function.MetricFunction {
	return function.MetricFunction{
		Name:         "transform." + name,
		MinArguments: 1,
		MaxArguments: 1,
		Compute: func(context function.EvaluationContext, arguments []function.Expression, groups function.Groups) (function.Value, error) {
			var err error
			// Calcuate the new timerange to include one extra point to the left
			newContext := context
			timerange := context.Timerange
			newContext.Timerange, err = api.NewSnappedTimerange(timerange.Start()-timerange.ResolutionMillis(), timerange.End(), timerange.ResolutionMillis())
			if err != nil {
				return nil, err
			}

			// The new context has a timerange which is extended beyond the query's.
			listValue, err := arguments[0].Evaluate(newContext)
			if err != nil {
				return nil, err
			}

			// This value must be a SeriesList.
			list, err := listValue.ToSeriesList(newContext.Timerange)
			if err != nil {
				return nil, err
			}

			// Reset the timerange
			list.Timerange = context.Timerange

			result, err := ApplyTransform(list, transformer, []function.Value{})
			if err != nil {
				return nil, err
			}

			// Validate our series are the correct length
			for i := range result.Series {
				if len(result.Series[i].Values) != len(list.Series[i].Values)-1 {
					return nil, fmt.Errorf("Expected transform to return %d values, received %d", len(list.Series[i].Values)-1, len(result.Series[i].Values))
				}
			}

			result.Query = fmt.Sprintf("transform.%s(%s)", name, listValue.GetName())
			result.Name = result.Query
			return result, nil
		},
	}
}
<commit_msg>Use panic since a user can't fix this issue<commit_after>// Copyright 2015 Square Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package transform

import (
	"fmt"
	"math"
	"strconv"

	"github.com/square/metrics/api"
	"github.com/square/metrics/function"
)

var Timeshift = function.MetricFunction{
	Name:         "transform.timeshift",
	MinArguments: 2,
	MaxArguments: 2,
	Compute: func(context function.EvaluationContext, arguments []function.Expression, groups function.Groups) (function.Value, error) {
		value, err := arguments[1].Evaluate(context)
		if err != nil {
			return nil, err
		}
		duration, err := value.ToDuration()
		if err != nil {
			return nil, err
		}
		newContext := context
		newContext.Timerange = newContext.Timerange.Shift(duration)

		result, err := arguments[0].Evaluate(newContext)
		if err != nil {
			return nil, err
		}

		if seriesValue, ok := result.(api.SeriesList); ok {
			seriesValue.Timerange = context.Timerange
			seriesValue.Query = fmt.Sprintf("transform.timeshift(%s,%s)", result.GetName(), value.GetName())
			seriesValue.Name = seriesValue.Query
			return seriesValue, nil
		}
		return result, nil
	},
}

var MovingAverage = function.MetricFunction{
	Name:         "transform.moving_average",
	MinArguments: 2,
	MaxArguments: 2,
	Compute: func(context function.EvaluationContext, arguments []function.Expression, groups function.Groups) (function.Value, error) {
		// Applying a similar trick as did TimeshiftFunction. It fetches data prior to the start of the timerange.

		sizeValue, err := arguments[1].Evaluate(context)
		if err != nil {
			return nil, err
		}
		size, err := sizeValue.ToDuration()
		if err != nil {
			return nil, err
		}
		limit := int(float64(size)/float64(context.Timerange.Resolution()) + 0.5) // Limit is the number of items to include in the average
		if limit < 1 {
			// At least one value must be included at all times
			limit = 1
		}

		newContext := context
		timerange := context.Timerange
		newContext.Timerange, err = api.NewSnappedTimerange(timerange.Start()-int64(limit-1)*timerange.ResolutionMillis(), timerange.End(), timerange.ResolutionMillis())
		if err != nil {
			return nil, err
		}
		// The new context has a timerange which is extended beyond the query's.
		listValue, err := arguments[0].Evaluate(newContext)
		if err != nil {
			return nil, err
		}

		// This value must be a SeriesList.
		list, err := listValue.ToSeriesList(newContext.Timerange)
		if err != nil {
			return nil, err
		}

		// The timerange must be reverted.
		list.Timerange = context.Timerange

		// Update each series in the list.
		for index, series := range list.Series {
			// The series will be given a (shorter) replaced list of values.
			results := make([]float64, context.Timerange.Slots())
			count := 0
			sum := 0.0
			for i := range series.Values {
				// Add the new element, if it isn't NaN.
				if !math.IsNaN(series.Values[i]) {
					sum += series.Values[i]
					count++
				}
				// Remove the oldest element, if it isn't NaN, and it's in range.
				// (e.g., if limit = 1, then this removes the previous element from the sum).
				if i >= limit && !math.IsNaN(series.Values[i-limit]) {
					sum -= series.Values[i-limit]
					count--
				}
				// Numerical error could (possibly) cause count == 0 but sum != 0.
				if i-limit+1 >= 0 {
					if count == 0 {
						results[i-limit+1] = math.NaN()
					} else {
						results[i-limit+1] = sum / float64(count)
					}
				}
			}
			list.Series[index].Values = results
		}
		list.Query = fmt.Sprintf("transform.moving_average(%s, %s)", listValue.GetName(), sizeValue.GetName())
		list.Name = list.Query
		return list, nil
	},
}

var Alias = function.MetricFunction{
	Name:         "transform.alias",
	MinArguments: 2,
	MaxArguments: 2,
	Compute: func(context function.EvaluationContext, arguments []function.Expression, groups function.Groups) (function.Value, error) {
		value, err := arguments[0].Evaluate(context)
		if err != nil {
			return nil, err
		}
		list, err := value.ToSeriesList(context.Timerange)
		if err != nil {
			return nil, err
		}
		nameValue, err := arguments[1].Evaluate(context)
		if err != nil {
			return nil, err
		}
		name, err := nameValue.ToString()
		if err != nil {
			return nil, err
		}
		list.Name = name
		list.Query = fmt.Sprintf("transform.alias(%s, %s)", value.GetName(), strconv.Quote(name))
		return list, nil
	},
}

// Derivative is special because it needs to get one extra data point to the left
// This transform estimates the "change per second" between the two samples (scaled consecutive difference)
var Derivative = newDerivativeBasedTransform("derivative", derivative)

func derivative(values []float64, parameters []function.Value, scale float64) ([]float64, error) {
	result := make([]float64, len(values)-1)
	for i := range values {
		if i == 0 {
			continue
		}
		// Scaled difference
		result[i-1] = (values[i] - values[i-1]) / scale
	}
	return result, nil
}

// Rate is special because it needs to get one extra data point to the left.
// This transform functions mostly like Derivative but bounds the result to be positive.
// Specifically this function is designed for strictly increasing counters that
// only decrease when reset to zero. That is, thie function returns consecutive
// differences which are at least 0, or math.Max of the newly reported value and 0
var Rate = newDerivativeBasedTransform("rate", rate)

func rate(values []float64, parameters []function.Value, scale float64) ([]float64, error) {
	result := make([]float64, len(values)-1)
	for i := range values {
		if i == 0 {
			continue
		}
		// Scaled difference
		result[i-1] = (values[i] - values[i-1]) / scale
		if result[i-1] < 0 {
			// values[i] is our best approximatation of the delta between i-1 and i
			// Why? This should only be used on counters, so if v[i] - v[i-1] < 0 then
			// the counter has reset, and we know *at least* v[i] increments have happened
			result[i-1] = math.Max(values[i], 0) / scale
		}
	}
	return result, nil
}

// newDerivativeBasedTransform returns a function.MetricFunction that performs
// a delta between two data points. The transform parameter is a function of type
// transform is expected to return an array of values whose length is 1 less
// than the given series
func newDerivativeBasedTransform(name string, transformer transform) function.MetricFunction {
	return function.MetricFunction{
		Name:         "transform." + name,
		MinArguments: 1,
		MaxArguments: 1,
		Compute: func(context function.EvaluationContext, arguments []function.Expression, groups function.Groups) (function.Value, error) {
			var err error
			// Calcuate the new timerange to include one extra point to the left
			newContext := context
			timerange := context.Timerange
			newContext.Timerange, err = api.NewSnappedTimerange(timerange.Start()-timerange.ResolutionMillis(), timerange.End(), timerange.ResolutionMillis())
			if err != nil {
				return nil, err
			}

			// The new context has a timerange which is extended beyond the query's.
			listValue, err := arguments[0].Evaluate(newContext)
			if err != nil {
				return nil, err
			}

			// This value must be a SeriesList.
			list, err := listValue.ToSeriesList(newContext.Timerange)
			if err != nil {
				return nil, err
			}

			// Reset the timerange
			list.Timerange = context.Timerange

			result, err := ApplyTransform(list, transformer, []function.Value{})
			if err != nil {
				return nil, err
			}

			// Validate our series are the correct length
			for i := range result.Series {
				if len(result.Series[i].Values) != len(list.Series[i].Values)-1 {
					panic(fmt.Sprintf("Expected transform to return %d values, received %d", len(list.Series[i].Values)-1, len(result.Series[i].Values)))
				}
			}

			result.Query = fmt.Sprintf("transform.%s(%s)", name, listValue.GetName())
			result.Name = result.Query
			return result, nil
		},
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"reflect"
	"strings"
	"testing"
)

func TestCurrentBranch(t *testing.T) {
	gt := newGitTest(t)
	defer gt.done()

	t.Logf("on master")
	checkCurrentBranch(t, "master", "origin/master", false, false, "", "")

	t.Logf("on newbranch")
	trun(t, gt.client, "git", "checkout", "-b", "newbranch")
	checkCurrentBranch(t, "newbranch", "origin/master", true, false, "", "")

	t.Logf("making change")
	write(t, gt.client+"/file", "i made a change")
	trun(t, gt.client, "git", "commit", "-a", "-m", "My change line.\n\nChange-Id: I0123456789abcdef0123456789abcdef\n")
	checkCurrentBranch(t, "newbranch", "origin/master", true, true, "I0123456789abcdef0123456789abcdef", "My change line.")

	t.Logf("on dev.branch")
	trun(t, gt.client, "git", "checkout", "-t", "-b", "dev.branch", "origin/dev.branch")
	checkCurrentBranch(t, "dev.branch", "origin/dev.branch", false, false, "", "")

	t.Logf("on newdev")
	trun(t, gt.client, "git", "checkout", "-t", "-b", "newdev", "origin/dev.branch")
	checkCurrentBranch(t, "newdev", "origin/dev.branch", true, false, "", "")

	t.Logf("making change")
	write(t, gt.client+"/file", "i made another change")
	trun(t, gt.client, "git", "commit", "-a", "-m", "My other change line.\n\nChange-Id: I1123456789abcdef0123456789abcdef\n")
	checkCurrentBranch(t, "newdev", "origin/dev.branch", true, true, "I1123456789abcdef0123456789abcdef", "My other change line.")

	t.Logf("detached head mode")
	trun(t, gt.client, "git", "checkout", "HEAD^0")
	checkCurrentBranch(t, "HEAD", "origin/HEAD", false, false, "", "")
}

func checkCurrentBranch(t *testing.T, name, origin string, isLocal, hasPending bool, changeID, subject string) {
	b := CurrentBranch()
	if b.Name != name {
		t.Errorf("b.Name = %q, want %q", b.Name, name)
	}
	if x := b.OriginBranch(); x != origin {
		t.Errorf("b.OriginBranch() = %q, want %q", x, origin)
	}
	if x := b.IsLocalOnly(); x != isLocal {
		t.Errorf("b.IsLocalOnly() = %v, want %v", x, isLocal)
	}
	if x := b.HasPendingCommit(); x != hasPending {
		t.Errorf("b.HasPendingCommit() = %v, want %v", x, isLocal)
	}
	if work := b.Pending(); len(work) > 0 {
		c := work[0]
		if x := c.ChangeID; x != changeID {
			t.Errorf("b.Pending()[0].ChangeID = %q, want %q", x, changeID)
		}
		if x := c.Subject; x != subject {
			t.Errorf("b.Pending()[0].Subject = %q, want %q", x, subject)
		}
	}
}

func TestLocalBranches(t *testing.T) {
	gt := newGitTest(t)
	defer gt.done()

	t.Logf("on master")
	checkLocalBranches(t, "master")

	t.Logf("on dev branch")
	trun(t, gt.client, "git", "checkout", "-b", "newbranch")
	checkLocalBranches(t, "master", "newbranch")

	t.Logf("detached head mode")
	trun(t, gt.client, "git", "checkout", "HEAD^0")
	checkLocalBranches(t, "HEAD", "master", "newbranch")
}

func checkLocalBranches(t *testing.T, want ...string) {
	var names []string
	branches := LocalBranches()
	for _, b := range branches {
		names = append(names, b.Name)
	}
	if !reflect.DeepEqual(names, want) {
		t.Errorf("LocalBranches() = %v, want %v", names, want)
	}
}

func TestAmbiguousRevision(t *testing.T) {
	gt := newGitTest(t)
	defer gt.done()
	gt.work(t)

	t.Logf("creating file paths that conflict with revision parameters")
	mkdir(t, gt.client+"/origin")
	write(t, gt.client+"/origin/master..work", "Uh-Oh! SpaghettiOs")
	mkdir(t, gt.client+"/work..origin")
	write(t, gt.client+"/work..origin/master", "Be sure to drink your Ovaltine")

	b := CurrentBranch()
	b.Submitted("I123456789")
}

func TestBranchpoint(t *testing.T) {
	gt := newGitTest(t)
	defer gt.done()

	// Get hash corresponding to checkout (known to server).
	hash := strings.TrimSpace(trun(t, gt.client, "git", "rev-parse", "HEAD"))

	// Any work we do after this point should find hash as branchpoint.
	for i := 0; i < 4; i++ {
		testMain(t, "branchpoint")
		t.Logf("numCommits=%d", i)
		testPrintedStdout(t, hash)
		testNoStderr(t)

		gt.work(t)
	}
}

func TestRebaseWork(t *testing.T) {
	gt := newGitTest(t)
	defer gt.done()

	// Get hash corresponding to checkout (known to server).
	// Any work we do after this point should find hash as branchpoint.
	hash := strings.TrimSpace(trun(t, gt.client, "git", "rev-parse", "HEAD"))

	testMainDied(t, "rebase-work", "-n")
	testPrintedStderr(t, "no pending work")

	write(t, gt.client+"/file", "uncommitted")
	testMainDied(t, "rebase-work", "-n")
	testPrintedStderr(t, "cannot rebase with uncommitted work")

	gt.work(t)

	for i := 0; i < 4; i++ {
		testMain(t, "rebase-work", "-n")
		t.Logf("numCommits=%d", i)
		testPrintedStderr(t, "git rebase -i "+hash)

		gt.work(t)
	}
}

func TestBranchpointMerge(t *testing.T) {
	gt := newGitTest(t)
	defer gt.done()

	// commit more work on master
	write(t, gt.server+"/file", "more work")
	trun(t, gt.server, "git", "commit", "-m", "work", "file")

	// update client
	trun(t, gt.client, "git", "checkout", "master")
	trun(t, gt.client, "git", "pull")

	hash := strings.TrimSpace(trun(t, gt.client, "git", "rev-parse", "HEAD"))

	// merge dev.branch
	testMain(t, "change", "work")
	trun(t, gt.client, "git", "merge", "-m", "merge", "origin/dev.branch")

	// check branchpoint is old head (despite this commit having two parents)
	bp := CurrentBranch().Branchpoint()
	if bp != hash {
		t.Logf("branches:\n%s", trun(t, gt.client, "git", "branch", "-a", "-v"))
		t.Logf("log:\n%s", trun(t, gt.client, "git", "log", "--graph", "--decorate"))
		t.Logf("log origin/master..HEAD:\n%s", trun(t, gt.client, "git", "log", "origin/master..HEAD"))
		t.Fatalf("branchpoint=%q, want %q", bp, hash)
	}
}
<commit_msg>git-codereview: specify --no-track on checkout<commit_after>// Copyright 2014 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"reflect"
	"strings"
	"testing"
)

func TestCurrentBranch(t *testing.T) {
	gt := newGitTest(t)
	defer gt.done()

	t.Logf("on master")
	checkCurrentBranch(t, "master", "origin/master", false, false, "", "")

	t.Logf("on newbranch")
	trun(t, gt.client, "git", "checkout", "--no-track", "-b", "newbranch")
	checkCurrentBranch(t, "newbranch", "origin/master", true, false, "", "")

	t.Logf("making change")
	write(t, gt.client+"/file", "i made a change")
	trun(t, gt.client, "git", "commit", "-a", "-m", "My change line.\n\nChange-Id: I0123456789abcdef0123456789abcdef\n")
	checkCurrentBranch(t, "newbranch", "origin/master", true, true, "I0123456789abcdef0123456789abcdef", "My change line.")

	t.Logf("on dev.branch")
	trun(t, gt.client, "git", "checkout", "-t", "-b", "dev.branch", "origin/dev.branch")
	checkCurrentBranch(t, "dev.branch", "origin/dev.branch", false, false, "", "")

	t.Logf("on newdev")
	trun(t, gt.client, "git", "checkout", "-t", "-b", "newdev", "origin/dev.branch")
	checkCurrentBranch(t, "newdev", "origin/dev.branch", true, false, "", "")

	t.Logf("making change")
	write(t, gt.client+"/file", "i made another change")
	trun(t, gt.client, "git", "commit", "-a", "-m", "My other change line.\n\nChange-Id: I1123456789abcdef0123456789abcdef\n")
	checkCurrentBranch(t, "newdev", "origin/dev.branch", true, true, "I1123456789abcdef0123456789abcdef", "My other change line.")

	t.Logf("detached head mode")
	trun(t, gt.client, "git", "checkout", "HEAD^0")
	checkCurrentBranch(t, "HEAD", "origin/HEAD", false, false, "", "")
}

func checkCurrentBranch(t *testing.T, name, origin string, isLocal, hasPending bool, changeID, subject string) {
	b := CurrentBranch()
	if b.Name != name {
		t.Errorf("b.Name = %q, want %q", b.Name, name)
	}
	if x := b.OriginBranch(); x != origin {
		t.Errorf("b.OriginBranch() = %q, want %q", x, origin)
	}
	if x := b.IsLocalOnly(); x != isLocal {
		t.Errorf("b.IsLocalOnly() = %v, want %v", x, isLocal)
	}
	if x := b.HasPendingCommit(); x != hasPending {
		t.Errorf("b.HasPendingCommit() = %v, want %v", x, isLocal)
	}
	if work := b.Pending(); len(work) > 0 {
		c := work[0]
		if x := c.ChangeID; x != changeID {
			t.Errorf("b.Pending()[0].ChangeID = %q, want %q", x, changeID)
		}
		if x := c.Subject; x != subject {
			t.Errorf("b.Pending()[0].Subject = %q, want %q", x, subject)
		}
	}
}

func TestLocalBranches(t *testing.T) {
	gt := newGitTest(t)
	defer gt.done()

	t.Logf("on master")
	checkLocalBranches(t, "master")

	t.Logf("on dev branch")
	trun(t, gt.client, "git", "checkout", "-b", "newbranch")
	checkLocalBranches(t, "master", "newbranch")

	t.Logf("detached head mode")
	trun(t, gt.client, "git", "checkout", "HEAD^0")
	checkLocalBranches(t, "HEAD", "master", "newbranch")
}

func checkLocalBranches(t *testing.T, want ...string) {
	var names []string
	branches := LocalBranches()
	for _, b := range branches {
		names = append(names, b.Name)
	}
	if !reflect.DeepEqual(names, want) {
		t.Errorf("LocalBranches() = %v, want %v", names, want)
	}
}

func TestAmbiguousRevision(t *testing.T) {
	gt := newGitTest(t)
	defer gt.done()
	gt.work(t)

	t.Logf("creating file paths that conflict with revision parameters")
	mkdir(t, gt.client+"/origin")
	write(t, gt.client+"/origin/master..work", "Uh-Oh! SpaghettiOs")
	mkdir(t, gt.client+"/work..origin")
	write(t, gt.client+"/work..origin/master", "Be sure to drink your Ovaltine")

	b := CurrentBranch()
	b.Submitted("I123456789")
}

func TestBranchpoint(t *testing.T) {
	gt := newGitTest(t)
	defer gt.done()

	// Get hash corresponding to checkout (known to server).
	hash := strings.TrimSpace(trun(t, gt.client, "git", "rev-parse", "HEAD"))

	// Any work we do after this point should find hash as branchpoint.
	for i := 0; i < 4; i++ {
		testMain(t, "branchpoint")
		t.Logf("numCommits=%d", i)
		testPrintedStdout(t, hash)
		testNoStderr(t)

		gt.work(t)
	}
}

func TestRebaseWork(t *testing.T) {
	gt := newGitTest(t)
	defer gt.done()

	// Get hash corresponding to checkout (known to server).
	// Any work we do after this point should find hash as branchpoint.
	hash := strings.TrimSpace(trun(t, gt.client, "git", "rev-parse", "HEAD"))

	testMainDied(t, "rebase-work", "-n")
	testPrintedStderr(t, "no pending work")

	write(t, gt.client+"/file", "uncommitted")
	testMainDied(t, "rebase-work", "-n")
	testPrintedStderr(t, "cannot rebase with uncommitted work")

	gt.work(t)

	for i := 0; i < 4; i++ {
		testMain(t, "rebase-work", "-n")
		t.Logf("numCommits=%d", i)
		testPrintedStderr(t, "git rebase -i "+hash)

		gt.work(t)
	}
}

func TestBranchpointMerge(t *testing.T) {
	gt := newGitTest(t)
	defer gt.done()

	// commit more work on master
	write(t, gt.server+"/file", "more work")
	trun(t, gt.server, "git", "commit", "-m", "work", "file")

	// update client
	trun(t, gt.client, "git", "checkout", "master")
	trun(t, gt.client, "git", "pull")

	hash := strings.TrimSpace(trun(t, gt.client, "git", "rev-parse", "HEAD"))

	// merge dev.branch
	testMain(t, "change", "work")
	trun(t, gt.client, "git", "merge", "-m", "merge", "origin/dev.branch")

	// check branchpoint is old head (despite this commit having two parents)
	bp := CurrentBranch().Branchpoint()
	if bp != hash {
		t.Logf("branches:\n%s", trun(t, gt.client, "git", "branch", "-a", "-v"))
		t.Logf("log:\n%s", trun(t, gt.client, "git", "log", "--graph", "--decorate"))
		t.Logf("log origin/master..HEAD:\n%s", trun(t, gt.client, "git", "log", "origin/master..HEAD"))
		t.Fatalf("branchpoint=%q, want %q", bp, hash)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package gcsfake

import (
	"errors"
	"io"
	"sort"
	"sync"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

// Create an in-memory bucket with the given name and empty contents.
func NewFakeBucket(name string) gcs.Bucket {
	return &bucket{name: name}
}

type object struct {
	// A storage.Object representing metadata for this object. Never changes.
	metadata *storage.Object

	// The contents of the object. These never change.
	contents []byte
}

// A slice of objects compared by name.
type objectSlice []object

func (s objectSlice) Len() int           { return len(s) }
func (s objectSlice) Less(i, j int) bool { return s[i].metadata.Name < s[j].metadata.Name }
func (s objectSlice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

// Return the smallest i such that s[i].metadata.Name >= name, or len(s) if
// there is no such i.
func (s objectSlice) lowerBound(name string) int {
	pred := func(i int) bool {
		return s[i].metadata.Name >= name
	}

	return sort.Search(len(s), pred)
}

type bucket struct {
	name string
	mu   sync.RWMutex

	// The set of extant objects.
	//
	// INVARIANT: Strictly increasing.
	objects objectSlice // GUARDED_BY(mu)
}

func (b *bucket) Name() string {
	return b.name
}

func (b *bucket) ListObjects(
	ctx context.Context,
	query *storage.Query) (*storage.Objects, error) {
	return nil, errors.New("TODO: Implement ListObjects.")
}

func (b *bucket) NewReader(
	ctx context.Context,
	objectName string) (io.ReadCloser, error) {
	return nil, errors.New("TODO: Implement NewReader.")
}

func (b *bucket) NewWriter(
	ctx context.Context,
	attrs *storage.ObjectAttrs) (gcs.ObjectWriter, error) {
	return newObjectWriter(b, attrs), nil
}

func (b *bucket) DeleteObject(
	ctx context.Context,
	name string) error {
	return errors.New("TODO: Implement DeleteObject.")
}

// Create an object struct for the given attributes and contents.
//
// EXCLUSIVE_LOCKS_REQUIRED(b.mu)
func (b *bucket) mintObject(
	attrs *storage.ObjectAttrs,
	contents []byte) (o object) {
	// Set up metadata.
	// TODO(jacobsa): Other fields.
	o.metadata = &storage.Object{
		Bucket: b.Name(),
		Name:   attrs.Name,
	}

	// Set up contents.
	o.contents = contents

	return
}

// Add a record for an object with the given attributes and contents, then
// return the minted metadata.
//
// LOCKS_EXCLUDED(mu)
func (b *bucket) addObject(
	attrs *storage.ObjectAttrs,
	contents []byte) *storage.Object {
	b.mu.Lock()
	defer b.mu.Unlock()

	// Create an object record from the given attributes.
	var o object = b.mintObject(attrs, contents)

	// Add it to our list of object.
	b.objects = append(b.objects, o)
	sort.Sort(b.objects)

	return o.metadata
}
<commit_msg>Implemented DeleteObject.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package gcsfake

import (
	"errors"
	"io"
	"sort"
	"sync"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

// Create an in-memory bucket with the given name and empty contents.
func NewFakeBucket(name string) gcs.Bucket {
	return &bucket{name: name}
}

type object struct {
	// A storage.Object representing metadata for this object. Never changes.
	metadata *storage.Object

	// The contents of the object. These never change.
	contents []byte
}

// A slice of objects compared by name.
type objectSlice []object

func (s objectSlice) Len() int           { return len(s) }
func (s objectSlice) Less(i, j int) bool { return s[i].metadata.Name < s[j].metadata.Name }
func (s objectSlice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

// Return the smallest i such that s[i].metadata.Name >= name, or len(s) if
// there is no such i.
func (s objectSlice) lowerBound(name string) int {
	pred := func(i int) bool {
		return s[i].metadata.Name >= name
	}

	return sort.Search(len(s), pred)
}

// Return the smallest i such that s[i].metadata.Name == name, or len(s) if
// there is no such i.
func (s objectSlice) find(name string) int {
	lb := s.lowerBound(name)
	if lb < len(s) && s[lb].metadata.Name == name {
		return lb
	}

	return len(s)
}

type bucket struct {
	name string
	mu   sync.RWMutex

	// The set of extant objects.
	//
	// INVARIANT: Strictly increasing.
	objects objectSlice // GUARDED_BY(mu)
}

func (b *bucket) Name() string {
	return b.name
}

func (b *bucket) ListObjects(
	ctx context.Context,
	query *storage.Query) (*storage.Objects, error) {
	return nil, errors.New("TODO: Implement ListObjects.")
}

func (b *bucket) NewReader(
	ctx context.Context,
	objectName string) (io.ReadCloser, error) {
	return nil, errors.New("TODO: Implement NewReader.")
}

func (b *bucket) NewWriter(
	ctx context.Context,
	attrs *storage.ObjectAttrs) (gcs.ObjectWriter, error) {
	return newObjectWriter(b, attrs), nil
}

// LOCKS_EXCLUDED(mu)
func (b *bucket) DeleteObject(
	ctx context.Context,
	name string) error {
	b.mu.Lock()
	defer b.mu.Unlock()

	// Do we possess the object with the given name?
	index := b.objects.find(name)
	if index == len(b.objects) {
		return errors.New("Object not found.")
	}

	// Remove the object.
	b.objects = append(b.objects[:index], b.objects[index+1:]...)

	return nil
}

// Create an object struct for the given attributes and contents.
//
// EXCLUSIVE_LOCKS_REQUIRED(b.mu)
func (b *bucket) mintObject(
	attrs *storage.ObjectAttrs,
	contents []byte) (o object) {
	// Set up metadata.
	// TODO(jacobsa): Other fields.
	o.metadata = &storage.Object{
		Bucket: b.Name(),
		Name:   attrs.Name,
	}

	// Set up contents.
	o.contents = contents

	return
}

// Add a record for an object with the given attributes and contents, then
// return the minted metadata.
//
// LOCKS_EXCLUDED(mu)
func (b *bucket) addObject(
	attrs *storage.ObjectAttrs,
	contents []byte) *storage.Object {
	b.mu.Lock()
	defer b.mu.Unlock()

	// Create an object record from the given attributes.
	var o object = b.mintObject(attrs, contents)

	// Add it to our list of object.
	b.objects = append(b.objects, o)
	sort.Sort(b.objects)

	return o.metadata
}
<|endoftext|>


<commit_before>package hosts

import (
	"fmt"
	"io"
	"text/template"

	mackerel "github.com/mackerelio/mackerel-client-go"

	"github.com/mackerelio/mkr/format"
	"github.com/mackerelio/mkr/logger"
	"github.com/mackerelio/mkr/mackerelclient"
)

type hostApp struct {
	client    mackerelclient.Client
	logger    *logger.Logger
	outStream io.Writer
}

type findHostsParam struct {
	verbose bool

	name     string
	service  string
	roles    []string
	statuses []string

	format string
}

func (ha *hostApp) findHosts(param findHostsParam) error {
	hosts, err := ha.client.FindHosts(&mackerel.FindHostsParam{
		Name:     param.name,
		Service:  param.service,
		Roles:    param.roles,
		Statuses: param.statuses,
	})
	if err != nil {
		return err
	}

	switch {
	case param.format != "":
		t, err := template.New("format").Parse(param.format)
		if err != nil {
			return err
		}
		return t.Execute(ha.outStream, hosts)
	case param.verbose:
		return format.PrettyPrintJSON(ha.outStream, hosts)
	default:
		var hostsFormat []*format.Host
		for _, host := range hosts {
			hostsFormat = append(hostsFormat, &format.Host{
				ID:            host.ID,
				Name:          host.Name,
				DisplayName:   host.DisplayName,
				Status:        host.Status,
				RoleFullnames: host.GetRoleFullnames(),
				IsRetired:     host.IsRetired,
				CreatedAt:     format.ISO8601Extended(host.DateFromCreatedAt()),
				IPAddresses:   host.IPAddresses(),
			})
		}
		return format.PrettyPrintJSON(ha.outStream, hostsFormat)
	}
}

type createHostParam struct {
	Name             string
	RoleFullnames    []string
	Status           string
	CustomIdentifier string
}

func (ha *hostApp) createHost(param createHostParam) error {
	hostID, err := ha.client.CreateHost(&mackerel.CreateHostParam{
		Name:             param.Name,
		RoleFullnames:    param.RoleFullnames,
		CustomIdentifier: param.CustomIdentifier,
	})
	ha.logger.DieIf(err)

	ha.logger.Log("created", hostID)

	if param.Status != "" {
		err := ha.client.UpdateHostStatus(hostID, param.Status)
		ha.logger.DieIf(err)
		ha.logger.Log("updated", fmt.Sprintf("%s %s", hostID, param.Status))
	}
	return nil
}
<commit_msg>safer logging<commit_after>package hosts

import (
	"fmt"
	"io"
	"text/template"

	mackerel "github.com/mackerelio/mackerel-client-go"

	"github.com/mackerelio/mkr/format"
	"github.com/mackerelio/mkr/logger"
	"github.com/mackerelio/mkr/mackerelclient"
)

type hostApp struct {
	client    mackerelclient.Client
	logger    *logger.Logger
	outStream io.Writer
}

type findHostsParam struct {
	verbose bool

	name     string
	service  string
	roles    []string
	statuses []string

	format string
}

func (ha *hostApp) findHosts(param findHostsParam) error {
	hosts, err := ha.client.FindHosts(&mackerel.FindHostsParam{
		Name:     param.name,
		Service:  param.service,
		Roles:    param.roles,
		Statuses: param.statuses,
	})
	if err != nil {
		return err
	}

	switch {
	case param.format != "":
		t, err := template.New("format").Parse(param.format)
		if err != nil {
			return err
		}
		return t.Execute(ha.outStream, hosts)
	case param.verbose:
		return format.PrettyPrintJSON(ha.outStream, hosts)
	default:
		var hostsFormat []*format.Host
		for _, host := range hosts {
			hostsFormat = append(hostsFormat, &format.Host{
				ID:            host.ID,
				Name:          host.Name,
				DisplayName:   host.DisplayName,
				Status:        host.Status,
				RoleFullnames: host.GetRoleFullnames(),
				IsRetired:     host.IsRetired,
				CreatedAt:     format.ISO8601Extended(host.DateFromCreatedAt()),
				IPAddresses:   host.IPAddresses(),
			})
		}
		return format.PrettyPrintJSON(ha.outStream, hostsFormat)
	}
}

type createHostParam struct {
	Name             string
	RoleFullnames    []string
	Status           string
	CustomIdentifier string
}

func (ha *hostApp) createHost(param createHostParam) error {
	hostID, err := ha.client.CreateHost(&mackerel.CreateHostParam{
		Name:             param.Name,
		RoleFullnames:    param.RoleFullnames,
		CustomIdentifier: param.CustomIdentifier,
	})
	ha.dieIf(err)

	ha.log("created", hostID)

	if param.Status != "" {
		err := ha.client.UpdateHostStatus(hostID, param.Status)
		ha.dieIf(err)
		ha.log("updated", fmt.Sprintf("%s %s", hostID, param.Status))
	}
	return nil
}

func (ha *hostApp) log(prefix, message string) {
	if ha.logger != nil {
		ha.logger.Log(prefix, message)
	}
}

func (ha *hostApp) dieIf(err error) {
	if ha.logger != nil {
		ha.logger.DieIf(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 RedHat, Inc.
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package sdjournal provides a low-level Go interface to the
// systemd journal wrapped around the sd-journal C API.
//
// All public read methods map closely to the sd-journal API functions. See the
// sd-journal.h documentation[1] for information about each function.
//
// To write to the journal, see the pure-Go "journal" package
//
// [1] http://www.freedesktop.org/software/systemd/man/sd-journal.html
package sdjournal

/*
#cgo pkg-config: libsystemd
#include <systemd/sd-journal.h>
#include <stdlib.h>
#include <syslog.h>
*/
import "C"
import (
	"fmt"
	"path/filepath"
	"sync"
	"time"
	"unsafe"
)

// Journal entry field strings which correspond to:
// http://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html
const (
	SD_JOURNAL_FIELD_SYSTEMD_UNIT = "_SYSTEMD_UNIT"
	SD_JOURNAL_FIELD_MESSAGE      = "MESSAGE"
	SD_JOURNAL_FIELD_PID          = "_PID"
	SD_JOURNAL_FIELD_UID          = "_UID"
	SD_JOURNAL_FIELD_GID          = "_GID"
	SD_JOURNAL_FIELD_HOSTNAME     = "_HOSTNAME"
	SD_JOURNAL_FIELD_MACHINE_ID   = "_MACHINE_ID"
)

// Journal event constants
const (
	SD_JOURNAL_NOP        = int(C.SD_JOURNAL_NOP)
	SD_JOURNAL_APPEND     = int(C.SD_JOURNAL_APPEND)
	SD_JOURNAL_INVALIDATE = int(C.SD_JOURNAL_INVALIDATE)
)

// A Journal is a Go wrapper of an sd_journal structure.
type Journal struct {
	cjournal *C.sd_journal
	mu       sync.Mutex
}

// A Match is a convenience wrapper to describe filters supplied to AddMatch.
type Match struct {
	Field string
	Value string
}

// String returns a string representation of a Match suitable for use with AddMatch.
func (m *Match) String() string {
	return m.Field + "=" + m.Value
}

// NewJournal returns a new Journal instance pointing to the local journal
func NewJournal() (*Journal, error) {
	j := &Journal{}
	r := C.sd_journal_open(&j.cjournal, C.SD_JOURNAL_LOCAL_ONLY)

	if r < 0 {
		return nil, fmt.Errorf("failed to open journal: %d", r)
	}

	return j, nil
}

// NewJournalFromDir returns a new Journal instance pointing to a journal residing
// in a given directory. The supplied path may be relative or absolute; if
// relative, it will be converted to an absolute path before being opened.
func NewJournalFromDir(path string) (*Journal, error) {
	path, err := filepath.Abs(path)
	if err != nil {
		return nil, err
	}

	p := C.CString(path)
	defer C.free(unsafe.Pointer(p))

	j := &Journal{}
	r := C.sd_journal_open_directory(&j.cjournal, p, 0)
	if r < 0 {
		return nil, fmt.Errorf("failed to open journal in directory %q: %d", path, r)
	}

	return j, nil
}

// Close closes a journal opened with NewJournal.
func (j *Journal) Close() error {
	j.mu.Lock()
	C.sd_journal_close(j.cjournal)
	j.mu.Unlock()

	return nil
}

// AddMatch adds a match by which to filter the entries of the journal.
func (j *Journal) AddMatch(match string) error {
	m := C.CString(match)
	defer C.free(unsafe.Pointer(m))

	j.mu.Lock()
	r := C.sd_journal_add_match(j.cjournal, unsafe.Pointer(m), C.size_t(len(match)))
	j.mu.Unlock()

	if r < 0 {
		return fmt.Errorf("failed to add match: %d", r)
	}

	return nil
}

// AddDisjunction inserts a logical OR in the match list.
func (j *Journal) AddDisjunction() error {
	j.mu.Lock()
	r := C.sd_journal_add_disjunction(j.cjournal)
	j.mu.Unlock()

	if r < 0 {
		return fmt.Errorf("failed to add a disjunction in the match list: %d", r)
	}

	return nil
}

// AddConjunction inserts a logical AND in the match list.
func (j *Journal) AddConjunction() error {
	j.mu.Lock()
	r := C.sd_journal_add_conjunction(j.cjournal)
	j.mu.Unlock()

	if r < 0 {
		return fmt.Errorf("failed to add a conjunction in the match list: %d", r)
	}

	return nil
}

// FlushMatches flushes all matches, disjunctions and conjunctions.
func (j *Journal) FlushMatches() {
	j.mu.Lock()
	C.sd_journal_flush_matches(j.cjournal)
	j.mu.Unlock()
}

// Next advances the read pointer into the journal by one entry.
func (j *Journal) Next() (int, error) {
	j.mu.Lock()
	r := C.sd_journal_next(j.cjournal)
	j.mu.Unlock()

	if r < 0 {
		return int(r), fmt.Errorf("failed to iterate journal: %d", r)
	}

	return int(r), nil
}

// NextSkip advances the read pointer by multiple entries at once,
// as specified by the skip parameter.
func (j *Journal) NextSkip(skip uint64) (uint64, error) {
	j.mu.Lock()
	r := C.sd_journal_next_skip(j.cjournal, C.uint64_t(skip))
	j.mu.Unlock()

	if r < 0 {
		return uint64(r), fmt.Errorf("failed to iterate journal: %d", r)
	}

	return uint64(r), nil
}

// Previous sets the read pointer into the journal back by one entry.
func (j *Journal) Previous() (uint64, error) {
	j.mu.Lock()
	r := C.sd_journal_previous(j.cjournal)
	j.mu.Unlock()

	if r < 0 {
		return uint64(r), fmt.Errorf("failed to iterate journal: %d", r)
	}

	return uint64(r), nil
}

// PreviousSkip sets back the read pointer by multiple entries at once,
// as specified by the skip parameter.
func (j *Journal) PreviousSkip(skip uint64) (uint64, error) {
	j.mu.Lock()
	r := C.sd_journal_previous_skip(j.cjournal, C.uint64_t(skip))
	j.mu.Unlock()

	if r < 0 {
		return uint64(r), fmt.Errorf("failed to iterate journal: %d", r)
	}

	return uint64(r), nil
}

// GetData gets the data object associated with a specific field from the
// current journal entry.
func (j *Journal) GetData(field string) (string, error) {
	f := C.CString(field)
	defer C.free(unsafe.Pointer(f))

	var d unsafe.Pointer
	var l C.size_t

	j.mu.Lock()
	r := C.sd_journal_get_data(j.cjournal, f, &d, &l)
	j.mu.Unlock()

	if r < 0 {
		return "", fmt.Errorf("failed to read message: %d", r)
	}

	msg := C.GoStringN((*C.char)(d), C.int(l))

	return msg, nil
}

// GetDataValue gets the data object associated with a specific field from the
// current journal entry, returning only the value of the object.
func (j *Journal) GetDataValue(field string) (string, error) {
	val, err := j.GetData(field)
	if err != nil {
		return "", err
	}
	return strings.SplitN(val, "=", 2)[1], nil
}

// SetDataThresold sets the data field size threshold for data returned by
// GetData. To retrieve the complete data fields this threshold should be
// turned off by setting it to 0, so that the library always returns the
// complete data objects.
func (j *Journal) SetDataThreshold(threshold uint64) error {
	j.mu.Lock()
	r := C.sd_journal_set_data_threshold(j.cjournal, C.size_t(threshold))
	j.mu.Unlock()

	if r < 0 {
		return fmt.Errorf("failed to set data threshold: %d", r)
	}

	return nil
}

// GetRealtimeUsec gets the realtime (wallclock) timestamp of the current
// journal entry.
func (j *Journal) GetRealtimeUsec() (uint64, error) {
	var usec C.uint64_t

	j.mu.Lock()
	r := C.sd_journal_get_realtime_usec(j.cjournal, &usec)
	j.mu.Unlock()

	if r < 0 {
		return 0, fmt.Errorf("error getting timestamp for entry: %d", r)
	}

	return uint64(usec), nil
}

// SeekTail may be used to seek to the end of the journal, i.e. the most recent
// available entry.
func (j *Journal) SeekTail() error {
	j.mu.Lock()
	r := C.sd_journal_seek_tail(j.cjournal)
	j.mu.Unlock()

	if r < 0 {
		return fmt.Errorf("failed to seek to tail of journal: %d", r)
	}

	return nil
}

// SeekRealtimeUsec seeks to the entry with the specified realtime (wallclock)
// timestamp, i.e. CLOCK_REALTIME.
func (j *Journal) SeekRealtimeUsec(usec uint64) error {
	j.mu.Lock()
	r := C.sd_journal_seek_realtime_usec(j.cjournal, C.uint64_t(usec))
	j.mu.Unlock()

	if r < 0 {
		return fmt.Errorf("failed to seek to %d: %d", usec, r)
	}

	return nil
}

// Wait will synchronously wait until the journal gets changed. The maximum time
// this call sleeps may be controlled with the timeout parameter.
func (j *Journal) Wait(timeout time.Duration) int {
	to := uint64(time.Now().Add(timeout).Unix() / 1000)
	j.mu.Lock()
	r := C.sd_journal_wait(j.cjournal, C.uint64_t(to))
	j.mu.Unlock()

	return int(r)
}

// GetUsage returns the journal disk space usage, in bytes.
func (j *Journal) GetUsage() (uint64, error) {
	var out C.uint64_t
	j.mu.Lock()
	r := C.sd_journal_get_usage(j.cjournal, &out)
	j.mu.Unlock()

	if r < 0 {
		return 0, fmt.Errorf("failed to get journal disk space usage: %d", r)
	}

	return uint64(out), nil
}
<commit_msg>sdjournal: add ability to wait indefinitely<commit_after>// Copyright 2015 RedHat, Inc.
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package sdjournal provides a low-level Go interface to the
// systemd journal wrapped around the sd-journal C API.
//
// All public read methods map closely to the sd-journal API functions. See the
// sd-journal.h documentation[1] for information about each function.
//
// To write to the journal, see the pure-Go "journal" package
//
// [1] http://www.freedesktop.org/software/systemd/man/sd-journal.html
package sdjournal

/*
#cgo pkg-config: libsystemd
#include <systemd/sd-journal.h>
#include <stdlib.h>
#include <syslog.h>
*/
import "C"
import (
	"fmt"
	"path/filepath"
	"sync"
	"time"
	"unsafe"
)

// Journal entry field strings which correspond to:
// http://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html
const (
	SD_JOURNAL_FIELD_SYSTEMD_UNIT = "_SYSTEMD_UNIT"
	SD_JOURNAL_FIELD_MESSAGE      = "MESSAGE"
	SD_JOURNAL_FIELD_PID          = "_PID"
	SD_JOURNAL_FIELD_UID          = "_UID"
	SD_JOURNAL_FIELD_GID          = "_GID"
	SD_JOURNAL_FIELD_HOSTNAME     = "_HOSTNAME"
	SD_JOURNAL_FIELD_MACHINE_ID   = "_MACHINE_ID"
)

// Journal event constants
const (
	SD_JOURNAL_NOP        = int(C.SD_JOURNAL_NOP)
	SD_JOURNAL_APPEND     = int(C.SD_JOURNAL_APPEND)
	SD_JOURNAL_INVALIDATE = int(C.SD_JOURNAL_INVALIDATE)
)

// The maximum value for a time.Duration.  Can be passed to sdjournal.Wait() to
// signal an indefinite wait for new journal events.
const (
	IndefiniteWait time.Duration = 2540400*time.Hour + 10*time.Minute + 10*time.Second
)

// A Journal is a Go wrapper of an sd_journal structure.
type Journal struct {
	cjournal *C.sd_journal
	mu       sync.Mutex
}

// A Match is a convenience wrapper to describe filters supplied to AddMatch.
type Match struct {
	Field string
	Value string
}

// String returns a string representation of a Match suitable for use with AddMatch.
func (m *Match) String() string {
	return m.Field + "=" + m.Value
}

// NewJournal returns a new Journal instance pointing to the local journal
func NewJournal() (*Journal, error) {
	j := &Journal{}
	r := C.sd_journal_open(&j.cjournal, C.SD_JOURNAL_LOCAL_ONLY)

	if r < 0 {
		return nil, fmt.Errorf("failed to open journal: %d", r)
	}

	return j, nil
}

// NewJournalFromDir returns a new Journal instance pointing to a journal residing
// in a given directory. The supplied path may be relative or absolute; if
// relative, it will be converted to an absolute path before being opened.
func NewJournalFromDir(path string) (*Journal, error) {
	path, err := filepath.Abs(path)
	if err != nil {
		return nil, err
	}

	p := C.CString(path)
	defer C.free(unsafe.Pointer(p))

	j := &Journal{}
	r := C.sd_journal_open_directory(&j.cjournal, p, 0)
	if r < 0 {
		return nil, fmt.Errorf("failed to open journal in directory %q: %d", path, r)
	}

	return j, nil
}

// Close closes a journal opened with NewJournal.
func (j *Journal) Close() error {
	j.mu.Lock()
	C.sd_journal_close(j.cjournal)
	j.mu.Unlock()

	return nil
}

// AddMatch adds a match by which to filter the entries of the journal.
func (j *Journal) AddMatch(match string) error {
	m := C.CString(match)
	defer C.free(unsafe.Pointer(m))

	j.mu.Lock()
	r := C.sd_journal_add_match(j.cjournal, unsafe.Pointer(m), C.size_t(len(match)))
	j.mu.Unlock()

	if r < 0 {
		return fmt.Errorf("failed to add match: %d", r)
	}

	return nil
}

// AddDisjunction inserts a logical OR in the match list.
func (j *Journal) AddDisjunction() error {
	j.mu.Lock()
	r := C.sd_journal_add_disjunction(j.cjournal)
	j.mu.Unlock()

	if r < 0 {
		return fmt.Errorf("failed to add a disjunction in the match list: %d", r)
	}

	return nil
}

// AddConjunction inserts a logical AND in the match list.
func (j *Journal) AddConjunction() error {
	j.mu.Lock()
	r := C.sd_journal_add_conjunction(j.cjournal)
	j.mu.Unlock()

	if r < 0 {
		return fmt.Errorf("failed to add a conjunction in the match list: %d", r)
	}

	return nil
}

// FlushMatches flushes all matches, disjunctions and conjunctions.
func (j *Journal) FlushMatches() {
	j.mu.Lock()
	C.sd_journal_flush_matches(j.cjournal)
	j.mu.Unlock()
}

// Next advances the read pointer into the journal by one entry.
func (j *Journal) Next() (int, error) {
	j.mu.Lock()
	r := C.sd_journal_next(j.cjournal)
	j.mu.Unlock()

	if r < 0 {
		return int(r), fmt.Errorf("failed to iterate journal: %d", r)
	}

	return int(r), nil
}

// NextSkip advances the read pointer by multiple entries at once,
// as specified by the skip parameter.
func (j *Journal) NextSkip(skip uint64) (uint64, error) {
	j.mu.Lock()
	r := C.sd_journal_next_skip(j.cjournal, C.uint64_t(skip))
	j.mu.Unlock()

	if r < 0 {
		return uint64(r), fmt.Errorf("failed to iterate journal: %d", r)
	}

	return uint64(r), nil
}

// Previous sets the read pointer into the journal back by one entry.
func (j *Journal) Previous() (uint64, error) {
	j.mu.Lock()
	r := C.sd_journal_previous(j.cjournal)
	j.mu.Unlock()

	if r < 0 {
		return uint64(r), fmt.Errorf("failed to iterate journal: %d", r)
	}

	return uint64(r), nil
}

// PreviousSkip sets back the read pointer by multiple entries at once,
// as specified by the skip parameter.
func (j *Journal) PreviousSkip(skip uint64) (uint64, error) {
	j.mu.Lock()
	r := C.sd_journal_previous_skip(j.cjournal, C.uint64_t(skip))
	j.mu.Unlock()

	if r < 0 {
		return uint64(r), fmt.Errorf("failed to iterate journal: %d", r)
	}

	return uint64(r), nil
}

// GetData gets the data object associated with a specific field from the
// current journal entry.
func (j *Journal) GetData(field string) (string, error) {
	f := C.CString(field)
	defer C.free(unsafe.Pointer(f))

	var d unsafe.Pointer
	var l C.size_t

	j.mu.Lock()
	r := C.sd_journal_get_data(j.cjournal, f, &d, &l)
	j.mu.Unlock()

	if r < 0 {
		return "", fmt.Errorf("failed to read message: %d", r)
	}

	msg := C.GoStringN((*C.char)(d), C.int(l))

	return msg, nil
}

// GetDataValue gets the data object associated with a specific field from the
// current journal entry, returning only the value of the object.
func (j *Journal) GetDataValue(field string) (string, error) {
	val, err := j.GetData(field)
	if err != nil {
		return "", err
	}
	return strings.SplitN(val, "=", 2)[1], nil
}

// SetDataThresold sets the data field size threshold for data returned by
// GetData. To retrieve the complete data fields this threshold should be
// turned off by setting it to 0, so that the library always returns the
// complete data objects.
func (j *Journal) SetDataThreshold(threshold uint64) error {
	j.mu.Lock()
	r := C.sd_journal_set_data_threshold(j.cjournal, C.size_t(threshold))
	j.mu.Unlock()

	if r < 0 {
		return fmt.Errorf("failed to set data threshold: %d", r)
	}

	return nil
}

// GetRealtimeUsec gets the realtime (wallclock) timestamp of the current
// journal entry.
func (j *Journal) GetRealtimeUsec() (uint64, error) {
	var usec C.uint64_t

	j.mu.Lock()
	r := C.sd_journal_get_realtime_usec(j.cjournal, &usec)
	j.mu.Unlock()

	if r < 0 {
		return 0, fmt.Errorf("error getting timestamp for entry: %d", r)
	}

	return uint64(usec), nil
}

// SeekTail may be used to seek to the end of the journal, i.e. the most recent
// available entry.
func (j *Journal) SeekTail() error {
	j.mu.Lock()
	r := C.sd_journal_seek_tail(j.cjournal)
	j.mu.Unlock()

	if r < 0 {
		return fmt.Errorf("failed to seek to tail of journal: %d", r)
	}

	return nil
}

// SeekRealtimeUsec seeks to the entry with the specified realtime (wallclock)
// timestamp, i.e. CLOCK_REALTIME.
func (j *Journal) SeekRealtimeUsec(usec uint64) error {
	j.mu.Lock()
	r := C.sd_journal_seek_realtime_usec(j.cjournal, C.uint64_t(usec))
	j.mu.Unlock()

	if r < 0 {
		return fmt.Errorf("failed to seek to %d: %d", usec, r)
	}

	return nil
}

// Wait will synchronously wait until the journal gets changed. The maximum time
// this call sleeps may be controlled with the timeout parameter.  If
// sdjournal.IndefiniteWait is passed as the timeout parameter, Wait will
// wait indefinitely for a journal change.
func (j *Journal) Wait(timeout time.Duration) int {
	var to uint64
	if timeout == IndefiniteWait {
		// sd_journal_wait(3) calls for a (uint64_t) -1 to be passed to signify
		// indefinite wait, but using a -1 overflows our C.uint64_t, so we use an
		// equivalent hex value.
		to = 0xffffffffffffffff
	} else {
		to = uint64(time.Now().Add(timeout).Unix() / 1000)
	}
	j.mu.Lock()
	r := C.sd_journal_wait(j.cjournal, C.uint64_t(to))
	j.mu.Unlock()

	return int(r)
}

// GetUsage returns the journal disk space usage, in bytes.
func (j *Journal) GetUsage() (uint64, error) {
	var out C.uint64_t
	j.mu.Lock()
	r := C.sd_journal_get_usage(j.cjournal, &out)
	j.mu.Unlock()

	if r < 0 {
		return 0, fmt.Errorf("failed to get journal disk space usage: %d", r)
	}

	return uint64(out), nil
}
<|endoftext|>


<commit_before>/*
 * ZAnnotate Copyright 2017 Regents of the University of Michigan
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy
 * of the License at http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

package zannotate

import (
	"bufio"
	"encoding/json"
	"net"
	"os"
	"sync"

	"github.com/oschwald/geoip2-golang"
	log "github.com/sirupsen/logrus"
)

type GlobalConf struct {
	InputFilePath           string
	InputFileType           string
	OutputFilePath          string
	MetadataFilePath        string
	LogFilePath             string
	Verbosity               int
	Threads                 int
	JSONIPFieldName         string
	JSONAnnotationFieldName string

	GeoIP2     bool
	GeoIP2Conf GeoIP2Conf

	Routing     bool
	RoutingConf RoutingConf
}

type Result struct {
	Ip      string         `json:"ip,omitempty"`
	GeoIP2  *GeoIP2Output  `json:"geoip2,omitempty"`
	Routing *RoutingOutput `json:"routing,omitempty"`
}

func AnnotateRead(path string, in chan<- string) {
	log.Debug("read thread started")
	var f *os.File
	if path == "" || path == "-" {
		log.Debug("reading input from stdin")
		f = os.Stdin
	} else {
		var err error
		f, err = os.Open(path)
		if err != nil {
			log.Fatal("unable to open input file:", err.Error())
		}
		log.Debug("reading input from ", path)
	}
	s := bufio.NewScanner(f)
	for s.Scan() {
		in <- s.Text()
	}
	if err := s.Err(); err != nil {
		log.Fatal("input unable to read file", err)
	}
	close(in)
	log.Debug("read thread finished")
}

func AnnotateWrite(path string, out <-chan string, wg *sync.WaitGroup) {
	log.Debug("write thread started")
	var f *os.File
	if path == "" || path == "-" {
		f = os.Stdout
	} else {
		var err error
		f, err = os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0666)
		if err != nil {
			log.Fatal("unable to open output file:", err.Error())
		}
		defer f.Close()
	}
	for n := range out {
		f.WriteString(n + "\n")
	}
	wg.Done()
	log.Debug("write thread finished")
}

func AnnotateWorker(conf *GlobalConf, in <-chan string, out chan<- string,
	wg *sync.WaitGroup, i int) {
	log.Debug("annotate worker ", i, " started")
	// not entirely sure if this geolocation library is thread safe
	// so for now we're just going to open the MaxMind database in every thraed
	var geoIP2db *geoip2.Reader
	if conf.GeoIP2 {
		geoIP2db = GeoIP2Open(&conf.GeoIP2Conf)
		defer geoIP2db.Close()
	}
	log.Debug("annotate worker ", i, " initialization finished")
	for line := range in {
		// all lookup operations performed off of IP, which we parse into
		// depending on the configuration type
		var ip net.IP
		// JSON use only, but must be accessible throughout the loop
		var jsonMap map[string]interface{}
		if conf.InputFileType == "json" {
			var inParsed interface{}
			if err := json.Unmarshal([]byte(line), &inParsed); err != nil {
				log.Fatal("unable to parse json: ", line)
			}
			jsonMap = inParsed.(map[string]interface{})
			if val, ok := jsonMap[conf.JSONIPFieldName]; ok {
				if valS, ok := val.(string); ok {
					ip = net.ParseIP(valS)
				} else {
					log.Fatal("ip is not a string in JSON for ", line)
				}
			} else {
				log.Fatal("unable to find IP address field in ", line)
			}
			if _, ok := jsonMap[conf.JSONAnnotationFieldName]; ok {
				log.Fatal("input record already contains annotation key ", line)
			}
		} else {
			ip = net.ParseIP(line)
		}
		if ip == nil {
			log.Fatal("invalid IP received: ", line)
		}
		var res Result
		if conf.GeoIP2 == true {
			record, err := geoIP2db.City(ip)
			if err != nil {
				log.Fatal(err)
			}
			res.GeoIP2 = GeoIP2FillStruct(record, &conf.GeoIP2Conf)
		}
		if conf.Routing {
			res.Routing = RoutingFillStruct(ip, &conf.RoutingConf)
		}
		if conf.InputFileType == "json" {
			jsonMap[conf.JSONAnnotationFieldName] = res
			jsonRes, err := json.Marshal(jsonMap)
			if err != nil {
				log.Fatal("Unable to marshal JSON result", err)
			}
			out <- string(jsonRes)

		} else {
			res.Ip = ip.String()
			jsonRes, err := json.Marshal(res)
			if err != nil {
				log.Fatal("Unable to marshal JSON result", err)
			}
			out <- string(jsonRes)
		}
	}
	wg.Done()
	log.Debug("annotate worker ", i, " finished")
}

func DoAnnotation(conf *GlobalConf) {
	outChan := make(chan string)
	inChan := make(chan string)

	var outputWG sync.WaitGroup
	outputWG.Add(1)

	//startTime := time.Now().Format(time.RFC3339)
	go AnnotateRead(conf.InputFilePath, inChan)
	go AnnotateWrite(conf.OutputFilePath, outChan, &outputWG)

	var annotateWG sync.WaitGroup
	annotateWG.Add(conf.Threads)
	for i := 0; i < conf.Threads; i++ {
		go AnnotateWorker(conf, inChan, outChan, &annotateWG, i)
	}
	annotateWG.Wait()
	close(outChan)
	outputWG.Wait()
	//endTime := time.Now().Format(time.RFC3339)
}
<commit_msg>Swap out bufio.Scanner for bufio.Reader<commit_after>/*
 * ZAnnotate Copyright 2017 Regents of the University of Michigan
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy
 * of the License at http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

package zannotate

import (
	"bufio"
	"encoding/json"
	"io"
	"net"
	"os"
	"sync"

	"github.com/oschwald/geoip2-golang"
	log "github.com/sirupsen/logrus"
)

type GlobalConf struct {
	InputFilePath           string
	InputFileType           string
	OutputFilePath          string
	MetadataFilePath        string
	LogFilePath             string
	Verbosity               int
	Threads                 int
	JSONIPFieldName         string
	JSONAnnotationFieldName string

	GeoIP2     bool
	GeoIP2Conf GeoIP2Conf

	Routing     bool
	RoutingConf RoutingConf
}

type Result struct {
	Ip      string         `json:"ip,omitempty"`
	GeoIP2  *GeoIP2Output  `json:"geoip2,omitempty"`
	Routing *RoutingOutput `json:"routing,omitempty"`
}

func AnnotateRead(path string, in chan<- string) {
	log.Debug("read thread started")
	var f *os.File
	if path == "" || path == "-" {
		log.Debug("reading input from stdin")
		f = os.Stdin
	} else {
		var err error
		f, err = os.Open(path)
		if err != nil {
			log.Fatal("unable to open input file:", err.Error())
		}
		log.Debug("reading input from ", path)
	}
	r := bufio.NewReader(f)
	line, err := Readln(r)
	for err == nil {
		in <- line
		line, err = Readln(r)
	}
	if err != nil && err != io.EOF {
		log.Fatal("input unable to read file", err)
	}
	close(in)
	log.Debug("read thread finished")
}

func Readln(r *bufio.Reader) (string, error) {
	var (
		isPrefix bool  = true
		err      error = nil
		line, ln []byte
	)
	for isPrefix && err == nil {
		line, isPrefix, err = r.ReadLine()
		ln = append(ln, line...)
	}
	return string(ln), err
}

func AnnotateWrite(path string, out <-chan string, wg *sync.WaitGroup) {
	log.Debug("write thread started")
	var f *os.File
	if path == "" || path == "-" {
		f = os.Stdout
	} else {
		var err error
		f, err = os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0666)
		if err != nil {
			log.Fatal("unable to open output file:", err.Error())
		}
		defer f.Close()
	}
	for n := range out {
		f.WriteString(n + "\n")
	}
	wg.Done()
	log.Debug("write thread finished")
}

func AnnotateWorker(conf *GlobalConf, in <-chan string, out chan<- string,
	wg *sync.WaitGroup, i int) {
	log.Debug("annotate worker ", i, " started")
	// not entirely sure if this geolocation library is thread safe
	// so for now we're just going to open the MaxMind database in every thraed
	var geoIP2db *geoip2.Reader
	if conf.GeoIP2 {
		geoIP2db = GeoIP2Open(&conf.GeoIP2Conf)
		defer geoIP2db.Close()
	}
	log.Debug("annotate worker ", i, " initialization finished")
	for line := range in {
		// all lookup operations performed off of IP, which we parse into
		// depending on the configuration type
		var ip net.IP
		// JSON use only, but must be accessible throughout the loop
		var jsonMap map[string]interface{}
		if conf.InputFileType == "json" {
			var inParsed interface{}
			if err := json.Unmarshal([]byte(line), &inParsed); err != nil {
				log.Fatal("unable to parse json: ", line)
			}
			jsonMap = inParsed.(map[string]interface{})
			if val, ok := jsonMap[conf.JSONIPFieldName]; ok {
				if valS, ok := val.(string); ok {
					ip = net.ParseIP(valS)
				} else {
					log.Fatal("ip is not a string in JSON for ", line)
				}
			} else {
				log.Fatal("unable to find IP address field in ", line)
			}
			if _, ok := jsonMap[conf.JSONAnnotationFieldName]; ok {
				log.Fatal("input record already contains annotation key ", line)
			}
		} else {
			ip = net.ParseIP(line)
		}
		if ip == nil {
			log.Fatal("invalid IP received: ", line)
		}
		var res Result
		if conf.GeoIP2 == true {
			record, err := geoIP2db.City(ip)
			if err != nil {
				log.Fatal(err)
			}
			res.GeoIP2 = GeoIP2FillStruct(record, &conf.GeoIP2Conf)
		}
		if conf.Routing {
			res.Routing = RoutingFillStruct(ip, &conf.RoutingConf)
		}
		if conf.InputFileType == "json" {
			jsonMap[conf.JSONAnnotationFieldName] = res
			jsonRes, err := json.Marshal(jsonMap)
			if err != nil {
				log.Fatal("Unable to marshal JSON result", err)
			}
			out <- string(jsonRes)

		} else {
			res.Ip = ip.String()
			jsonRes, err := json.Marshal(res)
			if err != nil {
				log.Fatal("Unable to marshal JSON result", err)
			}
			out <- string(jsonRes)
		}
	}
	wg.Done()
	log.Debug("annotate worker ", i, " finished")
}

func DoAnnotation(conf *GlobalConf) {
	outChan := make(chan string)
	inChan := make(chan string)

	var outputWG sync.WaitGroup
	outputWG.Add(1)

	//startTime := time.Now().Format(time.RFC3339)
	go AnnotateRead(conf.InputFilePath, inChan)
	go AnnotateWrite(conf.OutputFilePath, outChan, &outputWG)

	var annotateWG sync.WaitGroup
	annotateWG.Add(conf.Threads)
	for i := 0; i < conf.Threads; i++ {
		go AnnotateWorker(conf, inChan, outChan, &annotateWG, i)
	}
	annotateWG.Wait()
	close(outChan)
	outputWG.Wait()
	//endTime := time.Now().Format(time.RFC3339)
}
<|endoftext|>


<commit_before>/*
 * MinIO Go Library for Amazon S3 Compatible Cloud Storage
 * Copyright 2015-2020 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package minio

import (
	"context"
	"net/http"
	"net/url"

	"github.com/minio/minio-go/v7/pkg/s3utils"
)

// BucketExists verifies if bucket exists and you have permission to access it. Allows for a Context to
// control cancellations and timeouts.
func (c Client) BucketExists(ctx context.Context, bucketName string) (bool, error) {
	// Input validation.
	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
		return false, err
	}

	// Execute HEAD on bucketName.
	resp, err := c.executeMethod(ctx, http.MethodHead, requestMetadata{
		bucketName:       bucketName,
		contentSHA256Hex: emptySHA256Hex,
	})
	defer closeResponse(resp)
	if err != nil {
		if ToErrorResponse(err).Code == "NoSuchBucket" {
			return false, nil
		}
		return false, err
	}
	if resp != nil {
		resperr := httpRespToErrorResponse(resp, bucketName, "")
		if ToErrorResponse(resperr).Code == "NoSuchBucket" {
			return false, nil
		}
		if resp.StatusCode != http.StatusOK {
			return false, httpRespToErrorResponse(resp, bucketName, "")
		}
	}
	return true, nil
}

// StatObject verifies if object exists and you have permission to access.
func (c Client) StatObject(ctx context.Context, bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) {
	// Input validation.
	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
		return ObjectInfo{}, err
	}
	if err := s3utils.CheckValidObjectName(objectName); err != nil {
		return ObjectInfo{}, err
	}
	return c.statObject(ctx, bucketName, objectName, opts)
}

// Lower level API for statObject supporting pre-conditions and range headers.
func (c Client) statObject(ctx context.Context, bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) {
	// Input validation.
	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
		return ObjectInfo{}, err
	}
	if err := s3utils.CheckValidObjectName(objectName); err != nil {
		return ObjectInfo{}, err
	}
	headers := opts.Header()
	if opts.Internal.ReplicationDeleteMarker {
		headers.Set(minIOBucketReplicationDeleteMarker, "true")
	}

	urlValues := make(url.Values)
	if opts.VersionID != "" {
		urlValues.Set("versionId", opts.VersionID)
	}
	// Execute HEAD on objectName.
	resp, err := c.executeMethod(ctx, http.MethodHead, requestMetadata{
		bucketName:       bucketName,
		objectName:       objectName,
		queryValues:      urlValues,
		contentSHA256Hex: emptySHA256Hex,
		customHeader:     headers,
	})
	defer closeResponse(resp)
	if err != nil {
		return ObjectInfo{}, err
	}
	deleteMarker := resp.Header.Get(amzDeleteMarker) == "true"

	if resp != nil {
		if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
			if resp.StatusCode == http.StatusBadRequest && opts.VersionID != "" && deleteMarker {
				errResp := ErrorResponse{
					StatusCode: resp.StatusCode,
					Code:       "MethodNotAllowed",
					Message:    "The specified method is not allowed against this resource.",
					BucketName: bucketName,
					Key:        objectName,
				}
				return ObjectInfo{
					VersionID:      resp.Header.Get(amzVersionID),
					IsDeleteMarker: deleteMarker,
				}, errResp
			}
			return ObjectInfo{
				VersionID:      resp.Header.Get(amzVersionID),
				IsDeleteMarker: deleteMarker,
			}, httpRespToErrorResponse(resp, bucketName, objectName)
		}
	}

	return ToObjectInfo(bucketName, objectName, resp.Header)
}
<commit_msg>fix: error returned by statObject for delete markers (#1531)<commit_after>/*
 * MinIO Go Library for Amazon S3 Compatible Cloud Storage
 * Copyright 2015-2020 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package minio

import (
	"context"
	"net/http"
	"net/url"

	"github.com/minio/minio-go/v7/pkg/s3utils"
)

// BucketExists verifies if bucket exists and you have permission to access it. Allows for a Context to
// control cancellations and timeouts.
func (c Client) BucketExists(ctx context.Context, bucketName string) (bool, error) {
	// Input validation.
	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
		return false, err
	}

	// Execute HEAD on bucketName.
	resp, err := c.executeMethod(ctx, http.MethodHead, requestMetadata{
		bucketName:       bucketName,
		contentSHA256Hex: emptySHA256Hex,
	})
	defer closeResponse(resp)
	if err != nil {
		if ToErrorResponse(err).Code == "NoSuchBucket" {
			return false, nil
		}
		return false, err
	}
	if resp != nil {
		resperr := httpRespToErrorResponse(resp, bucketName, "")
		if ToErrorResponse(resperr).Code == "NoSuchBucket" {
			return false, nil
		}
		if resp.StatusCode != http.StatusOK {
			return false, httpRespToErrorResponse(resp, bucketName, "")
		}
	}
	return true, nil
}

// StatObject verifies if object exists and you have permission to access.
func (c Client) StatObject(ctx context.Context, bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) {
	// Input validation.
	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
		return ObjectInfo{}, err
	}
	if err := s3utils.CheckValidObjectName(objectName); err != nil {
		return ObjectInfo{}, err
	}
	return c.statObject(ctx, bucketName, objectName, opts)
}

// Lower level API for statObject supporting pre-conditions and range headers.
func (c Client) statObject(ctx context.Context, bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) {
	// Input validation.
	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
		return ObjectInfo{}, err
	}
	if err := s3utils.CheckValidObjectName(objectName); err != nil {
		return ObjectInfo{}, err
	}
	headers := opts.Header()
	if opts.Internal.ReplicationDeleteMarker {
		headers.Set(minIOBucketReplicationDeleteMarker, "true")
	}

	urlValues := make(url.Values)
	if opts.VersionID != "" {
		urlValues.Set("versionId", opts.VersionID)
	}
	// Execute HEAD on objectName.
	resp, err := c.executeMethod(ctx, http.MethodHead, requestMetadata{
		bucketName:       bucketName,
		objectName:       objectName,
		queryValues:      urlValues,
		contentSHA256Hex: emptySHA256Hex,
		customHeader:     headers,
	})
	defer closeResponse(resp)
	if err != nil {
		return ObjectInfo{}, err
	}

	if resp != nil {
		deleteMarker := resp.Header.Get(amzDeleteMarker) == "true"
		if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
			if resp.StatusCode == http.StatusMethodNotAllowed && opts.VersionID != "" && deleteMarker {
				errResp := ErrorResponse{
					StatusCode: resp.StatusCode,
					Code:       "MethodNotAllowed",
					Message:    "The specified method is not allowed against this resource.",
					BucketName: bucketName,
					Key:        objectName,
				}
				return ObjectInfo{
					VersionID:      resp.Header.Get(amzVersionID),
					IsDeleteMarker: deleteMarker,
				}, errResp
			}
			return ObjectInfo{
				VersionID:      resp.Header.Get(amzVersionID),
				IsDeleteMarker: deleteMarker,
			}, httpRespToErrorResponse(resp, bucketName, objectName)
		}
	}

	return ToObjectInfo(bucketName, objectName, resp.Header)
}
<|endoftext|>