<commit_before>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !windows

package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"

	"upspin.io/cmd/cacheserver/cacheutil"
	"upspin.io/config"
	"upspin.io/flags"
	"upspin.io/log"

	_ "upspin.io/pack/ee"
	_ "upspin.io/pack/eeintegrity"
	_ "upspin.io/pack/plain"

	"upspin.io/transports"
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s <mountpoint>\n", os.Args[0])
	flag.PrintDefaults()
	os.Exit(2)
}

func main() {
	flag.Usage = usage
	flags.Parse("config", "log")

	if flag.NArg() != 1 {
		usage()
	}

	// Normal setup, get configuration from file and push user cache onto config.
	cfg, err := config.FromFile(flags.Config)
	if err != nil {
		log.Debug.Fatal(err)
	}
	transports.Init(cfg)

	// Start the cache if needed.
	cacheutil.Start(cfg)

	// Mount the file system and start serving.
	mountpoint, err := filepath.Abs(flag.Arg(0))
	if err != nil {
		log.Fatalf("can't determine absolute path to mount point %s: %s", flag.Arg(0), err)
	}
	done := do(cfg, mountpoint, flags.CacheDir)
	<-done
}
<commit_msg>cmd/upspinfs: add 'cachedir' flags<commit_after>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !windows

package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"

	"upspin.io/cmd/cacheserver/cacheutil"
	"upspin.io/config"
	"upspin.io/flags"
	"upspin.io/log"

	_ "upspin.io/pack/ee"
	_ "upspin.io/pack/eeintegrity"
	_ "upspin.io/pack/plain"

	"upspin.io/transports"
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s <mountpoint>\n", os.Args[0])
	flag.PrintDefaults()
	os.Exit(2)
}

func main() {
	flag.Usage = usage
	flags.Parse("cachedir", "config", "log")

	if flag.NArg() != 1 {
		usage()
	}

	// Normal setup, get configuration from file and push user cache onto config.
	cfg, err := config.FromFile(flags.Config)
	if err != nil {
		log.Debug.Fatal(err)
	}
	transports.Init(cfg)

	// Start the cache if needed.
	cacheutil.Start(cfg)

	// Mount the file system and start serving.
	mountpoint, err := filepath.Abs(flag.Arg(0))
	if err != nil {
		log.Fatalf("can't determine absolute path to mount point %s: %s", flag.Arg(0), err)
	}
	done := do(cfg, mountpoint, flags.CacheDir)
	<-done
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/codeclimate/cc-engine-go/engine"
	"sourcegraph.com/sourcegraph/go-diff/diff"
	"os/exec"
	"strconv"
	"os"
	"strings"
	"fmt"
)

func main() {
	rootPath := "/code/"

	config, err := engine.LoadConfig()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error loading config: %s", err)
		os.Exit(1)
	}

	analysisFiles, err := engine.GoFileWalk(rootPath, engine.IncludePaths(rootPath, config))
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error initializing: %s", err)
		os.Exit(1)
	}

	for _, path := range analysisFiles {
		cmd := exec.Command("gofmt", "-d", path)

		out, err := cmd.CombinedOutput()
		if err != nil {
			return
		}

		diffs, err := diff.ParseMultiFileDiff(out)
		if err != nil {
			return
		}

		if diffs != nil && diffs[0] != nil && len(diffs[0].Hunks) > 0 {
			numHunks := strconv.Itoa((len(diffs[0].Hunks)))
			path := strings.SplitAfter(path, rootPath)[1]

			issue := &engine.Issue{
				Type:              "issue",
				Check:             "GoFmt/Style/GoFmt",
				Description:       "Your code does not pass gofmt in " + numHunks + " places. Go fmt your code!",
				RemediationPoints: int32(50000 * len(diffs[0].Hunks)),
				Categories:        []string{"Style"},
				Location: &engine.Location{
					Path: path,
					Lines: &engine.LinesOnlyPosition{
						Begin: 1,
						End:   1,
					},
				},
			}
			engine.PrintIssue(issue)
		}
	}

	os.Exit(0)
}
<commit_msg>Correctly pluralize 'places' in issue description<commit_after>package main

import (
	"github.com/codeclimate/cc-engine-go/engine"
	"sourcegraph.com/sourcegraph/go-diff/diff"
	"os/exec"
	"strconv"
	"os"
	"strings"
	"fmt"
)

func main() {
	rootPath := "/code/"

	config, err := engine.LoadConfig()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error loading config: %s", err)
		os.Exit(1)
	}

	analysisFiles, err := engine.GoFileWalk(rootPath, engine.IncludePaths(rootPath, config))
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error initializing: %s", err)
		os.Exit(1)
	}

	for _, path := range analysisFiles {
		cmd := exec.Command("gofmt", "-d", path)

		out, err := cmd.CombinedOutput()
		if err != nil {
			return
		}

		diffs, err := diff.ParseMultiFileDiff(out)
		if err != nil {
			return
		}

		if diffs != nil && diffs[0] != nil && len(diffs[0].Hunks) > 0 {
			numHunks := len(diffs[0].Hunks)

			description := "Your code does not pass gofmt in " + strconv.Itoa(numHunks) + " " + pluralizePlace(numHunks) + ". Go fmt your code!"
			path := strings.SplitAfter(path, rootPath)[1]

			issue := &engine.Issue{
				Type:              "issue",
				Check:             "GoFmt/Style/GoFmt",
				Description:       description,
				RemediationPoints: int32(50000 * numHunks),
				Categories:        []string{"Style"},
				Location: &engine.Location{
					Path: path,
					Lines: &engine.LinesOnlyPosition{
						Begin: 1,
						End:   1,
					},
				},
			}
			engine.PrintIssue(issue)
		}
	}

	os.Exit(0)
}

func pluralizePlace(quant int) string {
	if quant > 1 {
		return "places"
	} else {
		return "place"
	}
}
<|endoftext|>


<commit_before>package command

import (
	"io/ioutil"
	"os"
	"strings"
	"testing"

	"github.com/mitchellh/cli"
)

func TestPlanCommand_Implements(t *testing.T) {
	var _ cli.Command = &RunCommand{}
}

func TestPlanCommand_Fails(t *testing.T) {
	ui := new(cli.MockUi)
	cmd := &PlanCommand{Meta: Meta{Ui: ui}}

	// Fails on misuse
	if code := cmd.Run([]string{"some", "bad", "args"}); code != 1 {
		t.Fatalf("expected exit code 1, got: %d", code)
	}
	if out := ui.ErrorWriter.String(); !strings.Contains(out, cmd.Help()) {
		t.Fatalf("expected help output, got: %s", out)
	}
	ui.ErrorWriter.Reset()

	// Fails when specified file does not exist
	if code := cmd.Run([]string{"/unicorns/leprechauns"}); code != 1 {
		t.Fatalf("expect exit 1, got: %d", code)
	}
	if out := ui.ErrorWriter.String(); !strings.Contains(out, "Error opening") {
		t.Fatalf("expect parsing error, got: %s", out)
	}
	ui.ErrorWriter.Reset()

	// Fails on invalid HCL
	fh1, err := ioutil.TempFile("", "nomad")
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	defer os.Remove(fh1.Name())
	if _, err := fh1.WriteString("nope"); err != nil {
		t.Fatalf("err: %s", err)
	}
	if code := cmd.Run([]string{fh1.Name()}); code != 1 {
		t.Fatalf("expect exit 1, got: %d", code)
	}
	if out := ui.ErrorWriter.String(); !strings.Contains(out, "Error parsing") {
		t.Fatalf("expect parsing error, got: %s", err)
	}
	ui.ErrorWriter.Reset()

	// Fails on invalid job spec
	fh2, err := ioutil.TempFile("", "nomad")
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	defer os.Remove(fh2.Name())
	if _, err := fh2.WriteString(`job "job1" {}`); err != nil {
		t.Fatalf("err: %s", err)
	}
	if code := cmd.Run([]string{fh2.Name()}); code != 1 {
		t.Fatalf("expect exit 1, got: %d", code)
	}
	if out := ui.ErrorWriter.String(); !strings.Contains(out, "Error validating") {
		t.Fatalf("expect validation error, got: %s", out)
	}
	ui.ErrorWriter.Reset()

	// Fails on connection failure (requires a valid job)
	fh3, err := ioutil.TempFile("", "nomad")
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	defer os.Remove(fh3.Name())
	_, err = fh3.WriteString(`
job "job1" {
	type = "service"
	datacenters = [ "dc1" ]
	group "group1" {
		count = 1
		task "task1" {
			driver = "exec"
			resources = {
				cpu = 1000
				disk = 150
				memory = 512
			}
		}
	}
}`)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	if code := cmd.Run([]string{"-address=nope", fh3.Name()}); code != 1 {
		t.Fatalf("expected exit code 1, got: %d", code)
	}
	if out := ui.ErrorWriter.String(); !strings.Contains(out, "Error during plan") {
		t.Fatalf("expected failed query error, got: %s", out)
	}
}

func TestPlanCommand_From_STDIN(t *testing.T) {
	stdinR, stdinW, err := os.Pipe()
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	ui := new(cli.MockUi)
	cmd := &PlanCommand{
		Meta:      Meta{Ui: ui},
		testStdin: stdinR,
	}

	go func() {
		stdinW.WriteString(`
job "job1" {
  type = "service"
  datacenters = [ "dc1" ]
  group "group1" {
                count = 1
                task "task1" {
                        driver = "exec"
                        resources = {
                                cpu = 1000
                                disk = 150
                                memory = 512
                        }
                }
        }
}`)
		stdinW.Close()
	}()

	args := []string{"-"}
	if code := cmd.Run(args); code != 1 {
		t.Fatalf("expected exit code 1, got %d: %q", code, ui.ErrorWriter.String())
	}

	if out := ui.ErrorWriter.String(); !strings.Contains(out, "connection refused") {
		t.Fatalf("expected runtime error, got: %s", out)
	}
	ui.ErrorWriter.Reset()
}
<commit_msg>Fix tests<commit_after>package command

import (
	"io/ioutil"
	"os"
	"strings"
	"testing"

	"github.com/mitchellh/cli"
)

func TestPlanCommand_Implements(t *testing.T) {
	var _ cli.Command = &RunCommand{}
}

func TestPlanCommand_Fails(t *testing.T) {
	ui := new(cli.MockUi)
	cmd := &PlanCommand{Meta: Meta{Ui: ui}}

	// Fails on misuse
	if code := cmd.Run([]string{"some", "bad", "args"}); code != 255 {
		t.Fatalf("expected exit code 1, got: %d", code)
	}
	if out := ui.ErrorWriter.String(); !strings.Contains(out, cmd.Help()) {
		t.Fatalf("expected help output, got: %s", out)
	}
	ui.ErrorWriter.Reset()

	// Fails when specified file does not exist
	if code := cmd.Run([]string{"/unicorns/leprechauns"}); code != 255 {
		t.Fatalf("expect exit 1, got: %d", code)
	}
	if out := ui.ErrorWriter.String(); !strings.Contains(out, "Error opening") {
		t.Fatalf("expect parsing error, got: %s", out)
	}
	ui.ErrorWriter.Reset()

	// Fails on invalid HCL
	fh1, err := ioutil.TempFile("", "nomad")
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	defer os.Remove(fh1.Name())
	if _, err := fh1.WriteString("nope"); err != nil {
		t.Fatalf("err: %s", err)
	}
	if code := cmd.Run([]string{fh1.Name()}); code != 255 {
		t.Fatalf("expect exit 1, got: %d", code)
	}
	if out := ui.ErrorWriter.String(); !strings.Contains(out, "Error parsing") {
		t.Fatalf("expect parsing error, got: %s", err)
	}
	ui.ErrorWriter.Reset()

	// Fails on invalid job spec
	fh2, err := ioutil.TempFile("", "nomad")
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	defer os.Remove(fh2.Name())
	if _, err := fh2.WriteString(`job "job1" {}`); err != nil {
		t.Fatalf("err: %s", err)
	}
	if code := cmd.Run([]string{fh2.Name()}); code != 255 {
		t.Fatalf("expect exit 1, got: %d", code)
	}
	if out := ui.ErrorWriter.String(); !strings.Contains(out, "Error validating") {
		t.Fatalf("expect validation error, got: %s", out)
	}
	ui.ErrorWriter.Reset()

	// Fails on connection failure (requires a valid job)
	fh3, err := ioutil.TempFile("", "nomad")
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	defer os.Remove(fh3.Name())
	_, err = fh3.WriteString(`
job "job1" {
	type = "service"
	datacenters = [ "dc1" ]
	group "group1" {
		count = 1
		task "task1" {
			driver = "exec"
			resources = {
				cpu = 1000
				disk = 150
				memory = 512
			}
		}
	}
}`)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	if code := cmd.Run([]string{"-address=nope", fh3.Name()}); code != 255 {
		t.Fatalf("expected exit code 1, got: %d", code)
	}
	if out := ui.ErrorWriter.String(); !strings.Contains(out, "Error during plan") {
		t.Fatalf("expected failed query error, got: %s", out)
	}
}

func TestPlanCommand_From_STDIN(t *testing.T) {
	stdinR, stdinW, err := os.Pipe()
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	ui := new(cli.MockUi)
	cmd := &PlanCommand{
		Meta:      Meta{Ui: ui},
		testStdin: stdinR,
	}

	go func() {
		stdinW.WriteString(`
job "job1" {
  type = "service"
  datacenters = [ "dc1" ]
  group "group1" {
                count = 1
                task "task1" {
                        driver = "exec"
                        resources = {
                                cpu = 1000
                                disk = 150
                                memory = 512
                        }
                }
        }
}`)
		stdinW.Close()
	}()

	args := []string{"-"}
	if code := cmd.Run(args); code != 255 {
		t.Fatalf("expected exit code 1, got %d: %q", code, ui.ErrorWriter.String())
	}

	if out := ui.ErrorWriter.String(); !strings.Contains(out, "connection refused") {
		t.Fatalf("expected runtime error, got: %s", out)
	}
	ui.ErrorWriter.Reset()
}
<|endoftext|>


<commit_before>package debug

import (
	"log"
	"net/http"
	"time"
)

type debugResponsewriter struct {
	http.ResponseWriter
	code int
}

func (d debugResponsewriter) WriteHeader(n int) {
	d.ResponseWriter.WriteHeader(n)
	d.code = n
}

func New(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		drw := debugResponsewriter{w, 0}
		start := time.Now()
		h.ServeHTTP(drw, r)
		log.Printf("%s - %s - %s - %s - %d - %s", time.Since(start).String(), r.RemoteAddr, r.Method, r.RequestURI, drw.code, r.UserAgent())
	})
}
<commit_msg>WriteHeader should have a pointer receiver<commit_after>package debug

import (
	"log"
	"net/http"
	"time"
)

type debugResponsewriter struct {
	http.ResponseWriter
	code int
}

func (d *debugResponsewriter) WriteHeader(n int) {
	d.ResponseWriter.WriteHeader(n)
	d.code = n
}

func New(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		drw := &debugResponsewriter{w, 0}
		start := time.Now()
		h.ServeHTTP(drw, r)
		log.Printf("%s - %s - %s - %s - %d - %s", time.Since(start).String(), r.RemoteAddr, r.Method, r.RequestURI, drw.code, r.UserAgent())
	})
}
<|endoftext|>


<commit_before>package dec

// This file implements signed multi-precision decimals.

import (
	"math/big"
)

// Rounder represents a method for rounding the (possibly infinite decimal)
// result of a division to a finite Dec. It is used by Dec.Round() and
// Dec.Quo().
//
type Rounder interface {

	// When UseRemainder() returns true, the Round() method is passed the
	// remainder of the division, expressed as the numerator and denominator of
	// a rational.
	UseRemainder() bool

	// Round sets the rounded value of a quotient to z, and returns z.
	// quo is rounded down (truncated towards zero) to the scale obtained from
	// the Scaler in Quo().
	//
	// When the remainder is not used, remNum and remDen are nil.
	// When used, the remainder is normalized between -1 and 1; that is:
	//
	//  -|remDen| < remNum < |remDen|
	//
	// remDen has the same sign as y, and remNum is zero or has the same sign
	// as x.
	Round(z, quo *Dec, remNum, remDen *big.Int) *Dec
}

type rounder struct {
	useRem bool
	round  func(z, quo *Dec, remNum, remDen *big.Int) *Dec
}

func (r rounder) UseRemainder() bool {
	return r.useRem
}

func (r rounder) Round(z, quo *Dec, remNum, remDen *big.Int) *Dec {
	return r.round(z, quo, remNum, remDen)
}

// RoundExact returns quo if rem is zero, or nil otherwise. It is intended to
// be used with ScaleQuoExact when it is guaranteed that the result can be
// obtained without rounding. QuoExact is a shorthand for such a quotient
// operation.
//
var RoundExact Rounder = roundExact

// RoundDown rounds towards 0; that is, returns the Dec with the greatest
// absolute value not exceeding that of the result represented by quo and rem.
//
// The following table shows examples of the results for
// Quo(x, y, Scale(scale), RoundDown).
//
//      x      y    scale   result
//  ------------------------------
//    -1.8    10        1     -0.1
//    -1.5    10        1     -0.1
//    -1.2    10        1     -0.1
//    -1.0    10        1     -0.1
//    -0.8    10        1     -0.0
//    -0.5    10        1     -0.0
//    -0.2    10        1     -0.0
//     0.0    10        1      0.0
//     0.2    10        1      0.0
//     0.5    10        1      0.0
//     0.8    10        1      0.0
//     1.0    10        1      0.1
//     1.2    10        1      0.1
//     1.5    10        1      0.1
//     1.8    10        1      0.1
//
var RoundDown Rounder = roundDown

// RoundUp rounds away from 0; that is, returns the Dec with the smallest
// absolute value not smaller than that of the result represented by quo and
// rem.
//
// The following table shows examples of the results for
// Quo(x, y, Scale(scale), RoundUp).
//
//      x      y    scale   result
//  ------------------------------
//    -1.8    10        1     -0.2
//    -1.5    10        1     -0.2
//    -1.2    10        1     -0.2
//    -1.0    10        1     -0.1
//    -0.8    10        1     -0.1
//    -0.5    10        1     -0.1
//    -0.2    10        1     -0.1
//     0.0    10        1      0.0
//     0.2    10        1      0.1
//     0.5    10        1      0.1
//     0.8    10        1      0.1
//     1.0    10        1      0.1
//     1.2    10        1      0.2
//     1.5    10        1      0.2
//     1.8    10        1      0.2
//
var RoundUp Rounder = roundUp

// RoundHalfDown rounds to the nearest Dec, and when the remainder is 1/2, it
// rounds to the Dec with the lower absolute value.
//
// The following table shows examples of the results for
// Quo(x, y, Scale(scale), RoundHalfDown).
//
//      x      y    scale   result
//  ------------------------------
//    -1.8    10        1     -0.2
//    -1.5    10        1     -0.1
//    -1.2    10        1     -0.1
//    -1.0    10        1     -0.1
//    -0.8    10        1     -0.1
//    -0.5    10        1     -0.0
//    -0.2    10        1     -0.0
//     0.0    10        1      0.0
//     0.2    10        1      0.0
//     0.5    10        1      0.0
//     0.8    10        1      0.1
//     1.0    10        1      0.1
//     1.2    10        1      0.1
//     1.5    10        1      0.1
//     1.8    10        1      0.2
//
var RoundHalfDown Rounder = roundHalfDown

// RoundHalfUp rounds to the nearest Dec, and when the remainder is 1/2, it
// rounds to the Dec with the greater absolute value.
//
// The following table shows examples of the results for
// Quo(x, y, Scale(scale), RoundHalfUp).
//
//      x      y    scale   result
//  ------------------------------
//    -1.8    10        1     -0.2
//    -1.5    10        1     -0.2
//    -1.2    10        1     -0.1
//    -1.0    10        1     -0.1
//    -0.8    10        1     -0.1
//    -0.5    10        1     -0.1
//    -0.2    10        1     -0.0
//     0.0    10        1      0.0
//     0.2    10        1      0.0
//     0.5    10        1      0.1
//     0.8    10        1      0.1
//     1.0    10        1      0.1
//     1.2    10        1      0.1
//     1.5    10        1      0.2
//     1.8    10        1      0.2
//
var RoundHalfUp Rounder = roundHalfUp

// RoundHalfEven rounds to the nearest Dec, and when the remainder is 1/2, it
// rounds to the Dec with even last digit.
//
// The following table shows examples of the results for
// Quo(x, y, Scale(scale), RoundHalfEven).
//
//      x      y    scale   result
//  ------------------------------
//    -1.8    10        1     -0.2
//    -1.5    10        1     -0.2
//    -1.2    10        1     -0.1
//    -1.0    10        1     -0.1
//    -0.8    10        1     -0.1
//    -0.5    10        1     -0.0
//    -0.2    10        1     -0.0
//     0.0    10        1      0.0
//     0.2    10        1      0.0
//     0.5    10        1      0.0
//     0.8    10        1      0.1
//     1.0    10        1      0.1
//     1.2    10        1      0.1
//     1.5    10        1      0.2
//     1.8    10        1      0.2
//
var RoundHalfEven Rounder = roundHalfEven

// RoundFloor rounds towards negative infinity; that is, returns the greatest
// Dec not exceeding the result represented by quo and rem.
//
// The following table shows examples of the results for
// Quo(x, y, Scale(scale), RoundFloor).
//
//      x      y    scale   result
//  ------------------------------
//    -1.8    10        1     -0.2
//    -1.5    10        1     -0.2
//    -1.2    10        1     -0.2
//    -1.0    10        1     -0.1
//    -0.8    10        1     -0.1
//    -0.5    10        1     -0.1
//    -0.2    10        1     -0.1
//     0.0    10        1      0.0
//     0.2    10        1      0.0
//     0.5    10        1      0.0
//     0.8    10        1      0.0
//     1.0    10        1      0.1
//     1.2    10        1      0.1
//     1.5    10        1      0.1
//     1.8    10        1      0.1
//
var RoundFloor Rounder = roundFloor

// RoundCeil rounds towards positive infinity; that is, returns the
// smallest Dec not smaller than the result represented by quo and rem.
//
// The following table shows examples of the results for
// Quo(x, y, Scale(scale), RoundCeil).
//
//      x      y    scale   result
//  ------------------------------
//    -1.8    10        1     -0.1
//    -1.5    10        1     -0.1
//    -1.2    10        1     -0.1
//    -1.0    10        1     -0.1
//    -0.8    10        1     -0.0
//    -0.5    10        1     -0.0
//    -0.2    10        1     -0.0
//     0.0    10        1      0.0
//     0.2    10        1      0.1
//     0.5    10        1      0.1
//     0.8    10        1      0.1
//     1.0    10        1      0.1
//     1.2    10        1      0.2
//     1.5    10        1      0.2
//     1.8    10        1      0.2
//
var RoundCeil Rounder = roundCeil

var intSign = []*big.Int{big.NewInt(-1), big.NewInt(0), big.NewInt(1)}

var roundExact = rounder{true,
	func(z, q *Dec, rA, rB *big.Int) *Dec {
		if rA.Sign() != 0 {
			return nil
		}
		return z.move(q)
	}}

var roundDown = rounder{false,
	func(z, q *Dec, rA, rB *big.Int) *Dec {
		return z.move(q)
	}}

var roundUp = rounder{true,
	func(z, q *Dec, rA, rB *big.Int) *Dec {
		z.move(q)
		if rA.Sign() != 0 {
			z.Unscaled().Add(z.Unscaled(), intSign[rA.Sign()*rB.Sign()+1])
		}
		return z
	}}

var roundHalfDown = rounder{true,
	func(z, q *Dec, rA, rB *big.Int) *Dec {
		z.move(q)
		brA, brB := rA.BitLen(), rB.BitLen()
		if brA < brB-1 {
			// brA < brB-1 => |rA| < |rB/2|
			return z
		}
		adjust := false
		srA, srB := rA.Sign(), rB.Sign()
		s := srA * srB
		if brA == brB-1 {
			rA2 := new(big.Int).Lsh(rA, 1)
			if s < 0 {
				rA2.Neg(rA2)
			}
			if rA2.Cmp(rB)*srB > 0 {
				adjust = true
			}
		} else {
			// brA > brB-1 => |rA| > |rB/2|
			adjust = true
		}
		if adjust {
			z.Unscaled().Add(z.Unscaled(), intSign[s+1])
		}
		return z
	}}

var roundHalfUp = rounder{true,
	func(z, q *Dec, rA, rB *big.Int) *Dec {
		z.move(q)
		brA, brB := rA.BitLen(), rB.BitLen()
		if brA < brB-1 {
			// brA < brB-1 => |rA| < |rB/2|
			return z
		}
		adjust := false
		srA, srB := rA.Sign(), rB.Sign()
		s := srA * srB
		if brA == brB-1 {
			rA2 := new(big.Int).Lsh(rA, 1)
			if s < 0 {
				rA2.Neg(rA2)
			}
			if rA2.Cmp(rB)*srB >= 0 {
				adjust = true
			}
		} else {
			// brA > brB-1 => |rA| > |rB/2|
			adjust = true
		}
		if adjust {
			z.Unscaled().Add(z.Unscaled(), intSign[s+1])
		}
		return z
	}}

var roundHalfEven = rounder{true,
	func(z, q *Dec, rA, rB *big.Int) *Dec {
		z.move(q)
		brA, brB := rA.BitLen(), rB.BitLen()
		if brA < brB-1 {
			// brA < brB-1 => |rA| < |rB/2|
			return z
		}
		adjust := false
		srA, srB := rA.Sign(), rB.Sign()
		s := srA * srB
		if brA == brB-1 {
			rA2 := new(big.Int).Lsh(rA, 1)
			if s < 0 {
				rA2.Neg(rA2)
			}
			c := rA2.Cmp(rB) * srB
			if c > 0 || c == 0 && z.Unscaled().Bit(0) == 1 {
				adjust = true
			}
		} else {
			// brA > brB-1 => |rA| > |rB/2|
			adjust = true
		}
		if adjust {
			z.Unscaled().Add(z.Unscaled(), intSign[s+1])
		}
		return z
	}}

var roundFloor = rounder{true,
	func(z, q *Dec, rA, rB *big.Int) *Dec {
		z.move(q)
		if rA.Sign()*rB.Sign() < 0 {
			z.Unscaled().Add(z.Unscaled(), intSign[0])
		}
		return z
	}}

var roundCeil = rounder{true,
	func(z, q *Dec, rA, rB *big.Int) *Dec {
		z.move(q)
		if rA.Sign()*rB.Sign() > 0 {
			z.Unscaled().Add(z.Unscaled(), intSign[2])
		}
		return z
	}}
<commit_msg>refactor roundHalf*<commit_after>package dec

// This file implements signed multi-precision decimals.

import (
	"math/big"
)

// Rounder represents a method for rounding the (possibly infinite decimal)
// result of a division to a finite Dec. It is used by Dec.Round() and
// Dec.Quo().
//
type Rounder interface {

	// When UseRemainder() returns true, the Round() method is passed the
	// remainder of the division, expressed as the numerator and denominator of
	// a rational.
	UseRemainder() bool

	// Round sets the rounded value of a quotient to z, and returns z.
	// quo is rounded down (truncated towards zero) to the scale obtained from
	// the Scaler in Quo().
	//
	// When the remainder is not used, remNum and remDen are nil.
	// When used, the remainder is normalized between -1 and 1; that is:
	//
	//  -|remDen| < remNum < |remDen|
	//
	// remDen has the same sign as y, and remNum is zero or has the same sign
	// as x.
	Round(z, quo *Dec, remNum, remDen *big.Int) *Dec
}

type rounder struct {
	useRem bool
	round  func(z, quo *Dec, remNum, remDen *big.Int) *Dec
}

func (r rounder) UseRemainder() bool {
	return r.useRem
}

func (r rounder) Round(z, quo *Dec, remNum, remDen *big.Int) *Dec {
	return r.round(z, quo, remNum, remDen)
}

// RoundExact returns quo if rem is zero, or nil otherwise. It is intended to
// be used with ScaleQuoExact when it is guaranteed that the result can be
// obtained without rounding. QuoExact is a shorthand for such a quotient
// operation.
//
var RoundExact Rounder = roundExact

// RoundDown rounds towards 0; that is, returns the Dec with the greatest
// absolute value not exceeding that of the result represented by quo and rem.
//
// The following table shows examples of the results for
// Quo(x, y, Scale(scale), RoundDown).
//
//      x      y    scale   result
//  ------------------------------
//    -1.8    10        1     -0.1
//    -1.5    10        1     -0.1
//    -1.2    10        1     -0.1
//    -1.0    10        1     -0.1
//    -0.8    10        1     -0.0
//    -0.5    10        1     -0.0
//    -0.2    10        1     -0.0
//     0.0    10        1      0.0
//     0.2    10        1      0.0
//     0.5    10        1      0.0
//     0.8    10        1      0.0
//     1.0    10        1      0.1
//     1.2    10        1      0.1
//     1.5    10        1      0.1
//     1.8    10        1      0.1
//
var RoundDown Rounder = roundDown

// RoundUp rounds away from 0; that is, returns the Dec with the smallest
// absolute value not smaller than that of the result represented by quo and
// rem.
//
// The following table shows examples of the results for
// Quo(x, y, Scale(scale), RoundUp).
//
//      x      y    scale   result
//  ------------------------------
//    -1.8    10        1     -0.2
//    -1.5    10        1     -0.2
//    -1.2    10        1     -0.2
//    -1.0    10        1     -0.1
//    -0.8    10        1     -0.1
//    -0.5    10        1     -0.1
//    -0.2    10        1     -0.1
//     0.0    10        1      0.0
//     0.2    10        1      0.1
//     0.5    10        1      0.1
//     0.8    10        1      0.1
//     1.0    10        1      0.1
//     1.2    10        1      0.2
//     1.5    10        1      0.2
//     1.8    10        1      0.2
//
var RoundUp Rounder = roundUp

// RoundHalfDown rounds to the nearest Dec, and when the remainder is 1/2, it
// rounds to the Dec with the lower absolute value.
//
// The following table shows examples of the results for
// Quo(x, y, Scale(scale), RoundHalfDown).
//
//      x      y    scale   result
//  ------------------------------
//    -1.8    10        1     -0.2
//    -1.5    10        1     -0.1
//    -1.2    10        1     -0.1
//    -1.0    10        1     -0.1
//    -0.8    10        1     -0.1
//    -0.5    10        1     -0.0
//    -0.2    10        1     -0.0
//     0.0    10        1      0.0
//     0.2    10        1      0.0
//     0.5    10        1      0.0
//     0.8    10        1      0.1
//     1.0    10        1      0.1
//     1.2    10        1      0.1
//     1.5    10        1      0.1
//     1.8    10        1      0.2
//
var RoundHalfDown Rounder = roundHalfDown

// RoundHalfUp rounds to the nearest Dec, and when the remainder is 1/2, it
// rounds to the Dec with the greater absolute value.
//
// The following table shows examples of the results for
// Quo(x, y, Scale(scale), RoundHalfUp).
//
//      x      y    scale   result
//  ------------------------------
//    -1.8    10        1     -0.2
//    -1.5    10        1     -0.2
//    -1.2    10        1     -0.1
//    -1.0    10        1     -0.1
//    -0.8    10        1     -0.1
//    -0.5    10        1     -0.1
//    -0.2    10        1     -0.0
//     0.0    10        1      0.0
//     0.2    10        1      0.0
//     0.5    10        1      0.1
//     0.8    10        1      0.1
//     1.0    10        1      0.1
//     1.2    10        1      0.1
//     1.5    10        1      0.2
//     1.8    10        1      0.2
//
var RoundHalfUp Rounder = roundHalfUp

// RoundHalfEven rounds to the nearest Dec, and when the remainder is 1/2, it
// rounds to the Dec with even last digit.
//
// The following table shows examples of the results for
// Quo(x, y, Scale(scale), RoundHalfEven).
//
//      x      y    scale   result
//  ------------------------------
//    -1.8    10        1     -0.2
//    -1.5    10        1     -0.2
//    -1.2    10        1     -0.1
//    -1.0    10        1     -0.1
//    -0.8    10        1     -0.1
//    -0.5    10        1     -0.0
//    -0.2    10        1     -0.0
//     0.0    10        1      0.0
//     0.2    10        1      0.0
//     0.5    10        1      0.0
//     0.8    10        1      0.1
//     1.0    10        1      0.1
//     1.2    10        1      0.1
//     1.5    10        1      0.2
//     1.8    10        1      0.2
//
var RoundHalfEven Rounder = roundHalfEven

// RoundFloor rounds towards negative infinity; that is, returns the greatest
// Dec not exceeding the result represented by quo and rem.
//
// The following table shows examples of the results for
// Quo(x, y, Scale(scale), RoundFloor).
//
//      x      y    scale   result
//  ------------------------------
//    -1.8    10        1     -0.2
//    -1.5    10        1     -0.2
//    -1.2    10        1     -0.2
//    -1.0    10        1     -0.1
//    -0.8    10        1     -0.1
//    -0.5    10        1     -0.1
//    -0.2    10        1     -0.1
//     0.0    10        1      0.0
//     0.2    10        1      0.0
//     0.5    10        1      0.0
//     0.8    10        1      0.0
//     1.0    10        1      0.1
//     1.2    10        1      0.1
//     1.5    10        1      0.1
//     1.8    10        1      0.1
//
var RoundFloor Rounder = roundFloor

// RoundCeil rounds towards positive infinity; that is, returns the
// smallest Dec not smaller than the result represented by quo and rem.
//
// The following table shows examples of the results for
// Quo(x, y, Scale(scale), RoundCeil).
//
//      x      y    scale   result
//  ------------------------------
//    -1.8    10        1     -0.1
//    -1.5    10        1     -0.1
//    -1.2    10        1     -0.1
//    -1.0    10        1     -0.1
//    -0.8    10        1     -0.0
//    -0.5    10        1     -0.0
//    -0.2    10        1     -0.0
//     0.0    10        1      0.0
//     0.2    10        1      0.1
//     0.5    10        1      0.1
//     0.8    10        1      0.1
//     1.0    10        1      0.1
//     1.2    10        1      0.2
//     1.5    10        1      0.2
//     1.8    10        1      0.2
//
var RoundCeil Rounder = roundCeil

var intSign = []*big.Int{big.NewInt(-1), big.NewInt(0), big.NewInt(1)}

var roundExact = rounder{true,
	func(z, q *Dec, rA, rB *big.Int) *Dec {
		if rA.Sign() != 0 {
			return nil
		}
		return z.move(q)
	}}

var roundDown = rounder{false,
	func(z, q *Dec, rA, rB *big.Int) *Dec {
		return z.move(q)
	}}

var roundUp = rounder{true,
	func(z, q *Dec, rA, rB *big.Int) *Dec {
		z.move(q)
		if rA.Sign() != 0 {
			z.Unscaled().Add(z.Unscaled(), intSign[rA.Sign()*rB.Sign()+1])
		}
		return z
	}}

func roundHalf(f func(c int, odd uint) (roundUp bool)) func(z, q *Dec, rA, rB *big.Int) *Dec {
	return func(z, q *Dec, rA, rB *big.Int) *Dec {
		z.move(q)
		brA, brB := rA.BitLen(), rB.BitLen()
		if brA < brB-1 {
			// brA < brB-1 => |rA| < |rB/2|
			return z
		}
		roundUp := false
		srA, srB := rA.Sign(), rB.Sign()
		s := srA * srB
		if brA == brB-1 {
			rA2 := new(big.Int).Lsh(rA, 1)
			if s < 0 {
				rA2.Neg(rA2)
			}
			roundUp = f(rA2.Cmp(rB)*srB, z.Unscaled().Bit(0))
		} else {
			// brA > brB-1 => |rA| > |rB/2|
			roundUp = true
		}
		if roundUp {
			z.Unscaled().Add(z.Unscaled(), intSign[s+1])
		}
		return z
	}
}

var roundHalfDown = rounder{true, roundHalf(
	func(c int, odd uint) bool {
		return c > 0
	})}

var roundHalfUp = rounder{true, roundHalf(
	func(c int, odd uint) bool {
		return c >= 0
	})}

var roundHalfEven = rounder{true, roundHalf(
	func(c int, odd uint) bool {
		return c > 0 || c == 0 && odd == 1
	})}

var roundFloor = rounder{true,
	func(z, q *Dec, rA, rB *big.Int) *Dec {
		z.move(q)
		if rA.Sign()*rB.Sign() < 0 {
			z.Unscaled().Add(z.Unscaled(), intSign[0])
		}
		return z
	}}

var roundCeil = rounder{true,
	func(z, q *Dec, rA, rB *big.Int) *Dec {
		z.move(q)
		if rA.Sign()*rB.Sign() > 0 {
			z.Unscaled().Add(z.Unscaled(), intSign[2])
		}
		return z
	}}
<|endoftext|>


<commit_before>package sqlstore

import (
	"fmt"
	"time"

	"github.com/raintank/raintank-apps/task-server/model"
)

func ValidateMetrics(orgId int64, metrics map[string]int64) error {
	sess, err := newSession(false, "metric")
	if err != nil {
		return err
	}
	return validateMetrics(sess, orgId, metrics)
}

func validateMetrics(sess *session, orgId int64, metrics map[string]int64) error {
	for namespace, ver := range metrics {
		//validate metrics
		mQuery := &model.GetMetricsQuery{
			Namespace: namespace,
			OrgId:     orgId,
			Version:   ver,
		}
		matches, err := getMetrics(sess, mQuery)
		if err != nil {
			return err
		}
		if len(matches) == 0 {
			return fmt.Errorf("no matching metric found.")
		}
		//Use the latest version available.
		if ver == 0 {
			for _, m := range matches {
				if m.Version > ver {
					ver = m.Version
				}
			}
			metrics[namespace] = ver
		}
	}
	return nil
}

func GetMetrics(query *model.GetMetricsQuery) ([]*model.Metric, error) {
	sess, err := newSession(false, "metric")
	if err != nil {
		return nil, err
	}
	return getMetrics(sess, query)
}

func getMetrics(sess *session, query *model.GetMetricsQuery) ([]*model.Metric, error) {
	metrics := make([]*model.Metric, 0)
	sess.Where("(public=1 OR org_id = ?)", query.OrgId)
	if query.Namespace != "" {
		sess.And("namespace like ?", query.Namespace)
	}
	if query.Version != 0 {
		sess.And("version = ?", query.Version)
	}
	if query.OrderBy == "" {
		query.OrderBy = "namespace"
	}
	if query.Limit == 0 {
		query.Limit = 50
	}
	if query.Page == 0 {
		query.Page = 1
	}
	sess.Asc(query.OrderBy).Limit(query.Limit, (query.Page-1)*query.Limit)
	err := sess.Find(&metrics)
	if err != nil {
		return nil, err
	}
	return metrics, nil
}

func GetMetricById(id string, orgId int64) (*model.Metric, error) {
	sess, err := newSession(false, "metric")
	if err != nil {
		return nil, err
	}

	return getMetricById(sess, id, orgId)
}

func getMetricById(sess *session, id string, orgId int64) (*model.Metric, error) {
	m := &model.Metric{}
	exists, err := sess.Where("(public=1 OR org_id = ?) AND id=?", orgId, id).Get(m)
	if err != nil {
		return nil, err
	}
	if !exists {
		return nil, nil
	}
	return m, nil
}

func AddMissingMetricsForAgent(a *model.AgentDTO, m []*model.Metric) error {
	sess, err := newSession(true, "metric")
	if err != nil {
		return err
	}
	defer sess.Cleanup()
	if err = addMissingMetricsForAgent(sess, a, m); err != nil {
		return err
	}
	sess.Complete()
	return nil
}

func addMissingMetricsForAgent(sess *session, agent *model.AgentDTO, metrics []*model.Metric) error {
	existing, err := getAgentMetrics(sess, agent)
	if err != nil {
		return err
	}
	existingMap := make(map[string]*model.Metric)
	seenMap := make(map[string]*model.Metric)
	inserts := make([]*model.Metric, 0)
	agentMetrics := make([]*model.AgentMetric, 0)
	for _, m := range existing {
		key := fmt.Sprintf("%s:%d", m.Namespace, m.Version)
		existingMap[key] = m
	}
	for _, m := range metrics {
		key := fmt.Sprintf("%s:%d", m.Namespace, m.Version)
		seenMap[key] = m
		if e, ok := existingMap[key]; ok {
			if e.Public != m.Public {
				// public attribute has changed. need to update
				_, err := sess.Id(e.Id).Update(m)
				if err != nil {
					return err
				}
			}
		} else {
			inserts = append(inserts, m)
			agentMetrics = append(agentMetrics, &model.AgentMetric{
				AgentId:   agent.Id,
				Namespace: m.Namespace,
				Version:   m.Version,
				Created:   time.Now(),
			})
		}
	}
	for key, m := range existingMap {
		if _, ok := seenMap[key]; !ok {
			// need to delete agent_metric association.
			rawSql := "DELETE from agent_metric where agent_id=? and namespace=? and version=?"
			if _, err := sess.Exec(rawSql, agent.Id, m.Namespace, m.Version); err != nil {
				return err
			}
		}
	}

	if len(inserts) > 0 {
		if _, err := sess.Insert(&inserts); err != nil {
			return err
		}
	}
	if len(agentMetrics) > 0 {
		if _, err := sess.Insert(agentMetrics); err != nil {
			return err
		}
	}

	return nil
}

func GetAgentMetrics(agent *model.AgentDTO) ([]*model.Metric, error) {
	sess, err := newSession(true, "metric")
	if err != nil {
		return nil, err
	}
	defer sess.Cleanup()
	metrics, err := getAgentMetrics(sess, agent)
	if err != nil {
		return nil, err
	}
	sess.Complete()
	return metrics, nil
}

func getAgentMetrics(sess *session, agent *model.AgentDTO) ([]*model.Metric, error) {
	metrics := make([]*model.Metric, 0)
	sess.Table("metric")
	sess.Join("INNER", "agent_metric", "metric.namespace = agent_metric.namespace AND metric.version = agent_metric.version")
	sess.Where("agent_metric.agent_id=?", agent.Id)
	sess.Cols("`metric`.*")
	err := sess.Find(&metrics)
	if err != nil {
		return nil, err
	}
	return metrics, nil
}
<commit_msg>update validateMetrics to handle wildcards<commit_after>package sqlstore

import (
	"fmt"
	"strings"
	"time"

	"github.com/raintank/raintank-apps/task-server/model"
)

func ValidateMetrics(orgId int64, metrics map[string]int64) error {
	sess, err := newSession(false, "metric")
	if err != nil {
		return err
	}
	return validateMetrics(sess, orgId, metrics)
}

func validateMetrics(sess *session, orgId int64, metrics map[string]int64) error {
	for namespace, ver := range metrics {
		// for SQL "like" query, "%" is the wildcard character.
		ns := strings.Replace(namespace, "*", "%", -1)
		mQuery := &model.GetMetricsQuery{
			Namespace: ns,
			OrgId:     orgId,
			Version:   ver,
		}
		matches, err := getMetrics(sess, mQuery)
		if err != nil {
			return err
		}
		if len(matches) == 0 {
			return fmt.Errorf("no matching metric found.")
		}
		//Use the latest version available.
		if ver == 0 {
			for _, m := range matches {
				if m.Version > ver {
					ver = m.Version
				}
			}
			metrics[namespace] = ver
		}
	}
	return nil
}

func GetMetrics(query *model.GetMetricsQuery) ([]*model.Metric, error) {
	sess, err := newSession(false, "metric")
	if err != nil {
		return nil, err
	}
	return getMetrics(sess, query)
}

func getMetrics(sess *session, query *model.GetMetricsQuery) ([]*model.Metric, error) {
	metrics := make([]*model.Metric, 0)
	sess.Where("(public=1 OR org_id = ?)", query.OrgId)
	if query.Namespace != "" {
		sess.And("namespace like ?", query.Namespace)
	}
	if query.Version != 0 {
		sess.And("version = ?", query.Version)
	}
	if query.OrderBy == "" {
		query.OrderBy = "namespace"
	}
	if query.Limit == 0 {
		query.Limit = 50
	}
	if query.Page == 0 {
		query.Page = 1
	}
	sess.Asc(query.OrderBy).Limit(query.Limit, (query.Page-1)*query.Limit)
	err := sess.Find(&metrics)
	if err != nil {
		return nil, err
	}
	return metrics, nil
}

func GetMetricById(id string, orgId int64) (*model.Metric, error) {
	sess, err := newSession(false, "metric")
	if err != nil {
		return nil, err
	}

	return getMetricById(sess, id, orgId)
}

func getMetricById(sess *session, id string, orgId int64) (*model.Metric, error) {
	m := &model.Metric{}
	exists, err := sess.Where("(public=1 OR org_id = ?) AND id=?", orgId, id).Get(m)
	if err != nil {
		return nil, err
	}
	if !exists {
		return nil, nil
	}
	return m, nil
}

func AddMissingMetricsForAgent(a *model.AgentDTO, m []*model.Metric) error {
	sess, err := newSession(true, "metric")
	if err != nil {
		return err
	}
	defer sess.Cleanup()
	if err = addMissingMetricsForAgent(sess, a, m); err != nil {
		return err
	}
	sess.Complete()
	return nil
}

func addMissingMetricsForAgent(sess *session, agent *model.AgentDTO, metrics []*model.Metric) error {
	existing, err := getAgentMetrics(sess, agent)
	if err != nil {
		return err
	}
	existingMap := make(map[string]*model.Metric)
	seenMap := make(map[string]*model.Metric)
	inserts := make([]*model.Metric, 0)
	agentMetrics := make([]*model.AgentMetric, 0)
	for _, m := range existing {
		key := fmt.Sprintf("%s:%d", m.Namespace, m.Version)
		existingMap[key] = m
	}
	for _, m := range metrics {
		key := fmt.Sprintf("%s:%d", m.Namespace, m.Version)
		seenMap[key] = m
		if e, ok := existingMap[key]; ok {
			if e.Public != m.Public {
				// public attribute has changed. need to update
				_, err := sess.Id(e.Id).Update(m)
				if err != nil {
					return err
				}
			}
		} else {
			inserts = append(inserts, m)
			agentMetrics = append(agentMetrics, &model.AgentMetric{
				AgentId:   agent.Id,
				Namespace: m.Namespace,
				Version:   m.Version,
				Created:   time.Now(),
			})
		}
	}
	for key, m := range existingMap {
		if _, ok := seenMap[key]; !ok {
			// need to delete agent_metric association.
			rawSql := "DELETE from agent_metric where agent_id=? and namespace=? and version=?"
			if _, err := sess.Exec(rawSql, agent.Id, m.Namespace, m.Version); err != nil {
				return err
			}
		}
	}

	if len(inserts) > 0 {
		if _, err := sess.Insert(&inserts); err != nil {
			return err
		}
	}
	if len(agentMetrics) > 0 {
		if _, err := sess.Insert(agentMetrics); err != nil {
			return err
		}
	}

	return nil
}

func GetAgentMetrics(agent *model.AgentDTO) ([]*model.Metric, error) {
	sess, err := newSession(true, "metric")
	if err != nil {
		return nil, err
	}
	defer sess.Cleanup()
	metrics, err := getAgentMetrics(sess, agent)
	if err != nil {
		return nil, err
	}
	sess.Complete()
	return metrics, nil
}

func getAgentMetrics(sess *session, agent *model.AgentDTO) ([]*model.Metric, error) {
	metrics := make([]*model.Metric, 0)
	sess.Table("metric")
	sess.Join("INNER", "agent_metric", "metric.namespace = agent_metric.namespace AND metric.version = agent_metric.version")
	sess.Where("agent_metric.agent_id=?", agent.Id)
	sess.Cols("`metric`.*")
	err := sess.Find(&metrics)
	if err != nil {
		return nil, err
	}
	return metrics, nil
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"fmt"
	"io"
	"math/rand"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/getlantern/cloudflare"
	"github.com/getlantern/enproxy"
	"github.com/getlantern/tlsdialer"
	"github.com/getlantern/withtimeout"
)

var (
	// Set a short ttl on DNS entries
	ttl = 30 * time.Second

	// Test with a period of half the ttl
	testPeriod = ttl / 2

	// If we haven't had a successul test or reset after this amount of time,
	// pause testing until receipt of next register call.
	pauseAfter = 10 * time.Minute

	// Limit how long we're willing to wait for status
	statusTimeout = ttl * 2

	dialTimeout    = 3 * time.Second // how long to wait on connecting to host
	requestTimeout = 6 * time.Second // how long to wait for test requests to process
	proxyAttempts  = 1               // how many times to try a test request before considering host down

	// Sites to use for testing connectivity. WARNING - these should only be
	// sites with consistent fast response times, around the world, otherwise
	// tests may time out.
	testSites = []string{"www.google.com", "www.youtube.com", "www.facebook.com"}
)

type status struct {
	online            bool
	connectionRefused bool
}

// host is an actor that represents a host entry in CloudFlare and is
// responsible for checking connectivity to the host and updating CloudFlare DNS
// accordingly. Once a host has been created, it sticks around ad infinitum.
// If the host hasn't heard from the real-world host in over 10 minutes, it
// pauses its processing and only resumes once it hears from the client again.
type host struct {
	name        string
	ip          string
	record      *cloudflare.Record
	groups      map[string]*group
	lastSuccess time.Time
	lastTest    time.Time

	resetCh      chan string
	unregisterCh chan interface{}
	statusCh     chan chan *status

	proxiedClient     *http.Client
	reportedHost      string
	reportedHostMutex sync.Mutex
}

func (h *host) String() string {
	return fmt.Sprintf("%v (%v)", h.name, h.ip)
}

/*******************************************************************************
 * API for interacting with host
 ******************************************************************************/

// newHost creates a new host for the given name, ip and optional DNS record.
func newHost(name string, ip string, record *cloudflare.Record) *host {
	// Cache TLS sessions
	clientSessionCache := tls.NewLRUClientSessionCache(1000)

	h := &host{
		name:         name,
		ip:           ip,
		record:       record,
		resetCh:      make(chan string, 1000),
		unregisterCh: make(chan interface{}, 1),
		statusCh:     make(chan chan *status, 1000),
	}
	h.proxiedClient = &http.Client{
		Transport: &http.Transport{
			Dial: func(network, addr string) (net.Conn, error) {
				return enproxy.Dial(addr, &enproxy.Config{
					DialProxy: func(addr string) (net.Conn, error) {
						return tlsdialer.DialWithDialer(&net.Dialer{
							Timeout: dialTimeout,
						}, "tcp", ip+":443", true, &tls.Config{
							InsecureSkipVerify: true,
							ClientSessionCache: clientSessionCache,
						})
					},
					NewRequest: func(upstreamHost string, method string, body io.Reader) (req *http.Request, err error) {
						return http.NewRequest(method, "http://"+ip+"/", body)
					},
					OnFirstResponse: func(resp *http.Response) {
						h.reportedHostMutex.Lock()
						h.reportedHost = resp.Header.Get(enproxy.X_ENPROXY_PROXY_HOST)
						h.reportedHostMutex.Unlock()
					},
				})
			},
			DisableKeepAlives: true,
		},
		Timeout: requestTimeout,
	}

	if h.isFallback() {
		h.groups = map[string]*group{
			RoundRobin: &group{subdomain: RoundRobin},
			Fallbacks:  &group{subdomain: Fallbacks},
		}
	} else {
		h.groups = map[string]*group{
			Peers: &group{subdomain: Peers},
		}
	}

	return h
}

// status returns the status of this host as of the next scheduled check
func (h *host) status() (online bool, connectionRefused bool, timedOut bool) {
	// Buffer the channel so that if we time out, reportStatus can still report
	// without blocking.
	sch := make(chan *status, 1)
	h.statusCh <- sch
	select {
	case s := <-sch:
		return s.online, s.connectionRefused, false
	case <-time.After(statusTimeout):
		return false, false, true
	}
}

// reset resets this host's run loop in response to the host having reported in,
// which can include changing the name if the given name is new.
func (h *host) reset(newName string) {
	h.resetCh <- newName
}

// unregister unregisters this host in response to the host having requested
// unregistration.
func (h *host) unregister() {
	select {
	case h.unregisterCh <- nil:
		log.Tracef("Unregistering host %v", h)
	default:
		log.Tracef("Already unregistering host %v, ignoring new request", h)
	}
}

/*******************************************************************************
 * Implementation
 ******************************************************************************/

// run is the main run loop for this host
func (h *host) run() {
	checkImmediately := true
	h.lastSuccess = time.Now()
	h.lastTest = time.Now()
	periodTimer := time.NewTimer(0)
	pauseTimer := time.NewTimer(0)

	for {
		if !checkImmediately {
			// Limit the rate at which we run tests
			waitTime := h.lastTest.Add(testPeriod).Sub(time.Now())
			log.Tracef("Waiting %v until testing %v", waitTime, h)
			periodTimer.Reset(waitTime)
		}

		// Pause run loop after some largish amount of time
		pauseTimer.Reset(h.lastSuccess.Add(pauseAfter).Sub(time.Now()))

		select {
		case newName := <-h.resetCh:
			h.doReset(newName)
		case <-h.unregisterCh:
			log.Debugf("Unregistering %v and pausing", h)
			h.pause()
			checkImmediately = true
		case <-pauseTimer.C:
			log.Debugf("%v had no successful checks or resets in %v, pausing", h, pauseAfter)
			h.pause()
			checkImmediately = true
		case <-periodTimer.C:
			log.Tracef("Testing %v", h)
			_s, timedOut, err := withtimeout.Do(ttl, func() (interface{}, error) {
				online, connectionRefused, err := h.isAbleToProxy()
				return &status{online, connectionRefused}, err
			})
			s := &status{false, false}
			if timedOut {
				log.Debugf("Testing %v timed out unexpectedly", h)
			}
			if _s != nil {
				s = _s.(*status)
			}
			h.reportStatus(s)
			h.lastTest = time.Now()
			checkImmediately = false
			if s.online {
				log.Tracef("Test for %v successful", h)
				h.lastSuccess = time.Now()
				err := h.register()
				if err != nil {
					log.Error(err)
				}
			} else {
				log.Tracef("Test for %v failed with error: %v", h, err)
				// Deregister this host from its rotations. We leave the host
				// itself registered to support continued sticky routing in case
				// any clients still have connections open to it.
				h.deregisterFromRotations()
			}
		}
	}
}

// pause deregisters this host completely and then waits for the next reset
// before continuing
func (h *host) pause() {
	h.deregister()
	log.Debugf("%v paused", h)
	for {
		select {
		case newName := <-h.resetCh:
			log.Debugf("Unpausing checks for %v", h)
			h.doReset(newName)
			return
		case <-h.unregisterCh:
			log.Tracef("Ignoring unregister while paused")
		}
	}
}

// reportStatus reports the given status back to any callers that are waiting
// for it.
func (h *host) reportStatus(s *status) {
	for {
		select {
		case sch := <-h.statusCh:
			sch <- s
		default:
			return
		}
	}
}

func (h *host) doReset(newName string) {
	log.Tracef("Host notified us of its presence")
	h.lastSuccess = time.Now()
	h.lastTest = time.Time{}
	if newName != h.name {
		log.Debugf("Hostname for %v changed to %v", h, newName)
		if h.record != nil {
			log.Debugf("Deregistering old hostname %v", h.name)
			h.doDeregisterHost()
		}
		h.name = newName
	}
}

/*******************************************************************************
 * Functions for managing DNS
 ******************************************************************************/

func (h *host) register() error {
	err := h.registerHost()
	if err != nil {
		return fmt.Errorf("Unable to register host: %v", err)
	}
	err = h.registerToRotations()
	if err != nil {
		return fmt.Errorf("Unable to register rotations: %v", err)
	}
	return nil
}

func (h *host) registerHost() error {
	if h.record != nil {
		log.Tracef("Host already registered, no need to re-register: %v", h)
		return nil
	}

	log.Debugf("Registering %v", h)

	rec, err := cflutil.Register(h.name, h.ip)
	if err == nil || isDuplicateError(err) {
		h.record = rec
		err = nil
	}
	return err
}

func (h *host) registerToRotations() error {
	for _, group := range h.groups {
		err := group.register(h)
		if err != nil && !isDuplicateError(err) {
			return err
		}
	}
	return nil
}

func (h *host) deregister() {
	h.deregisterHost()
	h.deregisterFromRotations()
}

func (h *host) deregisterHost() {
	if h.record == nil {
		log.Debugf("Host not registered, no need to deregister: %v", h)
		return
	}

	if h.isFallback() {
		log.Debugf("Currently not deregistering fallbacks like %v", h)
		return
	}

	log.Debugf("Deregistering %v", h)
	h.doDeregisterHost()
}

func (h *host) doDeregisterHost() {
	err := cflutil.DestroyRecord(h.record)
	if err != nil {
		log.Errorf("Unable to deregister host %v: %v", h, err)
		return
	}

	h.record = nil
}

func (h *host) deregisterFromRotations() {
	for _, group := range h.groups {
		group.deregister(h)
	}
}

func (h *host) fullName() string {
	return h.name + ".getiantem.org"
}

func (h *host) isFallback() bool {
	return isFallback(h.name)
}

func (h *host) isAbleToProxy() (bool, bool, error) {
	// Check whether or not we can proxy a few times
	var lastErr error
	for i := 0; i < proxyAttempts; i++ {
		success, connectionRefused, err := h.doIsAbleToProxy()
		if err != nil {
			log.Debugf("Error testing %v: %v", h, err.Error())
		}
		lastErr = err
		if success || connectionRefused {
			// If we've succeeded, or our connection was flat-out refused, don't
			// bother trying to proxy again

			if success {
				// Make sure that the server is reporting the correct host name for sticky
				// routing.
				h.reportedHostMutex.Lock()
				defer h.reportedHostMutex.Unlock()
				if h.reportedHost != h.fullName() {
					success = false
					lastErr := fmt.Errorf("%v is reporting an unexpected host %v", h, h.reportedHost)
					log.Error(lastErr.Error())
				}
			}

			return success, connectionRefused, lastErr
		}
	}
	return false, false, lastErr
}

func (h *host) doIsAbleToProxy() (bool, bool, error) {
	// First just try a plain TCP connection. This is useful because the
	// underlying TCP-level error is consumed in the flashlight layer, and we
	// need that to be accessible on the client side in the logic for deciding
	// whether or not to display the port mapping message.
	addr := h.ip + ":443"
	conn, err := net.DialTimeout("tcp", addr, dialTimeout)
	if err != nil {
		err2 := fmt.Errorf("Unable to connect to %v: %v", addr, err)
		return false, strings.Contains(err.Error(), "connection refused"), err2
	}
	conn.Close()

	// Now actually try to proxy an http request
	site := testSites[rand.Intn(len(testSites))]
	resp, err := h.proxiedClient.Head("http://" + site)
	if err != nil {
		return false, false, fmt.Errorf("Unable to make proxied HEAD request to %v: %v", site, err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 && resp.StatusCode != 301 {
		err2 := fmt.Errorf("Proxying to %v via %v returned unexpected status %d,", site, h.ip, resp.StatusCode)
		return false, false, err2
	}

	return true, false, nil
}

func isDuplicateError(err error) bool {
	return strings.Contains(err.Error(), "The record already exists.")
}
<commit_msg>fix hardcoded domain<commit_after>package main

import (
	"crypto/tls"
	"fmt"
	"io"
	"math/rand"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/getlantern/cloudflare"
	"github.com/getlantern/enproxy"
	"github.com/getlantern/tlsdialer"
	"github.com/getlantern/withtimeout"
)

var (
	// Set a short ttl on DNS entries
	ttl = 30 * time.Second

	// Test with a period of half the ttl
	testPeriod = ttl / 2

	// If we haven't had a successul test or reset after this amount of time,
	// pause testing until receipt of next register call.
	pauseAfter = 10 * time.Minute

	// Limit how long we're willing to wait for status
	statusTimeout = ttl * 2

	dialTimeout    = 3 * time.Second // how long to wait on connecting to host
	requestTimeout = 6 * time.Second // how long to wait for test requests to process
	proxyAttempts  = 1               // how many times to try a test request before considering host down

	// Sites to use for testing connectivity. WARNING - these should only be
	// sites with consistent fast response times, around the world, otherwise
	// tests may time out.
	testSites = []string{"www.google.com", "www.youtube.com", "www.facebook.com"}
)

type status struct {
	online            bool
	connectionRefused bool
}

// host is an actor that represents a host entry in CloudFlare and is
// responsible for checking connectivity to the host and updating CloudFlare DNS
// accordingly. Once a host has been created, it sticks around ad infinitum.
// If the host hasn't heard from the real-world host in over 10 minutes, it
// pauses its processing and only resumes once it hears from the client again.
type host struct {
	name        string
	ip          string
	record      *cloudflare.Record
	groups      map[string]*group
	lastSuccess time.Time
	lastTest    time.Time

	resetCh      chan string
	unregisterCh chan interface{}
	statusCh     chan chan *status

	proxiedClient     *http.Client
	reportedHost      string
	reportedHostMutex sync.Mutex
}

func (h *host) String() string {
	return fmt.Sprintf("%v (%v)", h.name, h.ip)
}

/*******************************************************************************
 * API for interacting with host
 ******************************************************************************/

// newHost creates a new host for the given name, ip and optional DNS record.
func newHost(name string, ip string, record *cloudflare.Record) *host {
	// Cache TLS sessions
	clientSessionCache := tls.NewLRUClientSessionCache(1000)

	h := &host{
		name:         name,
		ip:           ip,
		record:       record,
		resetCh:      make(chan string, 1000),
		unregisterCh: make(chan interface{}, 1),
		statusCh:     make(chan chan *status, 1000),
	}
	h.proxiedClient = &http.Client{
		Transport: &http.Transport{
			Dial: func(network, addr string) (net.Conn, error) {
				return enproxy.Dial(addr, &enproxy.Config{
					DialProxy: func(addr string) (net.Conn, error) {
						return tlsdialer.DialWithDialer(&net.Dialer{
							Timeout: dialTimeout,
						}, "tcp", ip+":443", true, &tls.Config{
							InsecureSkipVerify: true,
							ClientSessionCache: clientSessionCache,
						})
					},
					NewRequest: func(upstreamHost string, method string, body io.Reader) (req *http.Request, err error) {
						return http.NewRequest(method, "http://"+ip+"/", body)
					},
					OnFirstResponse: func(resp *http.Response) {
						h.reportedHostMutex.Lock()
						h.reportedHost = resp.Header.Get(enproxy.X_ENPROXY_PROXY_HOST)
						h.reportedHostMutex.Unlock()
					},
				})
			},
			DisableKeepAlives: true,
		},
		Timeout: requestTimeout,
	}

	if h.isFallback() {
		h.groups = map[string]*group{
			RoundRobin: &group{subdomain: RoundRobin},
			Fallbacks:  &group{subdomain: Fallbacks},
		}
	} else {
		h.groups = map[string]*group{
			Peers: &group{subdomain: Peers},
		}
	}

	return h
}

// status returns the status of this host as of the next scheduled check
func (h *host) status() (online bool, connectionRefused bool, timedOut bool) {
	// Buffer the channel so that if we time out, reportStatus can still report
	// without blocking.
	sch := make(chan *status, 1)
	h.statusCh <- sch
	select {
	case s := <-sch:
		return s.online, s.connectionRefused, false
	case <-time.After(statusTimeout):
		return false, false, true
	}
}

// reset resets this host's run loop in response to the host having reported in,
// which can include changing the name if the given name is new.
func (h *host) reset(newName string) {
	h.resetCh <- newName
}

// unregister unregisters this host in response to the host having requested
// unregistration.
func (h *host) unregister() {
	select {
	case h.unregisterCh <- nil:
		log.Tracef("Unregistering host %v", h)
	default:
		log.Tracef("Already unregistering host %v, ignoring new request", h)
	}
}

/*******************************************************************************
 * Implementation
 ******************************************************************************/

// run is the main run loop for this host
func (h *host) run() {
	checkImmediately := true
	h.lastSuccess = time.Now()
	h.lastTest = time.Now()
	periodTimer := time.NewTimer(0)
	pauseTimer := time.NewTimer(0)

	for {
		if !checkImmediately {
			// Limit the rate at which we run tests
			waitTime := h.lastTest.Add(testPeriod).Sub(time.Now())
			log.Tracef("Waiting %v until testing %v", waitTime, h)
			periodTimer.Reset(waitTime)
		}

		// Pause run loop after some largish amount of time
		pauseTimer.Reset(h.lastSuccess.Add(pauseAfter).Sub(time.Now()))

		select {
		case newName := <-h.resetCh:
			h.doReset(newName)
		case <-h.unregisterCh:
			log.Debugf("Unregistering %v and pausing", h)
			h.pause()
			checkImmediately = true
		case <-pauseTimer.C:
			log.Debugf("%v had no successful checks or resets in %v, pausing", h, pauseAfter)
			h.pause()
			checkImmediately = true
		case <-periodTimer.C:
			log.Tracef("Testing %v", h)
			_s, timedOut, err := withtimeout.Do(ttl, func() (interface{}, error) {
				online, connectionRefused, err := h.isAbleToProxy()
				return &status{online, connectionRefused}, err
			})
			s := &status{false, false}
			if timedOut {
				log.Debugf("Testing %v timed out unexpectedly", h)
			}
			if _s != nil {
				s = _s.(*status)
			}
			h.reportStatus(s)
			h.lastTest = time.Now()
			checkImmediately = false
			if s.online {
				log.Tracef("Test for %v successful", h)
				h.lastSuccess = time.Now()
				err := h.register()
				if err != nil {
					log.Error(err)
				}
			} else {
				log.Tracef("Test for %v failed with error: %v", h, err)
				// Deregister this host from its rotations. We leave the host
				// itself registered to support continued sticky routing in case
				// any clients still have connections open to it.
				h.deregisterFromRotations()
			}
		}
	}
}

// pause deregisters this host completely and then waits for the next reset
// before continuing
func (h *host) pause() {
	h.deregister()
	log.Debugf("%v paused", h)
	for {
		select {
		case newName := <-h.resetCh:
			log.Debugf("Unpausing checks for %v", h)
			h.doReset(newName)
			return
		case <-h.unregisterCh:
			log.Tracef("Ignoring unregister while paused")
		}
	}
}

// reportStatus reports the given status back to any callers that are waiting
// for it.
func (h *host) reportStatus(s *status) {
	for {
		select {
		case sch := <-h.statusCh:
			sch <- s
		default:
			return
		}
	}
}

func (h *host) doReset(newName string) {
	log.Tracef("Host notified us of its presence")
	h.lastSuccess = time.Now()
	h.lastTest = time.Time{}
	if newName != h.name {
		log.Debugf("Hostname for %v changed to %v", h, newName)
		if h.record != nil {
			log.Debugf("Deregistering old hostname %v", h.name)
			h.doDeregisterHost()
		}
		h.name = newName
	}
}

/*******************************************************************************
 * Functions for managing DNS
 ******************************************************************************/

func (h *host) register() error {
	err := h.registerHost()
	if err != nil {
		return fmt.Errorf("Unable to register host: %v", err)
	}
	err = h.registerToRotations()
	if err != nil {
		return fmt.Errorf("Unable to register rotations: %v", err)
	}
	return nil
}

func (h *host) registerHost() error {
	if h.record != nil {
		log.Tracef("Host already registered, no need to re-register: %v", h)
		return nil
	}

	log.Debugf("Registering %v", h)

	rec, err := cflutil.Register(h.name, h.ip)
	if err == nil || isDuplicateError(err) {
		h.record = rec
		err = nil
	}
	return err
}

func (h *host) registerToRotations() error {
	for _, group := range h.groups {
		err := group.register(h)
		if err != nil && !isDuplicateError(err) {
			return err
		}
	}
	return nil
}

func (h *host) deregister() {
	h.deregisterHost()
	h.deregisterFromRotations()
}

func (h *host) deregisterHost() {
	if h.record == nil {
		log.Debugf("Host not registered, no need to deregister: %v", h)
		return
	}

	if h.isFallback() {
		log.Debugf("Currently not deregistering fallbacks like %v", h)
		return
	}

	log.Debugf("Deregistering %v", h)
	h.doDeregisterHost()
}

func (h *host) doDeregisterHost() {
	err := cflutil.DestroyRecord(h.record)
	if err != nil {
		log.Errorf("Unable to deregister host %v: %v", h, err)
		return
	}

	h.record = nil
}

func (h *host) deregisterFromRotations() {
	for _, group := range h.groups {
		group.deregister(h)
	}
}

func (h *host) fullName() string {
	return h.name + "." + *cfldomain
}

func (h *host) isFallback() bool {
	return isFallback(h.name)
}

func (h *host) isAbleToProxy() (bool, bool, error) {
	// Check whether or not we can proxy a few times
	var lastErr error
	for i := 0; i < proxyAttempts; i++ {
		success, connectionRefused, err := h.doIsAbleToProxy()
		if err != nil {
			log.Debugf("Error testing %v: %v", h, err.Error())
		}
		lastErr = err
		if success || connectionRefused {
			// If we've succeeded, or our connection was flat-out refused, don't
			// bother trying to proxy again

			if success {
				// Make sure that the server is reporting the correct host name for sticky
				// routing.
				h.reportedHostMutex.Lock()
				defer h.reportedHostMutex.Unlock()
				if h.reportedHost != h.fullName() {
					success = false
					lastErr := fmt.Errorf("%v is reporting an unexpected host %v", h, h.reportedHost)
					log.Error(lastErr.Error())
				}
			}

			return success, connectionRefused, lastErr
		}
	}
	return false, false, lastErr
}

func (h *host) doIsAbleToProxy() (bool, bool, error) {
	// First just try a plain TCP connection. This is useful because the
	// underlying TCP-level error is consumed in the flashlight layer, and we
	// need that to be accessible on the client side in the logic for deciding
	// whether or not to display the port mapping message.
	addr := h.ip + ":443"
	conn, err := net.DialTimeout("tcp", addr, dialTimeout)
	if err != nil {
		err2 := fmt.Errorf("Unable to connect to %v: %v", addr, err)
		return false, strings.Contains(err.Error(), "connection refused"), err2
	}
	conn.Close()

	// Now actually try to proxy an http request
	site := testSites[rand.Intn(len(testSites))]
	resp, err := h.proxiedClient.Head("http://" + site)
	if err != nil {
		return false, false, fmt.Errorf("Unable to make proxied HEAD request to %v: %v", site, err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 && resp.StatusCode != 301 {
		err2 := fmt.Errorf("Proxying to %v via %v returned unexpected status %d,", site, h.ip, resp.StatusCode)
		return false, false, err2
	}

	return true, false, nil
}

func isDuplicateError(err error) bool {
	return strings.Contains(err.Error(), "The record already exists.")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"

	"github.com/meisterluk/dupfiles-go/api"
	"github.com/meisterluk/dupfiles-go/match"
	"github.com/meisterluk/dupfiles-go/traversal"
)

func main() {
	var conf api.Config
	conf.HashAlgorithm = "sha512"
	conf.HashSpec.Content = true
	conf.HashSpec.Relpath = true

	bases := make([]api.Source, 0, 5)

	if len(os.Args)%2 == 0 {
		log.Println("Usage: ./dupfiles <NAME1> <PATH1> [<NAME2> <PATH2>]+")
		log.Fatal("Error: number of command-line arguments must be even")
	}

	if len(os.Args) == 1 {
		log.Print("usage: dupfiles <name> <path> [<name> <path>]*")
		os.Exit(0)
	}

	// create file system root instances
	for i := 1; i < len(os.Args); i = i + 2 {
		bases = append(bases, api.Source{Path: os.Args[i+1], Name: os.Args[i]})
	}

	// get ready for traversal
	trees := make([]api.Tree, 0, len(bases))
	treePtrs := make([]*api.Tree, 0, 5)
	for _, base := range bases {
		t := api.Tree{}
		trees = append(trees, t)
		treePtrs = append(treePtrs, &t)
		err := traversal.DFSTraverse(&conf, &base, &t)
		if err != nil {
			log.Fatal(err)
		}
	}

	done := make(chan bool)
	eqChan := make(chan []*api.Entry)

	go func() {
		for eq := range eqChan {
			fmt.Printf("{ ")
			for _, e := range eq {
				fmt.Printf(" %s %s  ", e.Base, e.Path)
			}
			fmt.Printf("}\n")
		}
		done <- true
	}()

	err := match.Match(&conf, treePtrs, eqChan)
	if err != nil {
		log.Fatal(err)
	}
	close(eqChan)
	<-done
}
<commit_msg>Introduce -v or --version<commit_after>package main

import (
	"fmt"
	"log"
	"os"

	"github.com/meisterluk/dupfiles-go/api"
	"github.com/meisterluk/dupfiles-go/match"
	"github.com/meisterluk/dupfiles-go/traversal"
)

func main() {
	var conf api.Config
	conf.HashAlgorithm = "sha512"
	conf.HashSpec.Content = true
	conf.HashSpec.Relpath = true

	bases := make([]api.Source, 0, 5)

	if os.Args[1] == "-v" || os.Args[1] == "--version" {
		log.Println("version 0.1 basic")
		os.Exit(0)
	}

	if len(os.Args)%2 == 0 {
		log.Println("Usage: ./dupfiles <NAME1> <PATH1> [<NAME2> <PATH2>]+")
		log.Fatal("Error: number of command-line arguments must be even")
	}

	if len(os.Args) == 1 {
		log.Print("usage: dupfiles <name> <path> [<name> <path>]*")
		os.Exit(0)
	}

	// create file system root instances
	for i := 1; i < len(os.Args); i = i + 2 {
		bases = append(bases, api.Source{Path: os.Args[i+1], Name: os.Args[i]})
	}

	// get ready for traversal
	trees := make([]api.Tree, 0, len(bases))
	treePtrs := make([]*api.Tree, 0, 5)
	for _, base := range bases {
		t := api.Tree{}
		trees = append(trees, t)
		treePtrs = append(treePtrs, &t)
		err := traversal.DFSTraverse(&conf, &base, &t)
		if err != nil {
			log.Fatal(err)
		}
	}

	done := make(chan bool)
	eqChan := make(chan []*api.Entry)

	go func() {
		for eq := range eqChan {
			fmt.Printf("{ ")
			for _, e := range eq {
				fmt.Printf(" %s %s  ", e.Base, e.Path)
			}
			fmt.Printf("}\n")
		}
		done <- true
	}()

	err := match.Match(&conf, treePtrs, eqChan)
	if err != nil {
		log.Fatal(err)
	}
	close(eqChan)
	<-done
}
<|endoftext|>


<commit_before>package db

import (
	"encoding/json"
	"fmt"
	"math/rand"
	"os"
	"strconv"
	"testing"
	"time"
)

const QUERY_BENCH_SIZE = 1000000 // Number of documents made available for query benchmark

func ensureMapHasKeys(m map[uint64]bool, keys ...uint64) bool {
	if len(m) != len(keys) {
		return false
	}
	for _, v := range keys {
		if _, ok := m[v]; !ok {
			return false
		}
	}
	return true
}

func runQuery(query string, col *Col) (map[uint64]bool, error) {
	result := make(map[uint64]bool)
	var jq interface{}
	if err := json.Unmarshal([]byte(query), &jq); err != nil {
		fmt.Println(err)
	}
	return result, EvalQuery(jq, col, &result)
}

func TestQuery(t *testing.T) {
	// prepare a collection of documents
	tmp := "/tmp/tiedot_query_test"
	os.RemoveAll(tmp)
	defer os.RemoveAll(tmp)
	col, err := OpenCol(tmp)
	if err != nil {
		t.Error(err)
		return
	}
	docs := []string{
		`{"a": {"b": [1]}, "c": 1, "d": 1, "special": {"thing": null} }`,
		`{"a": {"b": 1}, "c": [1], "d": 2}`,
		`{"a": {"b": [2]}, "c": 2, "d": 1}`,
		`{"a": {"b": 3}, "c": [3], "d": 2}`,
		`{"a": {"b": [4]}, "c": 4, "d": 1}`}
	ids := [5]uint64{}
	for i, doc := range docs {
		var jsonDoc interface{}
		json.Unmarshal([]byte(doc), &jsonDoc)
		if ids[i], err = col.Insert(jsonDoc); err != nil {
			fmt.Println(err)
			return
		}
	}
	col.Index([]string{"a", "b"})
	// hash scan
	q, err := runQuery(`["=", {"eq": 1, "in": ["a", "b"]}]`, col)
	if err != nil {
		t.Error(err)
	}
	if !ensureMapHasKeys(q, ids[0], ids[1]) {
		t.Error(q)
	}
	q, err = runQuery(`["=", {"eq": 1, "limit": 1, "in": ["a", "b"]}]`, col)
	if err != nil {
		fmt.Println(err)
	}
	if !ensureMapHasKeys(q, ids[0]) {
		t.Error(q)
	}
	// collection scan
	q, err = runQuery(`["=", {"eq": 1, "in": ["c"]}]`, col)
	if err != nil {
		t.Error(err)
	}
	if !ensureMapHasKeys(q, ids[0], ids[1]) {
		t.Error(q)
	}
	q, err = runQuery(`["=", {"eq": 1, "limit": 1, "in": ["c"]}]`, col)
	if err != nil {
		t.Error(err)
	}
	if !ensureMapHasKeys(q, ids[0]) {
		t.Error(q)
	}
	// all documents
	q, err = runQuery(`["all"]`, col)
	if err != nil {
		t.Error(err)
	}
	if !ensureMapHasKeys(q, ids[0], ids[1], ids[2], ids[3], ids[4]) {
		t.Error(q)
	}
	// union
	q, err = runQuery(`["u", ["=", {"eq": 4, "limit": 1, "in": ["a", "b"]}], ["=", {"eq": 1, "limit": 1, "in": ["c"]}]]`, col)
	if err != nil {
		t.Error(err)
	}
	if !ensureMapHasKeys(q, ids[0], ids[4]) {
		t.Error(q)
	}
	// intersection
	q, err = runQuery(`["n", ["=", {"eq": 2, "in": ["d"]}], ["all"]]`, col)
	if err != nil {
		t.Error(err)
	}
	if !ensureMapHasKeys(q, ids[1], ids[3]) {
		t.Error(q)
	}
	// complement
	q, err = runQuery(`["\\", ["=", {"eq": 4,  "in": ["c"]}], ["=", {"eq": 2, "in": ["d"]}], ["all"]]`, col)
	if err != nil {
		t.Error(err)
	}
	if !ensureMapHasKeys(q, ids[0], ids[2]) {
		t.Error(q)
	}
	// lookup on "special"
	q, err = runQuery(`["=", {"eq": {"thing": null},  "in": ["special"]}]`, col)
	if err != nil {
		t.Error(err)
	}
	if !ensureMapHasKeys(q, ids[0]) {
		t.Error(q)
	}
	col.Close()
}

func BenchmarkQuery(b *testing.B) {
	rand.Seed(time.Now().UTC().UnixNano())
	tmp := "/tmp/tiedot_query_bench"
	os.RemoveAll(tmp)
	defer os.RemoveAll(tmp)
	col, err := OpenCol(tmp)
	if err != nil {
		b.Errorf("Failed to open: %v", err)
		return
	}
	col.Index([]string{"a", "b", "c"})
	var jsonDoc interface{}
	var ids [QUERY_BENCH_SIZE]uint64
	for i := 0; i < QUERY_BENCH_SIZE; i++ {
		err := json.Unmarshal([]byte(`{"a": {"b": {"c": `+strconv.Itoa(rand.Intn(QUERY_BENCH_SIZE))+`}}, "d": "abcdefghijklmnopqrstuvwxyz"}`), &jsonDoc)
		ids[i], err = col.Insert(jsonDoc)
		if err != nil {
			b.Error(err)
		}
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		_, err := runQuery(fmt.Sprintf("[\"u\", [\"=\", {\"eq\": %d, \"limit\": 1, \"in\": [\"a\", \"b\", \"c\"]}], [\"=\", {\"eq\": %d, \"limit\": 1, \"in\": [\"a\", \"b\", \"c\"]}]]", rand.Intn(QUERY_BENCH_SIZE), rand.Intn(QUERY_BENCH_SIZE)), col)
		if err != nil {
			b.Error(err)
		}
	}
}
<commit_msg>query benchmark<commit_after>package db

import (
	"encoding/json"
	"fmt"
	"math/rand"
	"os"
	"strconv"
	"testing"
	"time"
)

const QUERY_BENCH_SIZE = 1000000 // Number of documents made available for query benchmark

func ensureMapHasKeys(m map[uint64]bool, keys ...uint64) bool {
	if len(m) != len(keys) {
		return false
	}
	for _, v := range keys {
		if _, ok := m[v]; !ok {
			return false
		}
	}
	return true
}

func runQuery(query string, col *Col) (map[uint64]bool, error) {
	result := make(map[uint64]bool)
	var jq interface{}
	if err := json.Unmarshal([]byte(query), &jq); err != nil {
		fmt.Println(err)
	}
	return result, EvalQuery(jq, col, &result)
}

func TestQuery(t *testing.T) {
	// prepare a collection of documents
	tmp := "/tmp/tiedot_query_test"
	os.RemoveAll(tmp)
	defer os.RemoveAll(tmp)
	col, err := OpenCol(tmp)
	if err != nil {
		t.Error(err)
		return
	}
	docs := []string{
		`{"a": {"b": [1]}, "c": 1, "d": 1, "special": {"thing": null} }`,
		`{"a": {"b": 1}, "c": [1], "d": 2}`,
		`{"a": {"b": [2]}, "c": 2, "d": 1}`,
		`{"a": {"b": 3}, "c": [3], "d": 2}`,
		`{"a": {"b": [4]}, "c": 4, "d": 1}`}
	ids := [5]uint64{}
	for i, doc := range docs {
		var jsonDoc interface{}
		json.Unmarshal([]byte(doc), &jsonDoc)
		if ids[i], err = col.Insert(jsonDoc); err != nil {
			fmt.Println(err)
			return
		}
	}
	col.Index([]string{"a", "b"})
	// hash scan
	q, err := runQuery(`["=", {"eq": 1, "in": ["a", "b"]}]`, col)
	if err != nil {
		t.Error(err)
	}
	if !ensureMapHasKeys(q, ids[0], ids[1]) {
		t.Error(q)
	}
	q, err = runQuery(`["=", {"eq": 1, "limit": 1, "in": ["a", "b"]}]`, col)
	if err != nil {
		fmt.Println(err)
	}
	if !ensureMapHasKeys(q, ids[0]) {
		t.Error(q)
	}
	// collection scan
	q, err = runQuery(`["=", {"eq": 1, "in": ["c"]}]`, col)
	if err != nil {
		t.Error(err)
	}
	if !ensureMapHasKeys(q, ids[0], ids[1]) {
		t.Error(q)
	}
	q, err = runQuery(`["=", {"eq": 1, "limit": 1, "in": ["c"]}]`, col)
	if err != nil {
		t.Error(err)
	}
	if !ensureMapHasKeys(q, ids[0]) {
		t.Error(q)
	}
	// all documents
	q, err = runQuery(`["all"]`, col)
	if err != nil {
		t.Error(err)
	}
	if !ensureMapHasKeys(q, ids[0], ids[1], ids[2], ids[3], ids[4]) {
		t.Error(q)
	}
	// union
	q, err = runQuery(`["u", ["=", {"eq": 4, "limit": 1, "in": ["a", "b"]}], ["=", {"eq": 1, "limit": 1, "in": ["c"]}]]`, col)
	if err != nil {
		t.Error(err)
	}
	if !ensureMapHasKeys(q, ids[0], ids[4]) {
		t.Error(q)
	}
	// intersection
	q, err = runQuery(`["n", ["=", {"eq": 2, "in": ["d"]}], ["all"]]`, col)
	if err != nil {
		t.Error(err)
	}
	if !ensureMapHasKeys(q, ids[1], ids[3]) {
		t.Error(q)
	}
	// complement
	q, err = runQuery(`["\\", ["=", {"eq": 4,  "in": ["c"]}], ["=", {"eq": 2, "in": ["d"]}], ["all"]]`, col)
	if err != nil {
		t.Error(err)
	}
	if !ensureMapHasKeys(q, ids[0], ids[2]) {
		t.Error(q)
	}
	// lookup on "special"
	q, err = runQuery(`["=", {"eq": {"thing": null},  "in": ["special"]}]`, col)
	if err != nil {
		t.Error(err)
	}
	if !ensureMapHasKeys(q, ids[0]) {
		t.Error(q)
	}
	col.Close()
}

func BenchmarkQuery(b *testing.B) {
	rand.Seed(time.Now().UTC().UnixNano())
	tmp := "/tmp/tiedot_query_bench"
	os.RemoveAll(tmp)
	defer os.RemoveAll(tmp)
	col, err := OpenCol(tmp)
	if err != nil {
		b.Errorf("Failed to open: %v", err)
		return
	}
	col.Index([]string{"a", "b", "c"})
	var jsonDoc interface{}
	var ids [QUERY_BENCH_SIZE]uint64
	for i := 0; i < QUERY_BENCH_SIZE; i++ {
		err := json.Unmarshal([]byte(`{"a": {"b": {"c": `+strconv.Itoa(rand.Intn(QUERY_BENCH_SIZE))+`}}, "d": "abcdefghijklmnopqrstuvwxyz"}`), &jsonDoc)
		ids[i], err = col.Insert(jsonDoc)
		if err != nil {
			b.Error(err)
		}
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		_, err := runQuery(fmt.Sprintf("[\"u\", [\"=\", {\"eq\": %d, \"limit\": 1, \"in\": [\"a\", \"b\", \"c\"]}], [\"=\", {\"eq\": %d, \"limit\": 1, \"in\": [\"a\", \"b\", \"c\"]}]]", rand.Intn(QUERY_BENCH_SIZE), rand.Intn(QUERY_BENCH_SIZE)), col)
		if err != nil {
			b.Error(err)
		}
	}
	col.Close()
}
<|endoftext|>


<commit_before>package id

import "testing"

func TestGenSafeUniqueSlug(t *testing.T) {
	slug := "slug"
	r := map[string]bool{}

	for i := 0; i < 1000; i++ {
		s := GenSafeUniqueSlug(slug)
		if s == slug {
			t.Errorf("Got same slug as inputted!")
		}
		if _, ok := r[s]; ok {
			t.Errorf("#%d: slug %s was already generated in testing.", i, s)
		}
		r[s] = true
	}
}
<commit_msg>Don't fail test when repeat slug generated<commit_after>package id

import "testing"

func TestGenSafeUniqueSlug(t *testing.T) {
	slug := "slug"
	r := map[string]bool{}

	for i := 0; i < 1000; i++ {
		s := GenSafeUniqueSlug(slug)
		if s == slug {
			t.Errorf("Got same slug as inputted!")
		}
		if _, ok := r[s]; ok {
			t.Logf("#%d: slug %s was already generated in testing.", i, s)
		}
		r[s] = true
	}
}
<|endoftext|>


<commit_before>package ftail

import (
	"bytes"
	"compress/zlib"
	"hash"
	"hash/fnv"
	"io"
	"log"
	"os"
	"strconv"
	"time"

	"github.com/masahide/ftailer/core"
	"github.com/masahide/ftailer/tail"
	"github.com/masahide/ftailer/tailex"
	"golang.org/x/net/context"
)

//const defaultMaxHeadHashSize = 1024

type Config struct {
	Name            string
	BufDir          string
	Period          time.Duration // 分割保存インターバル
	MaxHeadHashSize int64

	tailex.Config
}

type Ftail struct {
	rec       *core.Recorder
	lastSlice time.Time
	Pos       *core.Position
	Config

	buf bytes.Buffer
	*zlib.Writer
	lastTime time.Time
	headHash hash.Hash64
}

var tailDefaultConfig = tail.Config{
	Follow: true,
	ReOpen: true,
	Poll:   false,
	//OpenNotify:  true,
	//MaxLineSize:    16 * 1024 * 1024, // 16MB
	NotifyInterval: 1 * time.Second,
}

// ポジション情報がない場合に実ファイルから取得
func position(c Config) (pos *core.Position, err error) {
	var fi os.FileInfo
	filePath := c.Path
	if c.PathFmt != "" { // cronolog
		timeSlice := tailex.Truncate(c.Config.Time, c.RotatePeriod)
		searchPath := tailex.Time2Path(c.PathFmt, timeSlice)
		filePath, err = tailex.GlobSearch(searchPath)
		if err == tailex.ErrNoSuchFile {
			log.Printf("ftail position() GlobSearch(%s)  err: %s", searchPath, err)
			return &core.Position{}, nil
		} else if err != nil {
			log.Printf("ftail position() GlobSearch(%s)  err: %s", searchPath, err)
			return nil, err
		}
	}
	if fi, err = os.Stat(filePath); err != nil {
		log.Printf("Start os.Stat('%s')  err: %s,  ", filePath, err)
		return nil, err
	}
	pos = &core.Position{
		Name:     filePath,
		CreateAt: fi.ModTime(),
		Offset:   0,
	}
	return
}

func Start(ctx context.Context, c Config) error {
	f := &Ftail{
		Config:   c,
		headHash: fnv.New64(),
	}
	//if f.MaxHeadHashSize == 0 {
	//	f.MaxHeadHashSize = defaultMaxHeadHashSize
	//}
	var err error
	f.rec, err = core.NewRecorder(c.BufDir, c.Name, c.Period)
	if err != nil {
		log.Fatalln("NewRecorder err:", err)
	}
	defer f.rec.AllClose()

	f.Pos = f.rec.Position()
	if f.Pos == nil {
		if f.Pos, err = position(c); err != nil {
			log.Fatalln("position err:", err)
		}
	}
	f.Config.Config.Config = tailDefaultConfig
	f.ReOpenDelay = 5 * time.Second
	if f.Delay != 0 {
		f.ReOpenDelay = f.Delay
	}
	log.Printf("f.Pos: %s", f.Pos)

	if f.MaxHeadHashSize != 0 && f.Pos.Name != "" {
		hash, length, err := f.getHeadHash(f.Pos.Name, f.Pos.HashLength)
		if err != nil {
			log.Printf("getHeadHash err:%s", err)
		} else {
			if f.Pos.HeadHash == hash && f.Pos.HashLength == length { // ポジションファイルのハッシュ値と一致した場合はSeekInfoをセット
				log.Printf("match headHash: %s", f.Pos)
				f.Location = &tail.SeekInfo{Offset: f.Pos.Offset}
			} else {
				log.Printf("not match headHash: %s", f.Pos)
				f.Pos.HeadHash = hash
				f.Pos.HashLength = length
			}
		}
	} else {
		posTimeSlise := tailex.Truncate(f.Pos.CreateAt, c.RotatePeriod)
		nowTimeSlise := tailex.Truncate(time.Now(), c.RotatePeriod)
		if nowTimeSlise.Equal(posTimeSlise) { // 読み込んだポジションのcreateAtが現在のtimesliseと同じ場合
			f.Location = &tail.SeekInfo{Offset: f.Pos.Offset}
		}
	}
	t := tailex.TailFile(ctx, f.Config.Config)
	//var buf bytes.Buffer
	f.buf = bytes.Buffer{}
	f.Writer, err = zlib.NewWriterLevel(&f.buf, zlib.BestCompression)
	if err != nil {
		log.Fatalln("NewZlibWriter err:", err)
	}
	defer f.Flush()

	for {
		select {
		case <-ctx.Done(): // キャンセル処理
			return ctx.Err()
		case line, ok := <-t.Lines: // 新しい入力行の取得
			if !ok {
				return err
			}
			err := f.lineNotifyAction(ctx, line)
			if err != nil {
				return err
			}
		}

	}

}

// lineのNotifyType別に処理を分岐
func (f *Ftail) lineNotifyAction(ctx context.Context, line *tail.Line) error {
	var err error
	switch line.NotifyType {
	case tail.NewLineNotify: // 新しいライン
		err = f.Write(line)
		if err != nil {
			return err
		}
	case tail.TickerNotify, tailex.GlobLoopNotify: // 定期flush処理
		if err := f.Flush(); err != nil {
			return err
		}
		timeSlice := tailex.Truncate(line.Time, f.Period)
		if f.lastSlice.Sub(timeSlice) < 0 {
			// 新しいDBを開く
			if _, err = f.rec.CreateDB(timeSlice, f.Pos); err != nil {
				log.Printf("CreateDB err:%s", err)
				return err
			}
			f.lastSlice = timeSlice
		}
		// 古いDBを閉じる
		if _, err := f.rec.CloseOldDbs(line.Time); err != nil {
			log.Printf("CloseOldDbs err:%s", err)
			return err
		}
	case tail.NewFileNotify:
		f.lastTime = line.Time
		f.Pos.Name = line.Filename
		f.Pos.CreateAt = line.OpenTime
		f.Pos.Offset = line.Offset
		f.Pos.HeadHash, f.Pos.HashLength, err = f.getHeadHash(f.Pos.Name, f.MaxHeadHashSize)
		if err != nil {
			log.Printf("getHeadHash err:%s", err)
			return err
		}
		log.Printf("NewFileNotify getHeadHash :%s", f.Pos)
	}
	return nil
}

func (f *Ftail) addHash(line []byte) error {
	restSize := f.MaxHeadHashSize - f.Pos.HashLength
	if int64(len(line)) >= restSize {
		line = line[0:restSize]
	}
	written, err := f.headHash.Write(line)
	if err != nil {
		return err
	}
	f.Pos.HeadHash = strconv.FormatUint(f.headHash.Sum64(), 16)
	f.Pos.HashLength += int64(written)
	return nil
}

func (f *Ftail) Write(line *tail.Line) (err error) {
	f.lastTime = line.Time
	f.Pos.Name = line.Filename
	f.Pos.CreateAt = line.OpenTime
	f.Pos.Offset = line.Offset
	if f.Pos.HashLength < f.MaxHeadHashSize {
		if err := f.addHash(line.Text); err != nil {
			return nil
		}
	}
	_, err = f.Writer.Write(line.Text)
	return err
}

func (f *Ftail) Flush() error {
	if f.buf.Len() <= 0 {
		return nil
	}
	f.Writer.Close()
	err := f.rec.Put(core.Record{Time: f.lastTime, Data: f.buf.Bytes()}, f.Pos)
	f.buf.Reset()
	f.Reset(&f.buf)
	if err != nil {
		log.Printf("Flush %s err:%s", f.Pos.Name, err)
	}
	return err
}

func (f *Ftail) getHeadHash(fname string, getLength int64) (hash string, length int64, err error) {
	if f.MaxHeadHashSize == 0 || f.Pos.Name == "" {
		return "", 0, nil
	}
	var readFile *os.File
	readFile, err = os.Open(fname)
	if err != nil {
		return
	}
	defer readFile.Close()
	f.headHash = fnv.New64()
	length, err = io.CopyN(f.headHash, readFile, getLength)
	switch err {
	case nil:
	case io.EOF:
		err = nil
	default:
		return
	}
	hash = strconv.FormatUint(f.headHash.Sum64(), 16)
	return
}
<commit_msg>デバッグコードを追加<commit_after>package ftail

import (
	"bytes"
	"compress/zlib"
	"hash"
	"hash/fnv"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"time"

	"github.com/masahide/ftailer/core"
	"github.com/masahide/ftailer/tail"
	"github.com/masahide/ftailer/tailex"
	"golang.org/x/net/context"
)

//const defaultMaxHeadHashSize = 1024

type Config struct {
	Name            string
	BufDir          string
	Period          time.Duration // 分割保存インターバル
	MaxHeadHashSize int64

	tailex.Config
}

type Ftail struct {
	rec       *core.Recorder
	lastSlice time.Time
	Pos       *core.Position
	Config

	buf bytes.Buffer
	*zlib.Writer
	lastTime time.Time
	headHash hash.Hash64
	head     []byte
}

var tailDefaultConfig = tail.Config{
	Follow: true,
	ReOpen: true,
	Poll:   false,
	//OpenNotify:  true,
	//MaxLineSize:    16 * 1024 * 1024, // 16MB
	NotifyInterval: 1 * time.Second,
}

// ポジション情報がない場合に実ファイルから取得
func position(c Config) (pos *core.Position, err error) {
	var fi os.FileInfo
	filePath := c.Path
	if c.PathFmt != "" { // cronolog
		timeSlice := tailex.Truncate(c.Config.Time, c.RotatePeriod)
		searchPath := tailex.Time2Path(c.PathFmt, timeSlice)
		filePath, err = tailex.GlobSearch(searchPath)
		if err == tailex.ErrNoSuchFile {
			log.Printf("ftail position() GlobSearch(%s)  err: %s", searchPath, err)
			return &core.Position{}, nil
		} else if err != nil {
			log.Printf("ftail position() GlobSearch(%s)  err: %s", searchPath, err)
			return nil, err
		}
	}
	if fi, err = os.Stat(filePath); err != nil {
		log.Printf("Start os.Stat('%s')  err: %s,  ", filePath, err)
		return nil, err
	}
	pos = &core.Position{
		Name:     filePath,
		CreateAt: fi.ModTime(),
		Offset:   0,
	}
	return
}

func Start(ctx context.Context, c Config) error {
	f := &Ftail{
		Config:   c,
		headHash: fnv.New64(),
		head:     []byte{},
	}
	//if f.MaxHeadHashSize == 0 {
	//	f.MaxHeadHashSize = defaultMaxHeadHashSize
	//}
	var err error
	f.rec, err = core.NewRecorder(c.BufDir, c.Name, c.Period)
	if err != nil {
		log.Fatalln("NewRecorder err:", err)
	}
	defer f.rec.AllClose()

	f.Pos = f.rec.Position()
	if f.Pos == nil {
		if f.Pos, err = position(c); err != nil {
			log.Fatalln("position err:", err)
		}
	}
	f.Config.Config.Config = tailDefaultConfig
	f.ReOpenDelay = 5 * time.Second
	if f.Delay != 0 {
		f.ReOpenDelay = f.Delay
	}
	log.Printf("f.Pos: %s", f.Pos)

	if f.MaxHeadHashSize != 0 && f.Pos.Name != "" {
		oldhead := f.head
		hash, length, err := f.getHeadHash(f.Pos.Name, f.Pos.HashLength)
		if err != nil {
			log.Printf("getHeadHash err:%s", err)
		} else {
			if f.Pos.HeadHash == hash && f.Pos.HashLength == length { // ポジションファイルのハッシュ値と一致した場合はSeekInfoをセット
				log.Printf("match headHash: %s, head:%s", f.Pos, f.head)
				f.Location = &tail.SeekInfo{Offset: f.Pos.Offset}
			} else {
				log.Printf("not match headHash: %s, head:%s", oldhead)
				f.Pos.HeadHash = hash
				f.Pos.HashLength = length
				log.Printf("not match headHash new: %s, head:%s", f.Pos, f.head)
			}
		}
	} else {
		posTimeSlise := tailex.Truncate(f.Pos.CreateAt, c.RotatePeriod)
		nowTimeSlise := tailex.Truncate(time.Now(), c.RotatePeriod)
		if nowTimeSlise.Equal(posTimeSlise) { // 読み込んだポジションのcreateAtが現在のtimesliseと同じ場合
			f.Location = &tail.SeekInfo{Offset: f.Pos.Offset}
		}
	}
	t := tailex.TailFile(ctx, f.Config.Config)
	//var buf bytes.Buffer
	f.buf = bytes.Buffer{}
	f.Writer, err = zlib.NewWriterLevel(&f.buf, zlib.BestCompression)
	if err != nil {
		log.Fatalln("NewZlibWriter err:", err)
	}
	defer f.Flush()

	for {
		select {
		case <-ctx.Done(): // キャンセル処理
			return ctx.Err()
		case line, ok := <-t.Lines: // 新しい入力行の取得
			if !ok {
				return err
			}
			err := f.lineNotifyAction(ctx, line)
			if err != nil {
				return err
			}
		}

	}

}

// lineのNotifyType別に処理を分岐
func (f *Ftail) lineNotifyAction(ctx context.Context, line *tail.Line) error {
	var err error
	switch line.NotifyType {
	case tail.NewLineNotify: // 新しいライン
		err = f.Write(line)
		if err != nil {
			return err
		}
	case tail.TickerNotify, tailex.GlobLoopNotify: // 定期flush処理
		if err := f.Flush(); err != nil {
			return err
		}
		timeSlice := tailex.Truncate(line.Time, f.Period)
		if f.lastSlice.Sub(timeSlice) < 0 {
			// 新しいDBを開く
			if _, err = f.rec.CreateDB(timeSlice, f.Pos); err != nil {
				log.Printf("CreateDB err:%s", err)
				return err
			}
			f.lastSlice = timeSlice
		}
		// 古いDBを閉じる
		if _, err := f.rec.CloseOldDbs(line.Time); err != nil {
			log.Printf("CloseOldDbs err:%s", err)
			return err
		}
	case tail.NewFileNotify:
		f.lastTime = line.Time
		f.Pos.Name = line.Filename
		f.Pos.CreateAt = line.OpenTime
		f.Pos.Offset = line.Offset
		f.Pos.HeadHash, f.Pos.HashLength, err = f.getHeadHash(f.Pos.Name, f.MaxHeadHashSize)
		if err != nil {
			log.Printf("getHeadHash err:%s", err)
			return err
		}
		log.Printf("NewFileNotify getHeadHash :%s", f.Pos)
	}
	return nil
}

func (f *Ftail) addHash(line []byte) error {
	restSize := f.MaxHeadHashSize - f.Pos.HashLength
	if int64(len(line)) >= restSize {
		line = line[0:restSize]
	}
	written, err := f.headHash.Write(line)
	if err != nil {
		return err
	}
	f.head = append(f.head, line...)
	f.Pos.HeadHash = strconv.FormatUint(f.headHash.Sum64(), 16)
	f.Pos.HashLength += int64(written)
	log.Printf("addHash name:%s Pos:%s, head:%s", f.Name, f.Pos, string(f.head))
	return nil
}

func (f *Ftail) Write(line *tail.Line) (err error) {
	f.lastTime = line.Time
	f.Pos.Name = line.Filename
	f.Pos.CreateAt = line.OpenTime
	f.Pos.Offset = line.Offset
	if f.Pos.HashLength < f.MaxHeadHashSize {
		if err := f.addHash(line.Text); err != nil {
			return err
		}
	}
	_, err = f.Writer.Write(line.Text)
	return err
}

func (f *Ftail) Flush() error {
	if f.buf.Len() <= 0 {
		return nil
	}
	f.Writer.Close()
	err := f.rec.Put(core.Record{Time: f.lastTime, Data: f.buf.Bytes()}, f.Pos)
	f.buf.Reset()
	f.Reset(&f.buf)
	if err != nil {
		log.Printf("Flush %s err:%s", f.Pos.Name, err)
	}
	return err
}

func (f *Ftail) getHeadHash(fname string, getLength int64) (hash string, length int64, err error) {
	if f.MaxHeadHashSize == 0 || f.Pos.Name == "" {
		return "", 0, nil
	}
	var readFile *os.File
	readFile, err = os.Open(fname)
	if err != nil {
		return
	}
	defer readFile.Close()
	f.headHash = fnv.New64()
	tee := io.TeeReader(io.LimitReader(readFile, getLength), f.headHash)
	f.head, err = ioutil.ReadAll(tee)
	length = int64(len(f.head))
	//length, err = io.CopyN(f.headHash, readFile, getLength)
	switch err {
	case nil:
	case io.EOF:
		err = nil
	default:
		return
	}
	hash = strconv.FormatUint(f.headHash.Sum64(), 16)
	return
}
<|endoftext|>