<commit_before>package main

import (
	"encoding/json"
	"flag"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/thomersch/grandine/lib/cugdf"
	"github.com/thomersch/grandine/lib/mvt"
	"github.com/thomersch/grandine/lib/spatial"
	"github.com/thomersch/grandine/lib/tile"
)

var zoomlevels = []int{6, 7, 8, 9, 10, 11}

func main() {
	source := flag.String("src", "geo.geojson", "file to read from, supported formats: geojson, cugdf")
	target := flag.String("target", "tiles", "path where the tiles will be written")
	defaultLayer := flag.Bool("default-layer", true, "...")
	flag.Parse()

	f, err := os.Open(*source)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	err = os.MkdirAll(*target, 0777)
	if err != nil {
		log.Fatal(err)
	}

	log.Println("parsing input...")
	fc := spatial.FeatureCollection{}

	if strings.HasSuffix(strings.ToLower(*source), "geojson") {
		if err := json.NewDecoder(f).Decode(&fc); err != nil {
			log.Fatal(err)
		}
	} else {
		fc.Features, err = cugdf.Unmarshal(f)
		if err != nil {
			log.Fatal(err)
		}
	}

	log.Printf("read %d features", len(fc.Features))

	var bboxPts []spatial.Point
	for _, feat := range fc.Features {
		bb := feat.Geometry.BBox()
		bboxPts = append(bboxPts, bb.SW, bb.NE)
	}

	bbox := spatial.Line(bboxPts).BBox()
	log.Println("filtering features...")

	features := spatial.Filter(fc.Features, bbox)
	if len(features) == 0 {
		log.Println("no features to be processed, exiting.")
		os.Exit(2)
	}
	log.Printf("%d features to be processed", len(features))

	var tc []tile.ID
	for _, zoomlevel := range zoomlevels {
		tc = append(tc, tile.Coverage(bbox, zoomlevel)...)
	}
	log.Printf("attempting to generate %d tiles", len(tc))

	dtw := diskTileWriter{basedir: *target}
	dlm := defaultLayerMapper{defaultLayer: *defaultLayer}
	generateTiles(tc, features, &dtw, &dlm)
}

type diskTileWriter struct {
	basedir string
}

func (tw *diskTileWriter) WriteTile(tID tile.ID, buf []byte) error {
	err := os.MkdirAll(filepath.Join(tw.basedir, strconv.Itoa(tID.Z), strconv.Itoa(tID.X)), 0777)
	if err != nil {
		return err
	}
	tf, err := os.Create(filepath.Join(tw.basedir, strconv.Itoa(tID.Z), strconv.Itoa(tID.X), strconv.Itoa(tID.Y)+".mvt"))
	if err != nil {
		return err
	}
	defer tf.Close()
	_, err = tf.Write(buf)
	if err != nil {
		return err
	}
	return nil
}

type defaultLayerMapper struct {
	defaultLayer bool
}

func (dlm *defaultLayerMapper) LayerName(props map[string]interface{}) string {
	if _, ok := props["highway"]; ok {
		return "transportation"
	}
	if dlm.defaultLayer {
		return "default"
	}
	return ""
}

type layerMapper interface {
	LayerName(map[string]interface{}) string
}

type tileWriter interface {
	WriteTile(tile.ID, []byte) error
}

func generateTiles(tIDs []tile.ID, features []spatial.Feature, tw tileWriter, lm layerMapper) {
	for _, tID := range tIDs {
		log.Printf("Generating %v", tID)
		var layers = map[string][]spatial.Feature{}
		tileClipBBox := tID.BBox()
		for _, feat := range spatial.Filter(features, tileClipBBox) {
			for _, geom := range feat.Geometry.ClipToBBox(tileClipBBox) {
				feat.Geometry = geom
				ln := lm.LayerName(feat.Props)
				if len(ln) != 0 {
					if _, ok := layers[ln]; !ok {
						layers[ln] = []spatial.Feature{feat}
					} else {
						layers[ln] = append(layers[ln], feat)
					}
				}
			}
		}
		if !anyFeatures(layers) {
			continue
		}
		buf, err := mvt.EncodeTile(layers, tID)
		if err != nil {
			log.Fatal(err)
		}

		err = tw.WriteTile(tID, buf)
		if err != nil {
			log.Fatal(err)
		}
	}
}

func anyFeatures(layers map[string][]spatial.Feature) bool {
	for _, ly := range layers {
		if len(ly) > 0 {
			return true
		}
	}
	return false
}
<commit_msg>cmd/tiler: layer name from properties<commit_after>package main

import (
	"encoding/json"
	"flag"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/thomersch/grandine/lib/cugdf"
	"github.com/thomersch/grandine/lib/mvt"
	"github.com/thomersch/grandine/lib/spatial"
	"github.com/thomersch/grandine/lib/tile"
)

var zoomlevels = []int{6, 7, 8, 9, 10, 11}

func main() {
	source := flag.String("src", "geo.geojson", "file to read from, supported formats: geojson, cugdf")
	target := flag.String("target", "tiles", "path where the tiles will be written")
	defaultLayer := flag.Bool("default-layer", true, "...")
	flag.Parse()

	f, err := os.Open(*source)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	err = os.MkdirAll(*target, 0777)
	if err != nil {
		log.Fatal(err)
	}

	log.Println("parsing input...")
	fc := spatial.FeatureCollection{}

	if strings.HasSuffix(strings.ToLower(*source), "geojson") {
		if err := json.NewDecoder(f).Decode(&fc); err != nil {
			log.Fatal(err)
		}
	} else {
		fc.Features, err = cugdf.Unmarshal(f)
		if err != nil {
			log.Fatal(err)
		}
	}

	log.Printf("read %d features", len(fc.Features))

	var bboxPts []spatial.Point
	for _, feat := range fc.Features {
		bb := feat.Geometry.BBox()
		bboxPts = append(bboxPts, bb.SW, bb.NE)
	}

	bbox := spatial.Line(bboxPts).BBox()
	log.Println("filtering features...")

	features := spatial.Filter(fc.Features, bbox)
	if len(features) == 0 {
		log.Println("no features to be processed, exiting.")
		os.Exit(2)
	}
	log.Printf("%d features to be processed", len(features))

	var tc []tile.ID
	for _, zoomlevel := range zoomlevels {
		tc = append(tc, tile.Coverage(bbox, zoomlevel)...)
	}
	log.Printf("attempting to generate %d tiles", len(tc))

	dtw := diskTileWriter{basedir: *target}
	dlm := defaultLayerMapper{defaultLayer: *defaultLayer}
	generateTiles(tc, features, &dtw, &dlm)
}

type diskTileWriter struct {
	basedir string
}

func (tw *diskTileWriter) WriteTile(tID tile.ID, buf []byte) error {
	err := os.MkdirAll(filepath.Join(tw.basedir, strconv.Itoa(tID.Z), strconv.Itoa(tID.X)), 0777)
	if err != nil {
		return err
	}
	tf, err := os.Create(filepath.Join(tw.basedir, strconv.Itoa(tID.Z), strconv.Itoa(tID.X), strconv.Itoa(tID.Y)+".mvt"))
	if err != nil {
		return err
	}
	defer tf.Close()
	_, err = tf.Write(buf)
	if err != nil {
		return err
	}
	return nil
}

type defaultLayerMapper struct {
	defaultLayer bool
}

func (dlm *defaultLayerMapper) LayerName(props map[string]interface{}) string {
	if layerName, ok := props["@layer"]; ok {
		return layerName.(string)
	}
	if dlm.defaultLayer {
		return "default"
	}
	return ""
}

type layerMapper interface {
	LayerName(map[string]interface{}) string
}

type tileWriter interface {
	WriteTile(tile.ID, []byte) error
}

func generateTiles(tIDs []tile.ID, features []spatial.Feature, tw tileWriter, lm layerMapper) {
	for _, tID := range tIDs {
		log.Printf("Generating %v", tID)
		var layers = map[string][]spatial.Feature{}
		tileClipBBox := tID.BBox()
		for _, feat := range spatial.Filter(features, tileClipBBox) {
			for _, geom := range feat.Geometry.ClipToBBox(tileClipBBox) {
				feat.Geometry = geom
				ln := lm.LayerName(feat.Props)
				if len(ln) != 0 {
					if _, ok := layers[ln]; !ok {
						layers[ln] = []spatial.Feature{feat}
					} else {
						layers[ln] = append(layers[ln], feat)
					}
				}
			}
		}
		if !anyFeatures(layers) {
			continue
		}
		buf, err := mvt.EncodeTile(layers, tID)
		if err != nil {
			log.Fatal(err)
		}

		err = tw.WriteTile(tID, buf)
		if err != nil {
			log.Fatal(err)
		}
	}
}

func anyFeatures(layers map[string][]spatial.Feature) bool {
	for _, ly := range layers {
		if len(ly) > 0 {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/thomersch/grandine/lib/cugdf"
	"github.com/thomersch/grandine/lib/mvt"
	"github.com/thomersch/grandine/lib/spatial"
	"github.com/thomersch/grandine/lib/tile"
)

var zoomlevels = []int{6, 7, 8, 9, 10, 11}

func main() {
	source := flag.String("src", "geo.geojson", "file to read from, supported formats: geojson, cugdf")
	target := flag.String("target", "tiles", "path where the tiles will be written")
	defaultLayer := flag.Bool("default-layer", true, "...")
	flag.Parse()

	f, err := os.Open(*source)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	err = os.MkdirAll(*target, 0777)
	if err != nil {
		log.Fatal(err)
	}

	log.Println("parsing input...")
	fc := spatial.FeatureCollection{}

	if strings.HasSuffix(strings.ToLower(*source), "geojson") {
		if err := json.NewDecoder(f).Decode(&fc); err != nil {
			log.Fatal(err)
		}
	} else {
		fc.Features, err = cugdf.Unmarshal(f)
		if err != nil {
			log.Fatal(err)
		}
	}

	log.Printf("read %d features", len(fc.Features))

	var bboxPts []spatial.Point
	for _, feat := range fc.Features {
		bb := feat.Geometry.BBox()
		bboxPts = append(bboxPts, bb.SW, bb.NE)
	}

	bbox := spatial.Line(bboxPts).BBox()
	log.Println("filtering features...")

	// TODO: consider using rtree
	features := spatial.Filter(fc.Features, bbox)
	if len(features) == 0 {
		log.Println("no features to be processed, exiting.")
		os.Exit(2)
	}
	log.Printf("%d features to be processed", len(features))

	var tc []tile.ID
	for _, zoomlevel := range zoomlevels {
		tc = append(tc, tile.Coverage(bbox, zoomlevel)...)
	}
	log.Printf("attempting to generate %d tiles", len(tc))

	dtw := diskTileWriter{basedir: *target}
	dlm := defaultLayerMapper{defaultLayer: *defaultLayer}
	generateTiles(tc, features, &dtw, &dlm)
}

type diskTileWriter struct {
	basedir string
}

func (tw *diskTileWriter) WriteTile(tID tile.ID, buf []byte) error {
	err := os.MkdirAll(filepath.Join(tw.basedir, strconv.Itoa(tID.Z), strconv.Itoa(tID.X)), 0777)
	if err != nil {
		return err
	}
	tf, err := os.Create(filepath.Join(tw.basedir, strconv.Itoa(tID.Z), strconv.Itoa(tID.X), strconv.Itoa(tID.Y)+".mvt"))
	if err != nil {
		return err
	}
	defer tf.Close()
	_, err = tf.Write(buf)
	if err != nil {
		return err
	}
	return nil
}

type defaultLayerMapper struct {
	defaultLayer bool
}

func (dlm *defaultLayerMapper) LayerName(props map[string]interface{}) string {
	if layerName, ok := props["@layer"]; ok {
		return layerName.(string)
	}
	if dlm.defaultLayer {
		return "default"
	}
	return ""
}

type layerMapper interface {
	LayerName(map[string]interface{}) string
}

type tileWriter interface {
	WriteTile(tile.ID, []byte) error
}

func generateTiles(tIDs []tile.ID, features []spatial.Feature, tw tileWriter, lm layerMapper) {
	for _, tID := range tIDs {
		log.Printf("Generating %v", tID)
		var layers = map[string][]spatial.Feature{}
		tileClipBBox := tID.BBox()
		for _, feat := range spatial.Filter(features, tileClipBBox) {
			for _, geom := range feat.Geometry.ClipToBBox(tileClipBBox) {
				feat.Geometry = geom
				ln := lm.LayerName(feat.Props)
				if len(ln) != 0 {
					if _, ok := layers[ln]; !ok {
						layers[ln] = []spatial.Feature{feat}
					} else {
						layers[ln] = append(layers[ln], feat)
					}
				}
			}
		}
		if !anyFeatures(layers) {
			continue
		}
		buf, err := mvt.EncodeTile(layers, tID)
		if err != nil {
			log.Fatal(err)
		}

		err = tw.WriteTile(tID, buf)
		if err != nil {
			log.Fatal(err)
		}
	}
}

func anyFeatures(layers map[string][]spatial.Feature) bool {
	for _, ly := range layers {
		if len(ly) > 0 {
			return true
		}
	}
	return false
}
<commit_msg>tiler: check if there are any features<commit_after>package main

import (
	"encoding/json"
	"flag"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/thomersch/grandine/lib/cugdf"
	"github.com/thomersch/grandine/lib/mvt"
	"github.com/thomersch/grandine/lib/spatial"
	"github.com/thomersch/grandine/lib/tile"
)

var zoomlevels = []int{6, 7, 8, 9, 10, 11}

func main() {
	source := flag.String("src", "geo.geojson", "file to read from, supported formats: geojson, cugdf")
	target := flag.String("target", "tiles", "path where the tiles will be written")
	defaultLayer := flag.Bool("default-layer", true, "...")
	flag.Parse()

	f, err := os.Open(*source)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	err = os.MkdirAll(*target, 0777)
	if err != nil {
		log.Fatal(err)
	}

	log.Println("parsing input...")
	fc := spatial.FeatureCollection{}

	if strings.HasSuffix(strings.ToLower(*source), "geojson") {
		if err := json.NewDecoder(f).Decode(&fc); err != nil {
			log.Fatal(err)
		}
	} else {
		fc.Features, err = cugdf.Unmarshal(f)
		if err != nil {
			log.Fatal(err)
		}
	}

	if len(fc.Features) == 0 {
		log.Fatal("no features in input file")
	}

	log.Printf("read %d features", len(fc.Features))

	var bboxPts []spatial.Point
	for _, feat := range fc.Features {
		bb := feat.Geometry.BBox()
		bboxPts = append(bboxPts, bb.SW, bb.NE)
	}

	bbox := spatial.Line(bboxPts).BBox()
	log.Println("filtering features...")

	// TODO: consider using rtree
	features := spatial.Filter(fc.Features, bbox)
	if len(features) == 0 {
		log.Println("no features to be processed, exiting.")
		os.Exit(2)
	}
	log.Printf("%d features to be processed", len(features))

	var tc []tile.ID
	for _, zoomlevel := range zoomlevels {
		tc = append(tc, tile.Coverage(bbox, zoomlevel)...)
	}
	log.Printf("attempting to generate %d tiles", len(tc))

	dtw := diskTileWriter{basedir: *target}
	dlm := defaultLayerMapper{defaultLayer: *defaultLayer}
	generateTiles(tc, features, &dtw, &dlm)
}

type diskTileWriter struct {
	basedir string
}

func (tw *diskTileWriter) WriteTile(tID tile.ID, buf []byte) error {
	err := os.MkdirAll(filepath.Join(tw.basedir, strconv.Itoa(tID.Z), strconv.Itoa(tID.X)), 0777)
	if err != nil {
		return err
	}
	tf, err := os.Create(filepath.Join(tw.basedir, strconv.Itoa(tID.Z), strconv.Itoa(tID.X), strconv.Itoa(tID.Y)+".mvt"))
	if err != nil {
		return err
	}
	defer tf.Close()
	_, err = tf.Write(buf)
	if err != nil {
		return err
	}
	return nil
}

type defaultLayerMapper struct {
	defaultLayer bool
}

func (dlm *defaultLayerMapper) LayerName(props map[string]interface{}) string {
	if layerName, ok := props["@layer"]; ok {
		return layerName.(string)
	}
	if dlm.defaultLayer {
		return "default"
	}
	return ""
}

type layerMapper interface {
	LayerName(map[string]interface{}) string
}

type tileWriter interface {
	WriteTile(tile.ID, []byte) error
}

func generateTiles(tIDs []tile.ID, features []spatial.Feature, tw tileWriter, lm layerMapper) {
	for _, tID := range tIDs {
		log.Printf("Generating %v", tID)
		var layers = map[string][]spatial.Feature{}
		tileClipBBox := tID.BBox()
		for _, feat := range spatial.Filter(features, tileClipBBox) {
			for _, geom := range feat.Geometry.ClipToBBox(tileClipBBox) {
				feat.Geometry = geom
				ln := lm.LayerName(feat.Props)
				if len(ln) != 0 {
					if _, ok := layers[ln]; !ok {
						layers[ln] = []spatial.Feature{feat}
					} else {
						layers[ln] = append(layers[ln], feat)
					}
				}
			}
		}
		if !anyFeatures(layers) {
			continue
		}
		buf, err := mvt.EncodeTile(layers, tID)
		if err != nil {
			log.Fatal(err)
		}

		err = tw.WriteTile(tID, buf)
		if err != nil {
			log.Fatal(err)
		}
	}
}

func anyFeatures(layers map[string][]spatial.Feature) bool {
	for _, ly := range layers {
		if len(ly) > 0 {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2015-2018 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"
	"time"

	"github.com/u-root/u-root/pkg/testutil"
)

type makeit struct {
	n string      // name
	m os.FileMode // mode
	c []byte      // content
}

var old = makeit{
	n: "old.txt",
	m: 0777,
	c: []byte("old"),
}

var new = makeit{
	n: "new.txt",
	m: 0777,
	c: []byte("new"),
}

var tests = []makeit{
	{
		n: "hi1.txt",
		m: 0666,
		c: []byte("hi"),
	},
	{
		n: "hi2.txt",
		m: 0777,
		c: []byte("hi"),
	},
	old,
	new,
}

func setup() (string, error) {
	d, err := ioutil.TempDir(os.TempDir(), "hi.dir")
	if err != nil {
		return "", err
	}

	tmpdir := filepath.Join(d, "hi.sub.dir")
	if err := os.Mkdir(tmpdir, 0777); err != nil {
		return "", err
	}

	for _, t := range tests {
		if err := ioutil.WriteFile(filepath.Join(d, t.n), []byte(t.c), t.m); err != nil {
			return "", err
		}
	}

	return d, nil
}

func TestMv(t *testing.T) {
	d, err := setup()
	if err != nil {
		t.Fatal("err")
	}
	defer os.RemoveAll(d)

	t.Logf("Renaming file...")
	{
		files := []string{filepath.Join(d, "hi1.txt"), filepath.Join(d, "hi4.txt")}
		res := testutil.Command(t, files...)
		_, err = res.CombinedOutput()
		if err = testutil.IsExitCode(err, 0); err != nil {
			t.Error(err)
		}
	}

	dsub := filepath.Join(d, "hi.sub.dir")

	t.Logf("Moving files to directory...")
	{
		files := []string{filepath.Join(d, "hi2.txt"), filepath.Join(d, "hi4.txt"), dsub}
		res := testutil.Command(t, files...)
		_, err = res.CombinedOutput()
		if err = testutil.IsExitCode(err, 0); err != nil {
			t.Error(err)
		}
	}
}

func TestMvUpdate(t *testing.T) {
	*update = true
	d, err := setup()
	if err != nil {
		t.Error(err)
	}
	defer os.RemoveAll(d)
	t.Logf("Testing mv -u...")

	// Ensure that the newer file actually has a newer timestamp
	currentTime := time.Now().Local()
	oldTime := currentTime.Add(-10 * time.Second)
	err = os.Chtimes(filepath.Join(d, old.n), oldTime, oldTime)
	if err != nil {
		t.Error(err)
	}
	err = os.Chtimes(filepath.Join(d, new.n), currentTime, currentTime)
	if err != nil {
		t.Error(err)
	}

	// Check that it doesn't downgrade files with -u switch
	{
		files := []string{"-u", filepath.Join(d, old.n), filepath.Join(d, new.n)}
		res := testutil.Command(t, files...)
		_, err = res.CombinedOutput()
		if err = testutil.IsExitCode(err, 0); err != nil {
			t.Error(err)
		}
		newContent, err := ioutil.ReadFile(filepath.Join(d, new.n))
		if err != nil {
			t.Error(err)
		}
		if bytes.Equal(newContent, old.c) {
			t.Error("Newer file was overwritten by older file. Should not happen with -u.")
		}
	}

	// Check that it does update files with -u switch
	{
		files := []string{"-u", filepath.Join(d, new.n), filepath.Join(d, old.n)}
		res := testutil.Command(t, files...)
		_, err = res.CombinedOutput()
		if err = testutil.IsExitCode(err, 0); err != nil {
			t.Error(err)
		}
		newContent, err := ioutil.ReadFile(filepath.Join(d, old.n))
		if err != nil {
			t.Error(err)
		}
		if !bytes.Equal(newContent, new.c) {
			t.Error("Older file was not overwritten by newer file. Should happen with -u.")
		}
		if _, err := os.Lstat(filepath.Join(d, old.n)); err != nil {
			t.Error("The new file shouldn't be there anymore.")
		}
	}
}

func TestMain(m *testing.M) {
	testutil.Run(m, main)
}
<commit_msg>Update mv tests to check content and inode<commit_after>// Copyright 2015-2018 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"io/ioutil"
	"os"
	"path/filepath"
	"syscall"
	"testing"
	"time"

	"github.com/u-root/u-root/pkg/testutil"
)

type makeIt struct {
	n string      // name
	m os.FileMode // mode
	c []byte      // content
}

var hiFileContent = []byte("hi")

var old = makeIt{
	n: "old.txt",
	m: 0777,
	c: []byte("old"),
}

var new = makeIt{
	n: "new.txt",
	m: 0777,
	c: []byte("new"),
}

var tests = []makeIt{
	{
		n: "hi1.txt",
		m: 0666,
		c: hiFileContent,
	},
	{
		n: "hi2.txt",
		m: 0777,
		c: hiFileContent,
	},
	old,
	new,
}

func setup() (string, error) {
	d, err := ioutil.TempDir(os.TempDir(), "hi.dir")
	if err != nil {
		return "", err
	}

	tmpdir := filepath.Join(d, "hi.sub.dir")
	if err := os.Mkdir(tmpdir, 0777); err != nil {
		return "", err
	}

	for _, t := range tests {
		if err := ioutil.WriteFile(filepath.Join(d, t.n), []byte(t.c), t.m); err != nil {
			return "", err
		}
	}

	return d, nil
}

func getInode(file string) (uint64, error) {
	var stat syscall.Stat_t
	if err := syscall.Stat(file, &stat); err != nil {
		return 0, err
	}
	return stat.Ino, nil
}

func TestMv(t *testing.T) {
	d, err := setup()
	if err != nil {
		t.Fatal("err")
	}
	defer os.RemoveAll(d)

	t.Logf("Renaming file...")
	{
		originalInode, err := getInode(filepath.Join(d, "hi1.txt"))
		if err != nil {
			t.Error(err)
		}

		files := []string{filepath.Join(d, "hi1.txt"), filepath.Join(d, "hi4.txt")}
		res := testutil.Command(t, files...)
		_, err = res.CombinedOutput()
		if err = testutil.IsExitCode(err, 0); err != nil {
			t.Error(err)
		}

		t.Logf("Verify renamed file integrity...")
		{
			content, err := ioutil.ReadFile(filepath.Join(d, "hi4.txt"))
			if err != nil {
				t.Error(err)
			}

			if !bytes.Equal(hiFileContent, content) {
				t.Errorf("Expected file content to equal %s, got %s", hiFileContent, content)
			}

			movedInode, err := getInode(filepath.Join(d, "hi4.txt"))
			if err != nil {
				t.Error(err)
			}

			if originalInode != movedInode {
				t.Errorf("Expected inode to equal. Expected %d, got %d", originalInode, movedInode)
			}
		}
	}

	dsub := filepath.Join(d, "hi.sub.dir")

	t.Logf("Moving files to directory...")
	{
		originalInode, err := getInode(filepath.Join(d, "hi2.txt"))
		if err != nil {
			t.Error(err)
		}

		originalInodeFour, err := getInode(filepath.Join(d, "hi4.txt"))
		if err != nil {
			t.Error(err)
		}

		files := []string{filepath.Join(d, "hi2.txt"), filepath.Join(d, "hi4.txt"), dsub}
		res := testutil.Command(t, files...)
		_, err = res.CombinedOutput()
		if err = testutil.IsExitCode(err, 0); err != nil {
			t.Error(err)
		}

		t.Logf("Verify moved files into directory file integrity...")
		{
			content, err := ioutil.ReadFile(filepath.Join(dsub, "hi4.txt"))
			if err != nil {
				t.Error(err)
			}

			if !bytes.Equal(hiFileContent, content) {
				t.Errorf("Expected file content to equal %s, got %s", hiFileContent, content)
			}

			movedInode, err := getInode(filepath.Join(dsub, "hi2.txt"))
			if err != nil {
				t.Error(err)
			}

			movedInodeFour, err := getInode(filepath.Join(dsub, "hi4.txt"))
			if err != nil {
				t.Error(err)
			}

			if originalInode != movedInode {
				t.Errorf("Expected inode to equal. Expected %d, got %d", originalInode, movedInode)
			}

			if originalInodeFour != movedInodeFour {
				t.Errorf("Expected inode to equal. Expected %d, got %d", originalInodeFour, movedInodeFour)
			}
		}
	}
}

func TestMvUpdate(t *testing.T) {
	*update = true
	d, err := setup()
	if err != nil {
		t.Error(err)
	}
	defer os.RemoveAll(d)
	t.Logf("Testing mv -u...")

	// Ensure that the newer file actually has a newer timestamp
	currentTime := time.Now().Local()
	oldTime := currentTime.Add(-10 * time.Second)
	err = os.Chtimes(filepath.Join(d, old.n), oldTime, oldTime)
	if err != nil {
		t.Error(err)
	}
	err = os.Chtimes(filepath.Join(d, new.n), currentTime, currentTime)
	if err != nil {
		t.Error(err)
	}

	// Check that it doesn't downgrade files with -u switch
	{
		files := []string{"-u", filepath.Join(d, old.n), filepath.Join(d, new.n)}
		res := testutil.Command(t, files...)
		_, err = res.CombinedOutput()
		if err = testutil.IsExitCode(err, 0); err != nil {
			t.Error(err)
		}
		newContent, err := ioutil.ReadFile(filepath.Join(d, new.n))
		if err != nil {
			t.Error(err)
		}
		if bytes.Equal(newContent, old.c) {
			t.Error("Newer file was overwritten by older file. Should not happen with -u.")
		}
	}

	// Check that it does update files with -u switch
	{
		files := []string{"-u", filepath.Join(d, new.n), filepath.Join(d, old.n)}
		res := testutil.Command(t, files...)
		_, err = res.CombinedOutput()
		if err = testutil.IsExitCode(err, 0); err != nil {
			t.Error(err)
		}
		newContent, err := ioutil.ReadFile(filepath.Join(d, old.n))
		if err != nil {
			t.Error(err)
		}
		if !bytes.Equal(newContent, new.c) {
			t.Error("Older file was not overwritten by newer file. Should happen with -u.")
		}
		if _, err := os.Lstat(filepath.Join(d, old.n)); err != nil {
			t.Error("The new file shouldn't be there anymore.")
		}
	}
}

func TestMain(m *testing.M) {
	testutil.Run(m, main)
}
<|endoftext|>


<commit_before>package leveldb

import (
	"context"
	"github.com/chrislusf/seaweedfs/weed/filer2"
	"io/ioutil"
	"os"
	"testing"
)

func TestCreateAndFind(t *testing.T) {
	filer := filer2.NewFiler(nil, nil)
	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
	defer os.RemoveAll(dir)
	store := &LevelDB2Store{}
	store.initialize(dir)
	filer.SetStore(store)
	filer.DisableDirectoryCache()

	fullpath := filer2.FullPath("/home/chris/this/is/one/file1.jpg")

	ctx := context.Background()

	entry1 := &filer2.Entry{
		FullPath: fullpath,
		Attr: filer2.Attr{
			Mode: 0440,
			Uid:  1234,
			Gid:  5678,
		},
	}

	if err := filer.CreateEntry(ctx, entry1); err != nil {
		t.Errorf("create entry %v: %v", entry1.FullPath, err)
		return
	}

	entry, err := filer.FindEntry(ctx, fullpath)

	if err != nil {
		t.Errorf("find entry: %v", err)
		return
	}

	if entry.FullPath != entry1.FullPath {
		t.Errorf("find wrong entry: %v", entry.FullPath)
		return
	}

	// checking one upper directory
	entries, _ := filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is/one"), "", false, 100)
	if len(entries) != 1 {
		t.Errorf("list entries count: %v", len(entries))
		return
	}

	// checking one upper directory
	entries, _ = filer.ListDirectoryEntries(ctx, filer2.FullPath("/"), "", false, 100)
	if len(entries) != 1 {
		t.Errorf("list entries count: %v", len(entries))
		return
	}

}

func TestEmptyRoot(t *testing.T) {
	filer := filer2.NewFiler(nil, nil)
	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
	defer os.RemoveAll(dir)
	store := &LevelDB2Store{}
	store.initialize(dir)
	filer.SetStore(store)
	filer.DisableDirectoryCache()

	ctx := context.Background()

	// checking one upper directory
	entries, err := filer.ListDirectoryEntries(ctx, filer2.FullPath("/"), "", false, 100)
	if err != nil {
		t.Errorf("list entries: %v", err)
		return
	}
	if len(entries) != 0 {
		t.Errorf("list entries count: %v", len(entries))
		return
	}

}
<commit_msg>fix test<commit_after>package leveldb

import (
	"context"
	"github.com/chrislusf/seaweedfs/weed/filer2"
	"io/ioutil"
	"os"
	"testing"
)

func TestCreateAndFind(t *testing.T) {
	filer := filer2.NewFiler(nil, nil)
	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
	defer os.RemoveAll(dir)
	store := &LevelDB2Store{}
	store.initialize(dir,2)
	filer.SetStore(store)
	filer.DisableDirectoryCache()

	fullpath := filer2.FullPath("/home/chris/this/is/one/file1.jpg")

	ctx := context.Background()

	entry1 := &filer2.Entry{
		FullPath: fullpath,
		Attr: filer2.Attr{
			Mode: 0440,
			Uid:  1234,
			Gid:  5678,
		},
	}

	if err := filer.CreateEntry(ctx, entry1); err != nil {
		t.Errorf("create entry %v: %v", entry1.FullPath, err)
		return
	}

	entry, err := filer.FindEntry(ctx, fullpath)

	if err != nil {
		t.Errorf("find entry: %v", err)
		return
	}

	if entry.FullPath != entry1.FullPath {
		t.Errorf("find wrong entry: %v", entry.FullPath)
		return
	}

	// checking one upper directory
	entries, _ := filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is/one"), "", false, 100)
	if len(entries) != 1 {
		t.Errorf("list entries count: %v", len(entries))
		return
	}

	// checking one upper directory
	entries, _ = filer.ListDirectoryEntries(ctx, filer2.FullPath("/"), "", false, 100)
	if len(entries) != 1 {
		t.Errorf("list entries count: %v", len(entries))
		return
	}

}

func TestEmptyRoot(t *testing.T) {
	filer := filer2.NewFiler(nil, nil)
	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
	defer os.RemoveAll(dir)
	store := &LevelDB2Store{}
	store.initialize(dir,2)
	filer.SetStore(store)
	filer.DisableDirectoryCache()

	ctx := context.Background()

	// checking one upper directory
	entries, err := filer.ListDirectoryEntries(ctx, filer2.FullPath("/"), "", false, 100)
	if err != nil {
		t.Errorf("list entries: %v", err)
		return
	}
	if len(entries) != 0 {
		t.Errorf("list entries count: %v", len(entries))
		return
	}

}
<|endoftext|>


<commit_before>// Package collect provides functions for sending data to OpenTSDB.
//
// The "collect" namespace is used (i.e., <root>.collect) to collect
// program and queue metrics.
package collect // import "bosun.org/collect"

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strings"
	"sync"
	"time"

	"bosun.org/metadata"
	"bosun.org/opentsdb"
)

var (
	// Freq is how often metrics are sent to OpenTSDB.
	Freq = time.Second * 15

	// MaxQueueLen is the maximum size of the queue, above which incoming data will
	// be discarded. Defaults to about 150MB.
	MaxQueueLen = 200000

	// BatchSize is the maximum length of data points sent at once to OpenTSDB.
	BatchSize = 500

	// Debug enables debug logging.
	Debug = false

	// Print prints all datapoints to stdout instead of sending them.
	Print = false

	// DisableDefaultCollectors prevents the scollector self metrics from being
	// generated.
	DisableDefaultCollectors = false

	// Tags is an opentsdb.TagSet used when sending self metrics.
	Tags opentsdb.TagSet

	// Dropped is the number of dropped data points due to a full queue.
	dropped int64

	// Sent is the number of sent data points.
	sent int64

	tchan               chan *opentsdb.DataPoint
	tsdbURL             string
	osHostname          string
	metricRoot          string
	queue               []*opentsdb.DataPoint
	qlock, mlock, slock sync.Mutex // Locks for queues, maps, stats.
	counters            = make(map[string]*addMetric)
	sets                = make(map[string]*setMetric)
	puts                = make(map[string]*putMetric)
	client              = &http.Client{
		Transport: &timeoutTransport{Transport: new(http.Transport)},
		Timeout:   time.Minute,
	}
)

const (
	descCollectDropped      = "Counter of dropped data points due to the queue being full."
	descCollectSent         = "Counter of data points sent to the server."
	descCollectQueued       = "Total number of items currently queued and waiting to be sent to the server."
	descCollectAlloc        = "Total number of bytes allocated and still in use by the runtime (via runtime.ReadMemStats)."
	descCollectGoRoutines   = "Total number of goroutines that currently exist (via runtime.NumGoroutine)."
	descCollectPostDuration = "Total number of milliseconds it took to send an HTTP POST request to the server."
	descCollectPostCount    = "Counter of batches sent to the server."
	descCollectPostError    = "Counter of errors received when sending a batch to the server."
	descCollectPostBad      = "Counter of HTTP POST requests where resp.StatusCode != http.StatusNoContent."
	descCollectPostRestore  = "Counter of data points restored from batches that could not be sent to the server."
)

type timeoutTransport struct {
	*http.Transport
	Timeout time.Time
}

func (t *timeoutTransport) RoundTrip(r *http.Request) (*http.Response, error) {
	if time.Now().After(t.Timeout) {
		t.Transport.CloseIdleConnections()
		t.Timeout = time.Now().Add(time.Minute * 5)
	}
	return t.Transport.RoundTrip(r)
}

// InitChan is similar to Init, but uses the given channel instead of creating a
// new one.
func InitChan(tsdbhost *url.URL, root string, ch chan *opentsdb.DataPoint) error {
	if tchan != nil {
		return fmt.Errorf("cannot init twice")
	}
	if err := checkClean(root, "metric root"); err != nil {
		return err
	}
	u, err := tsdbhost.Parse("/api/put")
	if err != nil {
		return err
	}
	if strings.HasPrefix(u.Host, ":") {
		u.Host = "localhost" + u.Host
	}
	tsdbURL = u.String()
	metricRoot = root + "."
	tchan = ch
	go queuer()
	go send()
	go collect()
	if DisableDefaultCollectors {
		return nil
	}
	Set("collect.dropped", Tags, func() (i interface{}) {
		slock.Lock()
		i = dropped
		slock.Unlock()
		return
	})
	Set("collect.sent", Tags, func() (i interface{}) {
		slock.Lock()
		i = sent
		slock.Unlock()
		return
	})
	Set("collect.queued", Tags, func() (i interface{}) {
		qlock.Lock()
		i = len(queue)
		qlock.Unlock()
		return
	})
	Set("collect.alloc", Tags, func() interface{} {
		var ms runtime.MemStats
		runtime.ReadMemStats(&ms)
		return ms.Alloc
	})
	Set("collect.goroutines", Tags, func() interface{} {
		return runtime.NumGoroutine()
	})
	metadata.AddMetricMeta(metricRoot+"collect.dropped", metadata.Counter, metadata.PerSecond, descCollectDropped)
	metadata.AddMetricMeta(metricRoot+"collect.sent", metadata.Counter, metadata.PerSecond, descCollectSent)
	metadata.AddMetricMeta(metricRoot+"collect.queued", metadata.Gauge, metadata.Item, descCollectQueued)
	metadata.AddMetricMeta(metricRoot+"collect.alloc", metadata.Gauge, metadata.Bytes, descCollectAlloc)
	metadata.AddMetricMeta(metricRoot+"collect.goroutines", metadata.Gauge, metadata.Count, descCollectGoRoutines)
	metadata.AddMetricMeta(metricRoot+"collect.post.total_duration", metadata.Counter, metadata.MilliSecond, descCollectPostDuration)
	metadata.AddMetricMeta(metricRoot+"collect.post.count", metadata.Counter, metadata.PerSecond, descCollectPostCount)
	metadata.AddMetricMeta(metricRoot+"collect.post.error", metadata.Counter, metadata.PerSecond, descCollectPostError)
	metadata.AddMetricMeta(metricRoot+"collect.post.bad_status", metadata.Counter, metadata.PerSecond, descCollectPostBad)
	metadata.AddMetricMeta(metricRoot+"collect.post.restore", metadata.Counter, metadata.PerSecond, descCollectPostRestore)
	return nil
}

// Init sets up the channels and the queue for sending data to OpenTSDB. It also
// sets up the basename for all metrics.
func Init(tsdbhost *url.URL, root string) error {
	return InitChan(tsdbhost, root, make(chan *opentsdb.DataPoint))
}

func SetHostname(host string) error {
	if err := checkClean(host, "host tag"); err != nil {
		return err
	}
	osHostname = host
	return nil
}

func setHostName() error {
	h, err := os.Hostname()
	if err != nil {
		return err
	}
	osHostname = strings.ToLower(strings.SplitN(h, ".", 2)[0])
	if err := checkClean(osHostname, "host tag"); err != nil {
		return err
	}
	return nil
}

type setMetric struct {
	metric string
	ts     opentsdb.TagSet
	f      func() interface{}
}

// Set registers a callback for the given metric and tags, calling f immediately
// before queueing data for send.
func Set(metric string, ts opentsdb.TagSet, f func() interface{}) error {
	if err := check(metric, &ts); err != nil {
		return err
	}
	tss := metric + ts.String()
	mlock.Lock()
	sets[tss] = &setMetric{metric, ts.Copy(), f}
	mlock.Unlock()
	return nil
}

type addMetric struct {
	metric string
	ts     opentsdb.TagSet
	value  int64
}

// Add takes a metric and increments a counter for that metric. The metric name
// is appended to the basename specified in the Init function.
func Add(metric string, ts opentsdb.TagSet, inc int64) error {
	if err := check(metric, &ts); err != nil {
		return err
	}
	tss := metric + ts.String()
	mlock.Lock()
	if counters[tss] == nil {
		counters[tss] = &addMetric{
			metric: metric,
			ts:     ts.Copy(),
		}
	}
	counters[tss].value += inc
	mlock.Unlock()
	return nil
}

type putMetric struct {
	metric string
	ts     opentsdb.TagSet
	value  interface{}
}

// Put is useful for capturing "events" that have a gauge value. Subsequent
// calls between the sending interval will overwrite previous calls.
func Put(metric string, ts opentsdb.TagSet, v interface{}) error {
	if err := check(metric, &ts); err != nil {
		return err
	}
	tss := metric + ts.String()
	mlock.Lock()
	puts[tss] = &putMetric{metric, ts.Copy(), v}
	mlock.Unlock()
	return nil
}

func check(metric string, ts *opentsdb.TagSet) error {
	if err := checkClean(metric, "metric"); err != nil {
		return err
	}
	for k, v := range *ts {
		if err := checkClean(k, "tagk"); err != nil {
			return err
		}
		if err := checkClean(v, "tagv"); err != nil {
			return err
		}
	}
	if osHostname == "" {
		if err := setHostName(); err != nil {
			return err
		}
	}
	if *ts == nil {
		*ts = make(opentsdb.TagSet)
	}
	if host, present := (*ts)["host"]; !present {
		(*ts)["host"] = osHostname
	} else if host == "" {
		delete(*ts, "host")
	}
	return nil
}

func checkClean(s, t string) error {
	if sc, err := opentsdb.Clean(s); s != sc || err != nil {
		if err != nil {
			return err
		}
		return fmt.Errorf("%s %s may only contain a to z, A to Z, 0 to 9, -, _, ., / or Unicode letters and may not be empty", t, s)
	}
	return nil
}

func collect() {
	for {
		mlock.Lock()
		now := time.Now().Unix()
		for _, c := range counters {
			dp := &opentsdb.DataPoint{
				Metric:    metricRoot + c.metric,
				Timestamp: now,
				Value:     c.value,
				Tags:      c.ts,
			}
			tchan <- dp
		}
		for _, s := range sets {
			dp := &opentsdb.DataPoint{
				Metric:    metricRoot + s.metric,
				Timestamp: now,
				Value:     s.f(),
				Tags:      s.ts,
			}
			tchan <- dp
		}
		for _, s := range puts {
			dp := &opentsdb.DataPoint{
				Metric:    metricRoot + s.metric,
				Timestamp: now,
				Value:     s.value,
				Tags:      s.ts,
			}
			tchan <- dp
		}
		puts = make(map[string]*putMetric)
		mlock.Unlock()
		time.Sleep(Freq)
	}
}
<commit_msg>collect: Add aggregation support to collect<commit_after>// Package collect provides functions for sending data to OpenTSDB.
//
// The "collect" namespace is used (i.e., <root>.collect) to collect
// program and queue metrics.
package collect // import "bosun.org/collect"

import (
	"fmt"
	"math"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"sort"
	"strings"
	"sync"
	"time"

	"bosun.org/metadata"
	"bosun.org/opentsdb"
)

var (
	// Freq is how often metrics are sent to OpenTSDB.
	Freq = time.Second * 15

	// MaxQueueLen is the maximum size of the queue, above which incoming data will
	// be discarded. Defaults to about 150MB.
	MaxQueueLen = 200000

	// BatchSize is the maximum length of data points sent at once to OpenTSDB.
	BatchSize = 500

	// Debug enables debug logging.
	Debug = false

	// Print prints all datapoints to stdout instead of sending them.
	Print = false

	// DisableDefaultCollectors prevents the scollector self metrics from being
	// generated.
	DisableDefaultCollectors = false

	// Tags is an opentsdb.TagSet used when sending self metrics.
	Tags opentsdb.TagSet

	// Dropped is the number of dropped data points due to a full queue.
	dropped int64

	// Sent is the number of sent data points.
	sent int64

	tchan               chan *opentsdb.DataPoint
	tsdbURL             string
	osHostname          string
	metricRoot          string
	queue               []*opentsdb.DataPoint
	qlock, mlock, slock sync.Mutex // Locks for queues, maps, stats.
	counters            = make(map[string]*addMetric)
	sets                = make(map[string]*setMetric)
	puts                = make(map[string]*putMetric)
	aggs                = make(map[string]*agMetric)
	client              = &http.Client{
		Transport: &timeoutTransport{Transport: new(http.Transport)},
		Timeout:   time.Minute,
	}
)

const (
	descCollectDropped      = "Counter of dropped data points due to the queue being full."
	descCollectSent         = "Counter of data points sent to the server."
	descCollectQueued       = "Total number of items currently queued and waiting to be sent to the server."
	descCollectAlloc        = "Total number of bytes allocated and still in use by the runtime (via runtime.ReadMemStats)."
	descCollectGoRoutines   = "Total number of goroutines that currently exist (via runtime.NumGoroutine)."
	descCollectPostDuration = "Total number of milliseconds it took to send an HTTP POST request to the server."
	descCollectPostCount    = "Counter of batches sent to the server."
	descCollectPostError    = "Counter of errors received when sending a batch to the server."
	descCollectPostBad      = "Counter of HTTP POST requests where resp.StatusCode != http.StatusNoContent."
	descCollectPostRestore  = "Counter of data points restored from batches that could not be sent to the server."
)

type timeoutTransport struct {
	*http.Transport
	Timeout time.Time
}

func (t *timeoutTransport) RoundTrip(r *http.Request) (*http.Response, error) {
	if time.Now().After(t.Timeout) {
		t.Transport.CloseIdleConnections()
		t.Timeout = time.Now().Add(time.Minute * 5)
	}
	return t.Transport.RoundTrip(r)
}

// InitChan is similar to Init, but uses the given channel instead of creating a
// new one.
func InitChan(tsdbhost *url.URL, root string, ch chan *opentsdb.DataPoint) error {
	if tchan != nil {
		return fmt.Errorf("cannot init twice")
	}
	if err := checkClean(root, "metric root"); err != nil {
		return err
	}
	u, err := tsdbhost.Parse("/api/put")
	if err != nil {
		return err
	}
	if strings.HasPrefix(u.Host, ":") {
		u.Host = "localhost" + u.Host
	}
	tsdbURL = u.String()
	metricRoot = root + "."
	tchan = ch
	go queuer()
	go send()
	go collect()
	if DisableDefaultCollectors {
		return nil
	}
	Set("collect.dropped", Tags, func() (i interface{}) {
		slock.Lock()
		i = dropped
		slock.Unlock()
		return
	})
	Set("collect.sent", Tags, func() (i interface{}) {
		slock.Lock()
		i = sent
		slock.Unlock()
		return
	})
	Set("collect.queued", Tags, func() (i interface{}) {
		qlock.Lock()
		i = len(queue)
		qlock.Unlock()
		return
	})
	Set("collect.alloc", Tags, func() interface{} {
		var ms runtime.MemStats
		runtime.ReadMemStats(&ms)
		return ms.Alloc
	})
	Set("collect.goroutines", Tags, func() interface{} {
		return runtime.NumGoroutine()
	})
	metadata.AddMetricMeta(metricRoot+"collect.dropped", metadata.Counter, metadata.PerSecond, descCollectDropped)
	metadata.AddMetricMeta(metricRoot+"collect.sent", metadata.Counter, metadata.PerSecond, descCollectSent)
	metadata.AddMetricMeta(metricRoot+"collect.queued", metadata.Gauge, metadata.Item, descCollectQueued)
	metadata.AddMetricMeta(metricRoot+"collect.alloc", metadata.Gauge, metadata.Bytes, descCollectAlloc)
	metadata.AddMetricMeta(metricRoot+"collect.goroutines", metadata.Gauge, metadata.Count, descCollectGoRoutines)
	metadata.AddMetricMeta(metricRoot+"collect.post.total_duration", metadata.Counter, metadata.MilliSecond, descCollectPostDuration)
	metadata.AddMetricMeta(metricRoot+"collect.post.count", metadata.Counter, metadata.PerSecond, descCollectPostCount)
	metadata.AddMetricMeta(metricRoot+"collect.post.error", metadata.Counter, metadata.PerSecond, descCollectPostError)
	metadata.AddMetricMeta(metricRoot+"collect.post.bad_status", metadata.Counter, metadata.PerSecond, descCollectPostBad)
	metadata.AddMetricMeta(metricRoot+"collect.post.restore", metadata.Counter, metadata.PerSecond, descCollectPostRestore)
	return nil
}

// Init sets up the channels and the queue for sending data to OpenTSDB. It also
// sets up the basename for all metrics.
func Init(tsdbhost *url.URL, root string) error {
	return InitChan(tsdbhost, root, make(chan *opentsdb.DataPoint))
}

func SetHostname(host string) error {
	if err := checkClean(host, "host tag"); err != nil {
		return err
	}
	osHostname = host
	return nil
}

func setHostName() error {
	h, err := os.Hostname()
	if err != nil {
		return err
	}
	osHostname = strings.ToLower(strings.SplitN(h, ".", 2)[0])
	if err := checkClean(osHostname, "host tag"); err != nil {
		return err
	}
	return nil
}

type agMetric struct {
	metric string
	ts     opentsdb.TagSet
	values []float64
}

func AggregateMeta(metric string, desc string, rateType metadata.RateType, unit metadata.Unit) {
	agStrings := []string{"avg", "count", "min", "median", "max", "95", "99"}
	for _, ag := range agStrings {
		metadata.AddMetricMeta(metric+"_"+ag, rateType, unit, desc)
	}
}

func (am *agMetric) Process(now int64) {
	var avg float64
	for _, v := range am.values {
		avg += v
	}
	avg /= float64(len(am.values))
	extRoot := metricRoot + am.metric
	tchan <- &opentsdb.DataPoint{
		Metric:    extRoot + "_avg",
		Timestamp: now,
		Value:     avg,
		Tags:      am.ts,
	}
	tchan <- &opentsdb.DataPoint{
		Metric:    extRoot + "_count",
		Timestamp: now,
		Value:     len(am.values),
		Tags:      am.ts,
	}
	sort.Float64s(am.values)
	percentile := func(p float64) float64 {
		if p <= 0 {
			return am.values[0]
		}
		if p >= 1 {
			return am.values[len(am.values)-1]
		}
		i := p * float64(len(am.values)-1)
		i = math.Ceil(i)
		return am.values[int(i)]
	}
	tchan <- &opentsdb.DataPoint{
		Metric:    extRoot + "_min",
		Timestamp: now,
		Value:     percentile(0),
		Tags:      am.ts,
	}
	tchan <- &opentsdb.DataPoint{
		Metric:    extRoot + "_median",
		Timestamp: now,
		Value:     percentile(.5),
		Tags:      am.ts,
	}
	tchan <- &opentsdb.DataPoint{
		Metric:    extRoot + "_max",
		Timestamp: now,
		Value:     percentile(1),
		Tags:      am.ts,
	}
	tchan <- &opentsdb.DataPoint{
		Metric:    extRoot + "_95",
		Timestamp: now,
		Value:     percentile(.95),
		Tags:      am.ts,
	}
	tchan <- &opentsdb.DataPoint{
		Metric:    extRoot + "_99",
		Timestamp: now,
		Value:     percentile(.99),
		Tags:      am.ts,
	}
}

func Sample(metric string, ts opentsdb.TagSet, v float64) error {
	if err := check(metric, &ts); err != nil {
		return err
	}
	tss := metric + ts.String()
	mlock.Lock()
	if aggs[tss] == nil {
		aggs[tss] = &agMetric{
			metric: metric,
			ts:     ts.Copy(),
		}
	}
	aggs[tss].values = append(aggs[tss].values, v)
	mlock.Unlock()
	return nil
}

type setMetric struct {
	metric string
	ts     opentsdb.TagSet
	f      func() interface{}
}

// Set registers a callback for the given metric and tags, calling f immediately
// before queueing data for send.
func Set(metric string, ts opentsdb.TagSet, f func() interface{}) error {
	if err := check(metric, &ts); err != nil {
		return err
	}
	tss := metric + ts.String()
	mlock.Lock()
	sets[tss] = &setMetric{metric, ts.Copy(), f}
	mlock.Unlock()
	return nil
}

type addMetric struct {
	metric string
	ts     opentsdb.TagSet
	value  int64
}

// Add takes a metric and increments a counter for that metric. The metric name
// is appended to the basename specified in the Init function.
func Add(metric string, ts opentsdb.TagSet, inc int64) error {
	if err := check(metric, &ts); err != nil {
		return err
	}
	tss := metric + ts.String()
	mlock.Lock()
	if counters[tss] == nil {
		counters[tss] = &addMetric{
			metric: metric,
			ts:     ts.Copy(),
		}
	}
	counters[tss].value += inc
	mlock.Unlock()
	return nil
}

type putMetric struct {
	metric string
	ts     opentsdb.TagSet
	value  interface{}
}

// Put is useful for capturing "events" that have a gauge value. Subsequent
// calls between the sending interval will overwrite previous calls.
func Put(metric string, ts opentsdb.TagSet, v interface{}) error {
	if err := check(metric, &ts); err != nil {
		return err
	}
	tss := metric + ts.String()
	mlock.Lock()
	puts[tss] = &putMetric{metric, ts.Copy(), v}
	mlock.Unlock()
	return nil
}

func check(metric string, ts *opentsdb.TagSet) error {
	if err := checkClean(metric, "metric"); err != nil {
		return err
	}
	for k, v := range *ts {
		if err := checkClean(k, "tagk"); err != nil {
			return err
		}
		if err := checkClean(v, "tagv"); err != nil {
			return err
		}
	}
	if osHostname == "" {
		if err := setHostName(); err != nil {
			return err
		}
	}
	if *ts == nil {
		*ts = make(opentsdb.TagSet)
	}
	if host, present := (*ts)["host"]; !present {
		(*ts)["host"] = osHostname
	} else if host == "" {
		delete(*ts, "host")
	}
	return nil
}

func checkClean(s, t string) error {
	if sc, err := opentsdb.Clean(s); s != sc || err != nil {
		if err != nil {
			return err
		}
		return fmt.Errorf("%s %s may only contain a to z, A to Z, 0 to 9, -, _, ., / or Unicode letters and may not be empty", t, s)
	}
	return nil
}

func collect() {
	for {
		mlock.Lock()
		now := time.Now().Unix()
		for _, c := range counters {
			dp := &opentsdb.DataPoint{
				Metric:    metricRoot + c.metric,
				Timestamp: now,
				Value:     c.value,
				Tags:      c.ts,
			}
			tchan <- dp
		}
		for _, s := range sets {
			dp := &opentsdb.DataPoint{
				Metric:    metricRoot + s.metric,
				Timestamp: now,
				Value:     s.f(),
				Tags:      s.ts,
			}
			tchan <- dp
		}
		for _, s := range puts {
			dp := &opentsdb.DataPoint{
				Metric:    metricRoot + s.metric,
				Timestamp: now,
				Value:     s.value,
				Tags:      s.ts,
			}
			tchan <- dp
		}
		for _, am := range aggs {
			am.Process(now)
		}
		puts = make(map[string]*putMetric)
		aggs = make(map[string]*agMetric)
		mlock.Unlock()
		time.Sleep(Freq)
	}
}
<|endoftext|>


<commit_before>// DRAFT ONLY. THIS IS STILL IN DEVELOPEMENT. NO TESTS WHERE MADE

package collections

import (
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sync"

	tk "github.com/quintans/toolkit"
)

const (
	intByteSize = 4
)

var ErrShortRead = errors.New("short read")

// FileFifo stores some data in memory and after a threshold
// the data is written to disk.
type FileFifo struct {
	dir     string
	fileCap int64

	headFileSize int64
	headIdx      int64 // head position
	headFile     *os.File
	headFileIdx  int64

	tailIdx     int64 // tail position
	tailFile    *os.File
	tailFileIdx int64

	peekedData []byte
}

// NewFileFifo creates a FIFO supported by files.
// The supporting files will have a max size. Whenever that size is exceeded, a new file will be created.
// When all elements of a file are consumed (Pop) that file will be deleted.
//
// FileFifo is not safe for concurrent access.
func NewFileFifo(dir string, fileCap int64) (*FileFifo, error) {
	this := new(FileFifo)
	this.dir = dir
	this.fileCap = 1024 * 1024 * fileCap // MB to b

	err := this.Clear()
	if err != nil {
		return nil, err
	}

	return this, nil
}

func (this *FileFifo) Clear() error {
	this.headFileSize = 0
	this.headIdx = 0
	this.headFileIdx = 0
	if this.headFile != nil {
		err := this.headFile.Close()
		if err != nil {
			return err
		}
		this.headFile = nil
	}

	this.tailIdx = 0
	this.tailFileIdx = 0
	if this.tailFile != nil {
		err := this.tailFile.Close()
		if err != nil {
			return err
		}
		this.tailFile = nil
	}

	// (re)create dir
	err := os.RemoveAll(this.dir)
	if err != nil {
		return err
	}
	err = os.MkdirAll(this.dir, 0777)
	if err != nil {
		return err
	}

	// open file
	err = this.nextHeadFile()
	if err != nil {
		return err
	}
	err = this.nextTailFile()
	if err != nil {
		return err
	}

	return nil
}

func (this *FileFifo) nextHeadFile() error {
	var err error
	if this.headFile != nil {
		err = this.headFile.Close()
		if err != nil {
			return err
		}
	}
	this.headFileIdx++
	fp := filepath.Join(this.dir, fmt.Sprintf("%016X.dat", this.headFileIdx))
	this.headFile, err = os.Create(fp)
	if err != nil {
		return err
	}
	this.headFileSize = 0
	return nil
}

func (this *FileFifo) nextTailFile() error {
	var err error
	if this.tailFile != nil {
		err = this.tailFile.Close()
		if err != nil {
			return err
		}
		err = os.Remove(this.tailFile.Name())
		if err != nil {
			return err
		}
	}
	this.tailFileIdx++
	fp := filepath.Join(this.dir, fmt.Sprintf("%016X", this.tailFileIdx))
	this.tailFile, err = os.Open(fp)
	if err != nil {
		return err
	}
	return nil
}

func (this *FileFifo) Push(data []byte) error {
	if this.headFile == nil || this.headFileSize > this.fileCap {
		this.nextHeadFile()
	}

	// write data size
	var buf32 = make([]byte, intByteSize)
	size := len(data)
	binary.LittleEndian.PutUint32(buf32, uint32(size))
	n, err := this.headFile.Write(buf32)
	if err != nil {
		return err
	} else if n < size {
		return io.ErrShortWrite
	}

	// write data
	n, err = this.headFile.Write(data)
	if err != nil {
		return err
	} else if n < size {
		return io.ErrShortWrite
	}

	this.headFileSize += int64(intByteSize + size)
	this.headIdx++
	return nil
}

func (this *FileFifo) Pop() ([]byte, error) {
	data, err := this.Peek()
	this.peekedData = nil
	return data, err
}

func (this *FileFifo) Peek() ([]byte, error) {
	if this.peekedData != nil {
		return this.peekedData, nil
	} else if this.Size() > 0 {
		// read data size
		buf := make([]byte, intByteSize)
		n, err := this.tailFile.Read(buf)
		if err == io.EOF {
			this.nextTailFile()
			return this.Pop()
		} else if err != nil {
			return nil, err
		} else if n < intByteSize {
			return nil, ErrShortRead
		}
		size := int(binary.LittleEndian.Uint32(buf))
		// read data
		buf = make([]byte, size)
		n, err = this.tailFile.Read(buf)
		if err != nil {
			return nil, err
		} else if n < intByteSize {
			return nil, ErrShortRead
		}

		this.tailIdx++
		this.peekedData = buf
		return buf, nil

	} else {
		return nil, nil
	}
}

func (this *FileFifo) Size() int64 {
	return this.headIdx - this.tailIdx
}

type item struct {
	next  *item
	value interface{}
}

type BigFifo struct {
	fileFifo *FileFifo
	cond     *sync.Cond

	head      *item
	tail      *item
	size      int
	threshold int
	dir       string
	codec     tk.Codec
	factory   func() interface{}
}

// NewBigFifo creates a FIFO that after a certain number of elements will use disk files to store the elements.
//
// threshold: number after which will store to disk.
// dir: directory where the files will be created.
// codec: codec to convert between []byte and interface{}
//
// BigFifo is not safe for concurrent access.
func NewBigFifo(threshold int, dir string, fileCap int64, codec tk.Codec, factory func() interface{}) (*BigFifo, error) {
	// validate
	if threshold < 1 {
		return nil, errors.New("validate is less than 1")
	}
	if len(dir) == 0 {
		return nil, errors.New("dir is empty")
	}
	if codec == nil {
		return nil, errors.New("codec is nil")
	}
	if factory == nil {
		return nil, errors.New("factory is nil")
	}

	var err error
	this := new(BigFifo)
	this.fileFifo, err = NewFileFifo(dir, fileCap)
	if err != nil {
		return nil, err
	}
	this.threshold = threshold
	this.codec = codec
	this.factory = factory
	this.cond = sync.NewCond(&sync.Mutex{})
	return this, nil
}

func (this *BigFifo) Size() int64 {
	this.cond.L.Lock()
	defer this.cond.L.Unlock()

	return int64(this.size) + this.fileFifo.Size()
}

func (this *BigFifo) Clear() error {
	this.cond.L.Lock()
	defer this.cond.L.Unlock()

	this.head = nil
	this.tail = nil
	this.size = 0
	return this.fileFifo.Clear()
}

func (this *BigFifo) push(value interface{}) {
	e := &item{value: value}
	if this.head != nil {
		this.head.next = e
	}
	this.head = e

	if this.tail == nil {
		this.tail = e
	}

	this.size++
}

func (this *BigFifo) Push(value interface{}) error {
	this.cond.L.Lock()
	defer func() {
		this.cond.L.Unlock()
		this.cond.Signal()
	}()

	var err error
	if this.size < this.threshold {
		// still in the memory zone
		this.push(value)
	} else {
		// use disk, since the threshold was exceeded.
		data, err := this.codec.Encode(value)
		if err != nil {
			return err
		}
		err = this.fileFifo.Push(data)
	}

	return err
}

func (this *BigFifo) pop() (interface{}, error) {
	value := this.tail.value
	this.tail = this.tail.next
	this.size--

	// if there is data stored in file, get to memory
	data, err := this.fileFifo.Pop()
	if err != nil {
		return nil, err
	}

	if data != nil {
		v := this.factory()
		err = this.codec.Decode(data, v)
		if err != nil {
			return nil, err
		}
		this.push(v)
	}

	return value, nil
}

// PopOrWait returns the tail element removing it.
// If no element is available it will wait until one is added.
func (this *BigFifo) PopOrWait() (interface{}, error) {
	this.cond.L.Lock()
	defer this.cond.L.Unlock()

	// Pop will allways be executed from memory
	//If the queue is empty. will wait for an element
	for this.tail == nil {
		this.cond.Wait()
	}

	return this.pop()
}

// Pop returns the tail element removing it.
func (this *BigFifo) Pop() (interface{}, error) {
	this.cond.L.Lock()
	defer this.cond.L.Unlock()

	// Pop will allways be executed from memory
	if this.tail == nil {
		return nil, nil
	} else {
		return this.pop()
	}
}

// Pop returns the tail element without removing it.
func (this *BigFifo) Peek() interface{} {
	this.cond.L.Lock()
	defer this.cond.L.Unlock()

	if this.tail != nil {
		return this.tail.value
	}
	return nil
}

// the Idea is to have a FIFO with a windowing (circular) feature.
// If the max size is reached, the oldest element will be removed.
type Fifo struct {
	mu   sync.RWMutex
	lock bool

	head     *item
	tail     *item
	size     int
	capacity int
}

func NewFifo(capacity int) *Fifo {
	return newLockFifo(capacity, false)
}

// NewLockFifo creates a Queue to be accessed concurrently
func NewLockFifo(capacity int) *Fifo {
	return newLockFifo(capacity, true)
}

func newLockFifo(capacity int, lock bool) *Fifo {
	this := new(Fifo)
	this.capacity = capacity
	this.lock = lock
	return this
}

func (this *Fifo) Size() int {
	if this.lock {
		this.mu.RLock()
		defer this.mu.RUnlock()
	}

	return this.size
}

// Clear resets the queue.
func (this *Fifo) Clear() {
	if this.lock {
		this.mu.Lock()
		defer this.mu.Unlock()
	}

	this.head = nil
	this.tail = nil
	this.size = 0
}

// Push adds an element to the head of the fifo.
// If the capacity was exceeded returns the element that had to be pushed out, otherwise returns nil.
func (this *Fifo) Push(value interface{}) interface{} {
	if this.lock {
		this.mu.Lock()
		defer this.mu.Unlock()
	}

	var old interface{}
	// if capacity == 0 it will add until memory is exausted
	if this.capacity > 0 && this.size == this.capacity {
		old = this.pop()
	}
	// adds new element
	e := &item{value: value}
	if this.head != nil {
		this.head.next = e
	}
	this.head = e

	if this.tail == nil {
		this.tail = e
	}

	this.size++

	return old
}

func (this *Fifo) pop() interface{} {
	var value interface{}
	if this.tail != nil {
		value = this.tail.value
		this.tail = this.tail.next
		this.size--
	}
	return value
}

// Pop returns the tail element removing it.
func (this *Fifo) Pop() interface{} {
	if this.lock {
		this.mu.Lock()
		defer this.mu.Unlock()
	}

	return this.pop()
}

// Peek returns the tail element without removing it.
func (this *Fifo) Peek() interface{} {
	if this.lock {
		this.mu.RLock()
		defer this.mu.RUnlock()
	}

	if this.tail != nil {
		return this.tail.value
	}
	return nil
}
<commit_msg>tests and bug fixes<commit_after>// DRAFT ONLY. THIS IS STILL IN DEVELOPEMENT. NO TESTS WHERE MADE

package collections

import (
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sync"

	tk "github.com/quintans/toolkit"
	"github.com/quintans/toolkit/log"
)

const (
	intByteSize = 4
)

var ErrShortRead = errors.New("short read")
var fifoLogger = log.LoggerFor("github.com/quintans/toolkit/collection")

// FileFifo stores some data in memory and after a threshold
// the data is written to disk.
type FileFifo struct {
	dir     string
	fileCap int64

	headFileSize int64
	headIdx      int64 // head position
	headFile     *os.File
	headFileIdx  int64

	tailIdx     int64 // tail position
	tailFile    *os.File
	tailFileIdx int64

	peekedData []byte
}

// NewFileFifo creates a FIFO supported by files.
// The supporting files will have a max size. Whenever that size is exceeded, a new file will be created.
// When all elements of a file are consumed (Pop) that file will be deleted.
//
// FileFifo is not safe for concurrent access.
func NewFileFifo(dir string, fileCap int64) (*FileFifo, error) {
	this := new(FileFifo)
	this.dir = dir
	this.fileCap = 1024 * 1024 * fileCap // MB to b

	err := this.Clear()
	if err != nil {
		return nil, err
	}

	return this, nil
}

func (this *FileFifo) Clear() error {
	this.headFileSize = 0
	this.headIdx = 0
	this.headFileIdx = 0
	if this.headFile != nil {
		err := this.headFile.Close()
		if err != nil {
			return err
		}
		this.headFile = nil
	}

	this.tailIdx = 0
	this.tailFileIdx = 0
	if this.tailFile != nil {
		err := this.tailFile.Close()
		if err != nil {
			return err
		}
		this.tailFile = nil
	}

	// (re)create dir
	fifoLogger.Debugf("removing dir %s", this.dir)
	err := os.RemoveAll(this.dir)
	if err != nil {
		return err
	}

	fifoLogger.Debugf("creating dir %s", this.dir)
	err = os.MkdirAll(this.dir, 0777)
	if err != nil {
		return err
	}

	// open file
	err = this.nextHeadFile()
	if err != nil {
		return err
	}
	err = this.nextTailFile()
	if err != nil {
		return err
	}

	return nil
}

func (this *FileFifo) nextHeadFile() error {
	var err error
	if this.headFile != nil {
		err = this.headFile.Close()
		if err != nil {
			return err
		}
	}
	this.headFileIdx++
	fp := filepath.Join(this.dir, fmt.Sprintf("%016X", this.headFileIdx))
	fifoLogger.Debugf("creating file %s", fp)
	this.headFile, err = os.Create(fp)
	if err != nil {
		return err
	}
	this.headFileSize = 0
	return nil
}

func (this *FileFifo) nextTailFile() error {
	var err error
	if this.tailFile != nil {
		err = this.tailFile.Close()
		if err != nil {
			return err
		}
		fifoLogger.Debugf("removing file %s", this.tailFile.Name())
		err = os.Remove(this.tailFile.Name())
		if err != nil {
			return err
		}
	}
	this.tailFileIdx++
	fp := filepath.Join(this.dir, fmt.Sprintf("%016X", this.tailFileIdx))
	fifoLogger.Debugf("opening file %s", fp)
	this.tailFile, err = os.Open(fp)
	if err != nil {
		return err
	}
	return nil
}

func (this *FileFifo) Push(data []byte) error {
	if this.headFile == nil || this.headFileSize > this.fileCap {
		this.nextHeadFile()
	}

	// write data size
	var buf32 = make([]byte, intByteSize)
	size := len(data)
	binary.BigEndian.PutUint32(buf32, uint32(size))
	n, err := this.headFile.Write(buf32)
	if err != nil {
		return err
	} else if n < intByteSize {
		return io.ErrShortWrite
	}

	// write data
	n, err = this.headFile.Write(data)
	if err != nil {
		return err
	} else if n < size {
		return io.ErrShortWrite
	}

	this.headFileSize += int64(intByteSize + size)
	this.headIdx++
	return nil
}

func (this *FileFifo) Pop() ([]byte, error) {
	data, err := this.Peek()
	this.peekedData = nil
	this.tailIdx++
	return data, err
}

func (this *FileFifo) Peek() ([]byte, error) {
	if this.peekedData != nil {
		return this.peekedData, nil
	} else if this.Size() > 0 {
		// read data size
		buf := make([]byte, intByteSize)
		n, err := this.tailFile.Read(buf)
		if err == io.EOF {
			this.nextTailFile()
			return this.Pop()
		} else if err != nil {
			return nil, err
		} else if n < intByteSize {
			return nil, ErrShortRead
		}
		size := int(binary.BigEndian.Uint32(buf))
		// read data
		buf = make([]byte, size)
		n, err = this.tailFile.Read(buf)
		if err != nil {
			return nil, err
		} else if n < size {
			return nil, ErrShortRead
		}

		this.peekedData = buf
		return buf, nil

	} else {
		return nil, nil
	}
}

func (this *FileFifo) Size() int64 {
	return this.headIdx - this.tailIdx
}

type item struct {
	next  *item
	value interface{}
}

type BigFifo struct {
	fileFifo *FileFifo
	cond     *sync.Cond

	head      *item
	tail      *item
	size      int
	threshold int
	dir       string
	codec     tk.Codec
	factory   func() interface{}
}

// NewBigFifo creates a FIFO that after a certain number of elements will use disk files to store the elements.
//
// threshold: number after which will store to disk.
// dir: directory where the files will be created.
// codec: codec to convert between []byte and interface{}
//
// BigFifo is not safe for concurrent access.
func NewBigFifo(threshold int, dir string, fileCap int64, codec tk.Codec, factory func() interface{}) (*BigFifo, error) {
	// validate
	if threshold < 1 {
		return nil, errors.New("validate is less than 1")
	}
	if len(dir) == 0 {
		return nil, errors.New("dir is empty")
	}
	if codec == nil {
		return nil, errors.New("codec is nil")
	}
	if factory == nil {
		return nil, errors.New("factory is nil")
	}

	var err error
	this := new(BigFifo)
	this.fileFifo, err = NewFileFifo(dir, fileCap)
	if err != nil {
		return nil, err
	}
	this.threshold = threshold
	this.codec = codec
	this.factory = factory
	this.cond = sync.NewCond(&sync.Mutex{})
	return this, nil
}

func (this *BigFifo) Size() int64 {
	this.cond.L.Lock()
	defer this.cond.L.Unlock()

	return int64(this.size) + this.fileFifo.Size()
}

func (this *BigFifo) Clear() error {
	this.cond.L.Lock()
	defer this.cond.L.Unlock()

	this.head = nil
	this.tail = nil
	this.size = 0
	return this.fileFifo.Clear()
}

func (this *BigFifo) push(value interface{}) {
	e := &item{value: value}
	if this.head != nil {
		this.head.next = e
	}
	this.head = e

	if this.tail == nil {
		this.tail = e
	}

	this.size++
}

func (this *BigFifo) Push(value interface{}) error {
	this.cond.L.Lock()
	defer func() {
		this.cond.L.Unlock()
		this.cond.Signal()
	}()

	var err error
	if this.size < this.threshold {
		// still in the memory zone
		this.push(value)
	} else {
		// use disk, since the threshold was exceeded.
		data, err := this.codec.Encode(value)
		if err != nil {
			return err
		}
		err = this.fileFifo.Push(data)
	}

	return err
}

func (this *BigFifo) pop() (interface{}, error) {
	value := this.tail.value
	this.tail = this.tail.next
	this.size--

	// if there is data stored in file, get to memory
	data, err := this.fileFifo.Pop()
	if err != nil {
		return nil, err
	}

	if data != nil {
		v := this.factory()
		err = this.codec.Decode(data, v)
		if err != nil {
			return nil, err
		}
		this.push(v)
	}

	return value, nil
}

// PopOrWait returns the tail element removing it.
// If no element is available it will wait until one is added.
func (this *BigFifo) PopOrWait() (interface{}, error) {
	this.cond.L.Lock()
	defer this.cond.L.Unlock()

	// Pop will allways be executed from memory
	//If the queue is empty. will wait for an element
	for this.tail == nil {
		this.cond.Wait()
	}

	return this.pop()
}

// Pop returns the tail element removing it.
func (this *BigFifo) Pop() (interface{}, error) {
	this.cond.L.Lock()
	defer this.cond.L.Unlock()

	// Pop will allways be executed from memory
	if this.tail == nil {
		return nil, nil
	} else {
		return this.pop()
	}
}

// Pop returns the tail element without removing it.
func (this *BigFifo) Peek() interface{} {
	this.cond.L.Lock()
	defer this.cond.L.Unlock()

	if this.tail != nil {
		return this.tail.value
	}
	return nil
}

// the Idea is to have a FIFO with a windowing (circular) feature.
// If the max size is reached, the oldest element will be removed.
type Fifo struct {
	mu   sync.RWMutex
	lock bool

	head     *item
	tail     *item
	size     int
	capacity int
}

func NewFifo(capacity int) *Fifo {
	return newLockFifo(capacity, false)
}

// NewLockFifo creates a Queue to be accessed concurrently
func NewLockFifo(capacity int) *Fifo {
	return newLockFifo(capacity, true)
}

func newLockFifo(capacity int, lock bool) *Fifo {
	this := new(Fifo)
	this.capacity = capacity
	this.lock = lock
	return this
}

func (this *Fifo) Size() int {
	if this.lock {
		this.mu.RLock()
		defer this.mu.RUnlock()
	}

	return this.size
}

// Clear resets the queue.
func (this *Fifo) Clear() {
	if this.lock {
		this.mu.Lock()
		defer this.mu.Unlock()
	}

	this.head = nil
	this.tail = nil
	this.size = 0
}

// Push adds an element to the head of the fifo.
// If the capacity was exceeded returns the element that had to be pushed out, otherwise returns nil.
func (this *Fifo) Push(value interface{}) interface{} {
	if this.lock {
		this.mu.Lock()
		defer this.mu.Unlock()
	}

	var old interface{}
	// if capacity == 0 it will add until memory is exausted
	if this.capacity > 0 && this.size == this.capacity {
		old = this.pop()
	}
	// adds new element
	e := &item{value: value}
	if this.head != nil {
		this.head.next = e
	}
	this.head = e

	if this.tail == nil {
		this.tail = e
	}

	this.size++

	return old
}

func (this *Fifo) pop() interface{} {
	var value interface{}
	if this.tail != nil {
		value = this.tail.value
		this.tail = this.tail.next
		this.size--
	}
	return value
}

// Pop returns the tail element removing it.
func (this *Fifo) Pop() interface{} {
	if this.lock {
		this.mu.Lock()
		defer this.mu.Unlock()
	}

	return this.pop()
}

// Peek returns the tail element without removing it.
func (this *Fifo) Peek() interface{} {
	if this.lock {
		this.mu.RLock()
		defer this.mu.RUnlock()
	}

	if this.tail != nil {
		return this.tail.value
	}
	return nil
}
<|endoftext|>


<commit_before>package regexutil

import (
	"fmt"
	"reflect"
	"testing"

	"../db/sqlconstants"
)

const TEST_SQLITE3_CREATE_STATEMENT_A = "CREATE TABLE FOO ( X INTEGER, Y TEXT PRIMARY KEY)"
const TEST_SQLITE3_CREATE_STATEMENT_B = "CREATE TABLE FOO ( X INTEGER, Y TEXT PRIMARY KEY, Z BETA NOT NULL)"
const TEST_SQLITE3_CREATE_STATEMENT_C = "CREATE TABLE FOO ( X INTEGER, PRIMARY KEY Y, Z BETA NOT NULL)"

func TestReturnNameGroupValueMap(t *testing.T) {
	_, err := ParseAndReturnNameGroupValueMap(SQLITE3_CREATE_TABLE_FIRST_PASS_PARSER, TEST_SQLITE3_CREATE_STATEMENT_A)

	if err != nil {
		t.Error(err)
	}

}

func TestFirstPassParseCreateStatement(t *testing.T) {
	_, _, err := firstPassParseCreateStatement(TEST_SQLITE3_CREATE_STATEMENT_A)
	if err != nil {
		t.Error(err)
	}

}

func TestSecondPassParseCreateStatement(t *testing.T) {
	_, columns, err := firstPassParseCreateStatement(sqlconstants.SQLITE3_CREATE_BUCKET_SCHEMA)
	if err != nil {
		t.Error(err)
	}

	_, err = secondPassParseCreateStatement(columns)

	if err != nil {
		t.Error(err)
	}

	_, columns, err = firstPassParseCreateStatement(TEST_SQLITE3_CREATE_STATEMENT_B)
	if err != nil {
		t.Error(err)
	}

	_, err = secondPassParseCreateStatement(columns)

	if err != nil {
		t.Error(err)
	}

	_, columns, err = firstPassParseCreateStatement(TEST_SQLITE3_CREATE_STATEMENT_C)
	if err != nil {
		t.Error(err)
	}

	_, err = secondPassParseCreateStatement(columns)

	if err == nil {
		t.Error("Error should have been caught")
	}

}

func TestParseCreateStatement(t *testing.T) {

	table, columnArrayMap, err := ParseCreateStatement(TEST_SQLITE3_CREATE_STATEMENT_A)

	if err != nil {
		t.Error(err)
	}

	if table != "FOO" {
		t.Error("error table name is incorrect")
	}

	cam := []map[string]string{}
	c1 := map[string]string{"": "X INTEGER", "columntype": "INTEGER", "columnname": "X", "constraints": ""}
	c2 := map[string]string{"": "Y TEXT PRIMARY KEY", "columntype": "TEXT", "columnname": "Y", "constraints": "PRIMARY KEY"}

	cam = append(cam, c1)
	cam = append(cam, c2)

	if !reflect.DeepEqual(columnArrayMap, cam) {
		t.Error("ParseCreateStatement created unexpected results => test case ->", TEST_SQLITE3_CREATE_STATEMENT_A)
		t.Error("test data =>", cam)
		t.Error("returned data =>", columnArrayMap)
	}

	table, columnArrayMap, err = ParseCreateStatement(TEST_SQLITE3_CREATE_STATEMENT_B)

	if err != nil {
		t.Error(err)
	}

	if table != "FOO" {
		t.Error("error table name is incorrect")
	}

	cam = []map[string]string{}
	c1 = map[string]string{"": "X INTEGER", "columntype": "INTEGER", "columnname": "X", "constraints": ""}
	c2 = map[string]string{"": "Y TEXT PRIMARY KEY", "columntype": "TEXT", "columnname": "Y", "constraints": "PRIMARY KEY"}
	c3 := map[string]string{"": "Z BETA NOT NULL", "columntype": "BETA", "columnname": "Z", "constraints": "NOT NULL"}

	cam = append(cam, c1)
	cam = append(cam, c2)
	cam = append(cam, c3)

	if !reflect.DeepEqual(columnArrayMap, cam) {
		t.Error("ParseCreateStatement created unexpected results => test case ->", TEST_SQLITE3_CREATE_STATEMENT_B)
		t.Error("test data =>", cam)
		t.Error("returned data =>", columnArrayMap)
	}

	table, columnArrayMap, err = ParseCreateStatement(TEST_SQLITE3_CREATE_STATEMENT_C)

	if err == nil {
		t.Error("ERROR should be thrown")
	}

	fmt.Println(err)

}
<commit_msg>Added test case to test for DEFAULT CURRENT_TIMESTAMP, test case is TEST_SQLITE3_CREATE_STATEMENT_D<commit_after>package regexutil

import (
	"reflect"
	"testing"

	"../db/sqlconstants"
)

const TEST_SQLITE3_CREATE_STATEMENT_A = "CREATE TABLE FOO ( X INTEGER, Y TEXT PRIMARY KEY)"
const TEST_SQLITE3_CREATE_STATEMENT_B = "CREATE TABLE FOO ( X INTEGER, Y TEXT PRIMARY KEY, Z BETA NOT NULL)"
const TEST_SQLITE3_CREATE_STATEMENT_C = "CREATE TABLE FOO ( X INTEGER, PRIMARY KEY Y, Z BETA NOT NULL)"
const TEST_SQLITE3_CREATE_STATEMENT_D = "CREATE TABLE FOO ( X INTEGER, Y TEXT PRIMARY KEY, Z BETA NOT NULL, T DEFAULT  CURRENT_TIMESTAMP)"

func TestReturnNameGroupValueMap(t *testing.T) {
	_, err := ParseAndReturnNameGroupValueMap(SQLITE3_CREATE_TABLE_FIRST_PASS_PARSER, TEST_SQLITE3_CREATE_STATEMENT_A)

	if err != nil {
		t.Error(err)
	}

}

func TestFirstPassParseCreateStatement(t *testing.T) {
	_, _, err := firstPassParseCreateStatement(TEST_SQLITE3_CREATE_STATEMENT_A)
	if err != nil {
		t.Error(err)
	}

}

func TestSecondPassParseCreateStatement(t *testing.T) {
	_, columns, err := firstPassParseCreateStatement(sqlconstants.SQLITE3_CREATE_BUCKET_SCHEMA)
	if err != nil {
		t.Error(err)
	}

	_, err = secondPassParseCreateStatement(columns)

	if err != nil {
		t.Error(err)
	}

	_, columns, err = firstPassParseCreateStatement(TEST_SQLITE3_CREATE_STATEMENT_B)
	if err != nil {
		t.Error(err)
	}

	_, err = secondPassParseCreateStatement(columns)

	if err != nil {
		t.Error(err)
	}

	_, columns, err = firstPassParseCreateStatement(TEST_SQLITE3_CREATE_STATEMENT_C)
	if err != nil {
		t.Error(err)
	}

	_, err = secondPassParseCreateStatement(columns)

	if err == nil {
		t.Error("Error should have been caught")
	}

}

func TestParseCreateStatement(t *testing.T) {

	//CASE A

	table, columnArrayMap, err := ParseCreateStatement(TEST_SQLITE3_CREATE_STATEMENT_A)

	if err != nil {
		t.Error(err)
	}

	if table != "FOO" {
		t.Error("error table name is incorrect")
	}

	cam := []map[string]string{}
	c1 := map[string]string{"": "X INTEGER", "columntype": "INTEGER", "columnname": "X", "constraints": ""}
	c2 := map[string]string{"": "Y TEXT PRIMARY KEY", "columntype": "TEXT", "columnname": "Y", "constraints": "PRIMARY KEY"}

	cam = append(cam, c1)
	cam = append(cam, c2)

	if !reflect.DeepEqual(columnArrayMap, cam) {
		t.Error("ParseCreateStatement created unexpected results => test case ->", TEST_SQLITE3_CREATE_STATEMENT_A)
		t.Error("test data =>", cam)
		t.Error("returned data =>", columnArrayMap)
	}

	//CASE B

	table, columnArrayMap, err = ParseCreateStatement(TEST_SQLITE3_CREATE_STATEMENT_B)

	if err != nil {
		t.Error(err)
	}

	if table != "FOO" {
		t.Error("error table name is incorrect")
	}

	cam = []map[string]string{}
	c1 = map[string]string{"": "X INTEGER", "columntype": "INTEGER", "columnname": "X", "constraints": ""}
	c2 = map[string]string{"": "Y TEXT PRIMARY KEY", "columntype": "TEXT", "columnname": "Y", "constraints": "PRIMARY KEY"}
	c3 := map[string]string{"": "Z BETA NOT NULL", "columntype": "BETA", "columnname": "Z", "constraints": "NOT NULL"}

	cam = append(cam, c1)
	cam = append(cam, c2)
	cam = append(cam, c3)

	if !reflect.DeepEqual(columnArrayMap, cam) {
		t.Error("ParseCreateStatement created unexpected results => test case ->", TEST_SQLITE3_CREATE_STATEMENT_B)
		t.Error("test data =>", cam)
		t.Error("returned data =>", columnArrayMap)
	}

	//CASE C

	table, columnArrayMap, err = ParseCreateStatement(TEST_SQLITE3_CREATE_STATEMENT_C)

	if err == nil {
		t.Error("ERROR should be thrown")
	}

	//CASE D
	table, columnArrayMap, err = ParseCreateStatement(TEST_SQLITE3_CREATE_STATEMENT_D)

	if err != nil {
		t.Error(err)
	}

	if table != "FOO" {
		t.Error("error table name is incorrect")
	}

	cam = []map[string]string{}
	c1 = map[string]string{"": "X INTEGER", "columntype": "INTEGER", "columnname": "X", "constraints": ""}
	c2 = map[string]string{"": "Y TEXT PRIMARY KEY", "columntype": "TEXT", "columnname": "Y", "constraints": "PRIMARY KEY"}
	c3 = map[string]string{"": "Z BETA NOT NULL", "columntype": "BETA", "columnname": "Z", "constraints": "NOT NULL"}
	c4 := map[string]string{"": "T DEFAULT  CURRENT_TIMESTAMP", "columntype": "TIMESTAMP", "columnname": "T", "constraints": "DEFAULT  CURRENT_TIMESTAMP"}

	cam = append(cam, c1)
	cam = append(cam, c2)
	cam = append(cam, c3)
	cam = append(cam, c4)

	if !reflect.DeepEqual(columnArrayMap, cam) {
		t.Error("ParseCreateStatement created unexpected results => test case ->", TEST_SQLITE3_CREATE_STATEMENT_D)
		t.Error("test data =>", cam)
		t.Error("returned data =>", columnArrayMap)
	}

}
<|endoftext|>


<commit_before>package handler

import (
	"context"
	"strings"
	"time"

	"github.com/micro/go-micro/v2"
	"github.com/micro/go-micro/v2/auth"
	"github.com/micro/go-micro/v2/errors"
	log "github.com/micro/go-micro/v2/logger"
	"github.com/micro/go-micro/v2/registry"
	"github.com/micro/go-micro/v2/registry/service"
	pb "github.com/micro/go-micro/v2/registry/service/proto"
	"github.com/micro/micro/v2/internal/namespace"
)

type Registry struct {
	// service id
	Id string
	// the publisher
	Publisher micro.Publisher
	// internal registry
	Registry registry.Registry
	// auth to verify clients
	Auth auth.Auth
}

func ActionToEventType(action string) registry.EventType {
	switch action {
	case "create":
		return registry.Create
	case "delete":
		return registry.Delete
	default:
		return registry.Update
	}
}

func (r *Registry) publishEvent(action string, service *pb.Service) error {
	// TODO: timestamp should be read from received event
	// Right now registry.Result does not contain timestamp
	event := &pb.Event{
		Id:        r.Id,
		Type:      pb.EventType(ActionToEventType(action)),
		Timestamp: time.Now().UnixNano(),
		Service:   service,
	}

	log.Debugf("publishing event %s for action %s", event.Id, action)

	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()

	return r.Publisher.Publish(ctx, event)
}

// GetService from the registry with the name requested
func (r *Registry) GetService(ctx context.Context, req *pb.GetRequest, rsp *pb.GetResponse) error {
	// get the services in the requested namespace, e.g. the "foo" namespace. name
	// includes the namespace as the prefix, e.g. 'foo/go.micro.service.bar'
	name := namespace.FromContext(ctx) + nameSeperator + req.Service
	services, err := r.Registry.GetService(name)
	if err != nil {
		return errors.InternalServerError("go.micro.registry", err.Error())
	}

	// get the services in the default namespace if this wasn't the namespace
	// requested.
	if namespace.FromContext(ctx) != namespace.DefaultNamespace {
		name := namespace.DefaultNamespace + nameSeperator + req.Service
		defaultServices, err := r.Registry.GetService(name)
		if err != nil {
			return errors.InternalServerError("go.micro.registry", err.Error())
		}
		services = append(services, defaultServices...)
	}

	for _, srv := range services {
		rsp.Services = append(rsp.Services, service.ToProto(withoutNamespace(*srv)))
	}
	return nil
}

// Register a service
func (r *Registry) Register(ctx context.Context, req *pb.Service, rsp *pb.EmptyResponse) error {
	var regOpts []registry.RegisterOption
	if req.Options != nil {
		ttl := time.Duration(req.Options.Ttl) * time.Second
		regOpts = append(regOpts, registry.RegisterTTL(ttl))
	}

	service := service.ToService(withNamespace(*req, namespace.FromContext(ctx)))
	if err := r.Registry.Register(service, regOpts...); err != nil {
		return errors.InternalServerError("go.micro.registry", err.Error())
	}

	// publish the event
	go r.publishEvent("create", req)

	return nil
}

// Deregister a service
func (r *Registry) Deregister(ctx context.Context, req *pb.Service, rsp *pb.EmptyResponse) error {
	service := service.ToService(withNamespace(*req, namespace.FromContext(ctx)))
	if err := r.Registry.Deregister(service); err != nil {
		return errors.InternalServerError("go.micro.registry", err.Error())
	}

	// publish the event
	go r.publishEvent("delete", req)

	return nil
}

// ListServices returns all the services
func (r *Registry) ListServices(ctx context.Context, req *pb.ListRequest, rsp *pb.ListResponse) error {
	services, err := r.Registry.ListServices()
	if err != nil {
		return errors.InternalServerError("go.micro.registry", err.Error())
	}

	for _, srv := range services {
		// check to see if the service belongs to the defaut namespace
		// or the contexts namespace. TODO: think about adding a prefix
		//argument to ListServices
		if !canReadService(ctx, srv) {
			continue
		}

		rsp.Services = append(rsp.Services, service.ToProto(withoutNamespace(*srv)))
	}

	return nil
}

// Watch a service for changes
func (r *Registry) Watch(ctx context.Context, req *pb.WatchRequest, rsp pb.Registry_WatchStream) error {
	watcher, err := r.Registry.Watch(registry.WatchService(req.Service))
	if err != nil {
		return errors.InternalServerError("go.micro.registry", err.Error())
	}

	for {
		next, err := watcher.Next()
		if err != nil {
			return errors.InternalServerError("go.micro.registry", err.Error())
		}
		if !canReadService(ctx, next.Service) {
			continue
		}

		err = rsp.Send(&pb.Result{
			Action:  next.Action,
			Service: service.ToProto(withoutNamespace(*next.Service)),
		})
		if err != nil {
			return errors.InternalServerError("go.micro.registry", err.Error())
		}
	}
}

// canReadService is a helper function which returns a boolean indicating
// if a context can read a service.
func canReadService(ctx context.Context, srv *registry.Service) bool {
	// check if the service has no prefix which means it was written
	// directly to the store and is therefore assumed to be part of
	// the default namespace
	if len(strings.Split(srv.Name, nameSeperator)) == 1 {
		return true
	}

	// all users can read from the default namespace
	if strings.HasPrefix(srv.Name, namespace.DefaultNamespace+nameSeperator) {
		return true
	}

	// the service belongs to the contexts namespace
	if strings.HasPrefix(srv.Name, namespace.FromContext(ctx)+nameSeperator) {
		return true
	}

	return false
}

// nameSeperator is the string which is used as a seperator when joining
// namespace to the service name
const nameSeperator = "/"

// withoutNamespace returns the service with the namespace stripped from
// the name, e.g. 'default/go.micro.service.foo' => 'go.micro.service.foo'.
func withoutNamespace(srv registry.Service) *registry.Service {
	comps := strings.Split(srv.Name, nameSeperator)
	srv.Name = comps[len(comps)-1]
	return &srv
}

// withNamespace returns the service with the namespace prefixed to the
// name, e.g. 'go.micro.service.foo' => 'default/go.micro.service.foo'
func withNamespace(srv pb.Service, ns string) *pb.Service {
	srv.Name = strings.Join([]string{ns, srv.Name}, nameSeperator)
	return &srv
}
<commit_msg>Don't prefix default namespace<commit_after>package handler

import (
	"context"
	"strings"
	"time"

	"github.com/micro/go-micro/v2"
	"github.com/micro/go-micro/v2/auth"
	"github.com/micro/go-micro/v2/errors"
	log "github.com/micro/go-micro/v2/logger"
	"github.com/micro/go-micro/v2/registry"
	"github.com/micro/go-micro/v2/registry/service"
	pb "github.com/micro/go-micro/v2/registry/service/proto"
	"github.com/micro/micro/v2/internal/namespace"
)

type Registry struct {
	// service id
	Id string
	// the publisher
	Publisher micro.Publisher
	// internal registry
	Registry registry.Registry
	// auth to verify clients
	Auth auth.Auth
}

func ActionToEventType(action string) registry.EventType {
	switch action {
	case "create":
		return registry.Create
	case "delete":
		return registry.Delete
	default:
		return registry.Update
	}
}

func (r *Registry) publishEvent(action string, service *pb.Service) error {
	// TODO: timestamp should be read from received event
	// Right now registry.Result does not contain timestamp
	event := &pb.Event{
		Id:        r.Id,
		Type:      pb.EventType(ActionToEventType(action)),
		Timestamp: time.Now().UnixNano(),
		Service:   service,
	}

	log.Debugf("publishing event %s for action %s", event.Id, action)

	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()

	return r.Publisher.Publish(ctx, event)
}

// GetService from the registry with the name requested
func (r *Registry) GetService(ctx context.Context, req *pb.GetRequest, rsp *pb.GetResponse) error {
	// get the services in the requested namespace, e.g. the "foo" namespace. name
	// includes the namespace as the prefix, e.g. 'foo/go.micro.service.bar'
	name := namespace.FromContext(ctx) + nameSeperator + req.Service
	services, err := r.Registry.GetService(name)
	if err != nil {
		return errors.InternalServerError("go.micro.registry", err.Error())
	}

	// get the services in the default namespace if this wasn't the namespace
	// requested.
	if namespace.FromContext(ctx) != namespace.DefaultNamespace {
		name := namespace.DefaultNamespace + nameSeperator + req.Service
		defaultServices, err := r.Registry.GetService(name)
		if err != nil {
			return errors.InternalServerError("go.micro.registry", err.Error())
		}
		services = append(services, defaultServices...)
	}

	for _, srv := range services {
		rsp.Services = append(rsp.Services, service.ToProto(withoutNamespace(*srv)))
	}
	return nil
}

// Register a service
func (r *Registry) Register(ctx context.Context, req *pb.Service, rsp *pb.EmptyResponse) error {
	var regOpts []registry.RegisterOption
	if req.Options != nil {
		ttl := time.Duration(req.Options.Ttl) * time.Second
		regOpts = append(regOpts, registry.RegisterTTL(ttl))
	}

	service := service.ToService(withNamespace(*req, namespace.FromContext(ctx)))
	if err := r.Registry.Register(service, regOpts...); err != nil {
		return errors.InternalServerError("go.micro.registry", err.Error())
	}

	// publish the event
	go r.publishEvent("create", req)

	return nil
}

// Deregister a service
func (r *Registry) Deregister(ctx context.Context, req *pb.Service, rsp *pb.EmptyResponse) error {
	service := service.ToService(withNamespace(*req, namespace.FromContext(ctx)))
	if err := r.Registry.Deregister(service); err != nil {
		return errors.InternalServerError("go.micro.registry", err.Error())
	}

	// publish the event
	go r.publishEvent("delete", req)

	return nil
}

// ListServices returns all the services
func (r *Registry) ListServices(ctx context.Context, req *pb.ListRequest, rsp *pb.ListResponse) error {
	services, err := r.Registry.ListServices()
	if err != nil {
		return errors.InternalServerError("go.micro.registry", err.Error())
	}

	for _, srv := range services {
		// check to see if the service belongs to the defaut namespace
		// or the contexts namespace. TODO: think about adding a prefix
		//argument to ListServices
		if !canReadService(ctx, srv) {
			continue
		}

		rsp.Services = append(rsp.Services, service.ToProto(withoutNamespace(*srv)))
	}

	return nil
}

// Watch a service for changes
func (r *Registry) Watch(ctx context.Context, req *pb.WatchRequest, rsp pb.Registry_WatchStream) error {
	watcher, err := r.Registry.Watch(registry.WatchService(req.Service))
	if err != nil {
		return errors.InternalServerError("go.micro.registry", err.Error())
	}

	for {
		next, err := watcher.Next()
		if err != nil {
			return errors.InternalServerError("go.micro.registry", err.Error())
		}
		if !canReadService(ctx, next.Service) {
			continue
		}

		err = rsp.Send(&pb.Result{
			Action:  next.Action,
			Service: service.ToProto(withoutNamespace(*next.Service)),
		})
		if err != nil {
			return errors.InternalServerError("go.micro.registry", err.Error())
		}
	}
}

// canReadService is a helper function which returns a boolean indicating
// if a context can read a service.
func canReadService(ctx context.Context, srv *registry.Service) bool {
	// check if the service has no prefix which means it was written
	// directly to the store and is therefore assumed to be part of
	// the default namespace
	if len(strings.Split(srv.Name, nameSeperator)) == 1 {
		return true
	}

	// the service belongs to the contexts namespace
	if strings.HasPrefix(srv.Name, namespace.FromContext(ctx)+nameSeperator) {
		return true
	}

	return false
}

// nameSeperator is the string which is used as a seperator when joining
// namespace to the service name
const nameSeperator = "/"

// withoutNamespace returns the service with the namespace stripped from
// the name, e.g. 'bar/go.micro.service.foo' => 'go.micro.service.foo'.
func withoutNamespace(srv registry.Service) *registry.Service {
	comps := strings.Split(srv.Name, nameSeperator)
	srv.Name = comps[len(comps)-1]
	return &srv
}

// withNamespace returns the service with the namespace prefixed to the
// name, e.g. 'go.micro.service.foo' => 'bar/go.micro.service.foo'
func withNamespace(srv pb.Service, ns string) *pb.Service {
	// if the namespace is the default, don't append anything since this
	// means users not leveraging multi-tenancy won't experience any changes
	if ns == namespace.DefaultNamespace {
		return &srv
	}

	srv.Name = strings.Join([]string{ns, srv.Name}, nameSeperator)
	return &srv
}
<|endoftext|>


<commit_before>package dots

import "github.com/catorpilor/LeetCode/utils"

func minDistance(word1, word2 string) int {
	if word1 == word2 {
		return 0
	}
	m, n := len(word1), len(word2)
	dp := make([]int, n+1)
	for i := 1; i <= n; i++ {
		dp[i] = i
	}
	var upleft int
	for i := 1; i <= m; i++ {
		upleft = dp[0]
		dp[0] = i
		for j := 1; j <= n; j++ {
			pre, cur := dp[j-1], dp[j]
			if word1[i-1] == word2[j-1] {
				dp[j] = upleft
			} else {
				dp[j] = utils.Min(pre, cur) + 1
			}
			upleft = cur
		}
	}
	return dp[n]
}
<commit_msg>doc: add some comments<commit_after>package dots

import "github.com/catorpilor/LeetCode/utils"

// minDistance using dp to solve the problem.
// time complexity is O(mn)
// space complexity is o(n)
func minDistance(word1, word2 string) int {
	if word1 == word2 {
		return 0
	}
	m, n := len(word1), len(word2)
	dp := make([]int, n+1)
	for i := 1; i <= n; i++ {
		dp[i] = i
	}
	var upleft int
	for i := 1; i <= m; i++ {
		upleft = dp[0]
		dp[0] = i
		for j := 1; j <= n; j++ {
			pre, cur := dp[j-1], dp[j]
			if word1[i-1] == word2[j-1] {
				dp[j] = upleft
			} else {
				dp[j] = utils.Min(pre, cur) + 1
			}
			upleft = cur
		}
	}
	return dp[n]
}
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"strings"

	"github.com/hashicorp/nomad/api"
)

type InspectCommand struct {
	Meta
}

func (c *InspectCommand) Help() string {
	helpText := `
Usage: nomad inspect [options] <job>

  Inspect is used to see the specification of a submitted job.

General Options:

  ` + generalOptionsUsage() + `

Inspect Options:

  -json
    Output the job in its JSON format.

  -t
    Format and display job using a Go template.
`
	return strings.TrimSpace(helpText)
}

func (c *InspectCommand) Synopsis() string {
	return "Inspect a submitted job"
}

func (c *InspectCommand) Run(args []string) int {
	var json bool
	var tmpl string

	flags := c.Meta.FlagSet("inspect", FlagSetClient)
	flags.Usage = func() { c.Ui.Output(c.Help()) }
	flags.BoolVar(&json, "json", false, "")
	flags.StringVar(&tmpl, "t", "", "")

	if err := flags.Parse(args); err != nil {
		return 1
	}
	args = flags.Args()

	// Get the HTTP client
	client, err := c.Meta.Client()
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err))
		return 1
	}

	// If args not specified but output format is specified, format and output the jobs data list
	if len(args) == 0 && json || len(tmpl) > 0 {
		jobs, _, err := client.Jobs().List(nil)
		if err != nil {
			c.Ui.Error(fmt.Sprintf("Error querying jobs: %v", err))
			return 1
		}

		out, err := Format(json, tmpl, jobs)
		if err != nil {
			c.Ui.Error(err.Error())
			return 1
		}

		c.Ui.Output(out)
		return 0
	}

	// Check that we got exactly one job
	if len(args) != 1 {
		c.Ui.Error(c.Help())
		return 1
	}
	jobID := args[0]

	// Check if the job exists
	jobs, _, err := client.Jobs().PrefixList(jobID)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error inspecting job: %s", err))
		return 1
	}
	if len(jobs) == 0 {
		c.Ui.Error(fmt.Sprintf("No job(s) with prefix or id %q found", jobID))
		return 1
	}
	if len(jobs) > 1 && strings.TrimSpace(jobID) != jobs[0].ID {
		c.Ui.Output(fmt.Sprintf("Prefix matched multiple jobs\n\n%s", createStatusListOutput(jobs)))
		return 0
	}

	// Prefix lookup matched a single job
	job, _, err := client.Jobs().Info(jobs[0].ID, nil)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error inspecting job: %s", err))
		return 1
	}

	// If output format is specified, format and output the data
	if json || len(tmpl) > 0 {
		out, err := Format(json, tmpl, job)
		if err != nil {
			c.Ui.Error(err.Error())
			return 1
		}

		c.Ui.Output(out)
		return 0
	}

	// Print the contents of the job
	req := api.RegisterJobRequest{Job: job}
	f, err := DataFormat("json", "")
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error getting formatter: %s", err))
		return 1
	}

	out, err := f.TransformData(req)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error formatting the data: %s", err))
		return 1
	}
	c.Ui.Output(out)
	return 0
}
<commit_msg>Inspect job at a particular version<commit_after>package command

import (
	"fmt"
	"strings"

	"github.com/hashicorp/nomad/api"
)

type InspectCommand struct {
	Meta
}

func (c *InspectCommand) Help() string {
	helpText := `
Usage: nomad inspect [options] <job>

  Inspect is used to see the specification of a submitted job.

General Options:

  ` + generalOptionsUsage() + `

Inspect Options:

  -version <job version>
    Display only the history for the given job version.

  -json
    Output the job in its JSON format.

  -t
    Format and display job using a Go template.
`
	return strings.TrimSpace(helpText)
}

func (c *InspectCommand) Synopsis() string {
	return "Inspect a submitted job"
}

func (c *InspectCommand) Run(args []string) int {
	var json bool
	var tmpl, versionStr string

	flags := c.Meta.FlagSet("inspect", FlagSetClient)
	flags.Usage = func() { c.Ui.Output(c.Help()) }
	flags.BoolVar(&json, "json", false, "")
	flags.StringVar(&tmpl, "t", "", "")
	flags.StringVar(&versionStr, "version", "", "")

	if err := flags.Parse(args); err != nil {
		return 1
	}
	args = flags.Args()

	// Get the HTTP client
	client, err := c.Meta.Client()
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err))
		return 1
	}

	// If args not specified but output format is specified, format and output the jobs data list
	if len(args) == 0 && json || len(tmpl) > 0 {
		jobs, _, err := client.Jobs().List(nil)
		if err != nil {
			c.Ui.Error(fmt.Sprintf("Error querying jobs: %v", err))
			return 1
		}

		out, err := Format(json, tmpl, jobs)
		if err != nil {
			c.Ui.Error(err.Error())
			return 1
		}

		c.Ui.Output(out)
		return 0
	}

	// Check that we got exactly one job
	if len(args) != 1 {
		c.Ui.Error(c.Help())
		return 1
	}
	jobID := args[0]

	// Check if the job exists
	jobs, _, err := client.Jobs().PrefixList(jobID)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error inspecting job: %s", err))
		return 1
	}
	if len(jobs) == 0 {
		c.Ui.Error(fmt.Sprintf("No job(s) with prefix or id %q found", jobID))
		return 1
	}
	if len(jobs) > 1 && strings.TrimSpace(jobID) != jobs[0].ID {
		c.Ui.Output(fmt.Sprintf("Prefix matched multiple jobs\n\n%s", createStatusListOutput(jobs)))
		return 0
	}

	var version *uint64
	if versionStr != "" {
		v, _, err := parseVersion(versionStr)
		if err != nil {
			c.Ui.Error(fmt.Sprintf("Error parsing version value %q: %v", versionStr, err))
			return 1
		}

		version = &v
	}

	// Prefix lookup matched a single job
	job, err := getJob(client, jobs[0].ID, version)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error inspecting job: %s", err))
		return 1
	}

	// If output format is specified, format and output the data
	if json || len(tmpl) > 0 {
		out, err := Format(json, tmpl, job)
		if err != nil {
			c.Ui.Error(err.Error())
			return 1
		}

		c.Ui.Output(out)
		return 0
	}

	// Print the contents of the job
	req := api.RegisterJobRequest{Job: job}
	f, err := DataFormat("json", "")
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error getting formatter: %s", err))
		return 1
	}

	out, err := f.TransformData(req)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error formatting the data: %s", err))
		return 1
	}
	c.Ui.Output(out)
	return 0
}

// getJob retrieves the job optionally at a particular version.
func getJob(client *api.Client, jobID string, version *uint64) (*api.Job, error) {
	if version == nil {
		job, _, err := client.Jobs().Info(jobID, nil)
		return job, err
	}

	versions, _, _, err := client.Jobs().Versions(jobID, false, nil)
	if err != nil {
		return nil, err
	}

	for _, j := range versions {
		if *j.Version != *version {
			continue
		}
		return j, nil
	}

	return nil, fmt.Errorf("job %q with version %d couldn't be found", jobID, *version)
}
<|endoftext|>