<commit_before>package commands

import (
	"compress/gzip"
	"errors"
	"fmt"
	"io"
	"os"
	gopath "path"
	"strings"

	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb"

	cmds "github.com/ipfs/go-ipfs/commands"
	core "github.com/ipfs/go-ipfs/core"
	path "github.com/ipfs/go-ipfs/path"
	tar "github.com/ipfs/go-ipfs/thirdparty/tar"
	uarchive "github.com/ipfs/go-ipfs/unixfs/archive"
)

var ErrInvalidCompressionLevel = errors.New("Compression level must be between 1 and 9")

var GetCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Download IPFS objects",
		ShortDescription: `
Retrieves the object named by <ipfs-or-ipns-path> and stores the data to disk.

By default, the output will be stored at ./<ipfs-path>, but an alternate path
can be specified with '--output=<path>' or '-o=<path>'.

To output a TAR archive instead of unpacked files, use '--archive' or '-a'.

To compress the output with GZIP compression, use '--compress' or '-C'. You
may also specify the level of compression by specifying '-l=<1-9>'.
`,
	},

	Arguments: []cmds.Argument{
		cmds.StringArg("ipfs-path", true, false, "The path to the IPFS object(s) to be outputted").EnableStdin(),
	},
	Options: []cmds.Option{
		cmds.StringOption("output", "o", "The path where output should be stored"),
		cmds.BoolOption("archive", "a", "Output a TAR archive"),
		cmds.BoolOption("compress", "C", "Compress the output with GZIP compression"),
		cmds.IntOption("compression-level", "l", "The level of compression (1-9)"),
	},
	PreRun: func(req cmds.Request) error {
		_, err := getCompressOptions(req)
		return err
	},
	Run: func(req cmds.Request, res cmds.Response) {
		cmplvl, err := getCompressOptions(req)
		if err != nil {
			res.SetError(err, cmds.ErrClient)
			return
		}

		node, err := req.InvocContext().GetNode()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		p := path.Path(req.Arguments()[0])
		ctx := req.Context()
		dn, err := core.Resolve(ctx, node, p)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		archive, _, _ := req.Option("archive").Bool()
		reader, err := uarchive.DagArchive(ctx, dn, p.String(), node.DAG, archive, cmplvl)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		res.SetOutput(reader)
	},
	PostRun: func(req cmds.Request, res cmds.Response) {
		if res.Output() == nil {
			return
		}
		outReader := res.Output().(io.Reader)
		res.SetOutput(nil)

		outPath, _, _ := req.Option("output").String()
		if len(outPath) == 0 {
			_, outPath = gopath.Split(req.Arguments()[0])
			outPath = gopath.Clean(outPath)
		}

		cmplvl, err := getCompressOptions(req)
		if err != nil {
			res.SetError(err, cmds.ErrClient)
			return
		}

		archive, _, _ := req.Option("archive").Bool()

		gw := getWriter{
			Out:         os.Stdout,
			Err:         os.Stderr,
			Archive:     archive,
			Compression: cmplvl,
		}

		if err := gw.Write(outReader, outPath); err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
	},
}

func progressBarForReader(out io.Writer, r io.Reader) (*pb.ProgressBar, *pb.Reader) {
	// setup bar reader
	// TODO: get total length of files
	bar := pb.New(0).SetUnits(pb.U_BYTES)
	bar.Output = out
	barR := bar.NewProxyReader(r)
	return bar, barR
}

type getWriter struct {
	Out io.Writer // for output to user
	Err io.Writer // for progress bar output

	Archive     bool
	Compression int
}

func (gw *getWriter) Write(r io.Reader, fpath string) error {
	if gw.Archive || gw.Compression != gzip.NoCompression {
		return gw.writeArchive(r, fpath)
	}
	return gw.writeExtracted(r, fpath)
}

func (gw *getWriter) writeArchive(r io.Reader, fpath string) error {
	// adjust file name if tar
	if gw.Archive {
		if !strings.HasSuffix(fpath, ".tar") && !strings.HasSuffix(fpath, ".tar.gz") {
			fpath += ".tar"
		}
	}

	// adjust file name if gz
	if gw.Compression != gzip.NoCompression {
		if !strings.HasSuffix(fpath, ".gz") {
			fpath += ".gz"
		}
	}

	// create file
	file, err := os.Create(fpath)
	if err != nil {
		return err
	}
	defer file.Close()

	fmt.Fprintf(gw.Out, "Saving archive to %s\n", fpath)
	bar, barR := progressBarForReader(gw.Err, r)
	bar.Start()
	defer bar.Finish()

	_, err = io.Copy(file, barR)
	return err
}

func (gw *getWriter) writeExtracted(r io.Reader, fpath string) error {
	fmt.Fprintf(gw.Out, "Saving file(s) to %s\n", fpath)
	bar, barR := progressBarForReader(gw.Err, r)
	bar.Start()
	defer bar.Finish()

	extractor := &tar.Extractor{fpath}
	return extractor.Extract(barR)
}

func getCompressOptions(req cmds.Request) (int, error) {
	cmprs, _, _ := req.Option("compress").Bool()
	cmplvl, cmplvlFound, _ := req.Option("compression-level").Int()
	switch {
	case !cmprs:
		return gzip.NoCompression, nil
	case cmprs && !cmplvlFound:
		return gzip.DefaultCompression, nil
	case cmprs && cmplvlFound && (cmplvl < 1 || cmplvl > 9):
		return gzip.NoCompression, ErrInvalidCompressionLevel
	}
	return gzip.NoCompression, nil
}
<commit_msg>Fix gzip compression level not being set.<commit_after>package commands

import (
	"compress/gzip"
	"errors"
	"fmt"
	"io"
	"os"
	gopath "path"
	"strings"

	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb"

	cmds "github.com/ipfs/go-ipfs/commands"
	core "github.com/ipfs/go-ipfs/core"
	path "github.com/ipfs/go-ipfs/path"
	tar "github.com/ipfs/go-ipfs/thirdparty/tar"
	uarchive "github.com/ipfs/go-ipfs/unixfs/archive"
)

var ErrInvalidCompressionLevel = errors.New("Compression level must be between 1 and 9")

var GetCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Download IPFS objects",
		ShortDescription: `
Retrieves the object named by <ipfs-or-ipns-path> and stores the data to disk.

By default, the output will be stored at ./<ipfs-path>, but an alternate path
can be specified with '--output=<path>' or '-o=<path>'.

To output a TAR archive instead of unpacked files, use '--archive' or '-a'.

To compress the output with GZIP compression, use '--compress' or '-C'. You
may also specify the level of compression by specifying '-l=<1-9>'.
`,
	},

	Arguments: []cmds.Argument{
		cmds.StringArg("ipfs-path", true, false, "The path to the IPFS object(s) to be outputted").EnableStdin(),
	},
	Options: []cmds.Option{
		cmds.StringOption("output", "o", "The path where output should be stored"),
		cmds.BoolOption("archive", "a", "Output a TAR archive"),
		cmds.BoolOption("compress", "C", "Compress the output with GZIP compression"),
		cmds.IntOption("compression-level", "l", "The level of compression (1-9)"),
	},
	PreRun: func(req cmds.Request) error {
		_, err := getCompressOptions(req)
		return err
	},
	Run: func(req cmds.Request, res cmds.Response) {
		cmplvl, err := getCompressOptions(req)
		if err != nil {
			res.SetError(err, cmds.ErrClient)
			return
		}

		node, err := req.InvocContext().GetNode()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		p := path.Path(req.Arguments()[0])
		ctx := req.Context()
		dn, err := core.Resolve(ctx, node, p)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		archive, _, _ := req.Option("archive").Bool()
		reader, err := uarchive.DagArchive(ctx, dn, p.String(), node.DAG, archive, cmplvl)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		res.SetOutput(reader)
	},
	PostRun: func(req cmds.Request, res cmds.Response) {
		if res.Output() == nil {
			return
		}
		outReader := res.Output().(io.Reader)
		res.SetOutput(nil)

		outPath, _, _ := req.Option("output").String()
		if len(outPath) == 0 {
			_, outPath = gopath.Split(req.Arguments()[0])
			outPath = gopath.Clean(outPath)
		}

		cmplvl, err := getCompressOptions(req)
		if err != nil {
			res.SetError(err, cmds.ErrClient)
			return
		}

		archive, _, _ := req.Option("archive").Bool()

		gw := getWriter{
			Out:         os.Stdout,
			Err:         os.Stderr,
			Archive:     archive,
			Compression: cmplvl,
		}

		if err := gw.Write(outReader, outPath); err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
	},
}

func progressBarForReader(out io.Writer, r io.Reader) (*pb.ProgressBar, *pb.Reader) {
	// setup bar reader
	// TODO: get total length of files
	bar := pb.New(0).SetUnits(pb.U_BYTES)
	bar.Output = out
	barR := bar.NewProxyReader(r)
	return bar, barR
}

type getWriter struct {
	Out io.Writer // for output to user
	Err io.Writer // for progress bar output

	Archive     bool
	Compression int
}

func (gw *getWriter) Write(r io.Reader, fpath string) error {
	if gw.Archive || gw.Compression != gzip.NoCompression {
		return gw.writeArchive(r, fpath)
	}
	return gw.writeExtracted(r, fpath)
}

func (gw *getWriter) writeArchive(r io.Reader, fpath string) error {
	// adjust file name if tar
	if gw.Archive {
		if !strings.HasSuffix(fpath, ".tar") && !strings.HasSuffix(fpath, ".tar.gz") {
			fpath += ".tar"
		}
	}

	// adjust file name if gz
	if gw.Compression != gzip.NoCompression {
		if !strings.HasSuffix(fpath, ".gz") {
			fpath += ".gz"
		}
	}

	// create file
	file, err := os.Create(fpath)
	if err != nil {
		return err
	}
	defer file.Close()

	fmt.Fprintf(gw.Out, "Saving archive to %s\n", fpath)
	bar, barR := progressBarForReader(gw.Err, r)
	bar.Start()
	defer bar.Finish()

	_, err = io.Copy(file, barR)
	return err
}

func (gw *getWriter) writeExtracted(r io.Reader, fpath string) error {
	fmt.Fprintf(gw.Out, "Saving file(s) to %s\n", fpath)
	bar, barR := progressBarForReader(gw.Err, r)
	bar.Start()
	defer bar.Finish()

	extractor := &tar.Extractor{fpath}
	return extractor.Extract(barR)
}

func getCompressOptions(req cmds.Request) (int, error) {
	cmprs, _, _ := req.Option("compress").Bool()
	cmplvl, cmplvlFound, _ := req.Option("compression-level").Int()
	switch {
	case !cmprs:
		return gzip.NoCompression, nil
	case cmprs && !cmplvlFound:
		return gzip.DefaultCompression, nil
	case cmprs && cmplvlFound && (cmplvl < 1 || cmplvl > 9):
		return gzip.NoCompression, ErrInvalidCompressionLevel
	}
	return cmplvl, nil
}
<|endoftext|>


<commit_before>package core

import (
	"log"
	"testing"
)

func TestNewArkClient(t *testing.T) {
	arkapi := NewArkClient(nil)

	if arkapi == nil {
		t.Error("Error creating client")
	}
	log.Println(t.Name(), "Success")
}

func TestSwitchPeer(t *testing.T) {
	arkapi := NewArkClient(nil)

	if arkapi == nil {
		t.Error("Error creating client")
	}

	arkapi.SwitchPeer()
	log.Println(arkapi.GetActivePeer())
	log.Println(t.Name(), "Success")
}
<commit_msg>Test Update<commit_after>package core

import (
	"log"
	"testing"
)

func TestNewArkClient(t *testing.T) {
	arkapi := NewArkClient(nil)

	if arkapi == nil {
		t.Error("Error creating client")
	}
	log.Println(t.Name(), "Success")
}

func TestSwitchPeer(t *testing.T) {
	arkapi := NewArkClient(nil)

	if arkapi == nil {
		t.Error("Error creating client")
	}

	arkapi = arkapi.SwitchPeer()
	log.Println(arkapi.GetActivePeer())
	log.Println(t.Name(), "Success")
}
<|endoftext|>


<commit_before>// Copyright 2015 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !js,!windows

package audio

import (
	"fmt"
	"io"
	"runtime"
	"time"

	"golang.org/x/mobile/exp/audio/al"
)

const (
	maxBufferNum = 8
)

// TODO: This should be in player
var totalBufferNum = 0

type player struct {
	alSource   al.Source
	alBuffers  []al.Buffer
	source     io.Reader
	sampleRate int
	isClosed   bool
}

func startPlaying(src io.Reader, sampleRate int) (*player, error) {
	if e := al.OpenDevice(); e != nil {
		return nil, fmt.Errorf("audio: OpenAL initialization failed: %v", e)
	}
	s := al.GenSources(1)
	if err := al.Error(); err != 0 {
		panic(fmt.Sprintf("audio: al.GenSources error: %d", err))
	}
	p := &player{
		alSource:   s[0],
		alBuffers:  []al.Buffer{},
		source:     src,
		sampleRate: sampleRate,
	}
	runtime.SetFinalizer(p, (*player).close)
	if err := p.start(); err != nil {
		return nil, err
	}
	return p, nil
}

const (
	bufferSize = 1024
)

var (
	tmpBuffer    = make([]byte, bufferSize)
	tmpAlBuffers = make([]al.Buffer, maxBufferNum)
)

func (p *player) proceed() error {
	if err := al.Error(); err != 0 {
		panic(fmt.Sprintf("audio: before proceed: %d", err))
	}
	processedNum := p.alSource.BuffersProcessed()
	if 0 < processedNum {
		bufs := tmpAlBuffers[:processedNum]
		p.alSource.UnqueueBuffers(bufs...)
		if err := al.Error(); err != 0 {
			panic(fmt.Sprintf("audio: Unqueue in process: %d", err))
		}
		p.alBuffers = append(p.alBuffers, bufs...)
	}

	for 0 < len(p.alBuffers) {
		n, err := p.source.Read(tmpBuffer)
		if 0 < n {
			buf := p.alBuffers[0]
			p.alBuffers = p.alBuffers[1:]
			buf.BufferData(al.FormatStereo16, tmpBuffer[:n], int32(p.sampleRate))
			p.alSource.QueueBuffers(buf)
			if err := al.Error(); err != 0 {
				panic(fmt.Sprintf("audio: Queue in process: %d", err))
			}
		}
		if err != nil {
			return err
		}
		if n == 0 {
			time.Sleep(1)
		}
	}

	if p.alSource.State() == al.Stopped {
		al.RewindSources(p.alSource)
		al.PlaySources(p.alSource)
		if err := al.Error(); err != 0 {
			panic(fmt.Sprintf("audio: PlaySource in process: %d", err))
		}
	}

	return nil
}

func (p *player) start() error {
	n := maxBufferNum - int(p.alSource.BuffersQueued()) - len(p.alBuffers)
	if 0 < n {
		p.alBuffers = append(p.alBuffers, al.GenBuffers(n)...)
		totalBufferNum += n
		if maxBufferNum < totalBufferNum {
			panic("audio: too many buffers are created")
		}
	}
	if 0 < len(p.alBuffers) {
		emptyBytes := make([]byte, bufferSize)
		for _, buf := range p.alBuffers {
			// Note that the third argument of only the first buffer is used.
			buf.BufferData(al.FormatStereo16, emptyBytes, int32(p.sampleRate))
			p.alSource.QueueBuffers(buf)
		}
		p.alBuffers = []al.Buffer{}
	}
	al.PlaySources(p.alSource)

	go func() {
		// TODO: Is it OK to close asap?
		defer p.close()
		for {
			err := p.proceed()
			if err == io.EOF {
				break
			}
			if err != nil {
				// TODO: Record the last error
				panic(err)
			}
			runtime.Gosched()
		}
	}()
	return nil
}

// TODO: When is this called? Can we remove this?
func (p *player) close() error {
	if err := al.Error(); err != 0 {
		panic(fmt.Sprintf("audio: error before closing: %d", err))
	}
	if p.isClosed {
		return nil
	}
	var bs []al.Buffer
	al.RewindSources(p.alSource)
	al.StopSources(p.alSource)
	n := p.alSource.BuffersQueued()
	if 0 < n {
		bs = make([]al.Buffer, n)
		p.alSource.UnqueueBuffers(bs...)
		p.alBuffers = append(p.alBuffers, bs...)
	}
	p.isClosed = true
	if err := al.Error(); err != 0 {
		panic(fmt.Sprintf("audio: error after closing: %d", err))
	}
	runtime.SetFinalizer(p, nil)
	return nil
}
<commit_msg>audio: time.Sleep to avoid busy loop<commit_after>// Copyright 2015 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !js,!windows

package audio

import (
	"fmt"
	"io"
	"runtime"
	"time"

	"github.com/hajimehoshi/ebiten"
	"golang.org/x/mobile/exp/audio/al"
)

const (
	maxBufferNum = 8
)

// TODO: This should be in player
var totalBufferNum = 0

type player struct {
	alSource   al.Source
	alBuffers  []al.Buffer
	source     io.Reader
	sampleRate int
	isClosed   bool
}

func startPlaying(src io.Reader, sampleRate int) (*player, error) {
	if e := al.OpenDevice(); e != nil {
		return nil, fmt.Errorf("audio: OpenAL initialization failed: %v", e)
	}
	s := al.GenSources(1)
	if err := al.Error(); err != 0 {
		panic(fmt.Sprintf("audio: al.GenSources error: %d", err))
	}
	p := &player{
		alSource:   s[0],
		alBuffers:  []al.Buffer{},
		source:     src,
		sampleRate: sampleRate,
	}
	runtime.SetFinalizer(p, (*player).close)
	if err := p.start(); err != nil {
		return nil, err
	}
	return p, nil
}

const (
	bufferSize = 1024
)

var (
	tmpBuffer    = make([]byte, bufferSize)
	tmpAlBuffers = make([]al.Buffer, maxBufferNum)
)

func (p *player) proceed() error {
	if err := al.Error(); err != 0 {
		panic(fmt.Sprintf("audio: before proceed: %d", err))
	}
	processedNum := p.alSource.BuffersProcessed()
	if 0 < processedNum {
		bufs := tmpAlBuffers[:processedNum]
		p.alSource.UnqueueBuffers(bufs...)
		if err := al.Error(); err != 0 {
			panic(fmt.Sprintf("audio: Unqueue in process: %d", err))
		}
		p.alBuffers = append(p.alBuffers, bufs...)
	}

	for 0 < len(p.alBuffers) {
		n, err := p.source.Read(tmpBuffer)
		if 0 < n {
			buf := p.alBuffers[0]
			p.alBuffers = p.alBuffers[1:]
			buf.BufferData(al.FormatStereo16, tmpBuffer[:n], int32(p.sampleRate))
			p.alSource.QueueBuffers(buf)
			if err := al.Error(); err != 0 {
				panic(fmt.Sprintf("audio: Queue in process: %d", err))
			}
		}
		if err != nil {
			return err
		}
		if n == 0 {
			time.Sleep(1)
		}
	}

	if p.alSource.State() == al.Stopped {
		al.RewindSources(p.alSource)
		al.PlaySources(p.alSource)
		if err := al.Error(); err != 0 {
			panic(fmt.Sprintf("audio: PlaySource in process: %d", err))
		}
	}

	return nil
}

func (p *player) start() error {
	n := maxBufferNum - int(p.alSource.BuffersQueued()) - len(p.alBuffers)
	if 0 < n {
		p.alBuffers = append(p.alBuffers, al.GenBuffers(n)...)
		totalBufferNum += n
		if maxBufferNum < totalBufferNum {
			panic("audio: too many buffers are created")
		}
	}
	if 0 < len(p.alBuffers) {
		emptyBytes := make([]byte, bufferSize)
		for _, buf := range p.alBuffers {
			// Note that the third argument of only the first buffer is used.
			buf.BufferData(al.FormatStereo16, emptyBytes, int32(p.sampleRate))
			p.alSource.QueueBuffers(buf)
		}
		p.alBuffers = []al.Buffer{}
	}
	al.PlaySources(p.alSource)

	go func() {
		// TODO: Is it OK to close asap?
		defer p.close()
		for {
			err := p.proceed()
			if err == io.EOF {
				break
			}
			if err != nil {
				// TODO: Record the last error
				panic(err)
			}
			time.Sleep(1 * time.Second / ebiten.FPS / 2)
		}
	}()
	return nil
}

// TODO: When is this called? Can we remove this?
func (p *player) close() error {
	if err := al.Error(); err != 0 {
		panic(fmt.Sprintf("audio: error before closing: %d", err))
	}
	if p.isClosed {
		return nil
	}
	var bs []al.Buffer
	al.RewindSources(p.alSource)
	al.StopSources(p.alSource)
	n := p.alSource.BuffersQueued()
	if 0 < n {
		bs = make([]al.Buffer, n)
		p.alSource.UnqueueBuffers(bs...)
		p.alBuffers = append(p.alBuffers, bs...)
	}
	p.isClosed = true
	if err := al.Error(); err != 0 {
		panic(fmt.Sprintf("audio: error after closing: %d", err))
	}
	runtime.SetFinalizer(p, nil)
	return nil
}
<|endoftext|>


<commit_before>package main

import "C"

import (
	"flag"
	"fmt"
	"log"
	"os"
	"reflect"
	"strings"
	"unsafe"

	"github.com/bmatsuo/lmdb-go/exp/cmd/internal/lmdbcmd"
	"github.com/bmatsuo/lmdb-go/exp/lmdbscan"
	"github.com/bmatsuo/lmdb-go/lmdb"
)

func main() {
	opt := &Options{}
	flag.BoolVar(&opt.PrintInfo, "e", false, "Display information about the database environment")
	flag.BoolVar(&opt.PrintFree, "f", false, "Display freelist information")
	flag.BoolVar(&opt.PrintFreeSummary, "ff", false, "Display freelist information")
	flag.BoolVar(&opt.PrintFreeFull, "fff", false, "Display freelist information")
	flag.BoolVar(&opt.PrintReaders, "r", false, strings.Join([]string{
		"Display information about the environment reader table.",
		"Shows the process ID, thread ID, and transaction ID for each active reader slot.",
	}, "  "))
	flag.BoolVar(&opt.PrintReadersCheck, "rr", false, strings.Join([]string{
		"Implies -r.",
		"Check for stale entries in the reader table and clear them.",
		"The reader table is printed again after the check is performed.",
	}, "  "))
	flag.BoolVar(&opt.PrintStatAll, "a", false, "Display the status of all databases in the environment")
	flag.StringVar(&opt.PrintStatSub, "s", "", "Display the status of a specific subdatabase.")
	flag.BoolVar(&opt.Debug, "D", false, "print debug information")
	flag.Parse()

	lmdbcmd.PrintVersion()

	if opt.PrintStatAll && opt.PrintStatSub != "" {
		log.Fatal("only one of -a and -s may be provided")
	}

	if flag.NArg() > 1 {
		log.Fatalf("too many argument provided")
	}
	if flag.NArg() == 0 {
		log.Fatalf("missing argument")
	}
	opt.Path = flag.Arg(0)

	var failed bool
	defer func() {
		if e := recover(); e != nil {
			if opt.Debug {
				panic(e)
			}
			log.Print(e)
			failed = true
		}
		if failed {
			os.Exit(1)
		}
	}()

	err := doMain(opt)
	if err != nil {
		log.Print(err)
		failed = true
	}
}

type Options struct {
	Path string

	PrintInfo         bool
	PrintReaders      bool
	PrintReadersCheck bool
	PrintFree         bool
	PrintFreeSummary  bool
	PrintFreeFull     bool
	PrintStatAll      bool
	PrintStatSub      string

	Debug bool
}

func doMain(opt *Options) error {
	env, err := lmdb.NewEnv()
	if err != nil {
		return err
	}
	if opt.PrintStatAll || opt.PrintStatSub != "" {
		err = env.SetMaxDBs(1)
		if err != nil {
			return err
		}
	}
	err = env.Open(opt.Path, lmdbcmd.OpenFlag(), 0644)
	defer env.Close()
	if err != nil {
		return err
	}

	if opt.PrintInfo {
		err = doPrintInfo(env, opt)
		if err != nil {
			return err
		}
	}

	if opt.PrintFree || opt.PrintFreeSummary || opt.PrintFreeFull {
		err = doPrintFree(env, opt)
		if err != nil {
			return err
		}
	}

	err = doPrintStatRoot(env, opt)
	if err != nil {
		return err
	}

	if opt.PrintStatAll {
		err = doPrintStatAll(env, opt)
		if err != nil {
			return err
		}
	} else if opt.PrintStatSub != "" {
		err = doPrintStatDB(env, opt.PrintStatSub, opt)
		if err != nil {
			return err
		}
	}

	return nil
}

func doPrintInfo(env *lmdb.Env, opt *Options) error {
	info, err := env.Info()
	if err != nil {
		return err
	}

	pagesize := os.Getpagesize()

	fmt.Println("Environment Info")
	fmt.Println("  Map address:", nil)
	fmt.Println("  Map size:", info.MapSize)
	fmt.Println("  Page size:", pagesize)
	fmt.Println("  Max pages:", info.MapSize/int64(pagesize))
	fmt.Println("  Number of pages used:", info.LastPNO+1)
	fmt.Println("  Last transaction ID:", info.LastTxnID)
	fmt.Println("  Max readers:", info.MaxReaders)
	fmt.Println("  Number of readers used:", info.NumReaders)

	return nil
}

func doPrintFree(env *lmdb.Env, opt *Options) error {
	return env.View(func(txn *lmdb.Txn) (err error) {
		txn.RawRead = true

		fmt.Println("Freelist status")

		stat, err := txn.Stat(0)
		if err != nil {
			return err
		}
		printStat(stat, opt)

		var numpages int64
		s := lmdbscan.New(txn, 0)
		defer s.Close()
		for s.Scan() {
			key := s.Val()
			data := s.Val()
			txid := *(*C.size_t)(unsafe.Pointer(&key[0]))
			ipages := int64(*(*C.size_t)(unsafe.Pointer(&data[0])))
			numpages += ipages
			if opt.PrintFreeSummary || opt.PrintFreeFull {
				bad := ""
				hdr := reflect.SliceHeader{
					Data: uintptr(unsafe.Pointer(&data[0])),
					Len:  int(ipages) + 1,
					Cap:  int(ipages) + 1,
				}
				pages := *(*[]C.size_t)(unsafe.Pointer(&hdr))
				pages = pages[1:]
				var span C.size_t
				prev := C.size_t(1)
				for i := ipages - 1; i >= 0; i-- {
					pg := pages[i]
					if pg < prev {
						bad = " [bad sequence]"
					}
					prev = pg
					pg += span
					for i >= int64(span) && pages[i-int64(span)] == pg {
						span++
						pg++
					}
				}
				fmt.Printf("    Transaction %x, %d pages, maxspan %d%s\n", txid, ipages, span, bad)

				if opt.PrintFreeFull {
					for j := ipages - 1; j >= 0; {
						pg := pages[j]
						j--
						span := C.size_t(1)
						for j >= 0 && pages[j] == pg+span {
							j--
							span++
						}
						if span > 1 {
							fmt.Printf("     %9x[%d]\n", pg, span)
						} else {
							fmt.Printf("     %9x\n", pg)
						}
					}
				}
			}
		}
		err = s.Err()
		if err != nil {
			return err
		}

		fmt.Println("  Free pages:", numpages)

		return nil
	})
}

func doPrintStatRoot(env *lmdb.Env, opt *Options) error {
	stat, err := env.Stat()
	if err != nil {
		return err
	}

	fmt.Println("Status of Main DB")
	fmt.Println("  Tree depth:", stat.Depth)
	fmt.Println("  Branch pages:", stat.BranchPages)
	fmt.Println("  Lead pages:", stat.LeafPages)
	fmt.Println("  Overflow pages:", stat.OverflowPages)
	fmt.Println("  Entries:", stat.Entries)

	return nil
}

func doPrintStatDB(env *lmdb.Env, db string, opt *Options) error {
	err := env.View(func(txn *lmdb.Txn) (err error) {
		return printStatDB(env, txn, db, opt)
	})
	if err != nil {
		return fmt.Errorf("%v (%s)", err, db)
	}
	return nil
}

func printStatDB(env *lmdb.Env, txn *lmdb.Txn, db string, opt *Options) error {
	dbi, err := txn.OpenDBI(db, 0)
	if err != nil {
		return err
	}
	defer env.CloseDBI(dbi)

	stat, err := txn.Stat(dbi)
	if err != nil {
		return err
	}

	fmt.Println("Status of", db)
	printStat(stat, opt)

	return err
}

func printStat(stat *lmdb.Stat, opt *Options) error {
	fmt.Println("  Tree depth:", stat.Depth)
	fmt.Println("  Branch pages:", stat.BranchPages)
	fmt.Println("  Lead pages:", stat.LeafPages)
	fmt.Println("  Overflow pages:", stat.OverflowPages)
	fmt.Println("  Entries:", stat.Entries)

	return nil
}

func doPrintStatAll(env *lmdb.Env, opt *Options) error {
	return env.View(func(txn *lmdb.Txn) (err error) {
		dbi, err := txn.OpenRoot(0)
		if err != nil {
			return err
		}
		defer env.CloseDBI(dbi)

		s := lmdbscan.New(txn, dbi)
		defer s.Close()
		for s.Scan() {
			err = printStatDB(env, txn, string(s.Key()), opt)
			if e, ok := err.(*lmdb.OpError); ok {
				if e.Op == "mdb_dbi_open" {
					continue
				}
			}
			if err != nil {
				return fmt.Errorf("%v (%s)", err, s.Key())
			}
		}
		return s.Err()
	})
}
<commit_msg>add stub for handling `mdb_stat -r`<commit_after>package main

import "C"

import (
	"flag"
	"fmt"
	"log"
	"os"
	"reflect"
	"strings"
	"unsafe"

	"github.com/bmatsuo/lmdb-go/exp/cmd/internal/lmdbcmd"
	"github.com/bmatsuo/lmdb-go/exp/lmdbscan"
	"github.com/bmatsuo/lmdb-go/lmdb"
)

func main() {
	opt := &Options{}
	flag.BoolVar(&opt.PrintInfo, "e", false, "Display information about the database environment")
	flag.BoolVar(&opt.PrintFree, "f", false, "Display freelist information")
	flag.BoolVar(&opt.PrintFreeSummary, "ff", false, "Display freelist information")
	flag.BoolVar(&opt.PrintFreeFull, "fff", false, "Display freelist information")
	flag.BoolVar(&opt.PrintReaders, "r", false, strings.Join([]string{
		"Display information about the environment reader table.",
		"Shows the process ID, thread ID, and transaction ID for each active reader slot.",
	}, "  "))
	flag.BoolVar(&opt.PrintReadersCheck, "rr", false, strings.Join([]string{
		"Implies -r.",
		"Check for stale entries in the reader table and clear them.",
		"The reader table is printed again after the check is performed.",
	}, "  "))
	flag.BoolVar(&opt.PrintStatAll, "a", false, "Display the status of all databases in the environment")
	flag.StringVar(&opt.PrintStatSub, "s", "", "Display the status of a specific subdatabase.")
	flag.BoolVar(&opt.Debug, "D", false, "print debug information")
	flag.Parse()

	lmdbcmd.PrintVersion()

	if opt.PrintStatAll && opt.PrintStatSub != "" {
		log.Fatal("only one of -a and -s may be provided")
	}

	if flag.NArg() > 1 {
		log.Fatalf("too many argument provided")
	}
	if flag.NArg() == 0 {
		log.Fatalf("missing argument")
	}
	opt.Path = flag.Arg(0)

	var failed bool
	defer func() {
		if e := recover(); e != nil {
			if opt.Debug {
				panic(e)
			}
			log.Print(e)
			failed = true
		}
		if failed {
			os.Exit(1)
		}
	}()

	err := doMain(opt)
	if err != nil {
		log.Print(err)
		failed = true
	}
}

type Options struct {
	Path string

	PrintInfo         bool
	PrintReaders      bool
	PrintReadersCheck bool
	PrintFree         bool
	PrintFreeSummary  bool
	PrintFreeFull     bool
	PrintStatAll      bool
	PrintStatSub      string

	Debug bool
}

func doMain(opt *Options) error {
	env, err := lmdb.NewEnv()
	if err != nil {
		return err
	}
	if opt.PrintStatAll || opt.PrintStatSub != "" {
		err = env.SetMaxDBs(1)
		if err != nil {
			return err
		}
	}
	err = env.Open(opt.Path, lmdbcmd.OpenFlag(), 0644)
	defer env.Close()
	if err != nil {
		return err
	}

	if opt.PrintInfo {
		err = doPrintInfo(env, opt)
		if err != nil {
			return err
		}
	}

	if opt.PrintReaders || opt.PrintReadersCheck {
		err = doPrintReaders(env, opt)
		if err != nil {
			return err
		}
	}

	if opt.PrintFree || opt.PrintFreeSummary || opt.PrintFreeFull {
		err = doPrintFree(env, opt)
		if err != nil {
			return err
		}
	}

	err = doPrintStatRoot(env, opt)
	if err != nil {
		return err
	}

	if opt.PrintStatAll {
		err = doPrintStatAll(env, opt)
		if err != nil {
			return err
		}
	} else if opt.PrintStatSub != "" {
		err = doPrintStatDB(env, opt.PrintStatSub, opt)
		if err != nil {
			return err
		}
	}

	return nil
}

func doPrintInfo(env *lmdb.Env, opt *Options) error {
	info, err := env.Info()
	if err != nil {
		return err
	}

	pagesize := os.Getpagesize()

	fmt.Println("Environment Info")
	fmt.Println("  Map address:", nil)
	fmt.Println("  Map size:", info.MapSize)
	fmt.Println("  Page size:", pagesize)
	fmt.Println("  Max pages:", info.MapSize/int64(pagesize))
	fmt.Println("  Number of pages used:", info.LastPNO+1)
	fmt.Println("  Last transaction ID:", info.LastTxnID)
	fmt.Println("  Max readers:", info.MaxReaders)
	fmt.Println("  Number of readers used:", info.NumReaders)

	return nil
}

func doPrintReaders(env *lmdb.Env, opt *Options) error {
	return fmt.Errorf("TODO: implement Env.ReaderList")
}

func doPrintFree(env *lmdb.Env, opt *Options) error {
	return env.View(func(txn *lmdb.Txn) (err error) {
		txn.RawRead = true

		fmt.Println("Freelist status")

		stat, err := txn.Stat(0)
		if err != nil {
			return err
		}
		printStat(stat, opt)

		var numpages int64
		s := lmdbscan.New(txn, 0)
		defer s.Close()
		for s.Scan() {
			key := s.Val()
			data := s.Val()
			txid := *(*C.size_t)(unsafe.Pointer(&key[0]))
			ipages := int64(*(*C.size_t)(unsafe.Pointer(&data[0])))
			numpages += ipages
			if opt.PrintFreeSummary || opt.PrintFreeFull {
				bad := ""
				hdr := reflect.SliceHeader{
					Data: uintptr(unsafe.Pointer(&data[0])),
					Len:  int(ipages) + 1,
					Cap:  int(ipages) + 1,
				}
				pages := *(*[]C.size_t)(unsafe.Pointer(&hdr))
				pages = pages[1:]
				var span C.size_t
				prev := C.size_t(1)
				for i := ipages - 1; i >= 0; i-- {
					pg := pages[i]
					if pg < prev {
						bad = " [bad sequence]"
					}
					prev = pg
					pg += span
					for i >= int64(span) && pages[i-int64(span)] == pg {
						span++
						pg++
					}
				}
				fmt.Printf("    Transaction %x, %d pages, maxspan %d%s\n", txid, ipages, span, bad)

				if opt.PrintFreeFull {
					for j := ipages - 1; j >= 0; {
						pg := pages[j]
						j--
						span := C.size_t(1)
						for j >= 0 && pages[j] == pg+span {
							j--
							span++
						}
						if span > 1 {
							fmt.Printf("     %9x[%d]\n", pg, span)
						} else {
							fmt.Printf("     %9x\n", pg)
						}
					}
				}
			}
		}
		err = s.Err()
		if err != nil {
			return err
		}

		fmt.Println("  Free pages:", numpages)

		return nil
	})
}

func doPrintStatRoot(env *lmdb.Env, opt *Options) error {
	stat, err := env.Stat()
	if err != nil {
		return err
	}

	fmt.Println("Status of Main DB")
	fmt.Println("  Tree depth:", stat.Depth)
	fmt.Println("  Branch pages:", stat.BranchPages)
	fmt.Println("  Lead pages:", stat.LeafPages)
	fmt.Println("  Overflow pages:", stat.OverflowPages)
	fmt.Println("  Entries:", stat.Entries)

	return nil
}

func doPrintStatDB(env *lmdb.Env, db string, opt *Options) error {
	err := env.View(func(txn *lmdb.Txn) (err error) {
		return printStatDB(env, txn, db, opt)
	})
	if err != nil {
		return fmt.Errorf("%v (%s)", err, db)
	}
	return nil
}

func printStatDB(env *lmdb.Env, txn *lmdb.Txn, db string, opt *Options) error {
	dbi, err := txn.OpenDBI(db, 0)
	if err != nil {
		return err
	}
	defer env.CloseDBI(dbi)

	stat, err := txn.Stat(dbi)
	if err != nil {
		return err
	}

	fmt.Println("Status of", db)
	printStat(stat, opt)

	return err
}

func printStat(stat *lmdb.Stat, opt *Options) error {
	fmt.Println("  Tree depth:", stat.Depth)
	fmt.Println("  Branch pages:", stat.BranchPages)
	fmt.Println("  Lead pages:", stat.LeafPages)
	fmt.Println("  Overflow pages:", stat.OverflowPages)
	fmt.Println("  Entries:", stat.Entries)

	return nil
}

func doPrintStatAll(env *lmdb.Env, opt *Options) error {
	return env.View(func(txn *lmdb.Txn) (err error) {
		dbi, err := txn.OpenRoot(0)
		if err != nil {
			return err
		}
		defer env.CloseDBI(dbi)

		s := lmdbscan.New(txn, dbi)
		defer s.Close()
		for s.Scan() {
			err = printStatDB(env, txn, string(s.Key()), opt)
			if e, ok := err.(*lmdb.OpError); ok {
				if e.Op == "mdb_dbi_open" {
					continue
				}
			}
			if err != nil {
				return fmt.Errorf("%v (%s)", err, s.Key())
			}
		}
		return s.Err()
	})
}
<|endoftext|>


<commit_before>package store

import (
	"bytes"
	"encoding/json"
	// "fmt"
	"io"
	"os"
	"path/filepath"
	assetpath "path/filepath"

	c4db "github.com/avalanche-io/c4/db"
	c4 "github.com/avalanche-io/c4/id"
	slash "github.com/avalanche-io/path"
)

// Store represents a Asset storage location.
type Store struct {
	path string
	db   *c4db.DB
}

// writepath represents a writable folder for files prior to identification.
const writepath string = "scratch"

// OpenStorage opens the storage at the given path.  If the path doesn't already
// exist, OpenStorage will attempt to create it.
func Open(path string) (*Store, error) {
	// Make paths as necessary.
	err := makepaths(path, filepath.Join(path, writepath))
	if err != nil {
		return nil, err
	}

	// Open a C4 Database
	db_path := filepath.Join(path, "c4.db")
	db, err := c4db.Open(db_path, 0600, nil)
	if err != nil {
		return nil, err
	}

	// initialize and return a new Store
	s := &Store{path, db}
	if !s.Exists("/") {
		err = makeroot(path, db)
	}
	return s, err
}

// Create creates a new writable asset.
func (s *Store) Create(path string, ids ...*c4.ID) (Asset, error) {
	var id *c4.ID
	if len(ids) == 1 {
		id = ids[0]
	}

	temp_file, err := tmp(s.path)
	if err != nil {
		return nil, err
	}
	return NewFileAsset(path, (*storage)(s), os.O_RDWR, temp_file, id)
}

func (s *Store) Writer(path string, ids ...*c4.ID) (c4.WriteCloseIdentifier, error) {
	return s.Create(path, ids...)
}

func (s *Store) Reader(path string, ids ...*c4.ID) (c4.ReadCloseIdentifier, error) {
	return s.Open(path, ids...)
}

func (s *Store) Copy(src, dest string) error {
	id := s.db.Get([]byte(src))
	if id == nil {
		return ErrNotFound
	}
	return s.db.Set([]byte(dest), id)
}

func (s *Store) Move(src, dest string) error {
	err := s.Copy(src, dest)
	if err != nil {
		return err
	}
	s.db.Unset([]byte(src))
	return nil
}

//Open opens the named asset for reading.
func (s *Store) Open(name string, ids ...*c4.ID) (a Asset, err error) {
	var id *c4.ID
	if len(ids) == 1 {
		id = ids[0]
	}
	if len(name) > 0 {
		id = s.db.Get([]byte(name))
	}

	if id == nil {
		return nil, os.ErrNotExist
	}
	var file *os.File
	file_name := filepath.Join(pathtoasset(s.path, id), id.String())
	file, err = os.OpenFile(file_name, os.O_RDONLY, 0600)
	if err != nil {
		return nil, err
	}

	dir, filename := filepath.Split(name)
	if len(filename) == 0 {
		dir, filename = filepath.Split(dir)
		if len(dir) == 1 {
			filename = "/"
			dir = ""
		}
	}
	if slash.IsDir(name) {
		return NewDirAsset(name, (*storage)(s), os.O_RDONLY, file)
	}
	return NewFileAsset(name, (*storage)(s), os.O_RDONLY, file, id)
}

// Mkdir creates an empty directory entry for the given path if it does not already
// exist. Mkdir returns os.ErrExist if the path already exists.
func (s *Store) Mkdir(path string) error {
	if path == "/" {
		return nil
	}

	if path[len(path)-1:] != "/" {
		return dirError("directory must have \"/\" suffix")
	}
	if s.Exists(path) {
		return os.ErrExist
	}
	err := s.db.Set([]byte(path), c4.NIL_ID)
	if err != nil {
		return err
	}
	return s.update_directory([]byte(path))
}

// MkdirAll makes directories in the given bath that don't already exist.
func (s *Store) MkdirAll(path string) error {
	if path[len(path)-1:] != "/" {
		return dirError("directory must have \"/\" suffix")
	}
	p, err := slash.New(path)
	if err != nil {
		return err
	}
	for _, dir := range p.EveryPath() {
		if !s.Exists(dir) {
			err = s.Mkdir(dir)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

// AssetID returns the c4 id of file at the given path
func (s *Store) AssetID(name string) *c4.ID {
	return s.db.Get([]byte(name))
}

// Close closes the database, and any other cleanup as needed.
func (s *Store) Close() error {
	return s.db.Close()
}

// Exists tests if the path exists in the database, and if the identified file
// exists in the storage.
func (s *Store) Exists(path string) bool {
	id := s.db.Get([]byte(path))
	if id == nil {
		return false
	}
	e := exists(s.path, id)
	return e
}

func (s *Store) IDexists(id *c4.ID) bool {
	return s.db.IDexists(id)
}

// Add returns a copy of the Asset bound to the storage, or the unmodified Asset if it
// is already bound.
func (s *Store) Add(asset Asset) Asset {
	switch val := asset.Storage().(type) {
	case *storage:
		if s == (*Store)(val) {
			return asset
		}
	}

	return CopyAsset(asset, (*storage)(s))
}

func (s *Store) SetAttributes(key string, attrs map[string]interface{}) error {
	// convert to json
	data, err := json.Marshal(attrs)
	if err != nil {
		return err
	}
	// identify
	id := c4.Identify(bytes.NewReader(data))

	// Check if the id already exists.
	if exists(s.path, id) {
		s.db.SetAttributes([]byte(key), id)
		return nil
	}
	dir := pathtoasset(s.path, id)
	makepaths(dir)
	file_name := filepath.Join(dir, id.String())

	f, err := os.Create(file_name)
	if err != nil {
		return err
	}
	n, err := io.Copy(f, bytes.NewReader(data))
	if err != nil {
		return err
	}
	_ = n
	f.Close()
	s.db.SetAttributes([]byte(key), id)
	return nil
}

func (s *Store) GetAttributes(key string, attrs map[string]interface{}) error {
	id := s.db.GetAttributes([]byte(key))
	if id == nil {
		return ErrNotFound
	}
	file_name := filepath.Join(pathtoasset(s.path, id), id.String())
	f, err := os.Open(file_name)
	if err != nil {
		return err
	}
	defer f.Close()
	j := json.NewDecoder(f)
	err = j.Decode(&attrs)
	if err != nil {
		return err
	}
	return nil
}

// update_directory adds the file name in path to it's parent directory list and saves
func (s *Store) update_directory(key []byte) error {
	dir, name := assetpath.Split(string(key))
	if slash.IsDir(string(key)) {
		p, err := slash.New(string(key) + "/")
		if err != nil {
			return err
		}
		dir, name = p.Split()
	}

	var d Directory
	din, err := s.Open(dir)
	if err != nil {
		return err
	}
	defer din.Close()

	// read the file
	_, err = io.Copy(&d, din)
	if err != nil {
		return dirError("unable to read directory \"" + dir + "\" \"" + name + "\"" + err.Error())
	}

	// UPDATE
	// add the name to the directory
	d.Insert(name)

	// WRITE
	// create a new file
	dout, err := s.Create(dir)
	if err != nil {
		return dirError(err.Error())
	}

	// write data from the directory in ram
	_, err = io.Copy(dout, d)
	if err != nil {
		dout.Remove()
		return dirError(err.Error())
	}
	// commit changes.
	return dout.commit()
}
<commit_msg>fixed new c4.ReadCloser references in storage<commit_after>package store

import (
	"bytes"
	"encoding/json"
	// "fmt"
	"io"
	"os"
	"path/filepath"
	assetpath "path/filepath"

	c4db "github.com/avalanche-io/c4/db"
	c4 "github.com/avalanche-io/c4/id"
	slash "github.com/avalanche-io/path"
)

// Store represents a Asset storage location.
type Store struct {
	path string
	db   *c4db.DB
}

// writepath represents a writable folder for files prior to identification.
const writepath string = "scratch"

// OpenStorage opens the storage at the given path.  If the path doesn't already
// exist, OpenStorage will attempt to create it.
func Open(path string) (*Store, error) {
	// Make paths as necessary.
	err := makepaths(path, filepath.Join(path, writepath))
	if err != nil {
		return nil, err
	}

	// Open a C4 Database
	db_path := filepath.Join(path, "c4.db")
	db, err := c4db.Open(db_path, 0600, nil)
	if err != nil {
		return nil, err
	}

	// initialize and return a new Store
	s := &Store{path, db}
	if !s.Exists("/") {
		err = makeroot(path, db)
	}
	return s, err
}

// Create creates a new writable asset.
func (s *Store) Create(path string, ids ...*c4.ID) (Asset, error) {
	var id *c4.ID
	if len(ids) == 1 {
		id = ids[0]
	}

	temp_file, err := tmp(s.path)
	if err != nil {
		return nil, err
	}
	return NewFileAsset(path, (*storage)(s), os.O_RDWR, temp_file, id)
}

func (s *Store) Writer(path string, ids ...*c4.ID) (c4.WriteCloser, error) {
	return s.Create(path, ids...)
}

func (s *Store) Reader(path string, ids ...*c4.ID) (c4.ReadCloser, error) {
	return s.Open(path, ids...)
}

func (s *Store) Copy(src, dest string) error {
	id := s.db.Get([]byte(src))
	if id == nil {
		return ErrNotFound
	}
	return s.db.Set([]byte(dest), id)
}

func (s *Store) Move(src, dest string) error {
	err := s.Copy(src, dest)
	if err != nil {
		return err
	}
	s.db.Unset([]byte(src))
	return nil
}

//Open opens the named asset for reading.
func (s *Store) Open(name string, ids ...*c4.ID) (a Asset, err error) {
	var id *c4.ID
	if len(ids) == 1 {
		id = ids[0]
	}
	if len(name) > 0 {
		id = s.db.Get([]byte(name))
	}

	if id == nil {
		return nil, os.ErrNotExist
	}
	var file *os.File
	file_name := filepath.Join(pathtoasset(s.path, id), id.String())
	file, err = os.OpenFile(file_name, os.O_RDONLY, 0600)
	if err != nil {
		return nil, err
	}

	dir, filename := filepath.Split(name)
	if len(filename) == 0 {
		dir, filename = filepath.Split(dir)
		if len(dir) == 1 {
			filename = "/"
			dir = ""
		}
	}
	if slash.IsDir(name) {
		return NewDirAsset(name, (*storage)(s), os.O_RDONLY, file)
	}
	return NewFileAsset(name, (*storage)(s), os.O_RDONLY, file, id)
}

// Mkdir creates an empty directory entry for the given path if it does not already
// exist. Mkdir returns os.ErrExist if the path already exists.
func (s *Store) Mkdir(path string) error {
	if path == "/" {
		return nil
	}

	if path[len(path)-1:] != "/" {
		return dirError("directory must have \"/\" suffix")
	}
	if s.Exists(path) {
		return os.ErrExist
	}
	err := s.db.Set([]byte(path), c4.NIL_ID)
	if err != nil {
		return err
	}
	return s.update_directory([]byte(path))
}

// MkdirAll makes directories in the given bath that don't already exist.
func (s *Store) MkdirAll(path string) error {
	if path[len(path)-1:] != "/" {
		return dirError("directory must have \"/\" suffix")
	}
	p, err := slash.New(path)
	if err != nil {
		return err
	}
	for _, dir := range p.EveryPath() {
		if !s.Exists(dir) {
			err = s.Mkdir(dir)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

// AssetID returns the c4 id of file at the given path
func (s *Store) AssetID(name string) *c4.ID {
	return s.db.Get([]byte(name))
}

// Close closes the database, and any other cleanup as needed.
func (s *Store) Close() error {
	return s.db.Close()
}

// Exists tests if the path exists in the database, and if the identified file
// exists in the storage.
func (s *Store) Exists(path string) bool {
	id := s.db.Get([]byte(path))
	if id == nil {
		return false
	}
	e := exists(s.path, id)
	return e
}

func (s *Store) IDexists(id *c4.ID) bool {
	return s.db.IDexists(id)
}

// Add returns a copy of the Asset bound to the storage, or the unmodified Asset if it
// is already bound.
func (s *Store) Add(asset Asset) Asset {
	switch val := asset.Storage().(type) {
	case *storage:
		if s == (*Store)(val) {
			return asset
		}
	}

	return CopyAsset(asset, (*storage)(s))
}

func (s *Store) SetAttributes(key string, attrs map[string]interface{}) error {
	// convert to json
	data, err := json.Marshal(attrs)
	if err != nil {
		return err
	}
	// identify
	id := c4.Identify(bytes.NewReader(data))

	// Check if the id already exists.
	if exists(s.path, id) {
		s.db.SetAttributes([]byte(key), id)
		return nil
	}
	dir := pathtoasset(s.path, id)
	makepaths(dir)
	file_name := filepath.Join(dir, id.String())

	f, err := os.Create(file_name)
	if err != nil {
		return err
	}
	n, err := io.Copy(f, bytes.NewReader(data))
	if err != nil {
		return err
	}
	_ = n
	f.Close()
	s.db.SetAttributes([]byte(key), id)
	return nil
}

func (s *Store) GetAttributes(key string, attrs map[string]interface{}) error {
	id := s.db.GetAttributes([]byte(key))
	if id == nil {
		return ErrNotFound
	}
	file_name := filepath.Join(pathtoasset(s.path, id), id.String())
	f, err := os.Open(file_name)
	if err != nil {
		return err
	}
	defer f.Close()
	j := json.NewDecoder(f)
	err = j.Decode(&attrs)
	if err != nil {
		return err
	}
	return nil
}

// update_directory adds the file name in path to it's parent directory list and saves
func (s *Store) update_directory(key []byte) error {
	dir, name := assetpath.Split(string(key))
	if slash.IsDir(string(key)) {
		p, err := slash.New(string(key) + "/")
		if err != nil {
			return err
		}
		dir, name = p.Split()
	}

	var d Directory
	din, err := s.Open(dir)
	if err != nil {
		return err
	}
	defer din.Close()

	// read the file
	_, err = io.Copy(&d, din)
	if err != nil {
		return dirError("unable to read directory \"" + dir + "\" \"" + name + "\"" + err.Error())
	}

	// UPDATE
	// add the name to the directory
	d.Insert(name)

	// WRITE
	// create a new file
	dout, err := s.Create(dir)
	if err != nil {
		return dirError(err.Error())
	}

	// write data from the directory in ram
	_, err = io.Copy(dout, d)
	if err != nil {
		dout.Remove()
		return dirError(err.Error())
	}
	// commit changes.
	return dout.commit()
}
<|endoftext|>


<commit_before>package store

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"unicode"
)

type hasPrintf interface {
	Printf(fmt string, v ...interface{})
}

type sortByCommitId struct {
	data   []string
	logger hasPrintf
}

func (s sortByCommitId) Len() int {
	return len(s.data)
}
func (s sortByCommitId) Swap(i, j int) {
	s.data[i], s.data[j] = s.data[j], s.data[i]
}
func (s sortByCommitId) Less(i, j int) bool {
	s1 := s.data[i]
	id1, err1 := ExtractCommitIdFromFilename(s1)
	if err1 != nil {
		s.logger.Printf("sortByCommitId.Less: error parsing config file path: '%s': %v", s1, err1)
	}
	s2 := s.data[j]
	id2, err2 := ExtractCommitIdFromFilename(s2)
	if err2 != nil {
		s.logger.Printf("sortByCommitId.Less: error parsing config file path: '%s': %v", s2, err2)
	}
	return id1 < id2
}

func ExtractCommitIdFromFilename(filename string) (int, error) {
	lastDot := strings.LastIndexByte(filename, '.')
	commitId := filename[lastDot+1:]
	id, err := strconv.Atoi(commitId)
	if err != nil {
		return -1, fmt.Errorf("extractCommitIdFromFilename: error parsing filename [%s]: %v", filename, err)
	}

	return id, nil
}

func FindLastConfig(configPathPrefix string, logger hasPrintf) (string, error) {

	lastIdPath := getLastIdPath(configPathPrefix)
	f, openErr := os.Open(lastIdPath)
	if openErr == nil {
		defer f.Close()
		r := bufio.NewReader(f)
		line, _, readErr := r.ReadLine()
		if readErr == nil {
			id := string(line[:])
			path := getConfigPath(configPathPrefix, id)
			_, statErr := os.Stat(path)
			if statErr == nil {
				logger.Printf("FindLastConfig: found from shortcut: '%s'", path)
				return path, nil
			} else {
				logger.Printf("FindLastConfig: stat failure '%s': %v", lastIdPath, statErr)
			}
		} else {
			logger.Printf("FindLastConfig: read failure '%s': %v", lastIdPath, readErr)
		}
	}
	logger.Printf("FindLastConfig: last id file not found '%s': %v", lastIdPath, openErr)

	// search filesystem directory

	dirname, matches, err := ListConfig(configPathPrefix, logger)
	if err != nil {
		return "", err
	}

	size := len(matches)

	logger.Printf("FindLastConfig: found %d matching files: %v", size, matches)

	if size < 1 {
		return "", fmt.Errorf("FindLastConfig: no config file found for prefix: %s", configPathPrefix)
	}

	maxId := -1
	last := ""
	for _, m := range matches {
		id, idErr := ExtractCommitIdFromFilename(m)
		if idErr != nil {
			return "", fmt.Errorf("FindLastConfig: bad commit id: %s: %v", m, idErr)
		}
		if id >= maxId {
			maxId = id
			last = m
		}
	}

	lastPath := filepath.Join(dirname, last)

	logger.Printf("FindLastConfig: found: %s", lastPath)

	return lastPath, nil
}

func ListConfigSorted(configPathPrefix string, reverse bool, logger hasPrintf) (string, []string, error) {

	dirname, matches, err := ListConfig(configPathPrefix, logger)
	if err != nil {
		return dirname, matches, err
	}

	if reverse {
		sort.Sort(sort.Reverse(sortByCommitId{data: matches, logger: logger}))
	} else {
		sort.Sort(sortByCommitId{data: matches, logger: logger})
	}

	return dirname, matches, nil
}

func ListConfig(configPathPrefix string, logger hasPrintf) (string, []string, error) {

	dirname := filepath.Dir(configPathPrefix)

	dir, err := os.Open(dirname)
	if err != nil {
		return "", nil, fmt.Errorf("ListConfig: error opening dir '%s': %v", dirname, err)
	}

	names, e := dir.Readdirnames(0)
	if e != nil {
		return "", nil, fmt.Errorf("ListConfig: error reading dir '%s': %v", dirname, e)
	}

	dir.Close()

	basename := filepath.Base(configPathPrefix)

	// filter prefix
	matches := names[:0] // slice trick: Filtering without allocating
	for _, x := range names {
		lastByte := rune(x[len(x)-1])
		if unicode.IsDigit(lastByte) && strings.HasPrefix(x, basename) {
			matches = append(matches, x)
		}
	}

	return dirname, matches, nil
}

type HasWrite interface {
	Write(p []byte) (int, error)
}

func getLastIdPath(configPathPrefix string) string {
	return fmt.Sprintf("%slast", configPathPrefix)
}

func getConfigPath(configPathPrefix, id string) string {
	return fmt.Sprintf("%s%s", configPathPrefix, id)
}

func SaveNewConfig(configPathPrefix string, maxFiles int, logger hasPrintf, writeFunc func(HasWrite) error) (string, error) {

	lastConfig, err1 := FindLastConfig(configPathPrefix, logger)
	if err1 != nil {
		logger.Printf("SaveNewConfig: error reading config: [%s]: %v", configPathPrefix, err1)
	}

	id, err2 := ExtractCommitIdFromFilename(lastConfig)
	if err2 != nil {
		logger.Printf("SaveNewConfig: error parsing config path: [%s]: %v", lastConfig, err2)
	}

	newCommitId := id + 1
	newFilepath := getConfigPath(configPathPrefix, strconv.Itoa(newCommitId))

	logger.Printf("SaveNewConfig: newPath=[%s]", newFilepath)

	if _, err := os.Stat(newFilepath); err == nil {
		return "", fmt.Errorf("SaveNewConfig: new file exists: [%s]", newFilepath)
	}

	f, err3 := os.Create(newFilepath)
	if err3 != nil {
		return "", fmt.Errorf("SaveNewConfig: error creating file: [%s]: %v", newFilepath, err3)
	}

	w := bufio.NewWriter(f)

	if err := writeFunc(w); err != nil {
		return "", fmt.Errorf("SaveNewConfig: writeFunc error: [%s]: %v", newFilepath, err)
	}

	if err := w.Flush(); err != nil {
		return "", fmt.Errorf("SaveNewConfig: error flushing file: [%s]: %v", newFilepath, err)
	}

	if err := f.Close(); err != nil {
		return "", fmt.Errorf("SaveNewConfig: error closing file: [%s]: %v", newFilepath, err)
	}

	// write last id into shortcut file
	lastIdPath := getLastIdPath(configPathPrefix)
	if err := ioutil.WriteFile(lastIdPath, []byte(strconv.Itoa(newCommitId)), 0700); err != nil {
		logger.Printf("SaveNewConfig: error writing last id file '%s': %v", lastIdPath, err)
	}

	eraseOldFiles(configPathPrefix, maxFiles, logger)

	return newFilepath, nil
}

func eraseOldFiles(configPathPrefix string, maxFiles int, logger hasPrintf) {

	if maxFiles < 1 {
		return
	}

	dirname, matches, err := ListConfigSorted(configPathPrefix, false, logger)
	if err != nil {
		logger.Printf("eraseOldFiles: %v", err)
		return
	}

	totalFiles := len(matches)

	toDelete := totalFiles - maxFiles
	if toDelete < 1 {
		logger.Printf("eraseOldFiles: nothing to delete existing=%d <= max=%d", totalFiles, maxFiles)
		return
	}

	for i := 0; i < toDelete; i++ {
		path := filepath.Join(dirname, matches[i])
		logger.Printf("eraseOldFiles: delete: [%s]", path)
		if err := os.Remove(path); err != nil {
			logger.Printf("eraseOldFiles: delete: error: [%s]: %v", path, err)
		}
	}
}
<commit_msg>Save new configs to tmp file to avoid creating truncated files on failures.<commit_after>package store

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"unicode"
)

type hasPrintf interface {
	Printf(fmt string, v ...interface{})
}

type sortByCommitId struct {
	data   []string
	logger hasPrintf
}

func (s sortByCommitId) Len() int {
	return len(s.data)
}
func (s sortByCommitId) Swap(i, j int) {
	s.data[i], s.data[j] = s.data[j], s.data[i]
}
func (s sortByCommitId) Less(i, j int) bool {
	s1 := s.data[i]
	id1, err1 := ExtractCommitIdFromFilename(s1)
	if err1 != nil {
		s.logger.Printf("sortByCommitId.Less: error parsing config file path: '%s': %v", s1, err1)
	}
	s2 := s.data[j]
	id2, err2 := ExtractCommitIdFromFilename(s2)
	if err2 != nil {
		s.logger.Printf("sortByCommitId.Less: error parsing config file path: '%s': %v", s2, err2)
	}
	return id1 < id2
}

func ExtractCommitIdFromFilename(filename string) (int, error) {
	lastDot := strings.LastIndexByte(filename, '.')
	commitId := filename[lastDot+1:]
	id, err := strconv.Atoi(commitId)
	if err != nil {
		return -1, fmt.Errorf("extractCommitIdFromFilename: error parsing filename [%s]: %v", filename, err)
	}

	return id, nil
}

func FindLastConfig(configPathPrefix string, logger hasPrintf) (string, error) {

	lastIdPath := getLastIdPath(configPathPrefix)
	f, openErr := os.Open(lastIdPath)
	if openErr == nil {
		defer f.Close()
		r := bufio.NewReader(f)
		line, _, readErr := r.ReadLine()
		if readErr == nil {
			id := string(line[:])
			path := getConfigPath(configPathPrefix, id)
			_, statErr := os.Stat(path)
			if statErr == nil {
				logger.Printf("FindLastConfig: found from shortcut: '%s'", path)
				return path, nil
			} else {
				logger.Printf("FindLastConfig: stat failure '%s': %v", lastIdPath, statErr)
			}
		} else {
			logger.Printf("FindLastConfig: read failure '%s': %v", lastIdPath, readErr)
		}
	}
	logger.Printf("FindLastConfig: last id file not found '%s': %v", lastIdPath, openErr)

	// search filesystem directory

	dirname, matches, err := ListConfig(configPathPrefix, logger)
	if err != nil {
		return "", err
	}

	size := len(matches)

	logger.Printf("FindLastConfig: found %d matching files: %v", size, matches)

	if size < 1 {
		return "", fmt.Errorf("FindLastConfig: no config file found for prefix: %s", configPathPrefix)
	}

	maxId := -1
	last := ""
	for _, m := range matches {
		id, idErr := ExtractCommitIdFromFilename(m)
		if idErr != nil {
			return "", fmt.Errorf("FindLastConfig: bad commit id: %s: %v", m, idErr)
		}
		if id >= maxId {
			maxId = id
			last = m
		}
	}

	lastPath := filepath.Join(dirname, last)

	logger.Printf("FindLastConfig: found: %s", lastPath)

	return lastPath, nil
}

func ListConfigSorted(configPathPrefix string, reverse bool, logger hasPrintf) (string, []string, error) {

	dirname, matches, err := ListConfig(configPathPrefix, logger)
	if err != nil {
		return dirname, matches, err
	}

	if reverse {
		sort.Sort(sort.Reverse(sortByCommitId{data: matches, logger: logger}))
	} else {
		sort.Sort(sortByCommitId{data: matches, logger: logger})
	}

	return dirname, matches, nil
}

func ListConfig(configPathPrefix string, logger hasPrintf) (string, []string, error) {

	dirname := filepath.Dir(configPathPrefix)

	dir, err := os.Open(dirname)
	if err != nil {
		return "", nil, fmt.Errorf("ListConfig: error opening dir '%s': %v", dirname, err)
	}

	names, e := dir.Readdirnames(0)
	if e != nil {
		return "", nil, fmt.Errorf("ListConfig: error reading dir '%s': %v", dirname, e)
	}

	dir.Close()

	basename := filepath.Base(configPathPrefix)

	// filter prefix
	matches := names[:0] // slice trick: Filtering without allocating
	for _, x := range names {
		lastByte := rune(x[len(x)-1])
		if unicode.IsDigit(lastByte) && strings.HasPrefix(x, basename) {
			matches = append(matches, x)
		}
	}

	return dirname, matches, nil
}

type HasWrite interface {
	Write(p []byte) (int, error)
}

func getLastIdPath(configPathPrefix string) string {
	return fmt.Sprintf("%slast", configPathPrefix)
}

func getConfigPath(configPathPrefix, id string) string {
	return fmt.Sprintf("%s%s", configPathPrefix, id)
}

func SaveNewConfig(configPathPrefix string, maxFiles int, logger hasPrintf, writeFunc func(HasWrite) error) (string, error) {

	// get tmp file
	tmpPath := getConfigPath(configPathPrefix, "tmp")

	if _, err := os.Stat(tmpPath); err == nil {
		return "", fmt.Errorf("SaveNewConfig: tmp file exists: [%s]", tmpPath)
	}

	f, createErr := os.Create(tmpPath)
	if createErr != nil {
		return "", fmt.Errorf("SaveNewConfig: error creating tmp file: [%s]: %v", tmpPath, createErr)
	}

	defer os.Remove(tmpPath)

	// write to tmp file

	w := bufio.NewWriter(f)

	if err := writeFunc(w); err != nil {
		return "", fmt.Errorf("SaveNewConfig: writeFunc error: [%s]: %v", tmpPath, err)
	}

	if err := w.Flush(); err != nil {
		return "", fmt.Errorf("SaveNewConfig: error flushing file: [%s]: %v", tmpPath, err)
	}

	if err := f.Close(); err != nil {
		return "", fmt.Errorf("SaveNewConfig: error closing file: [%s]: %v", tmpPath, err)
	}

	// get new file

	lastConfig, err1 := FindLastConfig(configPathPrefix, logger)
	if err1 != nil {
		logger.Printf("SaveNewConfig: error reading config: [%s]: %v", configPathPrefix, err1)
	}

	id, err2 := ExtractCommitIdFromFilename(lastConfig)
	if err2 != nil {
		logger.Printf("SaveNewConfig: error parsing config path: [%s]: %v", lastConfig, err2)
	}

	newCommitId := id + 1
	newFilepath := getConfigPath(configPathPrefix, strconv.Itoa(newCommitId))

	logger.Printf("SaveNewConfig: newPath=[%s]", newFilepath)

	if _, err := os.Stat(newFilepath); err == nil {
		return "", fmt.Errorf("SaveNewConfig: new file exists: [%s]", newFilepath)
	}

	// rename tmp to new file

	if renameErr := os.Rename(tmpPath, newFilepath); renameErr != nil {
		return "", fmt.Errorf("SaveNewConfig: could not rename '%s' to '%s'; %v", tmpPath, newFilepath, renameErr)
	}

	// write shortcut file

	// write last id into shortcut file
	lastIdPath := getLastIdPath(configPathPrefix)
	if err := ioutil.WriteFile(lastIdPath, []byte(strconv.Itoa(newCommitId)), 0700); err != nil {
		logger.Printf("SaveNewConfig: error writing last id file '%s': %v", lastIdPath, err)
	}

	// erase old file

	eraseOldFiles(configPathPrefix, maxFiles, logger)

	return newFilepath, nil
}

func eraseOldFiles(configPathPrefix string, maxFiles int, logger hasPrintf) {

	if maxFiles < 1 {
		return
	}

	dirname, matches, err := ListConfigSorted(configPathPrefix, false, logger)
	if err != nil {
		logger.Printf("eraseOldFiles: %v", err)
		return
	}

	totalFiles := len(matches)

	toDelete := totalFiles - maxFiles
	if toDelete < 1 {
		logger.Printf("eraseOldFiles: nothing to delete existing=%d <= max=%d", totalFiles, maxFiles)
		return
	}

	for i := 0; i < toDelete; i++ {
		path := filepath.Join(dirname, matches[i])
		logger.Printf("eraseOldFiles: delete: [%s]", path)
		if err := os.Remove(path); err != nil {
			logger.Printf("eraseOldFiles: delete: error: [%s]: %v", path, err)
		}
	}
}
<|endoftext|>


<commit_before>package session

import (
	"github.com/gorilla/context"
	"github.com/gorilla/sessions"
	"github.com/labstack/echo"
	"github.com/labstack/echo/middleware"
)

type (
	// Config defines the config for CasbinAuth middleware.
	Config struct {
		// Skipper defines a function to skip middleware.
		Skipper middleware.Skipper

		// Session store.
		// Required.
		Store sessions.Store
	}
)

const (
	key = "_session_store"
)

var (
	// DefaultConfig is the default Session middleware config.
	DefaultConfig = Config{
		Skipper: middleware.DefaultSkipper,
	}
)

// Get returns a named session.
func Get(name string, c echo.Context) (*sessions.Session, error) {
	store := c.Get(key).(sessions.Store)
	return store.Get(c.Request(), name)
}

// Middleware returns a Session middleware.
func Middleware(store sessions.Store) echo.MiddlewareFunc {
	c := DefaultConfig
	c.Store = store
	return MiddlewareWithConfig(c)
}

// MiddlewareWithConfig returns a Sessions middleware with config.
// See `Middleware()`.
func MiddlewareWithConfig(config Config) echo.MiddlewareFunc {
	// Defaults
	if config.Skipper == nil {
		config.Skipper = DefaultConfig.Skipper
	}
	if config.Store == nil {
		panic("echo: session middleware requires store")
	}

	return func(next echo.HandlerFunc) echo.HandlerFunc {
		return func(c echo.Context) error {
			if config.Skipper(c) {
				return next(c)
			}
			defer context.Clear(c.Request())
			c.Set(key, config.Store)
			return next(c)
		}
	}
}
<commit_msg>Fixed the comment in Session middleware<commit_after>package session

import (
	"github.com/gorilla/context"
	"github.com/gorilla/sessions"
	"github.com/labstack/echo"
	"github.com/labstack/echo/middleware"
)

type (
	// Config defines the config for Session middleware.
	Config struct {
		// Skipper defines a function to skip middleware.
		Skipper middleware.Skipper

		// Session store.
		// Required.
		Store sessions.Store
	}
)

const (
	key = "_session_store"
)

var (
	// DefaultConfig is the default Session middleware config.
	DefaultConfig = Config{
		Skipper: middleware.DefaultSkipper,
	}
)

// Get returns a named session.
func Get(name string, c echo.Context) (*sessions.Session, error) {
	store := c.Get(key).(sessions.Store)
	return store.Get(c.Request(), name)
}

// Middleware returns a Session middleware.
func Middleware(store sessions.Store) echo.MiddlewareFunc {
	c := DefaultConfig
	c.Store = store
	return MiddlewareWithConfig(c)
}

// MiddlewareWithConfig returns a Sessions middleware with config.
// See `Middleware()`.
func MiddlewareWithConfig(config Config) echo.MiddlewareFunc {
	// Defaults
	if config.Skipper == nil {
		config.Skipper = DefaultConfig.Skipper
	}
	if config.Store == nil {
		panic("echo: session middleware requires store")
	}

	return func(next echo.HandlerFunc) echo.HandlerFunc {
		return func(c echo.Context) error {
			if config.Skipper(c) {
				return next(c)
			}
			defer context.Clear(c.Request())
			c.Set(key, config.Store)
			return next(c)
		}
	}
}
<|endoftext|>


<commit_before>package store

import (
	"github.com/cloudfoundry/hm9000/config"
	"github.com/cloudfoundry/hm9000/helpers/logger"
	"github.com/cloudfoundry/hm9000/models"
	"github.com/cloudfoundry/hm9000/storeadapter"
	"reflect"
	"time"
)

type Storeable interface {
	StoreKey() string
	ToJSON() []byte
}

type Store interface {
	BumpDesiredFreshness(timestamp time.Time) error
	BumpActualFreshness(timestamp time.Time) error

	IsDesiredStateFresh() (bool, error)
	IsActualStateFresh(time.Time) (bool, error)

	SaveDesiredState(desiredStates ...models.DesiredAppState) error
	GetDesiredState() ([]models.DesiredAppState, error)
	DeleteDesiredState(desiredStates ...models.DesiredAppState) error

	SaveActualState(actualStates ...models.InstanceHeartbeat) error
	GetActualState() ([]models.InstanceHeartbeat, error)
	DeleteActualState(actualStates ...models.InstanceHeartbeat) error

	SavePendingStartMessages(startMessages ...models.PendingStartMessage) error
	GetPendingStartMessages() ([]models.PendingStartMessage, error)
	DeletePendingStartMessages(startMessages ...models.PendingStartMessage) error

	SavePendingStopMessages(stopMessages ...models.PendingStopMessage) error
	GetPendingStopMessages() ([]models.PendingStopMessage, error)
	DeletePendingStopMessages(stopMessages ...models.PendingStopMessage) error

	SaveCrashCounts(crashCounts ...models.CrashCount) error
	GetCrashCounts() ([]models.CrashCount, error)
	DeleteCrashCounts(crashCounts ...models.CrashCount) error
}

type RealStore struct {
	config  config.Config
	adapter storeadapter.StoreAdapter
	logger  logger.Logger
}

func NewStore(config config.Config, adapter storeadapter.StoreAdapter, logger logger.Logger) *RealStore {
	return &RealStore{
		config:  config,
		adapter: adapter,
		logger:  logger,
	}
}

func (store *RealStore) fetchNodesUnderDir(dir string) ([]storeadapter.StoreNode, error) {
	nodes, err := store.adapter.List(dir)
	if err != nil {
		if err == storeadapter.ErrorKeyNotFound {
			return []storeadapter.StoreNode{}, nil
		}
		return []storeadapter.StoreNode{}, err
	}
	return nodes, nil
}

// buckle up, here be dragons...

func (store *RealStore) save(stuff interface{}, root string, ttl uint64) error {
	// t := time.Now()
	arrValue := reflect.ValueOf(stuff)

	nodes := make([]storeadapter.StoreNode, arrValue.Len())
	for i := 0; i < arrValue.Len(); i++ {
		item := arrValue.Index(i).Interface().(Storeable)
		nodes[i] = storeadapter.StoreNode{
			Key:   root + "/" + item.StoreKey(),
			Value: item.ToJSON(),
			TTL:   ttl,
		}
	}

	err := store.adapter.Set(nodes)

	// store.logger.Info("Save Duration", map[string]string{
	// 	"Number of Items": fmt.Sprintf("%d", arrValue.Len()),
	// 	"Duration":        fmt.Sprintf("%.4f seconds", time.Since(t).Seconds()),
	// })
	return err
}

func (store *RealStore) get(root string, sliceType reflect.Type, constructor reflect.Value) (reflect.Value, error) {
	// t := time.Now()

	nodes, err := store.fetchNodesUnderDir(root)
	if err != nil {
		return reflect.MakeSlice(sliceType, 0, 0), err
	}

	slice := reflect.MakeSlice(sliceType, 0, 0)
	for _, node := range nodes {
		out := constructor.Call([]reflect.Value{reflect.ValueOf(node.Value)})
		slice = reflect.Append(slice, out[0])
		if !out[1].IsNil() {
			return reflect.MakeSlice(sliceType, 0, 0), out[1].Interface().(error)
		}
	}

	// store.logger.Info("Get Duration", map[string]string{
	// 	"Number of Items": fmt.Sprintf("%d", slice.Len()),
	// 	"Duration":        fmt.Sprintf("%.4f seconds", time.Since(t).Seconds()),
	// })
	return slice, nil
}

func (store *RealStore) delete(stuff interface{}, root string) error {
	// t := time.Now()
	arrValue := reflect.ValueOf(stuff)

	for i := 0; i < arrValue.Len(); i++ {
		item := arrValue.Index(i).Interface().(Storeable)

		err := store.adapter.Delete(root + "/" + item.StoreKey())
		if err != nil {
			return err
		}
	}

	// store.logger.Info("Delete Duration", map[string]string{
	// 	"Number of Items": fmt.Sprintf("%d", arrValue.Len()),
	// 	"Duration":        fmt.Sprintf("%.4f seconds", time.Since(t).Seconds()),
	// })

	return nil
}
<commit_msg>restore performance logging<commit_after>package store

import (
	"fmt"
	"github.com/cloudfoundry/hm9000/config"
	"github.com/cloudfoundry/hm9000/helpers/logger"
	"github.com/cloudfoundry/hm9000/models"
	"github.com/cloudfoundry/hm9000/storeadapter"
	"reflect"
	"time"
)

type Storeable interface {
	StoreKey() string
	ToJSON() []byte
}

type Store interface {
	BumpDesiredFreshness(timestamp time.Time) error
	BumpActualFreshness(timestamp time.Time) error

	IsDesiredStateFresh() (bool, error)
	IsActualStateFresh(time.Time) (bool, error)

	SaveDesiredState(desiredStates ...models.DesiredAppState) error
	GetDesiredState() ([]models.DesiredAppState, error)
	DeleteDesiredState(desiredStates ...models.DesiredAppState) error

	SaveActualState(actualStates ...models.InstanceHeartbeat) error
	GetActualState() ([]models.InstanceHeartbeat, error)
	DeleteActualState(actualStates ...models.InstanceHeartbeat) error

	SavePendingStartMessages(startMessages ...models.PendingStartMessage) error
	GetPendingStartMessages() ([]models.PendingStartMessage, error)
	DeletePendingStartMessages(startMessages ...models.PendingStartMessage) error

	SavePendingStopMessages(stopMessages ...models.PendingStopMessage) error
	GetPendingStopMessages() ([]models.PendingStopMessage, error)
	DeletePendingStopMessages(stopMessages ...models.PendingStopMessage) error

	SaveCrashCounts(crashCounts ...models.CrashCount) error
	GetCrashCounts() ([]models.CrashCount, error)
	DeleteCrashCounts(crashCounts ...models.CrashCount) error
}

type RealStore struct {
	config  config.Config
	adapter storeadapter.StoreAdapter
	logger  logger.Logger
}

func NewStore(config config.Config, adapter storeadapter.StoreAdapter, logger logger.Logger) *RealStore {
	return &RealStore{
		config:  config,
		adapter: adapter,
		logger:  logger,
	}
}

func (store *RealStore) fetchNodesUnderDir(dir string) ([]storeadapter.StoreNode, error) {
	nodes, err := store.adapter.List(dir)
	if err != nil {
		if err == storeadapter.ErrorKeyNotFound {
			return []storeadapter.StoreNode{}, nil
		}
		return []storeadapter.StoreNode{}, err
	}
	return nodes, nil
}

// buckle up, here be dragons...

func (store *RealStore) save(stuff interface{}, root string, ttl uint64) error {
	t := time.Now()
	arrValue := reflect.ValueOf(stuff)

	nodes := make([]storeadapter.StoreNode, arrValue.Len())
	for i := 0; i < arrValue.Len(); i++ {
		item := arrValue.Index(i).Interface().(Storeable)
		nodes[i] = storeadapter.StoreNode{
			Key:   root + "/" + item.StoreKey(),
			Value: item.ToJSON(),
			TTL:   ttl,
		}
	}

	err := store.adapter.Set(nodes)

	store.logger.Info(fmt.Sprintf("Save Duration %s", root), map[string]string{
		"Number of Items": fmt.Sprintf("%d", arrValue.Len()),
		"Duration":        fmt.Sprintf("%.4f seconds", time.Since(t).Seconds()),
	})
	return err
}

func (store *RealStore) get(root string, sliceType reflect.Type, constructor reflect.Value) (reflect.Value, error) {
	t := time.Now()

	nodes, err := store.fetchNodesUnderDir(root)
	if err != nil {
		return reflect.MakeSlice(sliceType, 0, 0), err
	}

	slice := reflect.MakeSlice(sliceType, 0, 0)
	for _, node := range nodes {
		out := constructor.Call([]reflect.Value{reflect.ValueOf(node.Value)})
		slice = reflect.Append(slice, out[0])
		if !out[1].IsNil() {
			return reflect.MakeSlice(sliceType, 0, 0), out[1].Interface().(error)
		}
	}

	store.logger.Info(fmt.Sprintf("Get Duration %s", root), map[string]string{
		"Number of Items": fmt.Sprintf("%d", slice.Len()),
		"Duration":        fmt.Sprintf("%.4f seconds", time.Since(t).Seconds()),
	})
	return slice, nil
}

func (store *RealStore) delete(stuff interface{}, root string) error {
	t := time.Now()
	arrValue := reflect.ValueOf(stuff)

	for i := 0; i < arrValue.Len(); i++ {
		item := arrValue.Index(i).Interface().(Storeable)

		err := store.adapter.Delete(root + "/" + item.StoreKey())
		if err != nil {
			return err
		}
	}

	store.logger.Info(fmt.Sprintf("Delete Duration %s", root), map[string]string{
		"Number of Items": fmt.Sprintf("%d", arrValue.Len()),
		"Duration":        fmt.Sprintf("%.4f seconds", time.Since(t).Seconds()),
	})

	return nil
}
<|endoftext|>


<commit_before>package stringUtils

import (

)

func IsEmpty(s string) bool {
	return false
}
<commit_msg>fix bool<commit_after>package stringUtils

import (

)

func IsEmpty(s string) bool {
	return true
}
<|endoftext|>


<commit_before>/*
 * Minio Cloud Storage, (C) 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"net/rpc"
	"sync"
	"time"
)

// Attempt to retry only this many number of times before
// giving up on the remote RPC entirely.
const globalAuthRPCRetryThreshold = 1

// authConfig requires to make new AuthRPCClient.
type authConfig struct {
	accessKey        string // Access key (like username) for authentication.
	secretKey        string // Secret key (like Password) for authentication.
	serverAddr       string // RPC server address.
	serviceEndpoint  string // Endpoint on the server to make any RPC call.
	secureConn       bool   // Make TLS connection to RPC server or not.
	serviceName      string // Service name of auth server.
	disableReconnect bool   // Disable reconnect on failure or not.

	/// Retry configurable values.

	// Each retry unit multiplicative, measured in time.Duration.
	// This is the basic unit used for calculating backoffs.
	retryUnit time.Duration
	// Maximum retry duration i.e A caller would wait no more than this
	// duration to continue their loop.
	retryCap time.Duration

	// Maximum retries an call authRPC client would do for a failed
	// RPC call.
	retryAttemptThreshold int
}

// AuthRPCClient is a authenticated RPC client which does authentication before doing Call().
type AuthRPCClient struct {
	sync.Mutex            // Mutex to lock this object.
	rpcClient  *RPCClient // Reconnectable RPC client to make any RPC call.
	config     authConfig // Authentication configuration information.
	authToken  string     // Authentication token.
}

// newAuthRPCClient - returns a JWT based authenticated (go) rpc client, which does automatic reconnect.
func newAuthRPCClient(config authConfig) *AuthRPCClient {
	// Check if retry params are set properly if not default them.
	emptyDuration := time.Duration(int64(0))
	if config.retryUnit == emptyDuration {
		config.retryUnit = defaultRetryUnit
	}
	if config.retryCap == emptyDuration {
		config.retryCap = defaultRetryCap
	}
	if config.retryAttemptThreshold == 0 {
		config.retryAttemptThreshold = globalAuthRPCRetryThreshold
	}

	return &AuthRPCClient{
		rpcClient: newRPCClient(config.serverAddr, config.serviceEndpoint, config.secureConn),
		config:    config,
	}
}

// Login - a jwt based authentication is performed with rpc server.
func (authClient *AuthRPCClient) Login() (err error) {
	authClient.Lock()
	defer authClient.Unlock()

	// Return if already logged in.
	if authClient.authToken != "" {
		return nil
	}

	// Call login.
	args := LoginRPCArgs{
		Username:    authClient.config.accessKey,
		Password:    authClient.config.secretKey,
		Version:     Version,
		RequestTime: time.Now().UTC(),
	}

	reply := LoginRPCReply{}
	serviceMethod := authClient.config.serviceName + loginMethodName
	if err = authClient.rpcClient.Call(serviceMethod, &args, &reply); err != nil {
		return err
	}

	// Logged in successfully.
	authClient.authToken = reply.AuthToken

	return nil
}

// call makes a RPC call after logs into the server.
func (authClient *AuthRPCClient) call(serviceMethod string, args interface {
	SetAuthToken(authToken string)
	SetRequestTime(requestTime time.Time)
}, reply interface{}) (err error) {
	// On successful login, execute RPC call.
	if err = authClient.Login(); err == nil {
		// Set token and timestamp before the rpc call.
		args.SetAuthToken(authClient.authToken)
		args.SetRequestTime(time.Now().UTC())

		// Do RPC call.
		err = authClient.rpcClient.Call(serviceMethod, args, reply)
	}
	return err
}

// Call executes RPC call till success or globalAuthRPCRetryThreshold on ErrShutdown.
func (authClient *AuthRPCClient) Call(serviceMethod string, args interface {
	SetAuthToken(authToken string)
	SetRequestTime(requestTime time.Time)
}, reply interface{}) (err error) {

	// Done channel is used to close any lingering retry routine, as soon
	// as this function returns.
	doneCh := make(chan struct{})
	defer close(doneCh)

	for i := range newRetryTimer(authClient.config.retryUnit, authClient.config.retryCap, doneCh) {
		if err = authClient.call(serviceMethod, args, reply); err == rpc.ErrShutdown {
			// As connection at server side is closed, close the rpc client.
			authClient.Close()

			// Retry if reconnect is not disabled.
			if !authClient.config.disableReconnect {
				// Retry until threshold reaches.
				if i < authClient.config.retryAttemptThreshold {
					continue
				}
			}
		}
		break
	}
	return err
}

// Close closes underlying RPC Client.
func (authClient *AuthRPCClient) Close() error {
	authClient.Lock()
	defer authClient.Unlock()

	authClient.authToken = ""
	return authClient.rpcClient.Close()
}

// ServerAddr returns the serverAddr (network address) of the connection.
func (authClient *AuthRPCClient) ServerAddr() string {
	return authClient.config.serverAddr
}

// ServiceEndpoint returns the RPC service endpoint of the connection.
func (authClient *AuthRPCClient) ServiceEndpoint() string {
	return authClient.config.serviceEndpoint
}
<commit_msg>auth/rpc: Token can be concurrently edited protect it. (#3764)<commit_after>/*
 * Minio Cloud Storage, (C) 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"net/rpc"
	"sync"
	"time"
)

// Attempt to retry only this many number of times before
// giving up on the remote RPC entirely.
const globalAuthRPCRetryThreshold = 1

// authConfig requires to make new AuthRPCClient.
type authConfig struct {
	accessKey        string // Access key (like username) for authentication.
	secretKey        string // Secret key (like Password) for authentication.
	serverAddr       string // RPC server address.
	serviceEndpoint  string // Endpoint on the server to make any RPC call.
	secureConn       bool   // Make TLS connection to RPC server or not.
	serviceName      string // Service name of auth server.
	disableReconnect bool   // Disable reconnect on failure or not.

	/// Retry configurable values.

	// Each retry unit multiplicative, measured in time.Duration.
	// This is the basic unit used for calculating backoffs.
	retryUnit time.Duration
	// Maximum retry duration i.e A caller would wait no more than this
	// duration to continue their loop.
	retryCap time.Duration

	// Maximum retries an call authRPC client would do for a failed
	// RPC call.
	retryAttemptThreshold int
}

// AuthRPCClient is a authenticated RPC client which does authentication before doing Call().
type AuthRPCClient struct {
	sync.Mutex            // Mutex to lock this object.
	rpcClient  *RPCClient // Reconnectable RPC client to make any RPC call.
	config     authConfig // Authentication configuration information.
	authToken  string     // Authentication token.
}

// newAuthRPCClient - returns a JWT based authenticated (go) rpc client, which does automatic reconnect.
func newAuthRPCClient(config authConfig) *AuthRPCClient {
	// Check if retry params are set properly if not default them.
	emptyDuration := time.Duration(int64(0))
	if config.retryUnit == emptyDuration {
		config.retryUnit = defaultRetryUnit
	}
	if config.retryCap == emptyDuration {
		config.retryCap = defaultRetryCap
	}
	if config.retryAttemptThreshold == 0 {
		config.retryAttemptThreshold = globalAuthRPCRetryThreshold
	}

	return &AuthRPCClient{
		rpcClient: newRPCClient(config.serverAddr, config.serviceEndpoint, config.secureConn),
		config:    config,
	}
}

// Login - a jwt based authentication is performed with rpc server.
func (authClient *AuthRPCClient) Login() (err error) {
	authClient.Lock()
	defer authClient.Unlock()

	// Return if already logged in.
	if authClient.authToken != "" {
		return nil
	}

	// Call login.
	args := LoginRPCArgs{
		Username:    authClient.config.accessKey,
		Password:    authClient.config.secretKey,
		Version:     Version,
		RequestTime: time.Now().UTC(),
	}

	reply := LoginRPCReply{}
	serviceMethod := authClient.config.serviceName + loginMethodName
	if err = authClient.rpcClient.Call(serviceMethod, &args, &reply); err != nil {
		return err
	}

	// Logged in successfully.
	authClient.authToken = reply.AuthToken

	return nil
}

// call makes a RPC call after logs into the server.
func (authClient *AuthRPCClient) call(serviceMethod string, args interface {
	SetAuthToken(authToken string)
	SetRequestTime(requestTime time.Time)
}, reply interface{}) (err error) {
	// On successful login, execute RPC call.
	if err = authClient.Login(); err == nil {
		authClient.Lock()
		// Set token and timestamp before the rpc call.
		args.SetAuthToken(authClient.authToken)
		authClient.Unlock()
		args.SetRequestTime(time.Now().UTC())

		// Do RPC call.
		err = authClient.rpcClient.Call(serviceMethod, args, reply)
	}
	return err
}

// Call executes RPC call till success or globalAuthRPCRetryThreshold on ErrShutdown.
func (authClient *AuthRPCClient) Call(serviceMethod string, args interface {
	SetAuthToken(authToken string)
	SetRequestTime(requestTime time.Time)
}, reply interface{}) (err error) {

	// Done channel is used to close any lingering retry routine, as soon
	// as this function returns.
	doneCh := make(chan struct{})
	defer close(doneCh)

	for i := range newRetryTimer(authClient.config.retryUnit, authClient.config.retryCap, doneCh) {
		if err = authClient.call(serviceMethod, args, reply); err == rpc.ErrShutdown {
			// As connection at server side is closed, close the rpc client.
			authClient.Close()

			// Retry if reconnect is not disabled.
			if !authClient.config.disableReconnect {
				// Retry until threshold reaches.
				if i < authClient.config.retryAttemptThreshold {
					continue
				}
			}
		}
		break
	}
	return err
}

// Close closes underlying RPC Client.
func (authClient *AuthRPCClient) Close() error {
	authClient.Lock()
	defer authClient.Unlock()

	authClient.authToken = ""
	return authClient.rpcClient.Close()
}

// ServerAddr returns the serverAddr (network address) of the connection.
func (authClient *AuthRPCClient) ServerAddr() string {
	return authClient.config.serverAddr
}

// ServiceEndpoint returns the RPC service endpoint of the connection.
func (authClient *AuthRPCClient) ServiceEndpoint() string {
	return authClient.config.serviceEndpoint
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"os"

	"github.com/docopt/docopt-go"
	"github.com/luistm/banksaurus/app"
	"github.com/luistm/banksaurus/cmd/banksaurus/command"
)

var intro = "    \n    Your command line finance manager.\n\n"

var usage = `Usage:
	banksaurus -h | --help
	banksaurus report --input <file> [ --grouped ]
	banksaurus load --input <file>
	banksaurus seller change <id> --pretty <name>
	banksaurus seller new <name>
	banksaurus seller show
	banksaurus transaction show`

var options = `

Options:
	--grouped     The report is grouped by seller.
	--input       The path to the records list.
	--name        Specifies the name.
	-h --help     Show this screen.`

var errGeneric = errors.New("error while performing operation")

func main() {
	_, err := app.New("")
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to Setup application: %s\n", err.Error())
		os.Exit(2)
	}

	// TODO: Inject dependencies here
	// err := application.Add(aConstructor, "constructor.slug")
	// defer application.Close()

	arguments, err := docopt.Parse(intro+usage+options, nil, true, app.Version, false)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(2)
	}

	cmd, err := command.New(os.Args[1:])
	if err != nil {

		fmt.Fprintf(os.Stderr, errGeneric.Error())
		os.Exit(2)
	}

	err = cmd.Execute(arguments)
	if err != nil {
		fmt.Fprintf(os.Stderr, errGeneric.Error()+"\n")
		os.Exit(2)
	}
}
<commit_msg>Removes command which should not be available yet<commit_after>package main

import (
	"errors"
	"fmt"
	"os"

	"github.com/docopt/docopt-go"
	"github.com/luistm/banksaurus/app"
	"github.com/luistm/banksaurus/cmd/banksaurus/command"
)

var intro = "    \n    Your command line finance manager.\n\n"

var usage = `Usage:
	banksaurus -h | --help
	banksaurus report --input <file> [ --grouped ]
	banksaurus load --input <file>
	banksaurus seller change <id> --pretty <name>
	banksaurus seller new <name>
	banksaurus seller show`

var options = `

Options:
	--grouped     The report is grouped by seller.
	--input       The path to the records list.
	--name        Specifies the name.
	-h --help     Show this screen.`

var errGeneric = errors.New("error while performing operation")

func main() {
	_, err := app.New("")
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to Setup application: %s\n", err.Error())
		os.Exit(2)
	}

	// TODO: Inject dependencies here
	// err := application.Add(aConstructor, "constructor.slug")
	// defer application.Close()

	arguments, err := docopt.Parse(intro+usage+options, nil, true, app.Version, false)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(2)
	}

	cmd, err := command.New(os.Args[1:])
	if err != nil {

		fmt.Fprintf(os.Stderr, errGeneric.Error())
		os.Exit(2)
	}

	err = cmd.Execute(arguments)
	if err != nil {
		fmt.Fprintf(os.Stderr, errGeneric.Error()+"\n")
		os.Exit(2)
	}
}
<|endoftext|>