<commit_before>// Copyright 2016 The JoeFriday authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package mem gets and processes /proc/meminfo, returning the data in the
// appropriate format.
package mem

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strconv"
	"time"

	fb "github.com/google/flatbuffers/go"
	joe "github.com/mohae/joefriday"
)

type Info struct {
	Timestamp    int64
	MemTotal     int
	MemFree      int
	MemAvailable int
	Buffers      int
	Cached       int
	SwapCached   int
	Active       int
	Inactive     int
	SwapTotal    int
	SwapFree     int
}

// Serialize serializes the Info using flatbuffers.
func (i *Info) Serialize() []byte {
	bldr := fb.NewBuilder(0)
	DataStart(bldr)
	DataAddTimestamp(bldr, int64(i.Timestamp))
	DataAddMemTotal(bldr, int64(i.MemTotal))
	DataAddMemFree(bldr, int64(i.MemFree))
	DataAddMemAvailable(bldr, int64(i.MemAvailable))
	DataAddBuffers(bldr, int64(i.Buffers))
	DataAddCached(bldr, int64(i.Cached))
	DataAddSwapCached(bldr, int64(i.SwapCached))
	DataAddActive(bldr, int64(i.Active))
	DataAddInactive(bldr, int64(i.Inactive))
	DataAddSwapTotal(bldr, int64(i.SwapTotal))
	DataAddSwapFree(bldr, int64(i.SwapFree))
	bldr.Finish(DataEnd(bldr))
	return bldr.Bytes[bldr.Head():]
}

// Deserialize deserializes bytes representing flatbuffers serialized Data
// into *Info.  If the bytes are not from flatbuffers serialization of
// Data, it is a programmer error and a panic will occur.
func Deserialize(p []byte) *Info {
	data := GetRootAsData(p, 0)
	info := &Info{}
	info.Timestamp = data.Timestamp()
	info.MemTotal = int(data.MemTotal())
	info.MemFree = int(data.MemFree())
	info.MemAvailable = int(data.MemAvailable())
	info.Buffers = int(data.Buffers())
	info.Cached = int(data.Cached())
	info.SwapCached = int(data.SwapCached())
	info.Active = int(data.Active())
	info.Inactive = int(data.Inactive())
	info.SwapTotal = int(data.SwapTotal())
	info.SwapFree = int(data.SwapFree())
	return info
}

// GetInfo returns some of the results of /proc/meminfo.
func GetInfo() (*Info, error) {
	var l, i int
	var name string
	var err error
	var v byte
	t := time.Now().UTC().UnixNano()
	f, err := os.Open("/proc/meminfo")
	if err != nil {
		return nil, err
	}
	defer f.Close()
	buf := bufio.NewReader(f)
	inf := &Info{Timestamp: t}
	var pos int
	line := make([]byte, 0, 50)
	val := make([]byte, 0, 32)
	for {
		if l == 16 {
			break
		}
		line, err = buf.ReadSlice(joe.LF)
		if err != nil {
			if err == io.EOF {
				break
			}
			return nil, fmt.Errorf("error reading output bytes: %s", err)
		}
		l++
		if l > 8 && l < 15 {
			continue
		}
		// first grab the key name (everything up to the ':')
		for i, v = range line {
			if v == 0x3A {
				pos = i + 1
				break
			}
			val = append(val, v)
		}
		name = string(val[:])
		val = val[:0]
		// skip all spaces
		for i, v = range line[pos:] {
			if v != 0x20 {
				pos += i
				break
			}
		}

		// grab the numbers
		for _, v = range line[pos:] {
			if v == 0x20 || v == joe.CR {
				break
			}
			val = append(val, v)
		}
		// any conversion error results in 0
		i, err = strconv.Atoi(string(val[:]))
		if err != nil {
			return nil, fmt.Errorf("%s: %s", name, err)
		}
		val = val[:0]
		if name == "MemTotal" {
			inf.MemTotal = i
			continue
		}
		if name == "MemFree" {
			inf.MemFree = i
			continue
		}
		if name == "MemAvailable" {
			inf.MemAvailable = i
			continue
		}
		if name == "Buffers" {
			inf.Buffers = i
			continue
		}
		if name == "Cached" {
			inf.MemAvailable = i
			continue
		}
		if name == "SwapCached" {
			inf.SwapCached = i
			continue
		}
		if name == "Active" {
			inf.Active = i
			continue
		}
		if name == "Inactive" {
			inf.Inactive = i
			continue
		}
		if name == "SwapTotal" {
			inf.SwapTotal = i
			continue
		}
		if name == "SwapFree" {
			inf.SwapFree = i
			continue
		}
	}
	return inf, nil
}

// GetInfoR accepts a *bufio.Reader and returns some of the results of
// /proc/meminfo.  This is mainly here for benchmark purposes.
// TODO: decide whether this should be kept around.
func GetInfoR(buf *bufio.Reader) (*Info, error) {
	var l, i int
	var name string
	var err error
	var v byte
	t := time.Now().UTC().UnixNano()
	f, err := os.Open("/proc/meminfo")
	if err != nil {
		return nil, err
	}
	defer f.Close()
	buf.Reset(f)
	inf := &Info{Timestamp: t}
	var pos int
	line := make([]byte, 0, 50)
	val := make([]byte, 0, 32)
	for {
		if l == 16 {
			break
		}
		line, err = buf.ReadSlice(joe.LF)
		if err != nil {
			if err == io.EOF {
				break
			}
			return nil, fmt.Errorf("error reading output bytes: %s", err)
		}
		l++
		if l > 8 && l < 15 {
			continue
		}
		// first grab the key name (everything up to the ':')
		for i, v = range line {
			if v == 0x3A {
				pos = i + 1
				break
			}
			val = append(val, v)
		}
		name = string(val[:])
		val = val[:0]
		// skip all spaces
		for i, v = range line[pos:] {
			if v != 0x20 {
				pos += i
				break
			}
		}

		// grab the numbers
		for _, v = range line[pos:] {
			if v == 0x20 || v == joe.CR {
				break
			}
			val = append(val, v)
		}
		// any conversion error results in 0
		i, err = strconv.Atoi(string(val[:]))
		if err != nil {
			return nil, fmt.Errorf("%s: %s", name, err)
		}
		val = val[:0]
		if name == "MemTotal" {
			inf.MemTotal = i
			continue
		}
		if name == "MemFree" {
			inf.MemFree = i
			continue
		}
		if name == "MemAvailable" {
			inf.MemAvailable = i
			continue
		}
		if name == "Buffers" {
			inf.Buffers = i
			continue
		}
		if name == "Cached" {
			inf.MemAvailable = i
			continue
		}
		if name == "SwapCached" {
			inf.SwapCached = i
			continue
		}
		if name == "Active" {
			inf.Active = i
			continue
		}
		if name == "Inactive" {
			inf.Inactive = i
			continue
		}
		if name == "SwapTotal" {
			inf.SwapTotal = i
			continue
		}
		if name == "SwapFree" {
			inf.SwapFree = i
			continue
		}
	}
	return inf, nil
}

// GetData returns the current meminfo as flatbuffer serialized bytes.
// TODO: Benchmark to see if we should just create the flatbuffers w/o
// doing the intermediate step of to the data structure.
func GetData() ([]byte, error) {
	inf, err := GetInfo()
	if err != nil {
		return nil, err
	}
	return inf.Serialize(), nil
}

// DataTicker gathers the meminfo on a ticker, whose interval is defined by
// the received duration, and sends the results to the channel.  The output
// is Flatbuffers serialized Data.  Any error encountered during processing
// is sent to the error channel.  Processing will continue
//
// Either closing the done channel or sending struct{} to the done channel
// will result in function exit.  The out channel is closed on exit.
//
// This pre-allocates the builder and everything other than the []byte that
// gets sent to the out channel to reduce allocations, as this is expected
// to be both a frequent and a long-running process.  Doing so reduces
// byte allocations per tick just ~ 42%.
func DataTicker(interval time.Duration, outCh chan []byte, done chan struct{}, errCh chan error) {
	ticker := time.NewTicker(interval)
	defer ticker.Stop()
	defer close(outCh)
	// predeclare some vars
	var out bytes.Buffer
	var l, i, pos int
	var t int64
	var v byte
	var name string
	// premake some temp slices
	line := make([]byte, 0, 50)
	val := make([]byte, 0, 32)
	// just reset the bldr at the end of every ticker
	bldr := fb.NewBuilder(0)
	var buf bufio.Reader
	// ticker
	for {
		select {
		case <-done:
			return
		case <-ticker.C:
			f, err := os.Open("/proc/meminfo")
			if err != nil {
				errCh <- joe.Error{Type: "mem", Op: "cat /proc/meminfo", Err: err}
				continue
			}
			// The current timestamp is always in UTC
			t = time.Now().UTC().UnixNano()
			buf.Reset(f)
			DataStart(bldr)
			DataAddTimestamp(bldr, t)
			for {
				if l == 16 {
					break
				}
				line, err = buf.ReadSlice(joe.LF)
				if err != nil {
					if err == io.EOF {
						break
					}
					errCh <- joe.Error{Type: "mem", Op: "read command results", Err: err}
					continue
				}
				l++
				if l > 8 && l < 15 {
					continue
				}
				// first grab the key name (everything up to the ':')
				for i, v = range line {
					if v == 0x3A {
						pos = i + 1
						break
					}
					val = append(val, v)
				}
				name = string(val[:])
				val = val[:0]
				// skip all spaces
				for i, v = range line[pos:] {
					if v != 0x20 {
						pos += i
						break
					}
				}

				// grab the numbers
				for _, v = range line[pos:] {
					if v == 0x20 || v == joe.CR {
						break
					}
					val = append(val, v)
				}
				// any conversion error results in 0
				i, err = strconv.Atoi(string(val[:]))
				if err != nil {
					errCh <- joe.Error{Type: "mem", Op: "convert to int", Err: err}
					continue
				}
				val = val[:0]
				if name == "MemTotal" {
					DataAddMemTotal(bldr, int64(i))
					continue
				}
				if name == "MemFree" {
					DataAddMemFree(bldr, int64(i))
					continue
				}
				if name == "MemAvailable" {
					DataAddMemAvailable(bldr, int64(i))
					continue
				}
				if name == "Buffers" {
					DataAddBuffers(bldr, int64(i))
					continue
				}
				if name == "Cached" {
					DataAddMemAvailable(bldr, int64(i))
					continue
				}
				if name == "SwapCached" {
					DataAddSwapCached(bldr, int64(i))
					continue
				}
				if name == "Active" {
					DataAddActive(bldr, int64(i))
					continue
				}
				if name == "Inactive" {
					DataAddInactive(bldr, int64(i))
					continue
				}
				if name == "SwapTotal" {
					DataAddSwapTotal(bldr, int64(i))
					continue
				}
				if name == "SwapFree" {
					DataAddSwapFree(bldr, int64(i))
					continue
				}
			}
			f.Close()
			bldr.Finish(DataEnd(bldr))
			data := bldr.Bytes[bldr.Head():]
			outCh <- data
			bldr.Reset()
			out.Reset()
			l = 0
		}
	}
}

func (d *Data) String() string {
	return fmt.Sprintf("Timestamp: %v\nMemTotal:\t%d\tMemFree:\t%d\tMemAvailable:\t%d\tActive:\t%d\tInactive:\t%d\nCached:\t\t%d\tBuffers\t:%d\nSwapTotal:\t%d\tSwapCached:\t%d\tSwapFree:\t%d\n", time.Unix(0, d.Timestamp()).UTC(), d.MemTotal(), d.MemFree(), d.MemAvailable(), d.Active(), d.Inactive(), d.Cached(), d.Buffers(), d.SwapTotal(), d.SwapCached(), d.SwapFree())
}

func meminfo(buff *bytes.Buffer) error {
	cmd := exec.Command("cat", "/proc/meminfo")
	cmd.Stdout = buff
	return cmd.Run()
}
<commit_msg>read /proc/meminfo as a file instead of using cat and os.Exec()<commit_after>// Copyright 2016 The JoeFriday authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package mem gets and processes /proc/meminfo, returning the data in the
// appropriate format.
package mem

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
	"strconv"
	"time"

	fb "github.com/google/flatbuffers/go"
	joe "github.com/mohae/joefriday"
)

type Info struct {
	Timestamp    int64
	MemTotal     int
	MemFree      int
	MemAvailable int
	Buffers      int
	Cached       int
	SwapCached   int
	Active       int
	Inactive     int
	SwapTotal    int
	SwapFree     int
}

// Serialize serializes the Info using flatbuffers.
func (i *Info) Serialize() []byte {
	builder := fb.NewBuilder(0)
	DataStart(builder)
	DataAddTimestamp(builder, int64(i.Timestamp))
	DataAddMemTotal(builder, int64(i.MemTotal))
	DataAddMemFree(builder, int64(i.MemFree))
	DataAddMemAvailable(builder, int64(i.MemAvailable))
	DataAddBuffers(builder, int64(i.Buffers))
	DataAddCached(builder, int64(i.Cached))
	DataAddSwapCached(builder, int64(i.SwapCached))
	DataAddActive(builder, int64(i.Active))
	DataAddInactive(builder, int64(i.Inactive))
	DataAddSwapTotal(builder, int64(i.SwapTotal))
	DataAddSwapFree(builder, int64(i.SwapFree))
	builder.Finish(DataEnd(builder))
	return builder.Bytes[builder.Head():]
}

// Deserialize deserializes bytes representing flatbuffers serialized Data
// into *Info.  If the bytes are not from flatbuffers serialization of
// Data, it is a programmer error and a panic will occur.
func Deserialize(p []byte) *Info {
	data := GetRootAsData(p, 0)
	info := &Info{}
	info.Timestamp = data.Timestamp()
	info.MemTotal = int(data.MemTotal())
	info.MemFree = int(data.MemFree())
	info.MemAvailable = int(data.MemAvailable())
	info.Buffers = int(data.Buffers())
	info.Cached = int(data.Cached())
	info.SwapCached = int(data.SwapCached())
	info.Active = int(data.Active())
	info.Inactive = int(data.Inactive())
	info.SwapTotal = int(data.SwapTotal())
	info.SwapFree = int(data.SwapFree())
	return info
}

// GetInfo returns some of the results of /proc/meminfo.
func GetInfo() (*Info, error) {
	var l, i int
	var name string
	var err error
	var v byte
	t := time.Now().UTC().UnixNano()
	f, err := os.Open("/proc/meminfo")
	if err != nil {
		return nil, err
	}
	defer f.Close()
	buf := bufio.NewReader(f)
	inf := &Info{Timestamp: t}
	var pos int
	line := make([]byte, 0, 50)
	val := make([]byte, 0, 32)
	for {
		if l == 16 {
			break
		}
		line, err = buf.ReadSlice(joe.LF)
		if err != nil {
			if err == io.EOF {
				break
			}
			return nil, fmt.Errorf("error reading output bytes: %s", err)
		}
		l++
		if l > 8 && l < 15 {
			continue
		}
		// first grab the key name (everything up to the ':')
		for i, v = range line {
			if v == 0x3A {
				pos = i + 1
				break
			}
			val = append(val, v)
		}
		name = string(val[:])
		val = val[:0]
		// skip all spaces
		for i, v = range line[pos:] {
			if v != 0x20 {
				pos += i
				break
			}
		}

		// grab the numbers
		for _, v = range line[pos:] {
			if v == 0x20 || v == joe.CR {
				break
			}
			val = append(val, v)
		}
		// any conversion error results in 0
		i, err = strconv.Atoi(string(val[:]))
		if err != nil {
			return nil, fmt.Errorf("%s: %s", name, err)
		}
		val = val[:0]
		if name == "MemTotal" {
			inf.MemTotal = i
			continue
		}
		if name == "MemFree" {
			inf.MemFree = i
			continue
		}
		if name == "MemAvailable" {
			inf.MemAvailable = i
			continue
		}
		if name == "Buffers" {
			inf.Buffers = i
			continue
		}
		if name == "Cached" {
			inf.MemAvailable = i
			continue
		}
		if name == "SwapCached" {
			inf.SwapCached = i
			continue
		}
		if name == "Active" {
			inf.Active = i
			continue
		}
		if name == "Inactive" {
			inf.Inactive = i
			continue
		}
		if name == "SwapTotal" {
			inf.SwapTotal = i
			continue
		}
		if name == "SwapFree" {
			inf.SwapFree = i
			continue
		}
	}
	return inf, nil
}

// GetData returns the current meminfo as flatbuffer serialized bytes.
func GetData() ([]byte, error) {
	inf, err := GetInfo()
	if err != nil {
		return nil, err
	}
	return inf.Serialize(), nil
}

// DataTicker gathers the meminfo on a ticker, whose interval is defined by
// the received duration, and sends the results to the channel.  The output
// is Flatbuffers serialized Data.  Any error encountered during processing
// is sent to the error channel.  Processing will continue
//
// Either closing the done channel or sending struct{} to the done channel
// will result in function exit.  The out channel is closed on exit.
//
// This pre-allocates the builder and everything other than the []byte that
// gets sent to the out channel to reduce allocations, as this is expected
// to be both a frequent and a long-running process.  Doing so reduces
// byte allocations per tick just ~ 42%.
func DataTicker(interval time.Duration, outCh chan []byte, done chan struct{}, errCh chan error) {
	ticker := time.NewTicker(interval)
	defer ticker.Stop()
	defer close(outCh)
	// predeclare some vars
	var out bytes.Buffer
	var l, i, pos int
	var t int64
	var v byte
	var name string
	// premake some temp slices
	line := make([]byte, 0, 50)
	val := make([]byte, 0, 32)
	// just reset the bldr at the end of every ticker
	bldr := fb.NewBuilder(0)
	var buf bufio.Reader
	// ticker
	for {
		select {
		case <-done:
			return
		case <-ticker.C:
			f, err := os.Open("/proc/meminfo")
			if err != nil {
				errCh <- joe.Error{Type: "mem", Op: "cat /proc/meminfo", Err: err}
				continue
			}
			// The current timestamp is always in UTC
			t = time.Now().UTC().UnixNano()
			buf.Reset(f)
			DataStart(bldr)
			DataAddTimestamp(bldr, t)
			for {
				if l == 16 {
					break
				}
				line, err = buf.ReadSlice(joe.LF)
				if err != nil {
					if err == io.EOF {
						break
					}
					errCh <- joe.Error{Type: "mem", Op: "read command results", Err: err}
					continue
				}
				l++
				if l > 8 && l < 15 {
					continue
				}
				// first grab the key name (everything up to the ':')
				for i, v = range line {
					if v == 0x3A {
						pos = i + 1
						break
					}
					val = append(val, v)
				}
				name = string(val[:])
				val = val[:0]
				// skip all spaces
				for i, v = range line[pos:] {
					if v != 0x20 {
						pos += i
						break
					}
				}

				// grab the numbers
				for _, v = range line[pos:] {
					if v == 0x20 || v == joe.CR {
						break
					}
					val = append(val, v)
				}
				// any conversion error results in 0
				i, err = strconv.Atoi(string(val[:]))
				if err != nil {
					errCh <- joe.Error{Type: "mem", Op: "convert to int", Err: err}
					continue
				}
				val = val[:0]
				if name == "MemTotal" {
					DataAddMemTotal(bldr, int64(i))
					continue
				}
				if name == "MemFree" {
					DataAddMemFree(bldr, int64(i))
					continue
				}
				if name == "MemAvailable" {
					DataAddMemAvailable(bldr, int64(i))
					continue
				}
				if name == "Buffers" {
					DataAddBuffers(bldr, int64(i))
					continue
				}
				if name == "Cached" {
					DataAddMemAvailable(bldr, int64(i))
					continue
				}
				if name == "SwapCached" {
					DataAddSwapCached(bldr, int64(i))
					continue
				}
				if name == "Active" {
					DataAddActive(bldr, int64(i))
					continue
				}
				if name == "Inactive" {
					DataAddInactive(bldr, int64(i))
					continue
				}
				if name == "SwapTotal" {
					DataAddSwapTotal(bldr, int64(i))
					continue
				}
				if name == "SwapFree" {
					DataAddSwapFree(bldr, int64(i))
					continue
				}
			}
			f.Close()
			bldr.Finish(DataEnd(bldr))
			data := bldr.Bytes[bldr.Head():]
			outCh <- data
			bldr.Reset()
			out.Reset()
			l = 0
		}
	}
}

func (d *Data) String() string {
	return fmt.Sprintf("Timestamp: %v\nMemTotal:\t%d\tMemFree:\t%d\tMemAvailable:\t%d\tActive:\t%d\tInactive:\t%d\nCached:\t\t%d\tBuffers\t:%d\nSwapTotal:\t%d\tSwapCached:\t%d\tSwapFree:\t%d\n", time.Unix(0, d.Timestamp()).UTC(), d.MemTotal(), d.MemFree(), d.MemAvailable(), d.Active(), d.Inactive(), d.Cached(), d.Buffers(), d.SwapTotal(), d.SwapCached(), d.SwapFree())
}
<|endoftext|>


<commit_before>// Package memfs provides an interface for an in-memory filesystem.
package memfs

import (
	"bytes"
	"errors"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/rjeczalik/tools/fs"
)

const sep = string(os.PathSeparator)

// Directory represents an in-memory directory
type Directory map[string]interface{}

// File represents an in-memory file.
type File []byte

// FS provides an implementation for Filesystem interface, operating on
// an in-memory file tree.
// TODO(rjeczalik): sync.RWMutex
type FS struct {
	Tree Directory
}

var _ fs.Filesystem = FS{}

var (
	errDir       = errors.New("is a directory")
	errNotDir    = errors.New("not a directory")
	errCorrupted = errors.New("tree is corrupted")
)

// Cd gives new filesystem with a root starting at the path of the old filesystem.
func (fs FS) Cd(path string) (FS, error) {
	dir, perr := fs.lookup(path)
	if perr != nil {
		return FS{}, perr
	}
	return FS{Tree: dir}, nil
}

// Create creates an in-memory file under the given path.
func (fs FS) Create(name string) (fs.File, error) {
	dir, base, perr := fs.dirbase(name)
	if perr != nil {
		perr.Op = "Create"
		return nil, perr
	}
	if base == "" {
		return nil, &os.PathError{"Create", name, errDir}
	}
	if v, ok := dir[base]; ok {
		if _, ok = v.(Directory); ok {
			return nil, &os.PathError{"Create", name, errDir}
		}
	}
	dir[base] = File{}
	return file{s: name, f: fs.flushcb(dir, base), r: new(bytes.Reader)}, nil
}

// Mkdir creates an in-memory directory under the given path.
func (fs FS) Mkdir(name string, _ os.FileMode) error {
	dir, base, perr := fs.dirbase(name)
	if perr != nil {
		perr.Op = "Mkdir"
		return perr
	}
	if base == "" {
		return nil
	}
	if v, ok := dir[base]; ok {
		if _, ok = v.(File); ok {
			return &os.PathError{"Mkdir", name, errNotDir}
		}
	}
	dir[base] = Directory{}
	return nil
}

// MkdirAll creates new in-memory directory and all its parents, if needed.
func (fs FS) MkdirAll(name string, _ os.FileMode) error {
	var (
		dir = fs.Tree
		err error
	)
	fn := func(s string) bool {
		v, ok := dir[s]
		if !ok {
			d := Directory{}
			dir[s], dir = d, d
		} else if dir, ok = v.(Directory); !ok {
			err = &os.PathError{"MkdirAll", name, errNotDir}
			return false
		}
		return true
	}
	fs.dirwalk(name, fn)
	return err
}

// Open opens a file or directory given by the path.
func (fs FS) Open(name string) (fs.File, error) {
	dir, base, perr := fs.dirbase(name)
	if perr != nil {
		perr.Op = "Open"
		return nil, perr
	}
	if base == "" {
		return directory{s: name, d: dir}, nil
	}
	if _, ok := dir[base]; !ok {
		return nil, &os.PathError{"Open", name, os.ErrNotExist}
	}
	switch v := dir[base].(type) {
	case File:
		return file{s: name, f: fs.flushcb(dir, base), r: bytes.NewReader([]byte(v))}, nil
	case Directory:
		return directory{s: name, d: v}, nil
	}
	return nil, &os.PathError{"Open", name, errCorrupted}
}

// Remove removes a file from the tree given by the path.
func (fs FS) Remove(name string) error {
	dir, base, perr := fs.dirbase(name)
	if perr != nil {
		perr.Op = "Remove"
		return perr
	}
	if base == "" {
		return &os.PathError{"Remove", name, os.ErrPermission}
	}
	if _, ok := dir[base]; !ok {
		return &os.PathError{"Remove", name, os.ErrNotExist}
	}
	if _, ok := dir[base].(Directory); ok {
		return &os.PathError{"Remove", name, os.ErrPermission}
	}
	delete(dir, base)
	return nil
}

// Stat gives the details of a file or a directory given by the path.
func (fs FS) Stat(name string) (os.FileInfo, error) {
	f, err := fs.Open(name)
	if err != nil {
		return nil, err
	}
	return f.Stat()
}

func (fs FS) dirwalk(p string, fn func(string) bool) {
	if p == "" || p == "." {
		return
	}
	i := strings.Index(p, sep) + 1
	if i == 0 || i == len(p) {
		return
	}
	for i < len(p) {
		j := strings.Index(p[i:], sep)
		if j == -1 {
			j = len(p) - i
		}
		if !fn(p[i : i+j]) {
			return
		}
		i += j + 1
	}
}

func (fs FS) lookup(p string) (dir Directory, perr *os.PathError) {
	dir = fs.Tree
	fn := func(name string) bool {
		v, ok := dir[name]
		if !ok {
			perr = &os.PathError{Err: os.ErrNotExist}
			return false
		}
		if dir, ok = v.(Directory); !ok {
			perr = &os.PathError{Err: errNotDir}
			return false
		}
		return true
	}
	fs.dirwalk(p, fn)
	return
}

func (fs FS) dirbase(p string) (Directory, string, *os.PathError) {
	i := strings.LastIndex(p, sep)
	if i == -1 {
		return fs.Tree, "", nil
	}
	if i == 0 {
		return fs.Tree, p[1:], nil
	}
	dir, perr := fs.lookup(p[:i])
	if perr != nil {
		perr.Path = p
		return nil, "", perr
	}
	return dir, p[i+1:], nil
}

func (fs FS) flushcb(dir Directory, name string) func([]byte) {
	return func(p []byte) {
		dir[name] = File(p)
	}
}

type file struct {
	s string        // name
	f func([]byte)  // flush callback
	r *bytes.Reader // for reading (io.Seeker)
	w *bytes.Buffer // for writing - merge?
}

func (f file) Close() (err error) {
	if f.w != nil {
		f.f(f.w.Bytes())
		f.w = nil
	}
	return
}

func (f file) Read(p []byte) (int, error) {
	return f.r.Read(p)
}

func (f file) Readdir(int) ([]os.FileInfo, error) {
	return nil, &os.PathError{"Readdir", f.s, nil}
}

func (f file) Seek(offset int64, whence int) (int64, error) {
	return f.r.Seek(offset, whence)
}

func (f file) Stat() (os.FileInfo, error) {
	return fileinfo{f.s, int64(f.r.Len()), false}, nil
}

func (f file) Write(p []byte) (int, error) {
	if f.w == nil {
		f.w = new(bytes.Buffer)
	}
	return f.w.Write(p)
}

type directory struct {
	s string
	d Directory
}

func (d directory) Close() (err error) {
	return
}

func (d directory) Read(p []byte) (int, error) {
	return 0, &os.PathError{"Read", d.s, nil}
}

// TODO(rjeczalik): make it ordered so it actually works
func (d directory) Readdir(n int) (fi []os.FileInfo, err error) {
	if len(d.d) == 0 {
		return nil, errors.New("Readdir: directory is empty")
	}
	if n > 0 {
		return nil, errors.New("Readdir: not implemented")
	}
	fi = make([]os.FileInfo, 0, len(d.d))
	for k, v := range d.d {
		if f, ok := v.(File); ok {
			fi = append(fi, fileinfo{filepath.Join(d.s, k), int64(len(f)), false})
		} else {
			fi = append(fi, fileinfo{filepath.Join(d.s, k), 0, true})
		}
	}
	return
}

func (d directory) Seek(int64, int) (int64, error) {
	return 0, &os.PathError{"Seek", d.s, nil}
}

func (d directory) Stat() (os.FileInfo, error) {
	return fileinfo{d.s, 0, true}, nil
}

func (d directory) Write([]byte) (int, error) {
	return 0, &os.PathError{"Write", d.s, nil}
}

type fileinfo struct {
	s string
	n int64
	d bool
}

func (fi fileinfo) Name() string       { return fi.s }
func (fi fileinfo) Size() int64        { return fi.n }
func (fi fileinfo) Mode() os.FileMode  { return 0755 }
func (fi fileinfo) ModTime() time.Time { return time.Time{} }
func (fi fileinfo) IsDir() bool        { return fi.d }
func (fi fileinfo) Sys() interface{}   { return nil }
<commit_msg>fs/memfs: Add New<commit_after>// Package memfs provides an interface for an in-memory filesystem.
package memfs

import (
	"bytes"
	"errors"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/rjeczalik/tools/fs"
)

const sep = string(os.PathSeparator)

// Directory represents an in-memory directory
type Directory map[string]interface{}

// File represents an in-memory file.
type File []byte

// FS provides an implementation for Filesystem interface, operating on
// an in-memory file tree.
// TODO(rjeczalik): sync.RWMutex
type FS struct {
	Tree Directory
}

// New returns an empty filesystem.
func New() FS {
	return FS{
		Tree: Directory{},
	}
}

var _ fs.Filesystem = FS{}

var (
	errDir       = errors.New("is a directory")
	errNotDir    = errors.New("not a directory")
	errCorrupted = errors.New("tree is corrupted")
)

// Cd gives new filesystem with a root starting at the path of the old filesystem.
func (fs FS) Cd(path string) (FS, error) {
	dir, perr := fs.lookup(path)
	if perr != nil {
		return FS{}, perr
	}
	return FS{Tree: dir}, nil
}

// Create creates an in-memory file under the given path.
func (fs FS) Create(name string) (fs.File, error) {
	dir, base, perr := fs.dirbase(name)
	if perr != nil {
		perr.Op = "Create"
		return nil, perr
	}
	if base == "" {
		return nil, &os.PathError{"Create", name, errDir}
	}
	if v, ok := dir[base]; ok {
		if _, ok = v.(Directory); ok {
			return nil, &os.PathError{"Create", name, errDir}
		}
	}
	dir[base] = File{}
	return file{s: name, f: fs.flushcb(dir, base), r: new(bytes.Reader)}, nil
}

// Mkdir creates an in-memory directory under the given path.
func (fs FS) Mkdir(name string, _ os.FileMode) error {
	dir, base, perr := fs.dirbase(name)
	if perr != nil {
		perr.Op = "Mkdir"
		return perr
	}
	if base == "" {
		return nil
	}
	if v, ok := dir[base]; ok {
		if _, ok = v.(File); ok {
			return &os.PathError{"Mkdir", name, errNotDir}
		}
	}
	dir[base] = Directory{}
	return nil
}

// MkdirAll creates new in-memory directory and all its parents, if needed.
func (fs FS) MkdirAll(name string, _ os.FileMode) error {
	var (
		dir = fs.Tree
		err error
	)
	fn := func(s string) bool {
		v, ok := dir[s]
		if !ok {
			d := Directory{}
			dir[s], dir = d, d
		} else if dir, ok = v.(Directory); !ok {
			err = &os.PathError{"MkdirAll", name, errNotDir}
			return false
		}
		return true
	}
	fs.dirwalk(name, fn)
	return err
}

// Open opens a file or directory given by the path.
func (fs FS) Open(name string) (fs.File, error) {
	dir, base, perr := fs.dirbase(name)
	if perr != nil {
		perr.Op = "Open"
		return nil, perr
	}
	if base == "" {
		return directory{s: name, d: dir}, nil
	}
	if _, ok := dir[base]; !ok {
		return nil, &os.PathError{"Open", name, os.ErrNotExist}
	}
	switch v := dir[base].(type) {
	case File:
		return file{s: name, f: fs.flushcb(dir, base), r: bytes.NewReader([]byte(v))}, nil
	case Directory:
		return directory{s: name, d: v}, nil
	}
	return nil, &os.PathError{"Open", name, errCorrupted}
}

// Remove removes a file from the tree given by the path.
func (fs FS) Remove(name string) error {
	dir, base, perr := fs.dirbase(name)
	if perr != nil {
		perr.Op = "Remove"
		return perr
	}
	if base == "" {
		return &os.PathError{"Remove", name, os.ErrPermission}
	}
	if _, ok := dir[base]; !ok {
		return &os.PathError{"Remove", name, os.ErrNotExist}
	}
	if _, ok := dir[base].(Directory); ok {
		return &os.PathError{"Remove", name, os.ErrPermission}
	}
	delete(dir, base)
	return nil
}

// Stat gives the details of a file or a directory given by the path.
func (fs FS) Stat(name string) (os.FileInfo, error) {
	f, err := fs.Open(name)
	if err != nil {
		return nil, err
	}
	return f.Stat()
}

func (fs FS) dirwalk(p string, fn func(string) bool) {
	if p == "" || p == "." {
		return
	}
	i := strings.Index(p, sep) + 1
	if i == 0 || i == len(p) {
		return
	}
	for i < len(p) {
		j := strings.Index(p[i:], sep)
		if j == -1 {
			j = len(p) - i
		}
		if !fn(p[i : i+j]) {
			return
		}
		i += j + 1
	}
}

func (fs FS) lookup(p string) (dir Directory, perr *os.PathError) {
	dir = fs.Tree
	fn := func(name string) bool {
		v, ok := dir[name]
		if !ok {
			perr = &os.PathError{Err: os.ErrNotExist}
			return false
		}
		if dir, ok = v.(Directory); !ok {
			perr = &os.PathError{Err: errNotDir}
			return false
		}
		return true
	}
	fs.dirwalk(p, fn)
	return
}

func (fs FS) dirbase(p string) (Directory, string, *os.PathError) {
	i := strings.LastIndex(p, sep)
	if i == -1 {
		return fs.Tree, "", nil
	}
	if i == 0 {
		return fs.Tree, p[1:], nil
	}
	dir, perr := fs.lookup(p[:i])
	if perr != nil {
		perr.Path = p
		return nil, "", perr
	}
	return dir, p[i+1:], nil
}

func (fs FS) flushcb(dir Directory, name string) func([]byte) {
	return func(p []byte) {
		dir[name] = File(p)
	}
}

type file struct {
	s string        // name
	f func([]byte)  // flush callback
	r *bytes.Reader // for reading (io.Seeker)
	w *bytes.Buffer // for writing - merge?
}

func (f file) Close() (err error) {
	if f.w != nil {
		f.f(f.w.Bytes())
		f.w = nil
	}
	return
}

func (f file) Read(p []byte) (int, error) {
	return f.r.Read(p)
}

func (f file) Readdir(int) ([]os.FileInfo, error) {
	return nil, &os.PathError{"Readdir", f.s, nil}
}

func (f file) Seek(offset int64, whence int) (int64, error) {
	return f.r.Seek(offset, whence)
}

func (f file) Stat() (os.FileInfo, error) {
	return fileinfo{f.s, int64(f.r.Len()), false}, nil
}

func (f file) Write(p []byte) (int, error) {
	if f.w == nil {
		f.w = new(bytes.Buffer)
	}
	return f.w.Write(p)
}

type directory struct {
	s string
	d Directory
}

func (d directory) Close() (err error) {
	return
}

func (d directory) Read(p []byte) (int, error) {
	return 0, &os.PathError{"Read", d.s, nil}
}

// TODO(rjeczalik): make it ordered so it actually works
func (d directory) Readdir(n int) (fi []os.FileInfo, err error) {
	if len(d.d) == 0 {
		return nil, errors.New("Readdir: directory is empty")
	}
	if n > 0 {
		return nil, errors.New("Readdir: not implemented")
	}
	fi = make([]os.FileInfo, 0, len(d.d))
	for k, v := range d.d {
		if f, ok := v.(File); ok {
			fi = append(fi, fileinfo{filepath.Join(d.s, k), int64(len(f)), false})
		} else {
			fi = append(fi, fileinfo{filepath.Join(d.s, k), 0, true})
		}
	}
	return
}

func (d directory) Seek(int64, int) (int64, error) {
	return 0, &os.PathError{"Seek", d.s, nil}
}

func (d directory) Stat() (os.FileInfo, error) {
	return fileinfo{d.s, 0, true}, nil
}

func (d directory) Write([]byte) (int, error) {
	return 0, &os.PathError{"Write", d.s, nil}
}

type fileinfo struct {
	s string
	n int64
	d bool
}

func (fi fileinfo) Name() string       { return fi.s }
func (fi fileinfo) Size() int64        { return fi.n }
func (fi fileinfo) Mode() os.FileMode  { return 0755 }
func (fi fileinfo) ModTime() time.Time { return time.Time{} }
func (fi fileinfo) IsDir() bool        { return fi.d }
func (fi fileinfo) Sys() interface{}   { return nil }
<|endoftext|>


<commit_before>// Package memfs provides an interface for an in-memory filesystem.
package memfs

import (
	"bytes"
	"errors"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/rjeczalik/tools/fs"
)

// Directory represents an in-memory directory
type Directory map[string]interface{}

// File represents an in-memory file.
type File []byte

// FS provides an implementation for Filesystem interfaces, operating on
// an in-memory file tree.
// TODO(rjeczalik): sync.RWMutex
type FS struct {
	Tree Directory
}

var (
	errDir       = errors.New("is a directory")
	errNotDir    = errors.New("not a directory")
	errCorrupted = errors.New("tree is corrupted")
)

const sep = string(os.PathSeparator)

func (fs FS) lookup(p string) (Directory, *os.PathError) {
	if p == "" || p == "." {
		return fs.Tree, nil
	}
	j := strings.Index(p, sep) + 1
	if j == 0 || j == len(p) {
		return fs.Tree, nil
	}
	dir := fs.Tree
	for j < len(p) {
		i := strings.Index(p[j:], sep)
		if i == -1 {
			i = len(p) - j
		}
		v, ok := dir[p[j:j+i]]
		if !ok {
			return nil, &os.PathError{Err: os.ErrNotExist}
		}
		if dir, ok = v.(Directory); !ok {
			return nil, &os.PathError{Err: errNotDir}
		}
		j += i + 1
	}
	return dir, nil
}

func (fs FS) dirbase(p string) (Directory, string, *os.PathError) {
	i := strings.LastIndex(p, sep)
	if i == -1 {
		return fs.Tree, "", nil
	}
	if i == 0 {
		return fs.Tree, p[1:], nil
	}
	dir, perr := fs.lookup(p[:i])
	if perr != nil {
		perr.Path = p
		return nil, "", perr
	}
	return dir, p[i+1:], nil
}

func (fs FS) flushcb(dir Directory, name string) func([]byte) {
	return func(p []byte) {
		dir[name] = File(p)
	}
}

// Create creates an in-memory file under the given path.
func (fs FS) Create(name string) (fs.File, error) {
	dir, base, perr := fs.dirbase(name)
	if perr != nil {
		perr.Op = "Create"
		return nil, perr
	}
	if base == "" {
		return nil, &os.PathError{"Create", name, errDir}
	}
	if v, ok := dir[base]; ok {
		if _, ok = v.(Directory); ok {
			return nil, &os.PathError{"Create", name, errDir}
		}
	}
	dir[base] = File{}
	return file{s: name, f: fs.flushcb(dir, base), r: new(bytes.Reader)}, nil
}

// Mkdir creates an in-memory directory under the given path.
func (fs FS) Mkdir(name string, _ os.FileMode) error {
	dir, base, perr := fs.dirbase(name)
	if perr != nil {
		perr.Op = "Mkdir"
		return perr
	}
	if base == "" {
		return nil
	}
	if v, ok := dir[base]; ok {
		if _, ok = v.(File); ok {
			return &os.PathError{"Mkdir", name, errNotDir}
		}
	}
	dir[base] = Directory{}
	return nil
}

// Open opens a file or directory given by the path.
func (fs FS) Open(name string) (fs.File, error) {
	dir, base, perr := fs.dirbase(name)
	if perr != nil {
		perr.Op = "Open"
		return nil, perr
	}
	if base == "" {
		return directory{s: name, d: dir}, nil
	}
	if _, ok := dir[base]; !ok {
		return nil, &os.PathError{"Open", name, os.ErrNotExist}
	}
	switch v := dir[base].(type) {
	case File:
		return file{s: name, f: fs.flushcb(dir, base), r: bytes.NewReader([]byte(v))}, nil
	case Directory:
		return directory{s: name, d: v}, nil
	}
	return nil, &os.PathError{"Open", name, errCorrupted}
}

// Remove removes a file from the tree given by the path.
func (fs FS) Remove(name string) error {
	dir, base, perr := fs.dirbase(name)
	if perr != nil {
		perr.Op = "Remove"
		return perr
	}
	if base == "" {
		return &os.PathError{"Remove", name, os.ErrPermission}
	}
	if _, ok := dir[base]; !ok {
		return &os.PathError{"Remove", name, os.ErrNotExist}
	}
	if _, ok := dir[base].(Directory); ok {
		return &os.PathError{"Remove", name, os.ErrPermission}
	}
	delete(dir, base)
	return nil
}

// Stat gives the details of a file or a directory given by the path.
func (fs FS) Stat(name string) (os.FileInfo, error) {
	f, err := fs.Open(name)
	if err != nil {
		return nil, err
	}
	return f.Stat()
}

type file struct {
	s string        // name
	f func([]byte)  // flush callback
	r *bytes.Reader // for reading (io.Seeker)
	w *bytes.Buffer // for writing - merge?
}

func (f file) Close() (err error) {
	if f.w != nil {
		f.f(f.w.Bytes())
		f.w = nil
	}
	return
}

func (f file) Read(p []byte) (int, error) {
	return f.r.Read(p)
}

func (f file) Readdir(int) ([]os.FileInfo, error) {
	return nil, &os.PathError{"Readdir", f.s, nil}
}

func (f file) Seek(offset int64, whence int) (int64, error) {
	return f.r.Seek(offset, whence)
}

func (f file) Stat() (os.FileInfo, error) {
	return fileinfo{f.s, int64(f.r.Len()), false}, nil
}

func (f file) Write(p []byte) (int, error) {
	if f.w == nil {
		f.w = new(bytes.Buffer)
	}
	return f.w.Write(p)
}

type directory struct {
	s string
	d Directory
}

func (d directory) Close() (err error) {
	return
}

func (d directory) Read(p []byte) (int, error) {
	return 0, &os.PathError{"Read", d.s, nil}
}

// TODO(rjeczalik): make it ordered so it actually works
func (d directory) Readdir(n int) (fi []os.FileInfo, err error) {
	if len(d.d) == 0 {
		return nil, errors.New("Readdir: directory is empty")
	}
	if n > 0 {
		return nil, errors.New("Readdir: not implemented")
	}
	fi = make([]os.FileInfo, 0, len(d.d))
	for k, v := range d.d {
		if f, ok := v.(File); ok {
			fi = append(fi, fileinfo{filepath.Join(d.s, k), int64(len(f)), false})
		} else {
			fi = append(fi, fileinfo{filepath.Join(d.s, k), 0, true})
		}
	}
	return
}

func (d directory) Seek(int64, int) (int64, error) {
	return 0, &os.PathError{"Seek", d.s, nil}
}

func (d directory) Stat() (os.FileInfo, error) {
	return fileinfo{d.s, 0, true}, nil
}

func (d directory) Write([]byte) (int, error) {
	return 0, &os.PathError{"Write", d.s, nil}
}

type fileinfo struct {
	s string
	n int64
	d bool
}

func (fi fileinfo) Name() string       { return fi.s }
func (fi fileinfo) Size() int64        { return fi.n }
func (fi fileinfo) Mode() os.FileMode  { return 0 }
func (fi fileinfo) ModTime() time.Time { return time.Time{} }
func (fi fileinfo) IsDir() bool        { return fi.d }
func (fi fileinfo) Sys() interface{}   { return nil }
<commit_msg>netz: Add memnetz, a fake for the Network interface<commit_after>// Package memfs provides an interface for an in-memory filesystem.
package memfs

import (
	"bytes"
	"errors"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/rjeczalik/tools/fs"
)

// Directory represents an in-memory directory
type Directory map[string]interface{}

// File represents an in-memory file.
type File []byte

// FS provides an implementation for Filesystem interface, operating on
// an in-memory file tree.
// TODO(rjeczalik): sync.RWMutex
type FS struct {
	Tree Directory
}

var (
	errDir       = errors.New("is a directory")
	errNotDir    = errors.New("not a directory")
	errCorrupted = errors.New("tree is corrupted")
)

const sep = string(os.PathSeparator)

func (fs FS) lookup(p string) (Directory, *os.PathError) {
	if p == "" || p == "." {
		return fs.Tree, nil
	}
	j := strings.Index(p, sep) + 1
	if j == 0 || j == len(p) {
		return fs.Tree, nil
	}
	dir := fs.Tree
	for j < len(p) {
		i := strings.Index(p[j:], sep)
		if i == -1 {
			i = len(p) - j
		}
		v, ok := dir[p[j:j+i]]
		if !ok {
			return nil, &os.PathError{Err: os.ErrNotExist}
		}
		if dir, ok = v.(Directory); !ok {
			return nil, &os.PathError{Err: errNotDir}
		}
		j += i + 1
	}
	return dir, nil
}

func (fs FS) dirbase(p string) (Directory, string, *os.PathError) {
	i := strings.LastIndex(p, sep)
	if i == -1 {
		return fs.Tree, "", nil
	}
	if i == 0 {
		return fs.Tree, p[1:], nil
	}
	dir, perr := fs.lookup(p[:i])
	if perr != nil {
		perr.Path = p
		return nil, "", perr
	}
	return dir, p[i+1:], nil
}

func (fs FS) flushcb(dir Directory, name string) func([]byte) {
	return func(p []byte) {
		dir[name] = File(p)
	}
}

// Create creates an in-memory file under the given path.
func (fs FS) Create(name string) (fs.File, error) {
	dir, base, perr := fs.dirbase(name)
	if perr != nil {
		perr.Op = "Create"
		return nil, perr
	}
	if base == "" {
		return nil, &os.PathError{"Create", name, errDir}
	}
	if v, ok := dir[base]; ok {
		if _, ok = v.(Directory); ok {
			return nil, &os.PathError{"Create", name, errDir}
		}
	}
	dir[base] = File{}
	return file{s: name, f: fs.flushcb(dir, base), r: new(bytes.Reader)}, nil
}

// Mkdir creates an in-memory directory under the given path.
func (fs FS) Mkdir(name string, _ os.FileMode) error {
	dir, base, perr := fs.dirbase(name)
	if perr != nil {
		perr.Op = "Mkdir"
		return perr
	}
	if base == "" {
		return nil
	}
	if v, ok := dir[base]; ok {
		if _, ok = v.(File); ok {
			return &os.PathError{"Mkdir", name, errNotDir}
		}
	}
	dir[base] = Directory{}
	return nil
}

// Open opens a file or directory given by the path.
func (fs FS) Open(name string) (fs.File, error) {
	dir, base, perr := fs.dirbase(name)
	if perr != nil {
		perr.Op = "Open"
		return nil, perr
	}
	if base == "" {
		return directory{s: name, d: dir}, nil
	}
	if _, ok := dir[base]; !ok {
		return nil, &os.PathError{"Open", name, os.ErrNotExist}
	}
	switch v := dir[base].(type) {
	case File:
		return file{s: name, f: fs.flushcb(dir, base), r: bytes.NewReader([]byte(v))}, nil
	case Directory:
		return directory{s: name, d: v}, nil
	}
	return nil, &os.PathError{"Open", name, errCorrupted}
}

// Remove removes a file from the tree given by the path.
func (fs FS) Remove(name string) error {
	dir, base, perr := fs.dirbase(name)
	if perr != nil {
		perr.Op = "Remove"
		return perr
	}
	if base == "" {
		return &os.PathError{"Remove", name, os.ErrPermission}
	}
	if _, ok := dir[base]; !ok {
		return &os.PathError{"Remove", name, os.ErrNotExist}
	}
	if _, ok := dir[base].(Directory); ok {
		return &os.PathError{"Remove", name, os.ErrPermission}
	}
	delete(dir, base)
	return nil
}

// Stat gives the details of a file or a directory given by the path.
func (fs FS) Stat(name string) (os.FileInfo, error) {
	f, err := fs.Open(name)
	if err != nil {
		return nil, err
	}
	return f.Stat()
}

type file struct {
	s string        // name
	f func([]byte)  // flush callback
	r *bytes.Reader // for reading (io.Seeker)
	w *bytes.Buffer // for writing - merge?
}

func (f file) Close() (err error) {
	if f.w != nil {
		f.f(f.w.Bytes())
		f.w = nil
	}
	return
}

func (f file) Read(p []byte) (int, error) {
	return f.r.Read(p)
}

func (f file) Readdir(int) ([]os.FileInfo, error) {
	return nil, &os.PathError{"Readdir", f.s, nil}
}

func (f file) Seek(offset int64, whence int) (int64, error) {
	return f.r.Seek(offset, whence)
}

func (f file) Stat() (os.FileInfo, error) {
	return fileinfo{f.s, int64(f.r.Len()), false}, nil
}

func (f file) Write(p []byte) (int, error) {
	if f.w == nil {
		f.w = new(bytes.Buffer)
	}
	return f.w.Write(p)
}

type directory struct {
	s string
	d Directory
}

func (d directory) Close() (err error) {
	return
}

func (d directory) Read(p []byte) (int, error) {
	return 0, &os.PathError{"Read", d.s, nil}
}

// TODO(rjeczalik): make it ordered so it actually works
func (d directory) Readdir(n int) (fi []os.FileInfo, err error) {
	if len(d.d) == 0 {
		return nil, errors.New("Readdir: directory is empty")
	}
	if n > 0 {
		return nil, errors.New("Readdir: not implemented")
	}
	fi = make([]os.FileInfo, 0, len(d.d))
	for k, v := range d.d {
		if f, ok := v.(File); ok {
			fi = append(fi, fileinfo{filepath.Join(d.s, k), int64(len(f)), false})
		} else {
			fi = append(fi, fileinfo{filepath.Join(d.s, k), 0, true})
		}
	}
	return
}

func (d directory) Seek(int64, int) (int64, error) {
	return 0, &os.PathError{"Seek", d.s, nil}
}

func (d directory) Stat() (os.FileInfo, error) {
	return fileinfo{d.s, 0, true}, nil
}

func (d directory) Write([]byte) (int, error) {
	return 0, &os.PathError{"Write", d.s, nil}
}

type fileinfo struct {
	s string
	n int64
	d bool
}

func (fi fileinfo) Name() string       { return fi.s }
func (fi fileinfo) Size() int64        { return fi.n }
func (fi fileinfo) Mode() os.FileMode  { return 0 }
func (fi fileinfo) ModTime() time.Time { return time.Time{} }
func (fi fileinfo) IsDir() bool        { return fi.d }
func (fi fileinfo) Sys() interface{}   { return nil }
<|endoftext|>


<commit_before>package buffer_list

import (
	"testing"
	"time"
)

type TestData struct {
	a int64
	b int32
	c int64
}

func createList() *List {
	list := New(TestData{}, 4096)
	data := (*TestData)(list.Front().Value())
	data.a = 1
	data.b = 11

	return list
}

func TestBufferListiCreate(t *testing.T) {

	list := createList()

	if list.Len != 1 {
		t.Error("list.Len != 1")
	}
}

func TestBufferListInsertNewElem(t *testing.T) {

	list := createList()

	e := list.InsertNewElem(list.Front())
	data := (*TestData)(e.Value())

	data.a = 2
	data.b = 22

	if list.Len != 2 {
		t.Error("list.Len != 2")
	}

	data2 := (*TestData)(list.Back().Value())

	if data2.a != 2 {
		t.Error("data2.a != 2")
	}
}

func TestBufferListCreate10(t *testing.T) {

	list := createList()
	var data *TestData
	var e *Element
	for i := 1; i < 10; i++ {
		e = list.InsertNewElem(list.Back())
		data = (*TestData)(e.Value())
		data.a = int64(i) * 1
		data.b = int32(i) * 11
	}

	if list.Len != 10 {
		t.Error("list.len != 10")
	}

	data = (*TestData)(list.Back().Prev().Value())

	if data.b != 88 {
		t.Error("data.b != 88", data.b)
	}
}

func TestBufferListConcurrentCreate10(t *testing.T) {

	list := createList()
	for i := 1; i < 10; i++ {
		go func(list *List, i int) {
			ee := list.InsertNewElem(list.Back())
			tdata := (*TestData)(ee.Value())
			tdata.a = int64(i) * 1
			tdata.b = int32(i) * 11
		}(list, i)
	}

	time.Sleep(time.Second)

	if list.Len != 10 {
		t.Error("list.len != 10")
	}
}
<commit_msg> wait via channel on concurrent test<commit_after>package buffer_list

import (
	"testing"
)

type TestData struct {
	a int64
	b int32
	c int64
}

func createList() *List {
	list := New(TestData{}, 4096)
	data := (*TestData)(list.Front().Value())
	data.a = 1
	data.b = 11

	return list
}

func TestBufferListiCreate(t *testing.T) {

	list := createList()

	if list.Len != 1 {
		t.Error("list.Len != 1")
	}
}

func TestBufferListInsertNewElem(t *testing.T) {

	list := createList()

	e := list.InsertNewElem(list.Front())
	data := (*TestData)(e.Value())

	data.a = 2
	data.b = 22

	if list.Len != 2 {
		t.Error("list.Len != 2")
	}

	data2 := (*TestData)(list.Back().Value())

	if data2.a != 2 {
		t.Error("data2.a != 2")
	}
}

func TestBufferListCreate1000(t *testing.T) {

	list := createList()
	var data *TestData
	var e *Element
	for i := 1; i < 1000; i++ {
		e = list.InsertNewElem(list.Back())
		data = (*TestData)(e.Value())
		data.a = int64(i) * 1
		data.b = int32(i) * 11
	}

	if list.Len != 1000 {
		t.Error("list.len != 10")
	}

	data = (*TestData)(list.Back().Prev().Value())

	if data.b != 998*11 {
		t.Error("data.b != 998*11", data.b)
	}
}

func TestBufferListConcurrentCreate1000(t *testing.T) {

	list := createList()
	fin := make(chan bool, 10)

	c_elm := func(list *List, i int, fin chan bool) {
		ee := list.InsertNewElem(list.Back())
		tdata := (*TestData)(ee.Value())
		tdata.a = int64(i) * 1
		tdata.b = int32(i) * 11
		fin <- true
	}

	for i := 1; i < 1000; i++ {
		go c_elm(list, i, fin)
	}

	for i := 0; i < 999; i++ {
		select {
		case <-fin:
			continue
		}
	}

	if list.Len != 1000 {
		t.Error("list.len != 10", list.Len)
	}
}
<|endoftext|>


<commit_before>package settings

import (
	"encoding/json"
	"net/http"
	"net/url"
	"strings"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/instance/lifecycle"
	"github.com/cozy/cozy-stack/pkg/jsonapi"
	"github.com/cozy/cozy-stack/pkg/oauth"
	"github.com/cozy/cozy-stack/web/middlewares"
	"github.com/cozy/echo"
)

type apiContext struct {
	doc map[string]interface{}
}

func (c *apiContext) ID() string                             { return consts.ContextSettingsID }
func (c *apiContext) Rev() string                            { return "" }
func (c *apiContext) DocType() string                        { return consts.Settings }
func (c *apiContext) Clone() couchdb.Doc                     { return c }
func (c *apiContext) SetID(id string)                        {}
func (c *apiContext) SetRev(rev string)                      {}
func (c *apiContext) Relationships() jsonapi.RelationshipMap { return nil }
func (c *apiContext) Included() []jsonapi.Object             { return nil }
func (c *apiContext) Links() *jsonapi.LinksList {
	return &jsonapi.LinksList{Self: "/settings/context"}
}
func (c *apiContext) MarshalJSON() ([]byte, error) {
	return json.Marshal(c.doc)
}
func (c *apiContext) Match(field, expected string) bool {
	return false
}

func onboarded(c echo.Context) error {
	i := middlewares.GetInstance(c)
	if !middlewares.IsLoggedIn(c) {
		return c.Redirect(http.StatusSeeOther, i.PageURL("/auth/login", nil))
	}
	return finishOnboarding(c)
}

func finishOnboarding(c echo.Context) error {
	i := middlewares.GetInstance(c)
	if !i.OnboardingFinished {
		t := true
		err := lifecycle.Patch(i, &lifecycle.Options{OnboardingFinished: &t})
		if err != nil {
			return err
		}
	}
	redirect := i.OnboardedRedirection().String()

	// Retreiving client
	// If there is no onboarding client, we keep going
	client, err := oauth.FindOnboardingClient(i)

	// Redirect to permissions screen if we are in a mobile onboarding
	if err == nil && client.OnboardingSecret != "" {
		redirectURI := ""
		if len(client.RedirectURIs) > 0 {
			redirectURI = client.RedirectURIs[0]
		}

		// Create and adding a fallbackURI in case of no-supporting custom
		// protocol cozy<app>://
		// Basically, it parses the app slug and computes the web app url
		// Example: cozydrive:// => http://drive.alice.cozy.tools:8080/
		r, err := url.Parse(redirectURI)
		if err != nil {
			return err
		}
		// If the redirectURI scheme is not starting with cozy<app>://, it means
		// that we probably are on a recent mobile, handling universal/android
		// links. We won't provide a fallbackURI because the redirectURI should
		// be enough to handle the redirection on the mobile-side
		var fallbackURI string
		if strings.HasPrefix(r.Scheme, "cozy") {
			appSlug := strings.TrimLeft(r.Scheme, "cozy")
			fallbackURI = i.SubDomain(appSlug).String()
		}
		// Redirection
		queryParams := url.Values{
			"client_id":     {client.CouchID},
			"redirect_uri":  {redirectURI},
			"state":         {client.OnboardingState},
			"fallback_uri":  {fallbackURI},
			"response_type": {"code"},
			"scope":         {client.OnboardingPermissions},
		}
		redirect = i.PageURL("/auth/authorize", queryParams)

	}
	return c.Redirect(http.StatusSeeOther, redirect)
}

func context(c echo.Context) error {
	i := middlewares.GetInstance(c)
	ctx, err := i.SettingsContext()
	if err == instance.ErrContextNotFound {
		return jsonapi.NotFound(err)
	}
	if err != nil {
		return err
	}

	doc := &apiContext{ctx}
	if _, err = middlewares.GetPermission(c); err != nil {
		return echo.NewHTTPError(http.StatusForbidden)
	}

	return jsonapi.Data(c, http.StatusOK, doc, nil)
}
<commit_msg>Use SoftwareID instead of redirectURI scheme to calculate the fallback (#1891)<commit_after>package settings

import (
	"encoding/json"
	"net/http"
	"net/url"
	"strings"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/instance/lifecycle"
	"github.com/cozy/cozy-stack/pkg/jsonapi"
	"github.com/cozy/cozy-stack/pkg/oauth"
	"github.com/cozy/cozy-stack/web/middlewares"
	"github.com/cozy/echo"
)

type apiContext struct {
	doc map[string]interface{}
}

func (c *apiContext) ID() string                             { return consts.ContextSettingsID }
func (c *apiContext) Rev() string                            { return "" }
func (c *apiContext) DocType() string                        { return consts.Settings }
func (c *apiContext) Clone() couchdb.Doc                     { return c }
func (c *apiContext) SetID(id string)                        {}
func (c *apiContext) SetRev(rev string)                      {}
func (c *apiContext) Relationships() jsonapi.RelationshipMap { return nil }
func (c *apiContext) Included() []jsonapi.Object             { return nil }
func (c *apiContext) Links() *jsonapi.LinksList {
	return &jsonapi.LinksList{Self: "/settings/context"}
}
func (c *apiContext) MarshalJSON() ([]byte, error) {
	return json.Marshal(c.doc)
}
func (c *apiContext) Match(field, expected string) bool {
	return false
}

func onboarded(c echo.Context) error {
	i := middlewares.GetInstance(c)
	if !middlewares.IsLoggedIn(c) {
		return c.Redirect(http.StatusSeeOther, i.PageURL("/auth/login", nil))
	}
	return finishOnboarding(c)
}

func finishOnboarding(c echo.Context) error {
	i := middlewares.GetInstance(c)
	if !i.OnboardingFinished {
		t := true
		err := lifecycle.Patch(i, &lifecycle.Options{OnboardingFinished: &t})
		if err != nil {
			return err
		}
	}
	redirect := i.OnboardedRedirection().String()

	// Retreiving client
	// If there is no onboarding client, we keep going
	client, err := oauth.FindOnboardingClient(i)

	// Redirect to permissions screen if we are in a mobile onboarding
	if err == nil && client.OnboardingSecret != "" {
		redirectURI := ""
		if len(client.RedirectURIs) > 0 {
			redirectURI = client.RedirectURIs[0]
		}

		// Create and adding a fallbackURI in case of no-supporting custom
		// protocol cozy<app>://
		// Basically, it parses the app slug and computes the web app url
		// Example: cozydrive:// => http://drive.alice.cozy.tools:8080/
		r, err := url.Parse(redirectURI)
		if err != nil {
			return err
		}
		// If the redirectURI scheme is not starting with cozy<app>://, it means
		// that we probably are on a recent mobile, handling universal/android
		// links. We won't provide a fallbackURI because the redirectURI should
		// be enough to handle the redirection on the mobile-side
		var fallbackURI string
		if strings.HasPrefix(r.Scheme, "cozy") {
			appSlug := strings.TrimPrefix(client.SoftwareID, "registry://")
			fallbackURI = i.SubDomain(appSlug).String()
		}
		// Redirection
		queryParams := url.Values{
			"client_id":     {client.CouchID},
			"redirect_uri":  {redirectURI},
			"state":         {client.OnboardingState},
			"fallback_uri":  {fallbackURI},
			"response_type": {"code"},
			"scope":         {client.OnboardingPermissions},
		}
		redirect = i.PageURL("/auth/authorize", queryParams)

	}
	return c.Redirect(http.StatusSeeOther, redirect)
}

func context(c echo.Context) error {
	i := middlewares.GetInstance(c)
	ctx, err := i.SettingsContext()
	if err == instance.ErrContextNotFound {
		return jsonapi.NotFound(err)
	}
	if err != nil {
		return err
	}

	doc := &apiContext{ctx}
	if _, err = middlewares.GetPermission(c); err != nil {
		return echo.NewHTTPError(http.StatusForbidden)
	}

	return jsonapi.Data(c, http.StatusOK, doc, nil)
}
<|endoftext|>


<commit_before>package mesh

import (
	"fmt"
)

type GossipData interface {
	Encode() [][]byte
	Merge(GossipData) GossipData
}

type Gossip interface {
	// specific message from one peer to another
	// intermediate peers relay it using unicast topology.
	GossipUnicast(dstPeerName PeerName, msg []byte) error
	// send gossip to every peer, relayed using broadcast topology.
	GossipBroadcast(update GossipData) error
}

type Gossiper interface {
	OnGossipUnicast(sender PeerName, msg []byte) error
	// merge received data into state and return a representation of
	// the received data, for further propagation
	OnGossipBroadcast(sender PeerName, update []byte) (GossipData, error)
	// return state of everything we know; gets called periodically
	Gossip() GossipData
	// merge received data into state and return "everything new I've
	// just learnt", or nil if nothing in the received data was new
	OnGossip(update []byte) (GossipData, error)
}

// Accumulates GossipData that needs to be sent to one destination,
// and sends it when possible.
type GossipSender struct {
	send func(GossipData)
	cell chan GossipData
	// for testing
	sent    bool
	flushch chan chan bool
}

func NewGossipSender(send func(GossipData)) *GossipSender {
	cell := make(chan GossipData, 1)
	flushch := make(chan chan bool)
	sender := &GossipSender{
		send:    send,
		cell:    cell,
		flushch: flushch,
	}
	go sender.run()
	return sender
}

func (sender *GossipSender) run() {
	for {
		select {
		case pending := <-sender.cell:
			if pending == nil { // receive zero value when chan is closed
				return
			}
			sender.send(pending)
			sender.sent = true
		case ch := <-sender.flushch:
			// send anything pending, then reply back whether we sent
			// anything since previous flush
			select {
			case pending := <-sender.cell:
				sender.send(pending)
				sender.sent = true
			default:
			}
			ch <- sender.sent
			sender.sent = false
		}
	}
}

func (sender *GossipSender) Send(data GossipData) {
	// NB: this must not be invoked concurrently
	select {
	case pending := <-sender.cell:
		sender.cell <- pending.Merge(data)
	default:
		sender.cell <- data
	}
}

func (sender *GossipSender) Stop() {
	close(sender.cell)
}

type GossipChannels map[string]*GossipChannel

func (router *Router) NewGossip(channelName string, g Gossiper) Gossip {
	channel := NewGossipChannel(channelName, router.Ourself, router.Routes, g)
	router.gossipLock.Lock()
	defer router.gossipLock.Unlock()
	if _, found := router.gossipChannels[channelName]; found {
		checkFatal(fmt.Errorf("[gossip] duplicate channel %s", channelName))
	}
	router.gossipChannels[channelName] = channel
	return channel
}

func (router *Router) gossipChannel(channelName string) *GossipChannel {
	router.gossipLock.RLock()
	channel, found := router.gossipChannels[channelName]
	router.gossipLock.RUnlock()
	if found {
		return channel
	}
	router.gossipLock.Lock()
	defer router.gossipLock.Unlock()
	if channel, found = router.gossipChannels[channelName]; found {
		return channel
	}
	channel = NewGossipChannel(channelName, router.Ourself, router.Routes, &surrogateGossiper)
	channel.log("created surrogate channel")
	router.gossipChannels[channelName] = channel
	return channel
}

func (router *Router) gossipChannelSet() map[*GossipChannel]struct{} {
	channels := make(map[*GossipChannel]struct{})
	router.gossipLock.RLock()
	defer router.gossipLock.RUnlock()
	for _, channel := range router.gossipChannels {
		channels[channel] = void
	}
	return channels
}

func (router *Router) SendAllGossip() {
	for channel := range router.gossipChannelSet() {
		if gossip := channel.gossiper.Gossip(); gossip != nil {
			channel.Send(gossip)
		}
	}
}

func (router *Router) SendAllGossipDown(conn Connection) {
	for channel := range router.gossipChannelSet() {
		if gossip := channel.gossiper.Gossip(); gossip != nil {
			channel.SendDown(conn, gossip)
		}
	}
}

// for testing

func (router *Router) sendPendingGossip() bool {
	sentSomething := false
	for channel := range router.gossipChannelSet() {
		channel.Lock()
		for _, sender := range channel.senders {
			sentSomething = sender.flush() || sentSomething
		}
		for _, sender := range channel.broadcasters {
			sentSomething = sender.flush() || sentSomething
		}
		channel.Unlock()
	}
	return sentSomething
}

func (sender *GossipSender) flush() bool {
	ch := make(chan bool)
	sender.flushch <- ch
	return <-ch
}
<commit_msg>cosmetic<commit_after>package mesh

import (
	"fmt"
)

type GossipData interface {
	Encode() [][]byte
	Merge(GossipData) GossipData
}

type Gossip interface {
	// specific message from one peer to another
	// intermediate peers relay it using unicast topology.
	GossipUnicast(dstPeerName PeerName, msg []byte) error
	// send gossip to every peer, relayed using broadcast topology.
	GossipBroadcast(update GossipData) error
}

type Gossiper interface {
	OnGossipUnicast(sender PeerName, msg []byte) error
	// merge received data into state and return a representation of
	// the received data, for further propagation
	OnGossipBroadcast(sender PeerName, update []byte) (GossipData, error)
	// return state of everything we know; gets called periodically
	Gossip() GossipData
	// merge received data into state and return "everything new I've
	// just learnt", or nil if nothing in the received data was new
	OnGossip(update []byte) (GossipData, error)
}

// Accumulates GossipData that needs to be sent to one destination,
// and sends it when possible.
type GossipSender struct {
	send func(GossipData)
	cell chan GossipData
	// for testing
	sent    bool
	flushch chan chan bool
}

func NewGossipSender(send func(GossipData)) *GossipSender {
	cell := make(chan GossipData, 1)
	flushch := make(chan chan bool)
	s := &GossipSender{send: send, cell: cell, flushch: flushch}
	go s.run()
	return s
}

func (s *GossipSender) run() {
	for {
		select {
		case pending := <-s.cell:
			if pending == nil { // receive zero value when chan is closed
				return
			}
			s.send(pending)
			s.sent = true
		case ch := <-s.flushch:
			// send anything pending, then reply back whether we sent
			// anything since previous flush
			select {
			case pending := <-s.cell:
				s.send(pending)
				s.sent = true
			default:
			}
			ch <- s.sent
			s.sent = false
		}
	}
}

func (s *GossipSender) Send(data GossipData) {
	// NB: this must not be invoked concurrently
	select {
	case pending := <-s.cell:
		s.cell <- pending.Merge(data)
	default:
		s.cell <- data
	}
}

func (s *GossipSender) Stop() {
	close(s.cell)
}

type GossipChannels map[string]*GossipChannel

func (router *Router) NewGossip(channelName string, g Gossiper) Gossip {
	channel := NewGossipChannel(channelName, router.Ourself, router.Routes, g)
	router.gossipLock.Lock()
	defer router.gossipLock.Unlock()
	if _, found := router.gossipChannels[channelName]; found {
		checkFatal(fmt.Errorf("[gossip] duplicate channel %s", channelName))
	}
	router.gossipChannels[channelName] = channel
	return channel
}

func (router *Router) gossipChannel(channelName string) *GossipChannel {
	router.gossipLock.RLock()
	channel, found := router.gossipChannels[channelName]
	router.gossipLock.RUnlock()
	if found {
		return channel
	}
	router.gossipLock.Lock()
	defer router.gossipLock.Unlock()
	if channel, found = router.gossipChannels[channelName]; found {
		return channel
	}
	channel = NewGossipChannel(channelName, router.Ourself, router.Routes, &surrogateGossiper)
	channel.log("created surrogate channel")
	router.gossipChannels[channelName] = channel
	return channel
}

func (router *Router) gossipChannelSet() map[*GossipChannel]struct{} {
	channels := make(map[*GossipChannel]struct{})
	router.gossipLock.RLock()
	defer router.gossipLock.RUnlock()
	for _, channel := range router.gossipChannels {
		channels[channel] = void
	}
	return channels
}

func (router *Router) SendAllGossip() {
	for channel := range router.gossipChannelSet() {
		if gossip := channel.gossiper.Gossip(); gossip != nil {
			channel.Send(gossip)
		}
	}
}

func (router *Router) SendAllGossipDown(conn Connection) {
	for channel := range router.gossipChannelSet() {
		if gossip := channel.gossiper.Gossip(); gossip != nil {
			channel.SendDown(conn, gossip)
		}
	}
}

// for testing

func (router *Router) sendPendingGossip() bool {
	sentSomething := false
	for channel := range router.gossipChannelSet() {
		channel.Lock()
		for _, sender := range channel.senders {
			sentSomething = sender.flush() || sentSomething
		}
		for _, sender := range channel.broadcasters {
			sentSomething = sender.flush() || sentSomething
		}
		channel.Unlock()
	}
	return sentSomething
}

func (s *GossipSender) flush() bool {
	ch := make(chan bool)
	s.flushch <- ch
	return <-ch
}
<|endoftext|>


<commit_before>// This package implements the KVDB interface based on consul.
// Code from docker/libkv was leveraged to build parts of this module.
package consul

import (
	"encoding/json"
	"net/http"
	"strings"
	"time"

	api "github.com/hashicorp/consul/api"

	"github.com/portworx/kvdb"
)

const (
	Name    = "consul-kv"
	defHost = "127.0.0.1:8500"
)

type consulLock struct {
	lock    *api.Lock
	renewCh chan struct{}
}

type ConsulKV struct {
	client *api.Client
	config *api.Config
	domain string
}

func ConsulInit(domain string,
	machines []string,
	options map[string]string) (kvdb.Kvdb, error) {

	if len(machines) == 0 {
		machines = []string{defHost}
	}

	if domain != "" && !strings.HasSuffix(domain, "/") {
		domain = domain + "/"
	}

	// Create Consul client
	config := api.DefaultConfig()
	config.HttpClient = http.DefaultClient
	config.Address = machines[0]
	config.Scheme = "http"

	// Creates a new client
	client, err := api.NewClient(config)
	if err != nil {
		return nil, err
	}

	kv := &ConsulKV{
		config: config,
		client: client,
		domain: domain,
	}

	return kv, nil
}

func (kv *ConsulKV) String() string {
	return Name
}

func (kv *ConsulKV) createKv(pair *api.KVPair) *kvdb.KVPair {
	kvp := &kvdb.KVPair{
		Key:   pair.Key,
		Value: []byte(pair.Value),
	}

	kvp.Key = strings.TrimPrefix(kvp.Key, kv.domain)
	return kvp
}

func (kv *ConsulKV) pairToKv(action string, pair *api.KVPair, meta *api.QueryMeta) *kvdb.KVPair {
	kvp := kv.createKv(pair)
	switch action {
	case "create":
		kvp.Action = kvdb.KVCreate
	case "set", "update", "put":
		kvp.Action = kvdb.KVSet
	case "delete":
		kvp.Action = kvdb.KVDelete
	case "get":
		kvp.Action = kvdb.KVGet
	default:
		kvp.Action = kvdb.KVUknown
	}

	if meta != nil {
		kvp.KVDBIndex = meta.LastIndex
	}

	return kvp
}

func (kv *ConsulKV) pairToKvs(action string, pair []*api.KVPair, meta *api.QueryMeta) kvdb.KVPairs {
	kvs := make([]*kvdb.KVPair, len(pair))
	for i := range pair {
		kvs[i] = kv.pairToKv(action, pair[i], meta)
		if meta != nil {
			kvs[i].KVDBIndex = meta.LastIndex
		}
	}
	return kvs
}

func (kv *ConsulKV) toBytes(val interface{}) ([]byte, error) {
	var (
		b   []byte
		err error
	)

	switch val.(type) {
	case string:
		b = []byte(val.(string))
	case []byte:
		b = val.([]byte)
	default:
		b, err = json.Marshal(val)
		if err != nil {
			return nil, err
		}
	}

	return b, nil
}

func (kv *ConsulKV) Get(key string) (*kvdb.KVPair, error) {
	options := &api.QueryOptions{
		AllowStale:        false,
		RequireConsistent: true,
	}

	key = kv.domain + key
	pair, meta, err := kv.client.KV().Get(key, options)
	if err != nil {
		return nil, err
	}

	if pair == nil {
		return nil, kvdb.ErrNotFound
	}

	return kv.pairToKv("get", pair, meta), nil
}

func (kv *ConsulKV) GetVal(key string, val interface{}) (*kvdb.KVPair, error) {
	kvp, err := kv.Get(key)
	if err != nil {
		return nil, err
	}
	err = json.Unmarshal(kvp.Value, val)
	return kvp, err
}

func (kv *ConsulKV) Put(key string, val interface{}, ttl uint64) (*kvdb.KVPair, error) {
	key = kv.domain + key

	b, err := kv.toBytes(val)
	if err != nil {
		return nil, err
	}

	pair := &api.KVPair{
		Key:   key,
		Value: b,
	}

	_, err = kv.client.KV().Put(pair, nil)
	if err != nil {
		return nil, err
	}

	return kv.pairToKv("put", pair, nil), nil
}

func (kv *ConsulKV) Create(key string, val interface{}, ttl uint64) (*kvdb.KVPair, error) {
	_, err := kv.Get(key)
	if err == nil {
		return nil, kvdb.ErrExist
	}

	key = kv.domain + key

	b, err := kv.toBytes(val)
	if err != nil {
		return nil, err
	}

	pair := &api.KVPair{
		Key:   key,
		Value: b,
	}

	_, err = kv.client.KV().Put(pair, nil)
	if err != nil {
		return nil, err
	}

	return kv.pairToKv("create", pair, nil), nil
}

func (kv *ConsulKV) Update(key string, val interface{}, ttl uint64) (*kvdb.KVPair, error) {
	_, err := kv.Get(key)
	if err != nil {
		return nil, err
	}

	key = kv.domain + key

	b, err := kv.toBytes(val)
	if err != nil {
		return nil, err
	}

	pair := &api.KVPair{
		Key:   key,
		Value: b,
	}

	_, err = kv.client.KV().Put(pair, nil)
	if err != nil {
		return nil, err
	}

	return kv.pairToKv("update", pair, nil), nil
}

func (kv *ConsulKV) Enumerate(prefix string) (kvdb.KVPairs, error) {
	prefix = kv.domain + prefix

	pairs, _, err := kv.client.KV().List(prefix, nil)
	if err != nil {
		return nil, err
	}

	return kv.pairToKvs("enumerate", pairs, nil), nil
}

func (kv *ConsulKV) Delete(key string) (*kvdb.KVPair, error) {
	pair, err := kv.Get(key)
	if err != nil {
		return nil, err
	}

	key = kv.domain + key

	_, err = kv.client.KV().Delete(key, nil)
	if err != nil {
		return nil, err
	}

	return pair, nil
}

func (kv *ConsulKV) DeleteTree(key string) error {
	key = kv.domain + key

	_, err := kv.client.KV().DeleteTree(key, nil)
	if err != nil {
		return err
	}

	return nil
}

func (kv *ConsulKV) Keys(prefix, key string) ([]string, error) {
	return nil, kvdb.ErrNotSupported
}

func (kv *ConsulKV) CompareAndSet(kvp *kvdb.KVPair, flags kvdb.KVFlags, prevValue []byte) (*kvdb.KVPair, error) {
	return nil, kvdb.ErrNotSupported
}

func (kv *ConsulKV) CompareAndDelete(kvp *kvdb.KVPair, flags kvdb.KVFlags) (*kvdb.KVPair, error) {
	return nil, kvdb.ErrNotSupported
}

func (kv *ConsulKV) WatchKey(key string, waitIndex uint64, opaque interface{}, cb kvdb.WatchCB) error {
	return kvdb.ErrNotSupported
}

func (kv *ConsulKV) WatchTree(prefix string, waitIndex uint64, opaque interface{}, cb kvdb.WatchCB) error {
	return kvdb.ErrNotSupported
}

func (kv *ConsulKV) getLock(key string, ttl uint64) (*consulLock, error) {
	key = kv.domain + key

	lockOpts := &api.LockOptions{
		Key: key,
	}

	lock := &consulLock{}

	if ttl != 0 {
		TTL := time.Duration(0)
		entry := &api.SessionEntry{
			Behavior:  api.SessionBehaviorRelease, // Release the lock when the session expires
			TTL:       (TTL / 2).String(),         // Consul multiplies the TTL by 2x
			LockDelay: 1 * time.Millisecond,       // Virtually disable lock delay
		}

		// Create the key session
		session, _, err := kv.client.Session().Create(entry, nil)
		if err != nil {
			return nil, err
		}

		// Place the session on lock
		lockOpts.Session = session

		// Renew the session ttl lock periodically
		go kv.client.Session().RenewPeriodic(entry.TTL, session, nil, nil)
	}

	l, err := kv.client.LockOpts(lockOpts)
	if err != nil {
		return nil, err
	}

	lock.lock = l
	return lock, nil
}

func (kv *ConsulKV) Lock(key string, ttl uint64) (*kvdb.KVPair, error) {
	l, err := kv.getLock(key, ttl)
	if err != nil {
		return nil, err
	}

	_, err = l.lock.Lock(nil)
	if err != nil {
		return nil, err
	}

	pair := &kvdb.KVPair{
		Key:  key,
		Lock: l,
	}

	return pair, nil
}

func (kv *ConsulKV) Unlock(kvp *kvdb.KVPair) error {
	l := kvp.Lock.(*consulLock)

	return l.lock.Unlock()
}

func (kv *ConsulKV) TxNew() (kvdb.Tx, error) {
	return nil, kvdb.ErrNotSupported
}

func init() {
	kvdb.Register(Name, ConsulInit)
}
<commit_msg>strip http from the machines<commit_after>// This package implements the KVDB interface based on consul.
// Code from docker/libkv was leveraged to build parts of this module.
package consul

import (
	"encoding/json"
	"net/http"
	"strings"
	"time"

	api "github.com/hashicorp/consul/api"

	"github.com/portworx/kvdb"
)

const (
	Name    = "consul-kv"
	defHost = "127.0.0.1:8500"
)

type consulLock struct {
	lock    *api.Lock
	renewCh chan struct{}
}

type ConsulKV struct {
	client *api.Client
	config *api.Config
	domain string
}

func ConsulInit(domain string,
	machines []string,
	options map[string]string) (kvdb.Kvdb, error) {

	if len(machines) == 0 {
		machines = []string{defHost}
	} else {
		if strings.HasPrefix(machines[0], "http://") {
			machines[0] = strings.TrimPrefix(machines[0], "http://")
		} else if strings.HasPrefix(machines[0], "https://") {
			machines[0] = strings.TrimPrefix(machines[0], "https://")
		}
	}

	if domain != "" && !strings.HasSuffix(domain, "/") {
		domain = domain + "/"
	}

	// Create Consul client
	config := api.DefaultConfig()
	config.HttpClient = http.DefaultClient
	config.Address = machines[0]
	config.Scheme = "http"

	// Creates a new client
	client, err := api.NewClient(config)
	if err != nil {
		return nil, err
	}

	kv := &ConsulKV{
		config: config,
		client: client,
		domain: domain,
	}

	return kv, nil
}

func (kv *ConsulKV) String() string {
	return Name
}

func (kv *ConsulKV) createKv(pair *api.KVPair) *kvdb.KVPair {
	kvp := &kvdb.KVPair{
		Key:   pair.Key,
		Value: []byte(pair.Value),
	}

	kvp.Key = strings.TrimPrefix(kvp.Key, kv.domain)
	return kvp
}

func (kv *ConsulKV) pairToKv(action string, pair *api.KVPair, meta *api.QueryMeta) *kvdb.KVPair {
	kvp := kv.createKv(pair)
	switch action {
	case "create":
		kvp.Action = kvdb.KVCreate
	case "set", "update", "put":
		kvp.Action = kvdb.KVSet
	case "delete":
		kvp.Action = kvdb.KVDelete
	case "get":
		kvp.Action = kvdb.KVGet
	default:
		kvp.Action = kvdb.KVUknown
	}

	if meta != nil {
		kvp.KVDBIndex = meta.LastIndex
	}

	return kvp
}

func (kv *ConsulKV) pairToKvs(action string, pair []*api.KVPair, meta *api.QueryMeta) kvdb.KVPairs {
	kvs := make([]*kvdb.KVPair, len(pair))
	for i := range pair {
		kvs[i] = kv.pairToKv(action, pair[i], meta)
		if meta != nil {
			kvs[i].KVDBIndex = meta.LastIndex
		}
	}
	return kvs
}

func (kv *ConsulKV) toBytes(val interface{}) ([]byte, error) {
	var (
		b   []byte
		err error
	)

	switch val.(type) {
	case string:
		b = []byte(val.(string))
	case []byte:
		b = val.([]byte)
	default:
		b, err = json.Marshal(val)
		if err != nil {
			return nil, err
		}
	}

	return b, nil
}

func (kv *ConsulKV) Get(key string) (*kvdb.KVPair, error) {
	options := &api.QueryOptions{
		AllowStale:        false,
		RequireConsistent: true,
	}

	key = kv.domain + key
	pair, meta, err := kv.client.KV().Get(key, options)
	if err != nil {
		return nil, err
	}

	if pair == nil {
		return nil, kvdb.ErrNotFound
	}

	return kv.pairToKv("get", pair, meta), nil
}

func (kv *ConsulKV) GetVal(key string, val interface{}) (*kvdb.KVPair, error) {
	kvp, err := kv.Get(key)
	if err != nil {
		return nil, err
	}
	err = json.Unmarshal(kvp.Value, val)
	return kvp, err
}

func (kv *ConsulKV) Put(key string, val interface{}, ttl uint64) (*kvdb.KVPair, error) {
	key = kv.domain + key

	b, err := kv.toBytes(val)
	if err != nil {
		return nil, err
	}

	pair := &api.KVPair{
		Key:   key,
		Value: b,
	}

	_, err = kv.client.KV().Put(pair, nil)
	if err != nil {
		return nil, err
	}

	return kv.pairToKv("put", pair, nil), nil
}

func (kv *ConsulKV) Create(key string, val interface{}, ttl uint64) (*kvdb.KVPair, error) {
	_, err := kv.Get(key)
	if err == nil {
		return nil, kvdb.ErrExist
	}

	key = kv.domain + key

	b, err := kv.toBytes(val)
	if err != nil {
		return nil, err
	}

	pair := &api.KVPair{
		Key:   key,
		Value: b,
	}

	_, err = kv.client.KV().Put(pair, nil)
	if err != nil {
		return nil, err
	}

	return kv.pairToKv("create", pair, nil), nil
}

func (kv *ConsulKV) Update(key string, val interface{}, ttl uint64) (*kvdb.KVPair, error) {
	_, err := kv.Get(key)
	if err != nil {
		return nil, err
	}

	key = kv.domain + key

	b, err := kv.toBytes(val)
	if err != nil {
		return nil, err
	}

	pair := &api.KVPair{
		Key:   key,
		Value: b,
	}

	_, err = kv.client.KV().Put(pair, nil)
	if err != nil {
		return nil, err
	}

	return kv.pairToKv("update", pair, nil), nil
}

func (kv *ConsulKV) Enumerate(prefix string) (kvdb.KVPairs, error) {
	prefix = kv.domain + prefix

	pairs, _, err := kv.client.KV().List(prefix, nil)
	if err != nil {
		return nil, err
	}

	return kv.pairToKvs("enumerate", pairs, nil), nil
}

func (kv *ConsulKV) Delete(key string) (*kvdb.KVPair, error) {
	pair, err := kv.Get(key)
	if err != nil {
		return nil, err
	}

	key = kv.domain + key

	_, err = kv.client.KV().Delete(key, nil)
	if err != nil {
		return nil, err
	}

	return pair, nil
}

func (kv *ConsulKV) DeleteTree(key string) error {
	key = kv.domain + key

	_, err := kv.client.KV().DeleteTree(key, nil)
	if err != nil {
		return err
	}

	return nil
}

func (kv *ConsulKV) Keys(prefix, key string) ([]string, error) {
	return nil, kvdb.ErrNotSupported
}

func (kv *ConsulKV) CompareAndSet(kvp *kvdb.KVPair, flags kvdb.KVFlags, prevValue []byte) (*kvdb.KVPair, error) {
	return nil, kvdb.ErrNotSupported
}

func (kv *ConsulKV) CompareAndDelete(kvp *kvdb.KVPair, flags kvdb.KVFlags) (*kvdb.KVPair, error) {
	return nil, kvdb.ErrNotSupported
}

func (kv *ConsulKV) WatchKey(key string, waitIndex uint64, opaque interface{}, cb kvdb.WatchCB) error {
	return kvdb.ErrNotSupported
}

func (kv *ConsulKV) WatchTree(prefix string, waitIndex uint64, opaque interface{}, cb kvdb.WatchCB) error {
	return kvdb.ErrNotSupported
}

func (kv *ConsulKV) getLock(key string, ttl uint64) (*consulLock, error) {
	key = kv.domain + key

	lockOpts := &api.LockOptions{
		Key: key,
	}

	lock := &consulLock{}

	if ttl != 0 {
		TTL := time.Duration(0)
		entry := &api.SessionEntry{
			Behavior:  api.SessionBehaviorRelease, // Release the lock when the session expires
			TTL:       (TTL / 2).String(),         // Consul multiplies the TTL by 2x
			LockDelay: 1 * time.Millisecond,       // Virtually disable lock delay
		}

		// Create the key session
		session, _, err := kv.client.Session().Create(entry, nil)
		if err != nil {
			return nil, err
		}

		// Place the session on lock
		lockOpts.Session = session

		// Renew the session ttl lock periodically
		go kv.client.Session().RenewPeriodic(entry.TTL, session, nil, nil)
	}

	l, err := kv.client.LockOpts(lockOpts)
	if err != nil {
		return nil, err
	}

	lock.lock = l
	return lock, nil
}

func (kv *ConsulKV) Lock(key string, ttl uint64) (*kvdb.KVPair, error) {
	l, err := kv.getLock(key, ttl)
	if err != nil {
		return nil, err
	}

	_, err = l.lock.Lock(nil)
	if err != nil {
		return nil, err
	}

	pair := &kvdb.KVPair{
		Key:  key,
		Lock: l,
	}

	return pair, nil
}

func (kv *ConsulKV) Unlock(kvp *kvdb.KVPair) error {
	l := kvp.Lock.(*consulLock)

	return l.lock.Unlock()
}

func (kv *ConsulKV) TxNew() (kvdb.Tx, error) {
	return nil, kvdb.ErrNotSupported
}

func init() {
	kvdb.Register(Name, ConsulInit)
}
<|endoftext|>


<commit_before>// Package metric provides tools for comparing datasets.
package metric

import (
	"math"

	"github.com/ready-steady/linear/metric"
	"github.com/ready-steady/sort"
	"github.com/ready-steady/statistics/distribution"
)

// MSE computes the mean-square error.
//
// https://en.wikipedia.org/wiki/Mean_squared_error
func MSE(y, yhat []float64) float64 {
	var sum, Δ float64

	for i := range y {
		Δ = yhat[i] - y[i]
		sum += Δ * Δ
	}

	return sum / float64(len(y))
}

// RMSE computes the root-mean-square error.
//
// https://en.wikipedia.org/wiki/Root-mean-square_deviation
func RMSE(y, yhat []float64) float64 {
	return math.Sqrt(MSE(y, yhat))
}

// NRMSE computes the normalized root-mean-square error.
//
// https://en.wikipedia.org/wiki/Root-mean-square_deviation#Normalized_root-mean-square_deviation
func NRMSE(y, yhat []float64) float64 {
	count := len(y)

	min, max := y[0], y[0]
	for i := 1; i < count; i++ {
		if y[i] < min {
			min = y[i]
		}
		if y[i] > max {
			max = y[i]
		}
	}

	return RMSE(y, yhat) / (max - min)
}

// KolmogorovSmirnov computes the Kolmogorov–Smirnov statistic for two samples.
//
// https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test
func KolmogorovSmirnov(data1, data2 []float64) float64 {
	edges := detect(data1, data2)

	cdf1 := distribution.CDF(data1, edges)
	cdf2 := distribution.CDF(data2, edges)

	return metric.Uniform(cdf1, cdf2)
}

// KullbackLeibler computes the Kullback–Leibler divergence of q from p where p
// and q are two discrete probability distributions. The distribution p is
// assumed to be absolutely continuous with respect the distribution q, that is,
// q[i] = 0 implies that p[i] = 0.
//
// https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence
func KullbackLeibler(p, q []float64) float64 {
	divergence := 0.0
	for i := range p {
		if p[i] > 0 {
			divergence += p[i] * math.Log(p[i]/q[i])
		}
	}
	return divergence
}

func detect(data1, data2 []float64) []float64 {
	n1, n2 := len(data1), len(data2)

	edges := make([]float64, 1+n1+n2+1)
	edges[0] = math.Inf(-1)
	copy(edges[1:], data1)
	copy(edges[1+n1:], data2)
	edges[1+n1+n2] = -edges[0]

	return sort.Unique(edges)
}
<commit_msg>metric: a cosmetic change<commit_after>// Package metric provides tools for comparing datasets.
package metric

import (
	"math"

	"github.com/ready-steady/linear/metric"
	"github.com/ready-steady/sort"
	"github.com/ready-steady/statistics/distribution"
)

// MSE computes the mean-square error.
//
// https://en.wikipedia.org/wiki/Mean_squared_error
func MSE(y, yhat []float64) float64 {
	Σ := 0.0
	for i := range y {
		Δ := yhat[i] - y[i]
		Σ += Δ * Δ
	}
	return Σ / float64(len(y))
}

// RMSE computes the root-mean-square error.
//
// https://en.wikipedia.org/wiki/Root-mean-square_deviation
func RMSE(y, yhat []float64) float64 {
	return math.Sqrt(MSE(y, yhat))
}

// NRMSE computes the normalized root-mean-square error.
//
// https://en.wikipedia.org/wiki/Root-mean-square_deviation#Normalized_root-mean-square_deviation
func NRMSE(y, yhat []float64) float64 {
	count := len(y)

	min, max := y[0], y[0]
	for i := 1; i < count; i++ {
		if y[i] < min {
			min = y[i]
		}
		if y[i] > max {
			max = y[i]
		}
	}

	return RMSE(y, yhat) / (max - min)
}

// KolmogorovSmirnov computes the Kolmogorov–Smirnov statistic for two samples.
//
// https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test
func KolmogorovSmirnov(data1, data2 []float64) float64 {
	edges := detect(data1, data2)

	cdf1 := distribution.CDF(data1, edges)
	cdf2 := distribution.CDF(data2, edges)

	return metric.Uniform(cdf1, cdf2)
}

// KullbackLeibler computes the Kullback–Leibler divergence of q from p where p
// and q are two discrete probability distributions. The distribution p is
// assumed to be absolutely continuous with respect the distribution q, that is,
// q[i] = 0 implies that p[i] = 0.
//
// https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence
func KullbackLeibler(p, q []float64) float64 {
	divergence := 0.0
	for i := range p {
		if p[i] > 0 {
			divergence += p[i] * math.Log(p[i]/q[i])
		}
	}
	return divergence
}

func detect(data1, data2 []float64) []float64 {
	n1, n2 := len(data1), len(data2)

	edges := make([]float64, 1+n1+n2+1)
	edges[0] = math.Inf(-1)
	copy(edges[1:], data1)
	copy(edges[1+n1:], data2)
	edges[1+n1+n2] = -edges[0]

	return sort.Unique(edges)
}
<|endoftext|>