<commit_before>// Copyright (c) 2016-2017 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package logutils

import (
	"os"

	"github.com/prometheus/client_golang/prometheus"
	log "github.com/sirupsen/logrus"

	"github.com/projectcalico/felix/config"
	"github.com/projectcalico/libcalico-go/lib/logutils"
)

var (
	counterDroppedLogs = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "felix_logs_dropped",
		Help: "Number of logs dropped because the output stream was blocked.",
	})
	counterLogErrors = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "felix_log_errors",
		Help: "Number of errors encountered while logging.",
	})
)

func init() {
	prometheus.MustRegister(
		counterDroppedLogs,
		counterLogErrors,
	)
}

const logQueueSize = 100

// ConfigureEarlyLogging installs our logging adapters, and enables early logging to screen
// if it is enabled by either the FELIX_EARLYLOGSEVERITYSCREEN or FELIX_LOGSEVERITYSCREEN
// environment variable.
func ConfigureEarlyLogging() {
	// Replace logrus' formatter with a custom one using our time format,
	// shared with the Python code.
	log.SetFormatter(&logutils.Formatter{})

	// Install a hook that adds file/line no information.
	log.AddHook(&logutils.ContextHook{})

	// First try the early-only environment variable.  Since the normal
	// config processing doesn't know about that variable, normal config
	// will override it once it's loaded.
	rawLogLevel := os.Getenv("FELIX_EARLYLOGSEVERITYSCREEN")
	if rawLogLevel == "" {
		// Early-only flag not set, look for the normal config-owned
		// variable.
		rawLogLevel = os.Getenv("FELIX_LOGSEVERITYSCREEN")
	}

	// Default to logging errors.
	logLevelScreen := log.ErrorLevel
	if rawLogLevel != "" {
		parsedLevel, err := log.ParseLevel(rawLogLevel)
		if err == nil {
			logLevelScreen = parsedLevel
		} else {
			log.WithError(err).Error("Failed to parse early log level, defaulting to error.")
		}
	}
	log.SetLevel(logLevelScreen)
	log.Infof("Early screen log level set to %v", logLevelScreen)
}

// ConfigureLogging uses the resolved configuration to complete the logging
// configuration.  It creates hooks for the relevant logging targets and
// attaches them to logrus.
func ConfigureLogging(configParams *config.Config) {
	// Parse the log levels, defaulting to panic if in doubt.
	logLevelScreen := logutils.SafeParseLogLevel(configParams.LogSeverityScreen)
	logLevelFile := logutils.SafeParseLogLevel(configParams.LogSeverityFile)
	logLevelSyslog := logutils.SafeParseLogLevel(configParams.LogSeveritySys)

	// Work out the most verbose level that is being logged.
	mostVerboseLevel := logLevelScreen
	if logLevelFile > mostVerboseLevel {
		mostVerboseLevel = logLevelFile
	}
	if logLevelSyslog > mostVerboseLevel {
		mostVerboseLevel = logLevelScreen
	}
	// Disable all more-verbose levels using the global setting, this ensures that debug logs
	// are filtered out as early as possible.
	log.SetLevel(mostVerboseLevel)

	// Screen target.
	var dests []*logutils.Destination
	if configParams.LogSeverityScreen != "" {
		dests = append(dests, getScreenDestination(configParams, logLevelScreen))
	}

	// File target.  We record any errors so we can log them out below after finishing set-up
	// of the logger.
	var fileDirErr, fileOpenErr error
	if configParams.LogSeverityFile != "" && configParams.LogFilePath != "" {
		var destination *logutils.Destination
		destination, fileDirErr, fileOpenErr = getFileDestination(configParams, logLevelFile)
		if fileDirErr == nil && fileOpenErr == nil && destination != nil {
			dests = append(dests, destination)
		}
	}

	// Syslog target.  Again, we record the error if we fail to connect to syslog.
	var sysErr error
	if configParams.LogSeveritySys != "" {
		var destination *logutils.Destination
		destination, sysErr = getSyslogDestination(configParams, logLevelSyslog)
		if sysErr == nil && destination != nil {
			dests = append(dests, destination)
		}
	}

	hook := logutils.NewBackgroundHook(logutils.FilterLevels(mostVerboseLevel), logLevelSyslog, dests, counterDroppedLogs)
	hook.Start()
	log.AddHook(hook)

	// Disable logrus' default output, which only supports a single destination.  We use the
	// hook above to fan out logs to multiple destinations.
	log.SetOutput(&logutils.NullWriter{})

	// Since we push our logs onto a second thread via a channel, we can disable the
	// Logger's built-in mutex completely.
	log.StandardLogger().SetNoLock()

	// Do any deferred error logging.
	if fileDirErr != nil {
		log.WithError(fileDirErr).WithField("file", configParams.LogFilePath).
			Fatal("Failed to create log file directory.")
	}
	if fileOpenErr != nil {
		log.WithError(fileOpenErr).WithField("file", configParams.LogFilePath).
			Fatal("Failed to open log file.")
	}
	if sysErr != nil {
		// We don't bail out if we can't connect to syslog because our default is to try to
		// connect but it's very common for syslog to be disabled when we're run in a
		// container.
		log.WithError(sysErr).Error(
			"Failed to connect to syslog. To prevent this error, either set config " +
				"parameter LogSeveritySys=none or configure a local syslog service.")
	}
}

func getScreenDestination(configParams *config.Config, logLevel log.Level) *logutils.Destination {
	return logutils.NewStreamDestination(
		logLevel,
		os.Stderr,
		make(chan logutils.QueuedLog, logQueueSize),
		configParams.DebugDisableLogDropping,
		counterLogErrors,
	)
}
<commit_msg>Change to log to stdout by default.<commit_after>// Copyright (c) 2016-2018 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package logutils

import (
	"os"

	"github.com/prometheus/client_golang/prometheus"
	log "github.com/sirupsen/logrus"

	"github.com/projectcalico/felix/config"
	"github.com/projectcalico/libcalico-go/lib/logutils"
)

var (
	counterDroppedLogs = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "felix_logs_dropped",
		Help: "Number of logs dropped because the output stream was blocked.",
	})
	counterLogErrors = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "felix_log_errors",
		Help: "Number of errors encountered while logging.",
	})
)

func init() {
	prometheus.MustRegister(
		counterDroppedLogs,
		counterLogErrors,
	)
}

const logQueueSize = 100

// ConfigureEarlyLogging installs our logging adapters, and enables early logging to screen
// if it is enabled by either the FELIX_EARLYLOGSEVERITYSCREEN or FELIX_LOGSEVERITYSCREEN
// environment variable.
func ConfigureEarlyLogging() {
	// Log to stdout.  This prevents fluentd, for example, from interpreting all our logs as errors by default.
	log.SetOutput(os.Stdout)

	// Replace logrus' formatter with a custom one using our time format,
	// shared with the Python code.
	log.SetFormatter(&logutils.Formatter{})

	// Install a hook that adds file/line no information.
	log.AddHook(&logutils.ContextHook{})

	// First try the early-only environment variable.  Since the normal
	// config processing doesn't know about that variable, normal config
	// will override it once it's loaded.
	rawLogLevel := os.Getenv("FELIX_EARLYLOGSEVERITYSCREEN")
	if rawLogLevel == "" {
		// Early-only flag not set, look for the normal config-owned
		// variable.
		rawLogLevel = os.Getenv("FELIX_LOGSEVERITYSCREEN")
	}

	// Default to logging errors.
	logLevelScreen := log.ErrorLevel
	if rawLogLevel != "" {
		parsedLevel, err := log.ParseLevel(rawLogLevel)
		if err == nil {
			logLevelScreen = parsedLevel
		} else {
			log.WithError(err).Error("Failed to parse early log level, defaulting to error.")
		}
	}
	log.SetLevel(logLevelScreen)
	log.Infof("Early screen log level set to %v", logLevelScreen)
}

// ConfigureLogging uses the resolved configuration to complete the logging
// configuration.  It creates hooks for the relevant logging targets and
// attaches them to logrus.
func ConfigureLogging(configParams *config.Config) {
	// Parse the log levels, defaulting to panic if in doubt.
	logLevelScreen := logutils.SafeParseLogLevel(configParams.LogSeverityScreen)
	logLevelFile := logutils.SafeParseLogLevel(configParams.LogSeverityFile)
	logLevelSyslog := logutils.SafeParseLogLevel(configParams.LogSeveritySys)

	// Work out the most verbose level that is being logged.
	mostVerboseLevel := logLevelScreen
	if logLevelFile > mostVerboseLevel {
		mostVerboseLevel = logLevelFile
	}
	if logLevelSyslog > mostVerboseLevel {
		mostVerboseLevel = logLevelScreen
	}
	// Disable all more-verbose levels using the global setting, this ensures that debug logs
	// are filtered out as early as possible.
	log.SetLevel(mostVerboseLevel)

	// Screen target.
	var dests []*logutils.Destination
	if configParams.LogSeverityScreen != "" {
		dests = append(dests, getScreenDestination(configParams, logLevelScreen))
	}

	// File target.  We record any errors so we can log them out below after finishing set-up
	// of the logger.
	var fileDirErr, fileOpenErr error
	if configParams.LogSeverityFile != "" && configParams.LogFilePath != "" {
		var destination *logutils.Destination
		destination, fileDirErr, fileOpenErr = getFileDestination(configParams, logLevelFile)
		if fileDirErr == nil && fileOpenErr == nil && destination != nil {
			dests = append(dests, destination)
		}
	}

	// Syslog target.  Again, we record the error if we fail to connect to syslog.
	var sysErr error
	if configParams.LogSeveritySys != "" {
		var destination *logutils.Destination
		destination, sysErr = getSyslogDestination(configParams, logLevelSyslog)
		if sysErr == nil && destination != nil {
			dests = append(dests, destination)
		}
	}

	hook := logutils.NewBackgroundHook(logutils.FilterLevels(mostVerboseLevel), logLevelSyslog, dests, counterDroppedLogs)
	hook.Start()
	log.AddHook(hook)

	// Disable logrus' default output, which only supports a single destination.  We use the
	// hook above to fan out logs to multiple destinations.
	log.SetOutput(&logutils.NullWriter{})

	// Since we push our logs onto a second thread via a channel, we can disable the
	// Logger's built-in mutex completely.
	log.StandardLogger().SetNoLock()

	// Do any deferred error logging.
	if fileDirErr != nil {
		log.WithError(fileDirErr).WithField("file", configParams.LogFilePath).
			Fatal("Failed to create log file directory.")
	}
	if fileOpenErr != nil {
		log.WithError(fileOpenErr).WithField("file", configParams.LogFilePath).
			Fatal("Failed to open log file.")
	}
	if sysErr != nil {
		// We don't bail out if we can't connect to syslog because our default is to try to
		// connect but it's very common for syslog to be disabled when we're run in a
		// container.
		log.WithError(sysErr).Error(
			"Failed to connect to syslog. To prevent this error, either set config " +
				"parameter LogSeveritySys=none or configure a local syslog service.")
	}
}

func getScreenDestination(configParams *config.Config, logLevel log.Level) *logutils.Destination {
	return logutils.NewStreamDestination(
		logLevel,
		os.Stdout,
		make(chan logutils.QueuedLog, logQueueSize),
		configParams.DebugDisableLogDropping,
		counterLogErrors,
	)
}
<|endoftext|>


<commit_before>package system

import (
	"context"
	"io"
	"os"

	"github.com/docker/cli/cli"
	"github.com/docker/cli/cli/command"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
)

// newDialStdioCommand creates a new cobra.Command for `docker system dial-stdio`
func newDialStdioCommand(dockerCli command.Cli) *cobra.Command {
	cmd := &cobra.Command{
		Use:    "dial-stdio",
		Short:  "Proxy the stdio stream to the daemon connection. Should not be invoked manually.",
		Args:   cli.NoArgs,
		Hidden: true,
		RunE: func(cmd *cobra.Command, args []string) error {
			return runDialStdio(dockerCli)
		},
	}
	return cmd
}

func runDialStdio(dockerCli command.Cli) error {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	dialer := dockerCli.Client().Dialer()
	conn, err := dialer(ctx)
	if err != nil {
		return errors.Wrap(err, "failed to open the raw stream connection")
	}
	connHalfCloser, ok := conn.(halfCloser)
	if !ok {
		return errors.New("the raw stream connection does not implement halfCloser")
	}
	stdin2conn := make(chan error)
	conn2stdout := make(chan error)
	go func() {
		stdin2conn <- copier(connHalfCloser, &halfReadCloserWrapper{os.Stdin}, "stdin to stream")
	}()
	go func() {
		conn2stdout <- copier(&halfWriteCloserWrapper{os.Stdout}, connHalfCloser, "stream to stdout")
	}()
	select {
	case err = <-stdin2conn:
		if err != nil {
			return err
		}
		// wait for stdout
		err = <-conn2stdout
	case err = <-conn2stdout:
		// return immediately without waiting for stdin to be closed.
		// (stdin is never closed when tty)
	}
	return err
}

func copier(to halfWriteCloser, from halfReadCloser, debugDescription string) error {
	defer func() {
		if err := from.CloseRead(); err != nil {
			logrus.Errorf("error while CloseRead (%s): %v", debugDescription, err)
		}
		if err := to.CloseWrite(); err != nil {
			logrus.Errorf("error while CloseWrite (%s): %v", debugDescription, err)
		}
	}()
	if _, err := io.Copy(to, from); err != nil {
		return errors.Wrapf(err, "error while Copy (%s)", debugDescription)
	}
	return nil
}

type halfReadCloser interface {
	io.Reader
	CloseRead() error
}

type halfWriteCloser interface {
	io.Writer
	CloseWrite() error
}

type halfCloser interface {
	halfReadCloser
	halfWriteCloser
}

type halfReadCloserWrapper struct {
	io.ReadCloser
}

func (x *halfReadCloserWrapper) CloseRead() error {
	return x.Close()
}

type halfWriteCloserWrapper struct {
	io.WriteCloser
}

func (x *halfWriteCloserWrapper) CloseWrite() error {
	return x.Close()
}
<commit_msg>dial-stdio: handle connections which lack CloseRead method.<commit_after>package system

import (
	"context"
	"io"
	"os"

	"github.com/docker/cli/cli"
	"github.com/docker/cli/cli/command"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
)

// newDialStdioCommand creates a new cobra.Command for `docker system dial-stdio`
func newDialStdioCommand(dockerCli command.Cli) *cobra.Command {
	cmd := &cobra.Command{
		Use:    "dial-stdio",
		Short:  "Proxy the stdio stream to the daemon connection. Should not be invoked manually.",
		Args:   cli.NoArgs,
		Hidden: true,
		RunE: func(cmd *cobra.Command, args []string) error {
			return runDialStdio(dockerCli)
		},
	}
	return cmd
}

func runDialStdio(dockerCli command.Cli) error {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	dialer := dockerCli.Client().Dialer()
	conn, err := dialer(ctx)
	if err != nil {
		return errors.Wrap(err, "failed to open the raw stream connection")
	}

	var connHalfCloser halfCloser
	switch t := conn.(type) {
	case halfCloser:
		connHalfCloser = t
	case halfReadWriteCloser:
		connHalfCloser = &nopCloseReader{t}
	default:
		return errors.New("the raw stream connection does not implement halfCloser")
	}

	stdin2conn := make(chan error)
	conn2stdout := make(chan error)
	go func() {
		stdin2conn <- copier(connHalfCloser, &halfReadCloserWrapper{os.Stdin}, "stdin to stream")
	}()
	go func() {
		conn2stdout <- copier(&halfWriteCloserWrapper{os.Stdout}, connHalfCloser, "stream to stdout")
	}()
	select {
	case err = <-stdin2conn:
		if err != nil {
			return err
		}
		// wait for stdout
		err = <-conn2stdout
	case err = <-conn2stdout:
		// return immediately without waiting for stdin to be closed.
		// (stdin is never closed when tty)
	}
	return err
}

func copier(to halfWriteCloser, from halfReadCloser, debugDescription string) error {
	defer func() {
		if err := from.CloseRead(); err != nil {
			logrus.Errorf("error while CloseRead (%s): %v", debugDescription, err)
		}
		if err := to.CloseWrite(); err != nil {
			logrus.Errorf("error while CloseWrite (%s): %v", debugDescription, err)
		}
	}()
	if _, err := io.Copy(to, from); err != nil {
		return errors.Wrapf(err, "error while Copy (%s)", debugDescription)
	}
	return nil
}

type halfReadCloser interface {
	io.Reader
	CloseRead() error
}

type halfWriteCloser interface {
	io.Writer
	CloseWrite() error
}

type halfCloser interface {
	halfReadCloser
	halfWriteCloser
}

type halfReadWriteCloser interface {
	io.Reader
	halfWriteCloser
}

type nopCloseReader struct {
	halfReadWriteCloser
}

func (x *nopCloseReader) CloseRead() error {
	return nil
}

type halfReadCloserWrapper struct {
	io.ReadCloser
}

func (x *halfReadCloserWrapper) CloseRead() error {
	return x.Close()
}

type halfWriteCloserWrapper struct {
	io.WriteCloser
}

func (x *halfWriteCloserWrapper) CloseWrite() error {
	return x.Close()
}
<|endoftext|>


<commit_before>// Copyright ©2012 The bíogo.bam Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bgzf

import (
	"bufio"
	"bytes"
	"compress/flate"
	"compress/gzip"
	"io"
)

type Reader struct {
	gzip.Header
	r io.Reader

	lastChunk Chunk

	block *blockReader

	err error
}

type blockReader struct {
	owner *Reader

	cr *countReader
	gz *gzip.Reader

	decompressed Block
}

func newBlockReader(r io.Reader) (*blockReader, error) {
	cr := makeReader(r)
	gz, err := gzip.NewReader(cr)
	if err != nil {
		return nil, err
	}
	if expectedBlockSize(gz.Header) < 0 {
		return nil, ErrNoBlockSize
	}
	return &blockReader{cr: cr, gz: gz}, nil
}

func (b *blockReader) header() gzip.Header {
	return b.gz.Header
}

func (b *blockReader) reset(r io.Reader, off int64) (gzip.Header, error) {
	isNewBlock := b.decompressed == nil
	if isNewBlock {
		b.decompressed = &block{owner: b.owner}
	}

	if r != nil {
		switch cr := b.cr.r.(type) {
		case reseter:
			cr.Reset(r)
		default:
			b.cr = makeReader(r)
		}
		b.cr.n = off
		b.decompressed.setBase(off)
	}

	if isNewBlock {
		b.decompressed.setHeader(b.gz.Header)
		return b.gz.Header, b.fill()
	}

	b.decompressed.setBase(b.cr.n)

	err := b.gz.Reset(b.cr)
	if err == nil && expectedBlockSize(b.gz.Header) < 0 {
		err = ErrNoBlockSize
	}
	if err != nil {
		return b.gz.Header, err
	}

	b.decompressed.setHeader(b.gz.Header)
	return b.gz.Header, b.fill()
}

func (b *blockReader) fill() error {
	b.gz.Multistream(false)
	_, err := b.decompressed.readFrom(b.gz)
	return err
}

type Block interface {
	// Base returns the file offset of the start of
	// the gzip member from which the Block data was
	// decompressed.
	Base() int64

	io.Reader

	// header returns the gzip.Header of the gzip member
	// from which the Block data was decompressed.
	header() gzip.Header

	// ownedBy returns whether the Block is owned by
	// the given Reader.
	ownedBy(*Reader) bool

	// The following are unexported equivalents
	// of the io interfaces. seek is limited to
	// the file origin offset case and does not
	// return the new offset.
	seek(offset int64) error
	readFrom(io.Reader) (int64, error)

	// len returns the number of remaining
	// bytes that can be read from the Block.
	len() int

	// setBase sets the file offset of the start
	// and of the gzip member that the Block data
	// was decompressed from.
	setBase(int64)

	// setHeader sets the file header of of the gzip
	// member that the Block data was decompressed from.
	setHeader(gzip.Header)

	// beginTx marks the chunk beginning for a set
	// of reads.
	beginTx()

	// endTx returns the Chunk describing the chunk
	// the block read by a set of reads.
	endTx() Chunk
}

type block struct {
	owner *Reader

	base  int64
	h     gzip.Header
	valid bool

	chunk Chunk

	buf  *bytes.Reader
	data [MaxBlockSize]byte
}

func (b *block) Base() int64 { return b.base }

func (b *block) Read(p []byte) (int, error) {
	n, err := b.buf.Read(p)
	b.chunk.End.Block += uint16(n)
	return n, err
}

func (b *block) readFrom(r io.Reader) (int64, error) {
	o := b.owner
	b.owner = nil
	buf := bytes.NewBuffer(b.data[:0])
	n, err := io.Copy(buf, r)
	if err != nil {
		return n, err
	}
	b.buf = bytes.NewReader(buf.Bytes())
	b.owner = o
	return n, nil
}

func (b *block) seek(offset int64) error {
	_, err := b.buf.Seek(offset, 0)
	if err == nil {
		b.chunk.Begin.Block = uint16(offset)
		b.chunk.End.Block = uint16(offset)
	}
	return err
}

func (b *block) len() int {
	if b.buf == nil {
		return 0
	}
	return b.buf.Len()
}

func (b *block) setBase(n int64) {
	b.base = n
	b.chunk = Chunk{Begin: Offset{File: n}, End: Offset{File: n}}
}

func (b *block) setHeader(h gzip.Header) { b.h = h }

func (b *block) header() gzip.Header { return b.h }

func (b *block) ownedBy(r *Reader) bool { return b.owner == r }

func (b *block) beginTx() { b.chunk.Begin = b.chunk.End }

func (b *block) endTx() Chunk { return b.chunk }

func makeReader(r io.Reader) *countReader {
	switch r := r.(type) {
	case *countReader:
		panic("bgzf: illegal use of internal type")
	case flate.Reader:
		return &countReader{r: r}
	default:
		return &countReader{r: bufio.NewReader(r)}
	}
}

type countReader struct {
	r flate.Reader
	n int64
}

func (r *countReader) Read(p []byte) (int, error) {
	n, err := r.r.Read(p)
	r.n += int64(n)
	return n, err
}

func (r *countReader) ReadByte() (byte, error) {
	b, err := r.r.ReadByte()
	r.n++
	return b, err
}

func NewReader(r io.Reader) (*Reader, error) {
	b, err := newBlockReader(r)
	if err != nil {
		return nil, err
	}
	bg := &Reader{
		Header: b.header(),
		r:      r,
		block:  b,
	}
	b.owner = bg
	return bg, nil
}

type Offset struct {
	File  int64
	Block uint16
}

type Chunk struct {
	Begin Offset
	End   Offset
}

type reseter interface {
	Reset(io.Reader)
}

func (bg *Reader) Seek(off Offset) error {
	rs, ok := bg.r.(io.ReadSeeker)
	if !ok {
		return ErrNotASeeker
	}

	_, bg.err = rs.Seek(off.File, 0)
	if bg.err != nil {
		return bg.err
	}
	var h gzip.Header
	h, bg.err = bg.block.reset(bg.r, off.File)
	if bg.err != nil {
		return bg.err
	}
	bg.Header = h

	if off.Block > 0 {
		bg.err = bg.block.decompressed.seek(int64(off.Block))
	}

	return bg.err
}

func (bg *Reader) LastChunk() Chunk { return bg.lastChunk }

func (bg *Reader) Close() error {
	return bg.block.gz.Close()
}

func (bg *Reader) Read(p []byte) (int, error) {
	if bg.err != nil {
		return 0, bg.err
	}
	var h gzip.Header

	dec := bg.block.decompressed
	if dec != nil {
		dec.beginTx()
	}

	if dec == nil || dec.len() == 0 {
		h, bg.err = bg.block.reset(nil, 0)
		if bg.err != nil {
			return 0, bg.err
		}
		bg.Header = h
		dec = bg.block.decompressed
	}

	var n int
	for n < len(p) && bg.err == nil {
		var _n int
		_n, bg.err = dec.Read(p[n:])
		if _n > 0 {
			bg.lastChunk = dec.endTx()
		}
		n += _n
		if bg.err == io.EOF {
			if n == len(p) {
				bg.err = nil
				break
			}

			h, bg.err = bg.block.reset(nil, 0)
			if bg.err != nil {
				break
			}
			bg.Header = h
			dec = bg.block.decompressed
		}
	}

	return n, bg.err
}

func expectedBlockSize(h gzip.Header) int {
	i := bytes.Index(h.Extra, bgzfExtraPrefix)
	if i < 0 || i+5 >= len(h.Extra) {
		return -1
	}
	return (int(h.Extra[i+4]) | int(h.Extra[i+5])<<8) + 1
}
<commit_msg>Move reseter definition into private scope<commit_after>// Copyright ©2012 The bíogo.bam Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bgzf

import (
	"bufio"
	"bytes"
	"compress/flate"
	"compress/gzip"
	"io"
)

type Reader struct {
	gzip.Header
	r io.Reader

	lastChunk Chunk

	block *blockReader

	err error
}

type blockReader struct {
	owner *Reader

	cr *countReader
	gz *gzip.Reader

	decompressed Block
}

func newBlockReader(r io.Reader) (*blockReader, error) {
	cr := makeReader(r)
	gz, err := gzip.NewReader(cr)
	if err != nil {
		return nil, err
	}
	if expectedBlockSize(gz.Header) < 0 {
		return nil, ErrNoBlockSize
	}
	return &blockReader{cr: cr, gz: gz}, nil
}

func (b *blockReader) header() gzip.Header {
	return b.gz.Header
}

func (b *blockReader) reset(r io.Reader, off int64) (gzip.Header, error) {
	isNewBlock := b.decompressed == nil
	if isNewBlock {
		b.decompressed = &block{owner: b.owner}
	}

	if r != nil {
		type reseter interface {
			Reset(io.Reader)
		}
		switch cr := b.cr.r.(type) {
		case reseter:
			cr.Reset(r)
		default:
			b.cr = makeReader(r)
		}
		b.cr.n = off
		b.decompressed.setBase(off)
	}

	if isNewBlock {
		b.decompressed.setHeader(b.gz.Header)
		return b.gz.Header, b.fill()
	}

	b.decompressed.setBase(b.cr.n)

	err := b.gz.Reset(b.cr)
	if err == nil && expectedBlockSize(b.gz.Header) < 0 {
		err = ErrNoBlockSize
	}
	if err != nil {
		return b.gz.Header, err
	}

	b.decompressed.setHeader(b.gz.Header)
	return b.gz.Header, b.fill()
}

func (b *blockReader) fill() error {
	b.gz.Multistream(false)
	_, err := b.decompressed.readFrom(b.gz)
	return err
}

type Block interface {
	// Base returns the file offset of the start of
	// the gzip member from which the Block data was
	// decompressed.
	Base() int64

	io.Reader

	// header returns the gzip.Header of the gzip member
	// from which the Block data was decompressed.
	header() gzip.Header

	// ownedBy returns whether the Block is owned by
	// the given Reader.
	ownedBy(*Reader) bool

	// The following are unexported equivalents
	// of the io interfaces. seek is limited to
	// the file origin offset case and does not
	// return the new offset.
	seek(offset int64) error
	readFrom(io.Reader) (int64, error)

	// len returns the number of remaining
	// bytes that can be read from the Block.
	len() int

	// setBase sets the file offset of the start
	// and of the gzip member that the Block data
	// was decompressed from.
	setBase(int64)

	// setHeader sets the file header of of the gzip
	// member that the Block data was decompressed from.
	setHeader(gzip.Header)

	// beginTx marks the chunk beginning for a set
	// of reads.
	beginTx()

	// endTx returns the Chunk describing the chunk
	// the block read by a set of reads.
	endTx() Chunk
}

type block struct {
	owner *Reader

	base  int64
	h     gzip.Header
	valid bool

	chunk Chunk

	buf  *bytes.Reader
	data [MaxBlockSize]byte
}

func (b *block) Base() int64 { return b.base }

func (b *block) Read(p []byte) (int, error) {
	n, err := b.buf.Read(p)
	b.chunk.End.Block += uint16(n)
	return n, err
}

func (b *block) readFrom(r io.Reader) (int64, error) {
	o := b.owner
	b.owner = nil
	buf := bytes.NewBuffer(b.data[:0])
	n, err := io.Copy(buf, r)
	if err != nil {
		return n, err
	}
	b.buf = bytes.NewReader(buf.Bytes())
	b.owner = o
	return n, nil
}

func (b *block) seek(offset int64) error {
	_, err := b.buf.Seek(offset, 0)
	if err == nil {
		b.chunk.Begin.Block = uint16(offset)
		b.chunk.End.Block = uint16(offset)
	}
	return err
}

func (b *block) len() int {
	if b.buf == nil {
		return 0
	}
	return b.buf.Len()
}

func (b *block) setBase(n int64) {
	b.base = n
	b.chunk = Chunk{Begin: Offset{File: n}, End: Offset{File: n}}
}

func (b *block) setHeader(h gzip.Header) { b.h = h }

func (b *block) header() gzip.Header { return b.h }

func (b *block) ownedBy(r *Reader) bool { return b.owner == r }

func (b *block) beginTx() { b.chunk.Begin = b.chunk.End }

func (b *block) endTx() Chunk { return b.chunk }

func makeReader(r io.Reader) *countReader {
	switch r := r.(type) {
	case *countReader:
		panic("bgzf: illegal use of internal type")
	case flate.Reader:
		return &countReader{r: r}
	default:
		return &countReader{r: bufio.NewReader(r)}
	}
}

type countReader struct {
	r flate.Reader
	n int64
}

func (r *countReader) Read(p []byte) (int, error) {
	n, err := r.r.Read(p)
	r.n += int64(n)
	return n, err
}

func (r *countReader) ReadByte() (byte, error) {
	b, err := r.r.ReadByte()
	r.n++
	return b, err
}

func NewReader(r io.Reader) (*Reader, error) {
	b, err := newBlockReader(r)
	if err != nil {
		return nil, err
	}
	bg := &Reader{
		Header: b.header(),
		r:      r,
		block:  b,
	}
	b.owner = bg
	return bg, nil
}

type Offset struct {
	File  int64
	Block uint16
}

type Chunk struct {
	Begin Offset
	End   Offset
}

func (bg *Reader) Seek(off Offset) error {
	rs, ok := bg.r.(io.ReadSeeker)
	if !ok {
		return ErrNotASeeker
	}

	_, bg.err = rs.Seek(off.File, 0)
	if bg.err != nil {
		return bg.err
	}
	var h gzip.Header
	h, bg.err = bg.block.reset(bg.r, off.File)
	if bg.err != nil {
		return bg.err
	}
	bg.Header = h

	if off.Block > 0 {
		bg.err = bg.block.decompressed.seek(int64(off.Block))
	}

	return bg.err
}

func (bg *Reader) LastChunk() Chunk { return bg.lastChunk }

func (bg *Reader) Close() error {
	return bg.block.gz.Close()
}

func (bg *Reader) Read(p []byte) (int, error) {
	if bg.err != nil {
		return 0, bg.err
	}
	var h gzip.Header

	dec := bg.block.decompressed
	if dec != nil {
		dec.beginTx()
	}

	if dec == nil || dec.len() == 0 {
		h, bg.err = bg.block.reset(nil, 0)
		if bg.err != nil {
			return 0, bg.err
		}
		bg.Header = h
		dec = bg.block.decompressed
	}

	var n int
	for n < len(p) && bg.err == nil {
		var _n int
		_n, bg.err = dec.Read(p[n:])
		if _n > 0 {
			bg.lastChunk = dec.endTx()
		}
		n += _n
		if bg.err == io.EOF {
			if n == len(p) {
				bg.err = nil
				break
			}

			h, bg.err = bg.block.reset(nil, 0)
			if bg.err != nil {
				break
			}
			bg.Header = h
			dec = bg.block.decompressed
		}
	}

	return n, bg.err
}

func expectedBlockSize(h gzip.Header) int {
	i := bytes.Index(h.Extra, bgzfExtraPrefix)
	if i < 0 || i+5 >= len(h.Extra) {
		return -1
	}
	return (int(h.Extra[i+4]) | int(h.Extra[i+5])<<8) + 1
}
<|endoftext|>


<commit_before>// Copyright 2022 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package transport

import (
	"net/http"
	"net/http/httptest"
	"reflect"
	"testing"
)

func TestValidateSecureEndpoints(t *testing.T) {
	tlsInfo, err := createSelfCert(t)
	if err != nil {
		t.Fatalf("unable to create cert: %v", err)
	}

	remoteAddr := func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(r.RemoteAddr))
	}
	srv := httptest.NewServer(http.HandlerFunc(remoteAddr))
	defer srv.Close()

	tests := map[string]struct {
		endPoints         []string
		expectedEndpoints []string
		expectedErr       bool
	}{
		"invalidEndPoints": {
			endPoints: []string{
				"invalid endpoint",
			},
			expectedEndpoints: nil,
			expectedErr:       true,
		},
		"insecureEndpoints": {
			endPoints: []string{
				"http://127.0.0.1:8000",
				"http://" + srv.Listener.Addr().String(),
			},
			expectedEndpoints: nil,
			expectedErr:       true,
		},
		"secureEndPoints": {
			endPoints: []string{
				"https://" + srv.Listener.Addr().String(),
			},
			expectedEndpoints: []string{
				"https://" + srv.Listener.Addr().String(),
			},
			expectedErr: false,
		},
		"mixEndPoints": {
			endPoints: []string{
				"https://" + srv.Listener.Addr().String(),
				"http://" + srv.Listener.Addr().String(),
				"invalid end points",
			},
			expectedEndpoints: []string{
				"https://" + srv.Listener.Addr().String(),
			},
			expectedErr: true,
		},
	}
	for name, test := range tests {
		t.Run(name, func(t *testing.T) {
			secureEps, err := ValidateSecureEndpoints(*tlsInfo, test.endPoints)
			if test.expectedErr != (err != nil) {
				t.Errorf("Unexpected error, got: %v, want: %v", err, test.expectedError)
			}

			if err == nil && !test.expectedErr {
				if len(secureEps) != len(test.expectedEndpoints) {
					t.Errorf("expected %v endpoints, got %v", len(test.expectedEndpoints), len(secureEps))
				}
				if !reflect.DeepEqual(test.expectedEndpoints, secureEps) {
					t.Errorf("expected endpoints %v, got %v", test.expectedEndpoints, secureEps)
				}
			}
		})
	}
}
<commit_msg>remove endpoints length check in TestValidateSecureEndpoints()<commit_after>// Copyright 2022 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package transport

import (
	"net/http"
	"net/http/httptest"
	"reflect"
	"testing"
)

func TestValidateSecureEndpoints(t *testing.T) {
	tlsInfo, err := createSelfCert(t)
	if err != nil {
		t.Fatalf("unable to create cert: %v", err)
	}

	remoteAddr := func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(r.RemoteAddr))
	}
	srv := httptest.NewServer(http.HandlerFunc(remoteAddr))
	defer srv.Close()

	tests := map[string]struct {
		endPoints         []string
		expectedEndpoints []string
		expectedErr       bool
	}{
		"invalidEndPoints": {
			endPoints: []string{
				"invalid endpoint",
			},
			expectedEndpoints: nil,
			expectedErr:       true,
		},
		"insecureEndpoints": {
			endPoints: []string{
				"http://127.0.0.1:8000",
				"http://" + srv.Listener.Addr().String(),
			},
			expectedEndpoints: nil,
			expectedErr:       true,
		},
		"secureEndPoints": {
			endPoints: []string{
				"https://" + srv.Listener.Addr().String(),
			},
			expectedEndpoints: []string{
				"https://" + srv.Listener.Addr().String(),
			},
			expectedErr: false,
		},
		"mixEndPoints": {
			endPoints: []string{
				"https://" + srv.Listener.Addr().String(),
				"http://" + srv.Listener.Addr().String(),
				"invalid end points",
			},
			expectedEndpoints: []string{
				"https://" + srv.Listener.Addr().String(),
			},
			expectedErr: true,
		},
	}
	for name, test := range tests {
		t.Run(name, func(t *testing.T) {
			secureEps, err := ValidateSecureEndpoints(*tlsInfo, test.endPoints)
			if test.expectedErr != (err != nil) {
				t.Errorf("Unexpected error, got: %v, want: %v", err, test.expectedErr)
			}

			if !reflect.DeepEqual(test.expectedEndpoints, secureEps) {
				t.Errorf("expected endpoints %v, got %v", test.expectedEndpoints, secureEps)
			}
		})
	}
}
<|endoftext|>


<commit_before>package guidserver

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net"
	"net/url"
	"time"

	"github.com/cloudfoundry-incubator/garden"
	gclient "github.com/cloudfoundry-incubator/garden/client"
	gconn "github.com/cloudfoundry-incubator/garden/client/connection"
	"github.com/concourse/baggageclaim"
	bclient "github.com/concourse/baggageclaim/client"
	"github.com/concourse/go-concourse/concourse"
	"github.com/mgutz/ansi"
	"github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-golang/lager/lagertest"
)

const amazingRubyServer = `
require 'webrick'
require 'json'

server = WEBrick::HTTPServer.new :Port => 8080

registered = []
files = {}

server.mount_proc '/register' do |req, res|
  registered << req.body.chomp
  res.status = 200
end

server.mount_proc '/registrations' do |req, res|
  res.body = JSON.generate(registered)
end

trap('INT') {
  server.shutdown
}

server.start
`

type Server struct {
	gardenClient garden.Client

	container garden.Container
	rootfsVol baggageclaim.Volume

	addr string
}

func Start(client concourse.Client) *Server {
	logger := lagertest.NewTestLogger("guid-server")

	gLog := logger.Session("garden-connection")

	workers, err := client.ListWorkers()
	Expect(err).NotTo(HaveOccurred())

	var rootfsPath string
	var gardenClient garden.Client
	var baggageclaimClient baggageclaim.Client

	for _, w := range workers {
		if len(w.Tags) > 0 {
			continue
		}

		rootfsPath = ""

		for _, r := range w.ResourceTypes {
			if r.Type == "bosh-deployment" {
				rootfsPath = r.Image
			}
		}

		if rootfsPath != "" {
			gardenClient = gclient.New(gconn.NewWithLogger("tcp", w.GardenAddr, gLog))
			baggageclaimClient = bclient.New(w.BaggageclaimURL)
		}
	}

	if rootfsPath == "" {
		ginkgo.Fail("must have at least one worker that supports bosh-deployment resource type")
	}

	Eventually(gardenClient.Ping).Should(Succeed())

	rootfsVol, err := baggageclaimClient.CreateVolume(logger, baggageclaim.VolumeSpec{
		Strategy: baggageclaim.ImportStrategy{
			Path: rootfsPath,
		},
	})
	Expect(err).NotTo(HaveOccurred())

	container, err := gardenClient.Create(garden.ContainerSpec{
		RootFSPath: (&url.URL{Scheme: "raw", Path: rootfsVol.Path()}).String(),
		GraceTime:  time.Hour,
	})
	Expect(err).NotTo(HaveOccurred())

	_, err = container.Run(garden.ProcessSpec{
		Path: "ruby",
		Args: []string{"-e", amazingRubyServer},
		User: "root",
	}, garden.ProcessIO{
		Stdout: gexec.NewPrefixedWriter(
			fmt.Sprintf("%s%s ", ansi.Color("[o]", "green"), ansi.Color("[guid server]", "magenta")),
			ginkgo.GinkgoWriter,
		),
		Stderr: gexec.NewPrefixedWriter(
			fmt.Sprintf("%s%s ", ansi.Color("[e]", "red+bright"), ansi.Color("[guid server]", "magenta")),
			ginkgo.GinkgoWriter,
		),
	})
	Expect(err).NotTo(HaveOccurred())

	info, err := container.Info()
	Expect(err).NotTo(HaveOccurred())

	addr := fmt.Sprintf("%s:%d", info.ContainerIP, 8080)

	Eventually(func() (int, error) {
		get, err := container.Run(garden.ProcessSpec{
			Path: "ruby",
			Args: []string{"-rnet/http", "-e", `Net::HTTP.get(URI("http://127.0.0.1:8080/registrations"))`},
			User: "root",
		}, garden.ProcessIO{
			Stdout: gexec.NewPrefixedWriter(
				fmt.Sprintf("%s%s ", ansi.Color("[o]", "green"), ansi.Color("[guid server polling]", "magenta")),
				ginkgo.GinkgoWriter,
			),
			Stderr: gexec.NewPrefixedWriter(
				fmt.Sprintf("%s%s ", ansi.Color("[e]", "red+bright"), ansi.Color("[guid server polling]", "magenta")),
				ginkgo.GinkgoWriter,
			),
		})
		Expect(err).NotTo(HaveOccurred())

		return get.Wait()
	}).Should(Equal(0))

	return &Server{
		gardenClient: gardenClient,
		container:    container,
		rootfsVol:    rootfsVol,
		addr:         addr,
	}
}

func (server *Server) Stop() {
	err := server.gardenClient.Destroy(server.container.Handle())
	Expect(err).NotTo(HaveOccurred())

	server.rootfsVol.Release(baggageclaim.FinalTTL(time.Second))
}

func (server *Server) RegisterCommand() string {
	host, port, err := net.SplitHostPort(server.addr)
	Expect(err).ToNot(HaveOccurred())

	return fmt.Sprintf(`ruby -rnet/http -e 'Net::HTTP.start("%s", %s) { |http| puts http.post("/register", STDIN.read).body }'`, host, port)
}

func (server *Server) RegistrationsCommand() string {
	return fmt.Sprintf(`ruby -rnet/http -e 'puts Net::HTTP.get(URI("http://%s/registrations"))`, server.addr)
}

func (server *Server) ReportingGuids() []string {
	outBuf := new(bytes.Buffer)

	get, err := server.container.Run(garden.ProcessSpec{
		Path: "ruby",
		Args: []string{"-rnet/http", "-e", `puts Net::HTTP.get(URI("http://127.0.0.1:8080/registrations"))`},
		User: "root",
	}, garden.ProcessIO{
		Stdout: outBuf,
		Stderr: gexec.NewPrefixedWriter(
			fmt.Sprintf("%s%s ", ansi.Color("[e]", "red+bright"), ansi.Color("[guid server polling]", "magenta")),
			ginkgo.GinkgoWriter,
		),
	})
	Expect(err).NotTo(HaveOccurred())

	Expect(get.Wait()).To(Equal(0))

	var responses []string
	err = json.NewDecoder(outBuf).Decode(&responses)
	Expect(err).NotTo(HaveOccurred())

	return responses
}
<commit_msg>add missing quote<commit_after>package guidserver

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net"
	"net/url"
	"time"

	"github.com/cloudfoundry-incubator/garden"
	gclient "github.com/cloudfoundry-incubator/garden/client"
	gconn "github.com/cloudfoundry-incubator/garden/client/connection"
	"github.com/concourse/baggageclaim"
	bclient "github.com/concourse/baggageclaim/client"
	"github.com/concourse/go-concourse/concourse"
	"github.com/mgutz/ansi"
	"github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-golang/lager/lagertest"
)

const amazingRubyServer = `
require 'webrick'
require 'json'

server = WEBrick::HTTPServer.new :Port => 8080

registered = []
files = {}

server.mount_proc '/register' do |req, res|
  registered << req.body.chomp
  res.status = 200
end

server.mount_proc '/registrations' do |req, res|
  res.body = JSON.generate(registered)
end

trap('INT') {
  server.shutdown
}

server.start
`

type Server struct {
	gardenClient garden.Client

	container garden.Container
	rootfsVol baggageclaim.Volume

	addr string
}

func Start(client concourse.Client) *Server {
	logger := lagertest.NewTestLogger("guid-server")

	gLog := logger.Session("garden-connection")

	workers, err := client.ListWorkers()
	Expect(err).NotTo(HaveOccurred())

	var rootfsPath string
	var gardenClient garden.Client
	var baggageclaimClient baggageclaim.Client

	for _, w := range workers {
		if len(w.Tags) > 0 {
			continue
		}

		rootfsPath = ""

		for _, r := range w.ResourceTypes {
			if r.Type == "bosh-deployment" {
				rootfsPath = r.Image
			}
		}

		if rootfsPath != "" {
			gardenClient = gclient.New(gconn.NewWithLogger("tcp", w.GardenAddr, gLog))
			baggageclaimClient = bclient.New(w.BaggageclaimURL)
		}
	}

	if rootfsPath == "" {
		ginkgo.Fail("must have at least one worker that supports bosh-deployment resource type")
	}

	Eventually(gardenClient.Ping).Should(Succeed())

	rootfsVol, err := baggageclaimClient.CreateVolume(logger, baggageclaim.VolumeSpec{
		Strategy: baggageclaim.ImportStrategy{
			Path: rootfsPath,
		},
	})
	Expect(err).NotTo(HaveOccurred())

	container, err := gardenClient.Create(garden.ContainerSpec{
		RootFSPath: (&url.URL{Scheme: "raw", Path: rootfsVol.Path()}).String(),
		GraceTime:  time.Hour,
	})
	Expect(err).NotTo(HaveOccurred())

	_, err = container.Run(garden.ProcessSpec{
		Path: "ruby",
		Args: []string{"-e", amazingRubyServer},
		User: "root",
	}, garden.ProcessIO{
		Stdout: gexec.NewPrefixedWriter(
			fmt.Sprintf("%s%s ", ansi.Color("[o]", "green"), ansi.Color("[guid server]", "magenta")),
			ginkgo.GinkgoWriter,
		),
		Stderr: gexec.NewPrefixedWriter(
			fmt.Sprintf("%s%s ", ansi.Color("[e]", "red+bright"), ansi.Color("[guid server]", "magenta")),
			ginkgo.GinkgoWriter,
		),
	})
	Expect(err).NotTo(HaveOccurred())

	info, err := container.Info()
	Expect(err).NotTo(HaveOccurred())

	addr := fmt.Sprintf("%s:%d", info.ContainerIP, 8080)

	Eventually(func() (int, error) {
		get, err := container.Run(garden.ProcessSpec{
			Path: "ruby",
			Args: []string{"-rnet/http", "-e", `Net::HTTP.get(URI("http://127.0.0.1:8080/registrations"))`},
			User: "root",
		}, garden.ProcessIO{
			Stdout: gexec.NewPrefixedWriter(
				fmt.Sprintf("%s%s ", ansi.Color("[o]", "green"), ansi.Color("[guid server polling]", "magenta")),
				ginkgo.GinkgoWriter,
			),
			Stderr: gexec.NewPrefixedWriter(
				fmt.Sprintf("%s%s ", ansi.Color("[e]", "red+bright"), ansi.Color("[guid server polling]", "magenta")),
				ginkgo.GinkgoWriter,
			),
		})
		Expect(err).NotTo(HaveOccurred())

		return get.Wait()
	}).Should(Equal(0))

	return &Server{
		gardenClient: gardenClient,
		container:    container,
		rootfsVol:    rootfsVol,
		addr:         addr,
	}
}

func (server *Server) Stop() {
	err := server.gardenClient.Destroy(server.container.Handle())
	Expect(err).NotTo(HaveOccurred())

	server.rootfsVol.Release(baggageclaim.FinalTTL(time.Second))
}

func (server *Server) RegisterCommand() string {
	host, port, err := net.SplitHostPort(server.addr)
	Expect(err).ToNot(HaveOccurred())

	return fmt.Sprintf(`ruby -rnet/http -e 'Net::HTTP.start("%s", %s) { |http| puts http.post("/register", STDIN.read).body }'`, host, port)
}

func (server *Server) RegistrationsCommand() string {
	return fmt.Sprintf(`ruby -rnet/http -e 'puts Net::HTTP.get(URI("http://%s/registrations"))'`, server.addr)
}

func (server *Server) ReportingGuids() []string {
	outBuf := new(bytes.Buffer)

	get, err := server.container.Run(garden.ProcessSpec{
		Path: "ruby",
		Args: []string{"-rnet/http", "-e", `puts Net::HTTP.get(URI("http://127.0.0.1:8080/registrations"))`},
		User: "root",
	}, garden.ProcessIO{
		Stdout: outBuf,
		Stderr: gexec.NewPrefixedWriter(
			fmt.Sprintf("%s%s ", ansi.Color("[e]", "red+bright"), ansi.Color("[guid server polling]", "magenta")),
			ginkgo.GinkgoWriter,
		),
	})
	Expect(err).NotTo(HaveOccurred())

	Expect(get.Wait()).To(Equal(0))

	var responses []string
	err = json.NewDecoder(outBuf).Decode(&responses)
	Expect(err).NotTo(HaveOccurred())

	return responses
}
<|endoftext|>


<commit_before>// Copyright (c) 2018 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bolt

import (
	"bytes"
	"os"
	"testing"

	"github.com/boltdb/bolt"
	"github.com/ligato/cn-infra/datasync"
	"github.com/ligato/cn-infra/db/keyval"
	"github.com/ligato/cn-infra/logging"
	"github.com/ligato/cn-infra/logging/logrus"
	. "github.com/onsi/gomega"
)

func init() {
	logrus.DefaultLogger().SetLevel(logging.DebugLevel)
	boltLogger.SetLevel(logging.DebugLevel)
}

type testCtx struct {
	*testing.T
	client *Client
}

const testDbPath = "/tmp/bolt.db"

func setupTest(t *testing.T, newDB bool) *testCtx {
	RegisterTestingT(t)

	if newDB {
		err := os.Remove(testDbPath)
		if err != nil && !os.IsNotExist(err) {
			t.Fatal(err)
			return nil
		}
	}

	client, err := NewClient(&Config{
		DbPath:   testDbPath,
		FileMode: 432,
	})
	Expect(err).ToNot(HaveOccurred())
	if err != nil {
		return nil
	}

	return &testCtx{T: t, client: client}
}

func (tc *testCtx) teardownTest() {
	Expect(tc.client.Close()).To(Succeed())
}

func (tc *testCtx) isInDB(key string, expectedVal []byte) (exists bool) {
	err := tc.client.db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(rootBucket)
		if val := b.Get([]byte(key)); val != nil {
			exists = true
		}
		return nil
	})
	if err != nil {
		tc.Fatal(err)
	}
	return
}

func (tc *testCtx) populateDB(data map[string][]byte) {
	err := tc.client.db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket(rootBucket)
		for key, val := range data {
			if err := b.Put([]byte(key), val); err != nil {
				return err
			}
		}
		return nil
	})
	if err != nil {
		tc.Fatal(err)
	}
	return
}

func TestPut(t *testing.T) {
	tc := setupTest(t, true)
	defer tc.teardownTest()

	var key = "/agent/agent1/config/interface/iface0"
	var val = []byte("val")

	err := tc.client.Put(key, val)
	Expect(err).ToNot(HaveOccurred())
	Expect(tc.isInDB(key, val)).To(BeTrue())
}

func TestGet(t *testing.T) {
	tc := setupTest(t, true)
	defer tc.teardownTest()

	var key = "/agent/agent1/config/interface/iface0"
	var val = []byte("val")

	err := tc.client.Put(key, val)
	Expect(err).ToNot(HaveOccurred())
	Expect(tc.isInDB(key, val)).To(BeTrue())
}

func TestListKeys(t *testing.T) {
	tc := setupTest(t, true)
	defer tc.teardownTest()

	tc.populateDB(map[string][]byte{
		"/my/key/1":    []byte("val1"),
		"/my/key/2":    []byte("val2"),
		"/other/key/0": []byte("val0"),
	})

	kvi, err := tc.client.ListKeys("/my/key/")
	Expect(err).ToNot(HaveOccurred())
	Expect(kvi).NotTo(BeNil())

	expectedKeys := []string{"/my/key/1", "/my/key/2"}
	for i := 0; i <= len(expectedKeys); i++ {
		key, _, all := kvi.GetNext()
		if i == len(expectedKeys) {
			Expect(all).To(BeTrue())
			break
		}
		Expect(all).To(BeFalse())
		Expect(key).To(BeEquivalentTo(expectedKeys[i]))
	}
}

func TestListValues(t *testing.T) {
	tc := setupTest(t, true)
	defer tc.teardownTest()

	tc.populateDB(map[string][]byte{
		"/my/key/1":    []byte("val1"),
		"/my/key/2":    []byte("val2"),
		"/other/key/0": []byte("val0"),
	})

	kvi, err := tc.client.ListValues("/my/key/")
	Expect(err).ToNot(HaveOccurred())
	Expect(kvi).NotTo(BeNil())

	expectedKeys := []string{"/my/key/1", "/my/key/2"}
	expectedValues := [][]byte{[]byte("val1"), []byte("val2")}
	for i := 0; i <= len(expectedKeys); i++ {
		kv, all := kvi.GetNext()
		if i == len(expectedKeys) {
			Expect(all).To(BeTrue())
			break
		}
		Expect(all).To(BeFalse())
		Expect(kv.GetKey()).To(BeEquivalentTo(expectedKeys[i]))
		Expect(bytes.Compare(kv.GetValue(), expectedValues[i])).To(BeZero())
	}
}

func TestListKeysBroker(t *testing.T) {
	tc := setupTest(t, true)
	defer tc.teardownTest()

	tc.populateDB(map[string][]byte{
		"/my/key/1":    []byte("val1"),
		"/my/key/2":    []byte("val2"),
		"/my/keyx/xx":  []byte("x"),
		"/my/xkey/xx":  []byte("x"),
		"/other/key/0": []byte("val0"),
	})

	broker := tc.client.NewBroker("/my/")
	kvi, err := broker.ListKeys("key/")
	Expect(err).ToNot(HaveOccurred())
	Expect(kvi).NotTo(BeNil())

	expectedKeys := []string{"key/1", "key/2"}
	for i := 0; i <= len(expectedKeys); i++ {
		key, _, all := kvi.GetNext()
		if i == len(expectedKeys) {
			Expect(all).To(BeTrue())
			break
		}
		Expect(all).To(BeFalse())
		Expect(key).To(BeEquivalentTo(expectedKeys[i]))
	}
}

func TestListValuesBroker(t *testing.T) {
	tc := setupTest(t, true)
	defer tc.teardownTest()

	tc.populateDB(map[string][]byte{
		"/my/key/1":    []byte("val1"),
		"/my/key/2":    []byte("val2"),
		"/my/keyx/xx":  []byte("x"),
		"/my/xkey/xx":  []byte("x"),
		"/other/key/0": []byte("val0"),
	})

	broker := tc.client.NewBroker("/my/")
	kvi, err := broker.ListValues("key/")
	Expect(err).ToNot(HaveOccurred())
	Expect(kvi).NotTo(BeNil())

	expectedKeys := []string{"key/1", "key/2"}
	for i := 0; i <= len(expectedKeys); i++ {
		kv, all := kvi.GetNext()
		if i == len(expectedKeys) {
			Expect(all).To(BeTrue())
			break
		}
		Expect(all).To(BeFalse())
		Expect(kv.GetKey()).To(BeEquivalentTo(expectedKeys[i]))
	}
}

func TestDelete(t *testing.T) {
	tc := setupTest(t, true)
	defer tc.teardownTest()

	var key = "/agent/agent1/config/interface/iface0"
	var val = []byte("val")

	err := tc.client.Put(key, val)
	Expect(err).ToNot(HaveOccurred())
	existed, err := tc.client.Delete(key)
	Expect(err).ToNot(HaveOccurred())
	Expect(existed).To(BeTrue())

	existed, err = tc.client.Delete("/this/key/does/not/exists")
	Expect(err).To(HaveOccurred())
	Expect(existed).To(BeFalse())
	Expect(tc.isInDB(key, val)).To(BeFalse())
}

func TestPutInTxn(t *testing.T) {
	tc := setupTest(t, true)
	defer tc.teardownTest()

	txn := tc.client.NewTxn()
	Expect(txn).ToNot(BeNil())

	var key1 = "/agent/agent1/config/interface/iface0"
	var val1 = []byte("iface0")
	var key2 = "/agent/agent1/config/interface/iface1"
	var val2 = []byte("iface1")
	var key3 = "/agent/agent1/config/interface/iface2"
	var val3 = []byte("iface2")

	txn.Put(key1, val1).
		Put(key2, val2).
		Put(key3, val3)
	Expect(txn.Commit()).To(Succeed())
	Expect(tc.isInDB(key1, val1)).To(BeTrue())
	Expect(tc.isInDB(key2, val2)).To(BeTrue())
	Expect(tc.isInDB(key3, val3)).To(BeTrue())
}

func TestDeleteInTxn(t *testing.T) {
	tc := setupTest(t, true)
	defer tc.teardownTest()

	txn := tc.client.NewTxn()
	Expect(txn).ToNot(BeNil())

	var key1 = "/agent/agent1/config/interface/iface0"
	var val1 = []byte("iface0")
	var key2 = "/agent/agent1/config/interface/iface1"
	var val2 = []byte("iface1")
	var key3 = "/agent/agent1/config/interface/iface2"
	var val3 = []byte("iface2")

	txn.Put(key1, val1).
		Put(key2, val2).
		Put(key3, val3).
		Delete(key2)
	Expect(txn.Commit()).To(Succeed())
	Expect(tc.isInDB(key1, val1)).To(BeTrue())
	Expect(tc.isInDB(key2, val2)).To(BeFalse())
	Expect(tc.isInDB(key3, val3)).To(BeTrue())
}

func TestWatchPut(t *testing.T) {
	ctx := setupTest(t, true)
	defer ctx.teardownTest()

	const watchPrefix = "key/"
	const watchKey = watchPrefix + "val1"

	closeCh := make(chan string)
	watchCh := make(chan keyval.BytesWatchResp)
	err := ctx.client.Watch(keyval.ToChan(watchCh), closeCh, watchPrefix)
	Expect(err).To(BeNil())

	ctx.client.Put("/something/else/val1", []byte{0, 0, 7})
	ctx.client.Put(watchKey, []byte{1, 2, 3})

	var resp keyval.BytesWatchResp
	Eventually(watchCh).Should(Receive(&resp))
	Expect(resp.GetKey()).Should(Equal(watchKey))
	Expect(resp.GetValue()).Should(Equal([]byte{1, 2, 3}))
	Expect(resp.GetPrevValue()).Should(BeNil())
	Expect(resp.GetChangeType()).Should(Equal(datasync.Put))
}

func TestWatchDel(t *testing.T) {
	ctx := setupTest(t, true)
	defer ctx.teardownTest()

	const watchPrefix = "key/"
	const watchKey = watchPrefix + "val1"

	ctx.client.Put("/something/else/val1", []byte{0, 0, 7})
	ctx.client.Put(watchKey, []byte{1, 2, 3})

	closeCh := make(chan string)
	watchCh := make(chan keyval.BytesWatchResp)
	err := ctx.client.Watch(keyval.ToChan(watchCh), closeCh, watchKey)
	Expect(err).To(BeNil())

	ctx.client.Delete("/something/else/val1")
	ctx.client.Delete(watchKey)

	var resp keyval.BytesWatchResp
	Eventually(watchCh).Should(Receive(&resp))
	Expect(resp.GetKey()).Should(Equal(watchKey))
	Expect(resp.GetValue()).Should(BeNil())
	Expect(resp.GetPrevValue()).Should(Equal([]byte{1, 2, 3}))
	Expect(resp.GetChangeType()).Should(Equal(datasync.Delete))
}
<commit_msg>Add unit test for broker watcher<commit_after>// Copyright (c) 2018 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bolt

import (
	"bytes"
	"os"
	"testing"

	"github.com/boltdb/bolt"
	"github.com/ligato/cn-infra/datasync"
	"github.com/ligato/cn-infra/db/keyval"
	"github.com/ligato/cn-infra/logging"
	"github.com/ligato/cn-infra/logging/logrus"
	. "github.com/onsi/gomega"
)

func init() {
	logrus.DefaultLogger().SetLevel(logging.DebugLevel)
	boltLogger.SetLevel(logging.DebugLevel)
}

type testCtx struct {
	*testing.T
	client *Client
}

const testDbPath = "/tmp/bolt.db"

func setupTest(t *testing.T, newDB bool) *testCtx {
	RegisterTestingT(t)

	if newDB {
		err := os.Remove(testDbPath)
		if err != nil && !os.IsNotExist(err) {
			t.Fatal(err)
			return nil
		}
	}

	client, err := NewClient(&Config{
		DbPath:   testDbPath,
		FileMode: 432,
	})
	Expect(err).ToNot(HaveOccurred())
	if err != nil {
		return nil
	}

	return &testCtx{T: t, client: client}
}

func (tc *testCtx) teardownTest() {
	Expect(tc.client.Close()).To(Succeed())
}

func (tc *testCtx) isInDB(key string, expectedVal []byte) (exists bool) {
	err := tc.client.db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(rootBucket)
		if val := b.Get([]byte(key)); val != nil {
			exists = true
		}
		return nil
	})
	if err != nil {
		tc.Fatal(err)
	}
	return
}

func (tc *testCtx) populateDB(data map[string][]byte) {
	err := tc.client.db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket(rootBucket)
		for key, val := range data {
			if err := b.Put([]byte(key), val); err != nil {
				return err
			}
		}
		return nil
	})
	if err != nil {
		tc.Fatal(err)
	}
	return
}

func TestPut(t *testing.T) {
	tc := setupTest(t, true)
	defer tc.teardownTest()

	var key = "/agent/agent1/config/interface/iface0"
	var val = []byte("val")

	err := tc.client.Put(key, val)
	Expect(err).ToNot(HaveOccurred())
	Expect(tc.isInDB(key, val)).To(BeTrue())
}

func TestGet(t *testing.T) {
	tc := setupTest(t, true)
	defer tc.teardownTest()

	var key = "/agent/agent1/config/interface/iface0"
	var val = []byte("val")

	err := tc.client.Put(key, val)
	Expect(err).ToNot(HaveOccurred())
	Expect(tc.isInDB(key, val)).To(BeTrue())
}

func TestListKeys(t *testing.T) {
	tc := setupTest(t, true)
	defer tc.teardownTest()

	tc.populateDB(map[string][]byte{
		"/my/key/1":    []byte("val1"),
		"/my/key/2":    []byte("val2"),
		"/other/key/0": []byte("val0"),
	})

	kvi, err := tc.client.ListKeys("/my/key/")
	Expect(err).ToNot(HaveOccurred())
	Expect(kvi).NotTo(BeNil())

	expectedKeys := []string{"/my/key/1", "/my/key/2"}
	for i := 0; i <= len(expectedKeys); i++ {
		key, _, all := kvi.GetNext()
		if i == len(expectedKeys) {
			Expect(all).To(BeTrue())
			break
		}
		Expect(all).To(BeFalse())
		Expect(key).To(BeEquivalentTo(expectedKeys[i]))
	}
}

func TestListValues(t *testing.T) {
	tc := setupTest(t, true)
	defer tc.teardownTest()

	tc.populateDB(map[string][]byte{
		"/my/key/1":    []byte("val1"),
		"/my/key/2":    []byte("val2"),
		"/other/key/0": []byte("val0"),
	})

	kvi, err := tc.client.ListValues("/my/key/")
	Expect(err).ToNot(HaveOccurred())
	Expect(kvi).NotTo(BeNil())

	expectedKeys := []string{"/my/key/1", "/my/key/2"}
	expectedValues := [][]byte{[]byte("val1"), []byte("val2")}
	for i := 0; i <= len(expectedKeys); i++ {
		kv, all := kvi.GetNext()
		if i == len(expectedKeys) {
			Expect(all).To(BeTrue())
			break
		}
		Expect(all).To(BeFalse())
		Expect(kv.GetKey()).To(BeEquivalentTo(expectedKeys[i]))
		Expect(bytes.Compare(kv.GetValue(), expectedValues[i])).To(BeZero())
	}
}

func TestListKeysBroker(t *testing.T) {
	tc := setupTest(t, true)
	defer tc.teardownTest()

	tc.populateDB(map[string][]byte{
		"/my/key/1":    []byte("val1"),
		"/my/key/2":    []byte("val2"),
		"/my/keyx/xx":  []byte("x"),
		"/my/xkey/xx":  []byte("x"),
		"/other/key/0": []byte("val0"),
	})

	broker := tc.client.NewBroker("/my/")
	kvi, err := broker.ListKeys("key/")
	Expect(err).ToNot(HaveOccurred())
	Expect(kvi).NotTo(BeNil())

	expectedKeys := []string{"key/1", "key/2"}
	for i := 0; i <= len(expectedKeys); i++ {
		key, _, all := kvi.GetNext()
		if i == len(expectedKeys) {
			Expect(all).To(BeTrue())
			break
		}
		Expect(all).To(BeFalse())
		Expect(key).To(BeEquivalentTo(expectedKeys[i]))
	}
}

func TestListValuesBroker(t *testing.T) {
	tc := setupTest(t, true)
	defer tc.teardownTest()

	tc.populateDB(map[string][]byte{
		"/my/key/1":    []byte("val1"),
		"/my/key/2":    []byte("val2"),
		"/my/keyx/xx":  []byte("x"),
		"/my/xkey/xx":  []byte("x"),
		"/other/key/0": []byte("val0"),
	})

	broker := tc.client.NewBroker("/my/")
	kvi, err := broker.ListValues("key/")
	Expect(err).ToNot(HaveOccurred())
	Expect(kvi).NotTo(BeNil())

	expectedKeys := []string{"key/1", "key/2"}
	for i := 0; i <= len(expectedKeys); i++ {
		kv, all := kvi.GetNext()
		if i == len(expectedKeys) {
			Expect(all).To(BeTrue())
			break
		}
		Expect(all).To(BeFalse())
		Expect(kv.GetKey()).To(BeEquivalentTo(expectedKeys[i]))
	}
}

func TestDelete(t *testing.T) {
	tc := setupTest(t, true)
	defer tc.teardownTest()

	var key = "/agent/agent1/config/interface/iface0"
	var val = []byte("val")

	err := tc.client.Put(key, val)
	Expect(err).ToNot(HaveOccurred())
	existed, err := tc.client.Delete(key)
	Expect(err).ToNot(HaveOccurred())
	Expect(existed).To(BeTrue())

	existed, err = tc.client.Delete("/this/key/does/not/exists")
	Expect(err).To(HaveOccurred())
	Expect(existed).To(BeFalse())
	Expect(tc.isInDB(key, val)).To(BeFalse())
}

func TestPutInTxn(t *testing.T) {
	tc := setupTest(t, true)
	defer tc.teardownTest()

	txn := tc.client.NewTxn()
	Expect(txn).ToNot(BeNil())

	var key1 = "/agent/agent1/config/interface/iface0"
	var val1 = []byte("iface0")
	var key2 = "/agent/agent1/config/interface/iface1"
	var val2 = []byte("iface1")
	var key3 = "/agent/agent1/config/interface/iface2"
	var val3 = []byte("iface2")

	txn.Put(key1, val1).
		Put(key2, val2).
		Put(key3, val3)
	Expect(txn.Commit()).To(Succeed())
	Expect(tc.isInDB(key1, val1)).To(BeTrue())
	Expect(tc.isInDB(key2, val2)).To(BeTrue())
	Expect(tc.isInDB(key3, val3)).To(BeTrue())
}

func TestDeleteInTxn(t *testing.T) {
	tc := setupTest(t, true)
	defer tc.teardownTest()

	txn := tc.client.NewTxn()
	Expect(txn).ToNot(BeNil())

	var key1 = "/agent/agent1/config/interface/iface0"
	var val1 = []byte("iface0")
	var key2 = "/agent/agent1/config/interface/iface1"
	var val2 = []byte("iface1")
	var key3 = "/agent/agent1/config/interface/iface2"
	var val3 = []byte("iface2")

	txn.Put(key1, val1).
		Put(key2, val2).
		Put(key3, val3).
		Delete(key2)
	Expect(txn.Commit()).To(Succeed())
	Expect(tc.isInDB(key1, val1)).To(BeTrue())
	Expect(tc.isInDB(key2, val2)).To(BeFalse())
	Expect(tc.isInDB(key3, val3)).To(BeTrue())
}

func TestWatchPut(t *testing.T) {
	ctx := setupTest(t, true)
	defer ctx.teardownTest()

	const watchPrefix = "/key/"
	const watchKey = watchPrefix + "val1"

	closeCh := make(chan string)
	watchCh := make(chan keyval.BytesWatchResp)
	err := ctx.client.Watch(keyval.ToChan(watchCh), closeCh, watchPrefix)
	Expect(err).To(BeNil())

	ctx.client.Put("/something/else/val1", []byte{0, 0, 7})
	ctx.client.Put(watchKey, []byte{1, 2, 3})

	var resp keyval.BytesWatchResp
	Eventually(watchCh).Should(Receive(&resp))
	Expect(resp.GetKey()).Should(Equal(watchKey))
	Expect(resp.GetValue()).Should(Equal([]byte{1, 2, 3}))
	Expect(resp.GetPrevValue()).Should(BeNil())
	Expect(resp.GetChangeType()).Should(Equal(datasync.Put))
}

func TestWatchDel(t *testing.T) {
	ctx := setupTest(t, true)
	defer ctx.teardownTest()

	const watchPrefix = "/key/"
	const watchKey = watchPrefix + "val1"

	ctx.client.Put("/something/else/val1", []byte{0, 0, 7})
	ctx.client.Put(watchKey, []byte{1, 2, 3})

	closeCh := make(chan string)
	watchCh := make(chan keyval.BytesWatchResp)
	err := ctx.client.Watch(keyval.ToChan(watchCh), closeCh, watchKey)
	Expect(err).To(BeNil())

	ctx.client.Delete("/something/else/val1")
	ctx.client.Delete(watchKey)

	var resp keyval.BytesWatchResp
	Eventually(watchCh).Should(Receive(&resp))
	Expect(resp.GetKey()).Should(Equal(watchKey))
	Expect(resp.GetValue()).Should(BeNil())
	Expect(resp.GetPrevValue()).Should(Equal([]byte{1, 2, 3}))
	Expect(resp.GetChangeType()).Should(Equal(datasync.Delete))
}

func TestWatchPutBroker(t *testing.T) {
	ctx := setupTest(t, true)
	defer ctx.teardownTest()

	const brokerPrefix = "/my/prefix/"
	const watchPrefix = "key/"
	const watchKey = watchPrefix + "val1"

	broker := ctx.client.NewWatcher(brokerPrefix)

	closeCh := make(chan string)
	watchCh := make(chan keyval.BytesWatchResp)

	err := broker.Watch(keyval.ToChan(watchCh), closeCh, watchPrefix)
	Expect(err).To(BeNil())

	ctx.client.Put(brokerPrefix+"something/else/val1", []byte{0, 0, 7})
	ctx.client.Put(brokerPrefix+watchKey, []byte{1, 2, 3})

	var resp keyval.BytesWatchResp
	Eventually(watchCh).Should(Receive(&resp))
	Expect(resp.GetKey()).Should(Equal(watchKey))
	Expect(resp.GetValue()).Should(Equal([]byte{1, 2, 3}))
	Expect(resp.GetPrevValue()).Should(BeNil())
	Expect(resp.GetChangeType()).Should(Equal(datasync.Put))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/lxc/lxd/lxd/db"
	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	projecthelpers "github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/pkg/errors"
)

func doProfileUpdate(d *Daemon, projectName string, name string, id int64, profile *api.Profile, req api.ProfilePut) error {
	// Check project limits.
	err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
		return projecthelpers.AllowProfileUpdate(tx, projectName, name, req)
	})
	if err != nil {
		return err
	}

	// Sanity checks
	err = instance.ValidConfig(d.os, req.Config, true, false)
	if err != nil {
		return err
	}

	// At this point we don't know the instance type, so just use instancetype.Any type for validation.
	err = instance.ValidDevices(d.State(), d.cluster, projectName, instancetype.Any, deviceConfig.NewDevices(req.Devices), false)
	if err != nil {
		return err
	}

	containers, err := getProfileContainersInfo(d.cluster, projectName, name)
	if err != nil {
		return errors.Wrapf(err, "failed to query instances associated with profile '%s'", name)
	}

	// Check if the root device is supposed to be changed or removed.
	oldProfileRootDiskDeviceKey, oldProfileRootDiskDevice, _ := shared.GetRootDiskDevice(profile.Devices)
	_, newProfileRootDiskDevice, _ := shared.GetRootDiskDevice(req.Devices)
	if len(containers) > 0 && oldProfileRootDiskDevice["pool"] != "" && newProfileRootDiskDevice["pool"] == "" || (oldProfileRootDiskDevice["pool"] != newProfileRootDiskDevice["pool"]) {
		// Check for containers using the device
		for _, container := range containers {
			// Check if the device is locally overridden
			k, v, _ := shared.GetRootDiskDevice(container.Devices.CloneNative())
			if k != "" && v["pool"] != "" {
				continue
			}

			// Check what profile the device comes from
			profiles := container.Profiles
			for i := len(profiles) - 1; i >= 0; i-- {
				_, profile, err := d.cluster.GetProfile(projecthelpers.Default, profiles[i])
				if err != nil {
					return err
				}

				// Check if we find a match for the device
				_, ok := profile.Devices[oldProfileRootDiskDeviceKey]
				if ok {
					// Found the profile
					if profiles[i] == name {
						// If it's the current profile, then we can't modify that root device
						return fmt.Errorf("At least one instance relies on this profile's root disk device")
					} else {
						// If it's not, then move on to the next container
						break
					}
				}
			}
		}
	}

	// Update the database
	err = d.cluster.Transaction(func(tx *db.ClusterTx) error {
		return tx.UpdateProfile(projectName, name, db.Profile{
			Project:     projectName,
			Name:        name,
			Description: req.Description,
			Config:      req.Config,
			Devices:     req.Devices,
		})
	})
	if err != nil {
		return err
	}

	// Update all the containers on this node using the profile. Must be
	// done after db.TxCommit due to DB lock.
	nodeName := ""
	err = d.cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		nodeName, err = tx.GetLocalNodeName()
		return err
	})
	if err != nil {
		return errors.Wrap(err, "failed to query local node name")
	}
	failures := map[string]error{}
	for _, args := range containers {
		err := doProfileUpdateContainer(d, name, profile.ProfilePut, nodeName, args)
		if err != nil {
			failures[args.Name] = err
		}
	}

	if len(failures) != 0 {
		msg := "The following containers failed to update (profile change still saved):\n"
		for cname, err := range failures {
			msg += fmt.Sprintf(" - %s: %s\n", cname, err)
		}
		return fmt.Errorf("%s", msg)
	}

	return nil
}

// Like doProfileUpdate but does not update the database, since it was already
// updated by doProfileUpdate itself, called on the notifying node.
func doProfileUpdateCluster(d *Daemon, project, name string, old api.ProfilePut) error {
	nodeName := ""
	err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		nodeName, err = tx.GetLocalNodeName()
		return err
	})
	if err != nil {
		return errors.Wrap(err, "Failed to query local node name")
	}

	containers, err := getProfileContainersInfo(d.cluster, project, name)
	if err != nil {
		return errors.Wrapf(err, "Failed to query instances associated with profile '%s'", name)
	}

	failures := map[string]error{}
	for _, args := range containers {
		err := doProfileUpdateContainer(d, name, old, nodeName, args)
		if err != nil {
			failures[args.Name] = err
		}
	}

	if len(failures) != 0 {
		msg := "The following containers failed to update (profile change still saved):\n"
		for cname, err := range failures {
			msg += fmt.Sprintf(" - %s: %s\n", cname, err)
		}
		return fmt.Errorf("%s", msg)
	}

	return nil
}

// Profile update of a single container.
func doProfileUpdateContainer(d *Daemon, name string, old api.ProfilePut, nodeName string, args db.InstanceArgs) error {
	if args.Node != "" && args.Node != nodeName {
		// No-op, this container does not belong to this node.
		return nil
	}

	profiles, err := d.cluster.GetProfiles(args.Project, args.Profiles)
	if err != nil {
		return err
	}

	for i, profileName := range args.Profiles {
		if profileName == name {
			// Overwrite the new config from the database with the old config and devices.
			profiles[i].Config = old.Config
			profiles[i].Devices = old.Devices
			break
		}
	}

	// Load the instance using the old profile config.
	inst, err := instance.Load(d.State(), args, profiles)
	if err != nil {
		return err
	}

	// Update will internally load the new profile configs and detect the changes to apply.
	return inst.Update(db.InstanceArgs{
		Architecture: inst.Architecture(),
		Config:       inst.LocalConfig(),
		Description:  inst.Description(),
		Devices:      inst.LocalDevices(),
		Ephemeral:    inst.IsEphemeral(),
		Profiles:     inst.Profiles(),
		Project:      inst.Project(),
		Type:         inst.Type(),
		Snapshot:     inst.IsSnapshot(),
	}, true)
}

// Query the db for information about containers associated with the given
// profile.
func getProfileContainersInfo(cluster *db.Cluster, project, profile string) ([]db.InstanceArgs, error) {
	// Query the db for information about containers associated with the
	// given profile.
	names, err := cluster.GetInstancesWithProfile(project, profile)
	if err != nil {
		return nil, errors.Wrapf(err, "Failed to query instances with profile '%s'", profile)
	}

	containers := []db.InstanceArgs{}
	err = cluster.Transaction(func(tx *db.ClusterTx) error {
		for ctProject, ctNames := range names {
			for _, ctName := range ctNames {
				container, err := tx.GetInstance(ctProject, ctName)
				if err != nil {
					return err
				}

				containers = append(containers, db.InstanceToArgs(container))
			}
		}

		return nil
	})
	if err != nil {
		return nil, errors.Wrapf(err, "Failed to fetch instances")
	}

	return containers, nil
}
<commit_msg>lxd/profiles/utils: Comment on doProfileUpdateContainer for clarity<commit_after>package main

import (
	"fmt"

	"github.com/lxc/lxd/lxd/db"
	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	projecthelpers "github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/pkg/errors"
)

func doProfileUpdate(d *Daemon, projectName string, name string, id int64, profile *api.Profile, req api.ProfilePut) error {
	// Check project limits.
	err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
		return projecthelpers.AllowProfileUpdate(tx, projectName, name, req)
	})
	if err != nil {
		return err
	}

	// Sanity checks
	err = instance.ValidConfig(d.os, req.Config, true, false)
	if err != nil {
		return err
	}

	// At this point we don't know the instance type, so just use instancetype.Any type for validation.
	err = instance.ValidDevices(d.State(), d.cluster, projectName, instancetype.Any, deviceConfig.NewDevices(req.Devices), false)
	if err != nil {
		return err
	}

	containers, err := getProfileContainersInfo(d.cluster, projectName, name)
	if err != nil {
		return errors.Wrapf(err, "failed to query instances associated with profile '%s'", name)
	}

	// Check if the root device is supposed to be changed or removed.
	oldProfileRootDiskDeviceKey, oldProfileRootDiskDevice, _ := shared.GetRootDiskDevice(profile.Devices)
	_, newProfileRootDiskDevice, _ := shared.GetRootDiskDevice(req.Devices)
	if len(containers) > 0 && oldProfileRootDiskDevice["pool"] != "" && newProfileRootDiskDevice["pool"] == "" || (oldProfileRootDiskDevice["pool"] != newProfileRootDiskDevice["pool"]) {
		// Check for containers using the device
		for _, container := range containers {
			// Check if the device is locally overridden
			k, v, _ := shared.GetRootDiskDevice(container.Devices.CloneNative())
			if k != "" && v["pool"] != "" {
				continue
			}

			// Check what profile the device comes from
			profiles := container.Profiles
			for i := len(profiles) - 1; i >= 0; i-- {
				_, profile, err := d.cluster.GetProfile(projecthelpers.Default, profiles[i])
				if err != nil {
					return err
				}

				// Check if we find a match for the device
				_, ok := profile.Devices[oldProfileRootDiskDeviceKey]
				if ok {
					// Found the profile
					if profiles[i] == name {
						// If it's the current profile, then we can't modify that root device
						return fmt.Errorf("At least one instance relies on this profile's root disk device")
					} else {
						// If it's not, then move on to the next container
						break
					}
				}
			}
		}
	}

	// Update the database
	err = d.cluster.Transaction(func(tx *db.ClusterTx) error {
		return tx.UpdateProfile(projectName, name, db.Profile{
			Project:     projectName,
			Name:        name,
			Description: req.Description,
			Config:      req.Config,
			Devices:     req.Devices,
		})
	})
	if err != nil {
		return err
	}

	// Update all the containers on this node using the profile. Must be
	// done after db.TxCommit due to DB lock.
	nodeName := ""
	err = d.cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		nodeName, err = tx.GetLocalNodeName()
		return err
	})
	if err != nil {
		return errors.Wrap(err, "failed to query local node name")
	}
	failures := map[string]error{}
	for _, args := range containers {
		err := doProfileUpdateContainer(d, name, profile.ProfilePut, nodeName, args)
		if err != nil {
			failures[args.Name] = err
		}
	}

	if len(failures) != 0 {
		msg := "The following containers failed to update (profile change still saved):\n"
		for cname, err := range failures {
			msg += fmt.Sprintf(" - %s: %s\n", cname, err)
		}
		return fmt.Errorf("%s", msg)
	}

	return nil
}

// Like doProfileUpdate but does not update the database, since it was already
// updated by doProfileUpdate itself, called on the notifying node.
func doProfileUpdateCluster(d *Daemon, project, name string, old api.ProfilePut) error {
	nodeName := ""
	err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		nodeName, err = tx.GetLocalNodeName()
		return err
	})
	if err != nil {
		return errors.Wrap(err, "Failed to query local node name")
	}

	containers, err := getProfileContainersInfo(d.cluster, project, name)
	if err != nil {
		return errors.Wrapf(err, "Failed to query instances associated with profile '%s'", name)
	}

	failures := map[string]error{}
	for _, args := range containers {
		err := doProfileUpdateContainer(d, name, old, nodeName, args)
		if err != nil {
			failures[args.Name] = err
		}
	}

	if len(failures) != 0 {
		msg := "The following containers failed to update (profile change still saved):\n"
		for cname, err := range failures {
			msg += fmt.Sprintf(" - %s: %s\n", cname, err)
		}
		return fmt.Errorf("%s", msg)
	}

	return nil
}

// Profile update of a single container.
func doProfileUpdateContainer(d *Daemon, name string, old api.ProfilePut, nodeName string, args db.InstanceArgs) error {
	if args.Node != "" && args.Node != nodeName {
		// No-op, this container does not belong to this node.
		return nil
	}

	profiles, err := d.cluster.GetProfiles(args.Project, args.Profiles)
	if err != nil {
		return err
	}

	for i, profileName := range args.Profiles {
		if profileName == name {
			// Overwrite the new config from the database with the old config and devices.
			profiles[i].Config = old.Config
			profiles[i].Devices = old.Devices
			break
		}
	}

	// Load the instance using the old profile config.
	inst, err := instance.Load(d.State(), args, profiles)
	if err != nil {
		return err
	}

	// Update will internally load the new profile configs and detect the changes to apply.
	return inst.Update(db.InstanceArgs{
		Architecture: inst.Architecture(),
		Config:       inst.LocalConfig(),
		Description:  inst.Description(),
		Devices:      inst.LocalDevices(),
		Ephemeral:    inst.IsEphemeral(),
		Profiles:     inst.Profiles(), // List of profile names to load from DB.
		Project:      inst.Project(),
		Type:         inst.Type(),
		Snapshot:     inst.IsSnapshot(),
	}, true)
}

// Query the db for information about containers associated with the given
// profile.
func getProfileContainersInfo(cluster *db.Cluster, project, profile string) ([]db.InstanceArgs, error) {
	// Query the db for information about containers associated with the
	// given profile.
	names, err := cluster.GetInstancesWithProfile(project, profile)
	if err != nil {
		return nil, errors.Wrapf(err, "Failed to query instances with profile '%s'", profile)
	}

	containers := []db.InstanceArgs{}
	err = cluster.Transaction(func(tx *db.ClusterTx) error {
		for ctProject, ctNames := range names {
			for _, ctName := range ctNames {
				container, err := tx.GetInstance(ctProject, ctName)
				if err != nil {
					return err
				}

				containers = append(containers, db.InstanceToArgs(container))
			}
		}

		return nil
	})
	if err != nil {
		return nil, errors.Wrapf(err, "Failed to fetch instances")
	}

	return containers, nil
}
<|endoftext|>


<commit_before>package device

import (
	"fmt"
)

// ErrUnsupportedDevType is the error that occurs when an unsupported device type is created.
var ErrUnsupportedDevType = fmt.Errorf("Unsupported device type")
<commit_msg>lxd/device/errors: Adds ErrCannotUpdate error<commit_after>package device

import (
	"fmt"
)

// ErrUnsupportedDevType is the error that occurs when an unsupported device type is created.
var ErrUnsupportedDevType = fmt.Errorf("Unsupported device type")

// ErrCannotUpdate is the error that occurs when a device cannot be updated.
var ErrCannotUpdate = fmt.Errorf("Device does not support updates")
<|endoftext|>


<commit_before>// $G $D/$F.go || echo BUG: should compile

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

type T struct {
	children []T;
}

/*
uetli:/home/gri/go/test/bugs gri$ 6g bug210.go
bug210.go:10: invalid recursive type []T
*/
<commit_msg>duplicate of bug190<commit_after><|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"
	"strings"

	"github.com/gorilla/mux"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/version"
)

// /1.0/storage-pools
// List all storage pools.
func storagePoolsGet(d *Daemon, r *http.Request) Response {
	recursionStr := r.FormValue("recursion")
	recursion, err := strconv.Atoi(recursionStr)
	if err != nil {
		recursion = 0
	}

	pools, err := db.StoragePools(d.db)
	if err != nil && err != db.NoSuchObjectError {
		return SmartError(err)
	}

	resultString := []string{}
	resultMap := []api.StoragePool{}
	for _, pool := range pools {
		if recursion == 0 {
			resultString = append(resultString, fmt.Sprintf("/%s/storage-pools/%s", version.APIVersion, pool))
		} else {
			plID, pl, err := db.StoragePoolGet(d.db, pool)
			if err != nil {
				continue
			}

			// Get all users of the storage pool.
			poolUsedBy, err := storagePoolUsedByGet(d.db, plID, pool)
			if err != nil {
				return SmartError(err)
			}
			pl.UsedBy = poolUsedBy

			resultMap = append(resultMap, *pl)
		}
	}

	if recursion == 0 {
		return SyncResponse(true, resultString)
	}

	return SyncResponse(true, resultMap)
}

// /1.0/storage-pools
// Create a storage pool.
func storagePoolsPost(d *Daemon, r *http.Request) Response {
	req := api.StoragePoolsPost{}

	// Parse the request.
	err := json.NewDecoder(r.Body).Decode(&req)
	if err != nil {
		return BadRequest(err)
	}

	// Sanity checks.
	if req.Name == "" {
		return BadRequest(fmt.Errorf("No name provided"))
	}

	if req.Driver == "" {
		return BadRequest(fmt.Errorf("No driver provided"))
	}

	err = storagePoolCreateInternal(d.State(), req.Name, req.Description, req.Driver, req.Config)
	if err != nil {
		return InternalError(err)
	}

	return SyncResponseLocation(true, nil, fmt.Sprintf("/%s/storage-pools/%s", version.APIVersion, req.Name))
}

var storagePoolsCmd = Command{name: "storage-pools", get: storagePoolsGet, post: storagePoolsPost}

// /1.0/storage-pools/{name}
// Get a single storage pool.
func storagePoolGet(d *Daemon, r *http.Request) Response {
	poolName := mux.Vars(r)["name"]

	// Get the existing storage pool.
	poolID, pool, err := db.StoragePoolGet(d.db, poolName)
	if err != nil {
		return SmartError(err)
	}

	// Get all users of the storage pool.
	poolUsedBy, err := storagePoolUsedByGet(d.db, poolID, poolName)
	if err != nil && err != db.NoSuchObjectError {
		return SmartError(err)
	}
	pool.UsedBy = poolUsedBy

	etag := []interface{}{pool.Name, pool.Driver, pool.Config}

	return SyncResponseETag(true, &pool, etag)
}

// /1.0/storage-pools/{name}
// Replace pool properties.
func storagePoolPut(d *Daemon, r *http.Request) Response {
	poolName := mux.Vars(r)["name"]

	// Get the existing storage pool.
	_, dbInfo, err := db.StoragePoolGet(d.db, poolName)
	if err != nil {
		return SmartError(err)
	}

	// Validate the ETag
	etag := []interface{}{dbInfo.Name, dbInfo.Driver, dbInfo.Config}

	err = util.EtagCheck(r, etag)
	if err != nil {
		return PreconditionFailed(err)
	}

	req := api.StoragePoolPut{}
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		return BadRequest(err)
	}

	// Validate the configuration
	err = storagePoolValidateConfig(poolName, dbInfo.Driver, req.Config)
	if err != nil {
		return BadRequest(err)
	}

	err = storagePoolUpdate(d.State(), poolName, req.Description, req.Config)
	if err != nil {
		return InternalError(err)
	}

	return EmptySyncResponse
}

// /1.0/storage-pools/{name}
// Change pool properties.
func storagePoolPatch(d *Daemon, r *http.Request) Response {
	poolName := mux.Vars(r)["name"]

	// Get the existing network
	_, dbInfo, err := db.StoragePoolGet(d.db, poolName)
	if dbInfo != nil {
		return SmartError(err)
	}

	// Validate the ETag
	etag := []interface{}{dbInfo.Name, dbInfo.Driver, dbInfo.Config}

	err = util.EtagCheck(r, etag)
	if err != nil {
		return PreconditionFailed(err)
	}

	req := api.StoragePoolPut{}
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		return BadRequest(err)
	}

	// Config stacking
	if req.Config == nil {
		req.Config = map[string]string{}
	}

	for k, v := range dbInfo.Config {
		_, ok := req.Config[k]
		if !ok {
			req.Config[k] = v
		}
	}

	// Validate the configuration
	err = storagePoolValidateConfig(poolName, dbInfo.Driver, req.Config)
	if err != nil {
		return BadRequest(err)
	}

	err = storagePoolUpdate(d.State(), poolName, req.Description, req.Config)
	if err != nil {
		return InternalError(fmt.Errorf("failed to update the storage pool configuration"))
	}

	return EmptySyncResponse
}

// /1.0/storage-pools/{name}
// Delete storage pool.
func storagePoolDelete(d *Daemon, r *http.Request) Response {
	poolName := mux.Vars(r)["name"]

	poolID, err := db.StoragePoolGetID(d.db, poolName)
	if err != nil {
		return NotFound
	}

	// Check if the storage pool has any volumes associated with it, if so
	// error out.
	volumeCount, err := db.StoragePoolVolumesGetNames(d.db, poolID)
	if volumeCount > 0 {
		return BadRequest(fmt.Errorf("storage pool \"%s\" has volumes attached to it", poolName))
	}

	// Check if the storage pool is still referenced in any profiles.
	profiles, err := profilesUsingPoolGetNames(d.db, poolName)
	if err != nil {
		return SmartError(err)
	}
	if len(profiles) > 0 {
		return BadRequest(fmt.Errorf("Storage pool \"%s\" has profiles using it:\n%s", poolName, strings.Join(profiles, "\n")))
	}

	s, err := storagePoolInit(d.State(), poolName)
	if err != nil {
		return InternalError(err)
	}

	err = s.StoragePoolDelete()
	if err != nil {
		return InternalError(err)
	}

	err = dbStoragePoolDeleteAndUpdateCache(d.db, poolName)
	if err != nil {
		return SmartError(err)
	}

	return EmptySyncResponse
}

var storagePoolCmd = Command{name: "storage-pools/{name}", get: storagePoolGet, put: storagePoolPut, patch: storagePoolPatch, delete: storagePoolDelete}
<commit_msg>storage: Don't mask error messages<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"
	"strings"

	"github.com/gorilla/mux"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/version"
)

// /1.0/storage-pools
// List all storage pools.
func storagePoolsGet(d *Daemon, r *http.Request) Response {
	recursionStr := r.FormValue("recursion")
	recursion, err := strconv.Atoi(recursionStr)
	if err != nil {
		recursion = 0
	}

	pools, err := db.StoragePools(d.db)
	if err != nil && err != db.NoSuchObjectError {
		return SmartError(err)
	}

	resultString := []string{}
	resultMap := []api.StoragePool{}
	for _, pool := range pools {
		if recursion == 0 {
			resultString = append(resultString, fmt.Sprintf("/%s/storage-pools/%s", version.APIVersion, pool))
		} else {
			plID, pl, err := db.StoragePoolGet(d.db, pool)
			if err != nil {
				continue
			}

			// Get all users of the storage pool.
			poolUsedBy, err := storagePoolUsedByGet(d.db, plID, pool)
			if err != nil {
				return SmartError(err)
			}
			pl.UsedBy = poolUsedBy

			resultMap = append(resultMap, *pl)
		}
	}

	if recursion == 0 {
		return SyncResponse(true, resultString)
	}

	return SyncResponse(true, resultMap)
}

// /1.0/storage-pools
// Create a storage pool.
func storagePoolsPost(d *Daemon, r *http.Request) Response {
	req := api.StoragePoolsPost{}

	// Parse the request.
	err := json.NewDecoder(r.Body).Decode(&req)
	if err != nil {
		return BadRequest(err)
	}

	// Sanity checks.
	if req.Name == "" {
		return BadRequest(fmt.Errorf("No name provided"))
	}

	if req.Driver == "" {
		return BadRequest(fmt.Errorf("No driver provided"))
	}

	err = storagePoolCreateInternal(d.State(), req.Name, req.Description, req.Driver, req.Config)
	if err != nil {
		return InternalError(err)
	}

	return SyncResponseLocation(true, nil, fmt.Sprintf("/%s/storage-pools/%s", version.APIVersion, req.Name))
}

var storagePoolsCmd = Command{name: "storage-pools", get: storagePoolsGet, post: storagePoolsPost}

// /1.0/storage-pools/{name}
// Get a single storage pool.
func storagePoolGet(d *Daemon, r *http.Request) Response {
	poolName := mux.Vars(r)["name"]

	// Get the existing storage pool.
	poolID, pool, err := db.StoragePoolGet(d.db, poolName)
	if err != nil {
		return SmartError(err)
	}

	// Get all users of the storage pool.
	poolUsedBy, err := storagePoolUsedByGet(d.db, poolID, poolName)
	if err != nil && err != db.NoSuchObjectError {
		return SmartError(err)
	}
	pool.UsedBy = poolUsedBy

	etag := []interface{}{pool.Name, pool.Driver, pool.Config}

	return SyncResponseETag(true, &pool, etag)
}

// /1.0/storage-pools/{name}
// Replace pool properties.
func storagePoolPut(d *Daemon, r *http.Request) Response {
	poolName := mux.Vars(r)["name"]

	// Get the existing storage pool.
	_, dbInfo, err := db.StoragePoolGet(d.db, poolName)
	if err != nil {
		return SmartError(err)
	}

	// Validate the ETag
	etag := []interface{}{dbInfo.Name, dbInfo.Driver, dbInfo.Config}

	err = util.EtagCheck(r, etag)
	if err != nil {
		return PreconditionFailed(err)
	}

	req := api.StoragePoolPut{}
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		return BadRequest(err)
	}

	// Validate the configuration
	err = storagePoolValidateConfig(poolName, dbInfo.Driver, req.Config)
	if err != nil {
		return BadRequest(err)
	}

	err = storagePoolUpdate(d.State(), poolName, req.Description, req.Config)
	if err != nil {
		return InternalError(err)
	}

	return EmptySyncResponse
}

// /1.0/storage-pools/{name}
// Change pool properties.
func storagePoolPatch(d *Daemon, r *http.Request) Response {
	poolName := mux.Vars(r)["name"]

	// Get the existing network
	_, dbInfo, err := db.StoragePoolGet(d.db, poolName)
	if dbInfo != nil {
		return SmartError(err)
	}

	// Validate the ETag
	etag := []interface{}{dbInfo.Name, dbInfo.Driver, dbInfo.Config}

	err = util.EtagCheck(r, etag)
	if err != nil {
		return PreconditionFailed(err)
	}

	req := api.StoragePoolPut{}
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		return BadRequest(err)
	}

	// Config stacking
	if req.Config == nil {
		req.Config = map[string]string{}
	}

	for k, v := range dbInfo.Config {
		_, ok := req.Config[k]
		if !ok {
			req.Config[k] = v
		}
	}

	// Validate the configuration
	err = storagePoolValidateConfig(poolName, dbInfo.Driver, req.Config)
	if err != nil {
		return BadRequest(err)
	}

	err = storagePoolUpdate(d.State(), poolName, req.Description, req.Config)
	if err != nil {
		return InternalError(err)
	}

	return EmptySyncResponse
}

// /1.0/storage-pools/{name}
// Delete storage pool.
func storagePoolDelete(d *Daemon, r *http.Request) Response {
	poolName := mux.Vars(r)["name"]

	poolID, err := db.StoragePoolGetID(d.db, poolName)
	if err != nil {
		return NotFound
	}

	// Check if the storage pool has any volumes associated with it, if so
	// error out.
	volumeCount, err := db.StoragePoolVolumesGetNames(d.db, poolID)
	if volumeCount > 0 {
		return BadRequest(fmt.Errorf("storage pool \"%s\" has volumes attached to it", poolName))
	}

	// Check if the storage pool is still referenced in any profiles.
	profiles, err := profilesUsingPoolGetNames(d.db, poolName)
	if err != nil {
		return SmartError(err)
	}
	if len(profiles) > 0 {
		return BadRequest(fmt.Errorf("Storage pool \"%s\" has profiles using it:\n%s", poolName, strings.Join(profiles, "\n")))
	}

	s, err := storagePoolInit(d.State(), poolName)
	if err != nil {
		return InternalError(err)
	}

	err = s.StoragePoolDelete()
	if err != nil {
		return InternalError(err)
	}

	err = dbStoragePoolDeleteAndUpdateCache(d.db, poolName)
	if err != nil {
		return SmartError(err)
	}

	return EmptySyncResponse
}

var storagePoolCmd = Command{name: "storage-pools/{name}", get: storagePoolGet, put: storagePoolPut, patch: storagePoolPatch, delete: storagePoolDelete}
<|endoftext|>


<commit_before>/*

*/
package mandrake

import (
	"log"
	"encoding/json"

	"golang.org/x/exp/inotify"
	"github.com/hosom/gomandrake/config"
	"github.com/hosom/gomandrake/filemeta"
	"github.com/hosom/gomandrake/plugin"
)

// Mandrake is a wrapper struct for the bulk of the application logic
type Mandrake struct {
	AnalysisPipeline	chan string
	MonitoredDirectory	string
	Analyzers			[]plugin.AnalyzerCaller
	AnalyzerFilter		map[string][]plugin.AnalyzerCaller
}

// NewMandrake creates and returns a Mandrake struct utilizing a passed
// parsed configuration file to create the correct fields.
func NewMandrake(c config.Config) Mandrake {
	analyzers := []plugin.AnalyzerCaller{}
	filter := make(map[string][]plugin.AnalyzerCaller)
	for _, plug := range c.Analyzers {
		analyzer := plugin.NewAnalyzerCaller(plug)
		// Build a slice of all AnalyzerCaller structs
		analyzers = append(analyzers, analyzer)

		// Create a map to function as a mime_type filter for analyzers
		for _, mime := range analyzer.MimeFilter {
			filter[mime] = append(filter[mime], analyzer)
		}
	}

	return Mandrake{make(chan string), c.MonitoredDirectory, analyzers, filter}
}

// ListenAndServe starts the goroutines that perform all of the heavy lifting
// including Monitor() and DispatchAnalysis().
func (m Mandrake) ListenAndServe() {
	log.SetPrefix("[mandrake] ")
	log.Println(m.Analyzers[0])
	go m.DispatchAnalysis()
	m.Monitor()
}

// DispatchAnalysis intelligently sends a new file to registered plugins so
// that it can be analyzed.
func (m Mandrake) DispatchAnalysis() {
	for fpath := range m.AnalysisPipeline {
		fmeta, err := filemeta.NewFileMeta(fpath)
		if err != nil {
			log.Println(err)
		}

		// Create JSON filemeta object to pass to plugins so that plugins
		// receive basic contextual information about the file.
		fs, err := json.Marshal(fmeta)

		var analysis map[string]map[string]interface{}

		for _, analyzer := range m.AnalyzerFilter["all"] {
			result, err := analyzer.Analyze(string(fs))
			if err != nil {
				log.Print(err)
			}
			analysis[analyzer.Name] = MapFromJSON(result)
		}

		for _, analyzer := range m.AnalyzerFilter[fmeta.Mime] {
			result, err := analyzer.Analyze(string(fs))
			if err != nil {
				log.Print(err)
			}
			analysis[analyzer.Name] = MapFromJSON(result)
		}

		log.Println(json.Marshal(analysis))
		log.Println(string(fs))
		log.Printf("%s", fpath)
	}
}

// Monitor uses inotify to monitor the MonitoredDirectory for IN_CLOSE_WRITE
// events. Files written to the MonitoredDirectory will be sent to the
// analysis pipeline to be analyzed.
func (m Mandrake) Monitor() {
	log.Println("starting inotify watcher")
	watcher, err := inotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	log.Printf("adding watcher to %s directory", m.MonitoredDirectory)
	err = watcher.AddWatch(m.MonitoredDirectory, inotify.IN_CLOSE_WRITE)
	if err != nil {
		log.Fatal(err)
	}

	for {
		select {
		case ev := <- watcher.Event:
			m.AnalysisPipeline <- ev.Name
		case err := <- watcher.Error:
			log.Printf("inotify error: %s", err)
		}
	}
}

// MapFromJSON accepts an anonymous JSON object as a string and returns the
// resulting Map
func MapFromJSON(s string) map[string]interface{} {
	log.Printf("Performing mapping with string: %s", s)
	var f interface{}
	json.Unmarshal([]byte(s), &f)
	m := f.(map[string]interface{})
	log.Printf("Mapping complete: %s", m)
	return m
}<commit_msg>don't think this will work<commit_after>/*

*/
package mandrake

import (
	"log"
	"encoding/json"

	"golang.org/x/exp/inotify"
	"github.com/hosom/gomandrake/config"
	"github.com/hosom/gomandrake/filemeta"
	"github.com/hosom/gomandrake/plugin"
)

// Mandrake is a wrapper struct for the bulk of the application logic
type Mandrake struct {
	AnalysisPipeline	chan string
	MonitoredDirectory	string
	Analyzers			[]plugin.AnalyzerCaller
	AnalyzerFilter		map[string][]plugin.AnalyzerCaller
}

// NewMandrake creates and returns a Mandrake struct utilizing a passed
// parsed configuration file to create the correct fields.
func NewMandrake(c config.Config) Mandrake {
	analyzers := []plugin.AnalyzerCaller{}
	filter := make(map[string][]plugin.AnalyzerCaller)
	for _, plug := range c.Analyzers {
		analyzer := plugin.NewAnalyzerCaller(plug)
		// Build a slice of all AnalyzerCaller structs
		analyzers = append(analyzers, analyzer)

		// Create a map to function as a mime_type filter for analyzers
		for _, mime := range analyzer.MimeFilter {
			filter[mime] = append(filter[mime], analyzer)
		}
	}

	return Mandrake{make(chan string), c.MonitoredDirectory, analyzers, filter}
}

// ListenAndServe starts the goroutines that perform all of the heavy lifting
// including Monitor() and DispatchAnalysis().
func (m Mandrake) ListenAndServe() {
	log.SetPrefix("[mandrake] ")
	log.Println(m.Analyzers[0])
	go m.DispatchAnalysis()
	m.Monitor()
}

// DispatchAnalysis intelligently sends a new file to registered plugins so
// that it can be analyzed.
func (m Mandrake) DispatchAnalysis() {
	for fpath := range m.AnalysisPipeline {
		fmeta, err := filemeta.NewFileMeta(fpath)
		if err != nil {
			log.Println(err)
		}

		// Create JSON filemeta object to pass to plugins so that plugins
		// receive basic contextual information about the file.
		fs, err := json.Marshal(fmeta)

		var analysis map[string]map[string]interface{}

		for _, analyzer := range m.AnalyzerFilter["all"] {
			result, err := analyzer.Analyze(string(fs))
			if err != nil {
				log.Print(err)
			}
			analysis[analyzer.Name] = make(map[string]interface{})
			analysis[analyzer.Name] = MapFromJSON(result)
		}

		for _, analyzer := range m.AnalyzerFilter[fmeta.Mime] {
			result, err := analyzer.Analyze(string(fs))
			if err != nil {
				log.Print(err)
			}
			analysis[analyzer.Name] = make(map[string]interface{})
			analysis[analyzer.Name] = MapFromJSON(result)
		}

		log.Println(json.Marshal(analysis))
		log.Println(string(fs))
		log.Printf("%s", fpath)
	}
}

// Monitor uses inotify to monitor the MonitoredDirectory for IN_CLOSE_WRITE
// events. Files written to the MonitoredDirectory will be sent to the
// analysis pipeline to be analyzed.
func (m Mandrake) Monitor() {
	log.Println("starting inotify watcher")
	watcher, err := inotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	log.Printf("adding watcher to %s directory", m.MonitoredDirectory)
	err = watcher.AddWatch(m.MonitoredDirectory, inotify.IN_CLOSE_WRITE)
	if err != nil {
		log.Fatal(err)
	}

	for {
		select {
		case ev := <- watcher.Event:
			m.AnalysisPipeline <- ev.Name
		case err := <- watcher.Error:
			log.Printf("inotify error: %s", err)
		}
	}
}

// MapFromJSON accepts an anonymous JSON object as a string and returns the
// resulting Map
func MapFromJSON(s string) map[string]interface{} {
	log.Printf("Performing mapping with string: %s", s)
	var f interface{}
	json.Unmarshal([]byte(s), &f)
	m := f.(map[string]interface{})
	log.Printf("Mapping complete: %s", m)
	return m
}<|endoftext|>