<commit_before>// Copyright 2010  The "go-linoise" Authors
//
// Use of this source code is governed by the Simplified BSD License
// that can be found in the LICENSE file.
//
// This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
// OR CONDITIONS OF ANY KIND, either express or implied. See the License
// for more details.

package linoise

import (
	"bufio"
	"fmt"
	"os"
	"utf8"
)

// Values by default
var (
	Input  = os.Stdin
	Output = os.Stdout
)

// ASCII codes
const (
	_ESC       = 27 // Escape: Ctrl-[ (033 in octal)
	_L_BRACKET = 91 // Left square bracket: [
)

// ANSI terminal escape controls
var (
	cursorToleft   = []byte("\033[0G")        // Cursor to left edge.
	delScreen      = []byte("\033[2J")        // Erase the screen.
	toleftDelRight = []byte("\033[0K\033[0G") // Cursor to left; erase to right.
)

// === Type
// ===

// Represents a line.
type line struct {
	promptLen int      // Prompt size
	prompt    string   // Primary prompt
	*buffer            // Text buffer
	hist      *history // History file
}

func NewLine(hist *history, prompt string) *line {
	return &line{
		len(prompt),
		prompt,
		newBuffer(),
		hist,
	}
}

// === Output
// ===

// Returns a slice of the contents of the buffer.
func (ln *line) Bytes() []byte {
	chars := make([]byte, ln.size*utf8.UTFMax)
	var end, runeLen int

	// === Each character (as integer) is encoded to []byte
	for i := 0; i < ln.size; i++ {
		if i != 0 {
			runeLen = utf8.EncodeRune(ln.data[i], chars[end:])
			end += runeLen
		} else {
			runeLen = utf8.EncodeRune(ln.data[i], chars)
			end = runeLen
		}
	}
	return chars[:end]
}

// Returns the contents of the buffer as a string.
func (ln *line) String() string { return string(ln.data[:ln.size]) }

// Refreshes the line.
func (ln *line) Refresh() (err os.Error) {
	if _, err = Output.Write(cursorToleft); err != nil {
		return
	}
	if _, err = fmt.Fprint(Output, ln.prompt); err != nil {
		return
	}
	if _, err = Output.Write(ln.Bytes()); err != nil {
		return
	}
	if _, err = Output.Write(toleftDelRight); err != nil {
		return
	}
	// Move cursor to original position.
	_, err = fmt.Fprintf(Output, "\033[%dC", ln.promptLen+ln.cursor)

	return
}

// === Get
// ===

func (ln *line) Run() (err os.Error) {
	// Print the primary prompt.
	if _, err = fmt.Fprint(Output, ln.prompt); err != nil {
		return
	}

	in := bufio.NewReader(Input) // Read input.
	seq := make([]byte, 2)       // For escape sequences.
	seq2 := make([]byte, 2)      // Extended escape sequences.

	for {
		rune, _, err := in.ReadRune()
		if err != nil {
			return
		}

		switch rune {
		default:
			useRefresh, err := ln.Insert(rune)
			if err != nil {
				return
			}

			if useRefresh {
				if err = ln.Refresh(); err != nil {
					return
				}
			}

		case 9: // horizontal tab
			//!!! disabled by now
			continue

		case 13: // enter
			ln.hist.Add(ln.String())
			goto _deleteLine

		case 127, 8: // backspace, Ctrl-h
			if ln.DeleteLast() {
				goto _refresh
			}

		case 20: // Ctrl-t //!!! add
			continue

		case 2: // Ctrl-b
			goto _leftArrow

		case 6: // Ctrl-f
			goto _rightArrow

		case 16: // Ctrl-p
			seq[1] = 65
			goto _upDownArrow

		case 14: // Ctrl-n
			seq[1] = 66
			goto _upDownArrow

		case 3, 4: // Ctrl-c, Ctrl-d
			if rune == 3 {
				ln.Insert('^')
				ln.Insert('C')
				err = ErrCtrlC
			} else {
				ln.Insert('^')
				ln.Insert('D')
				err = ErrCtrlD
			}

			ln.Refresh()
			return

		// Escape sequence
		case _ESC:
			if _, err = in.Read(seq); err != nil {
				return
			}

			if seq[0] == _L_BRACKET {
				switch seq[1] {
				case 68:
					goto _leftArrow
				case 67:
					goto _rightArrow
				case 65, 66:
					goto _upDownArrow
				}

				// Extended escape.
				if seq[1] > 48 && seq[1] < 55 {
					if _, err = in.Read(seq2); err != nil {
						return
					}
					//!!! doesn't works
					if seq[1] == 51 && seq2[0] == 126 { // Delete
						if ln.Delete() {
							goto _refresh
						}
					}
				}
			}

		case 21: // Ctrl+u, delete the whole line.
			goto _deleteLine

		case 11: // Ctrl+k, delete from current to end of line.
			ln.size = ln.cursor
			goto _refresh

		case 1: // Ctrl+a, go to the start of the line.
			ln.cursor = 0
			goto _refresh

		case 5: // Ctrl+e, go to the end of the line.
			ln.cursor = ln.size
			goto _refresh
		}
		continue

	_leftArrow:
		if ln.Left() {
			goto _refresh
		}
		continue

	_rightArrow:
		if ln.Right() {
			goto _refresh
		}
		continue

	_upDownArrow:
		if ln.hist.Len > 1 {
			// Update the current history entry before to overwrite it with tne
			// next one.

		}
		continue

	_deleteLine:
		ln.cursor, ln.size = 0, 0
		//goto _refresh

	_refresh:
		if err = ln.Refresh(); err != nil {
			return
		}
		continue
	}

	return
}

<commit_msg>Added function for keys: home, end up-down arrow, Ctrl-C.<commit_after>// Copyright 2010  The "go-linoise" Authors
//
// Use of this source code is governed by the Simplified BSD License
// that can be found in the LICENSE file.
//
// This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
// OR CONDITIONS OF ANY KIND, either express or implied. See the License
// for more details.

package linoise

import (
	"bufio"
	"fmt"
	"os"
	"utf8"
)

// Values by default
var (
	Input  = os.Stdin
	Output = os.Stdout
)

// ASCII codes
const (
	_ESC       = 27 // Escape: Ctrl-[ (033 in octal)
	_L_BRACKET = 91 // Left square bracket: [
)

// ANSI terminal escape controls
var (
	cursorToleft   = []byte("\033[0G")        // Cursor to left edge.
	delScreen      = []byte("\033[2J")        // Erase the screen.
	toleftDelRight = []byte("\033[0K\033[0G") // Cursor to left; erase to right.
)

// === Type
// ===

// Represents a line.
type line struct {
	promptLen int      // Prompt size
	prompt    string   // Primary prompt
	*buffer            // Text buffer
	hist      *history // History file
}

func NewLine(hist *history, prompt string) *line {
	return &line{
		len(prompt),
		prompt,
		newBuffer(),
		hist,
	}
}

// === Output
// ===

// Returns a slice of the contents of the buffer.
func (ln *line) Bytes() []byte {
	chars := make([]byte, ln.size*utf8.UTFMax)
	var end, runeLen int

	// === Each character (as integer) is encoded to []byte
	for i := 0; i < ln.size; i++ {
		if i != 0 {
			runeLen = utf8.EncodeRune(ln.data[i], chars[end:])
			end += runeLen
		} else {
			runeLen = utf8.EncodeRune(ln.data[i], chars)
			end = runeLen
		}
	}
	return chars[:end]
}

// Returns the contents of the buffer as a string.
func (ln *line) String() string { return string(ln.data[:ln.size]) }

// Refreshes the line.
func (ln *line) Refresh() (err os.Error) {
	if _, err = Output.Write(cursorToleft); err != nil {
		return
	}
	if _, err = fmt.Fprint(Output, ln.prompt); err != nil {
		return
	}
	if _, err = Output.Write(ln.Bytes()); err != nil {
		return
	}
	if _, err = Output.Write(toleftDelRight); err != nil {
		return
	}
	// Move cursor to original position.
	_, err = fmt.Fprintf(Output, "\033[%dC", ln.promptLen+ln.cursor)

	return
}

// === Get
// ===

func (ln *line) Run() (err os.Error) {
	var anotherLine []int // For lines got from history.
	var isHistoryUsed bool // If the history has been accessed.

	in := bufio.NewReader(Input) // Read input.
	seq := make([]byte, 2)       // For escape sequences.
	seq2 := make([]byte, 2)      // Extended escape sequences.

	// Print the primary prompt.
	if _, err = fmt.Fprint(Output, ln.prompt); err != nil {
		return
	}

	for {
		rune, _, err := in.ReadRune()
		if err != nil {
			return
		}

		switch rune {
		default:
			useRefresh, err := ln.Insert(rune)
			if err != nil {
				return
			}

			if useRefresh {
				if err = ln.Refresh(); err != nil {
					return
				}
			}

		case 9: // horizontal tab
			//!!! disabled by now
			continue

		case 13: // enter
			ln.hist.Add(ln.String())
			goto _deleteLine

		case 127, 8: // backspace, Ctrl-h
			if ln.DeletePrev() {
				goto _refresh
			}

		case 20: // Ctrl-t //!!! add
			continue

		case 2: // Ctrl-b
			goto _leftArrow

		case 6: // Ctrl-f
			goto _rightArrow

		case 16: // Ctrl-p
			seq[1] = 65
			goto _upDownArrow

		case 14: // Ctrl-n
			seq[1] = 66
			goto _upDownArrow

		case 3: // Ctrl-c
			ln.Insert('^')
			ln.Insert('C')

			if _, err = fmt.Fprint(Output, "\n"); err != nil {
				return
			}
			goto _deleteLine

		case 4: // Ctrl-d
			ln.Insert('^')
			ln.Insert('D')
			ln.Refresh()
			return ErrCtrlD

		// Escape sequence
		case _ESC:
			if _, err = in.Read(seq); err != nil {
				return
			}
			//fmt.Print(" >", seq) //!!! DEBUG

			if seq[0] == _L_BRACKET {
				switch seq[1] {
				case 68:
					goto _leftArrow
				case 67:
					goto _rightArrow
				case 65, 66: // Up, Down
					goto _upDownArrow
				}

				// Extended escape.
				if seq[1] > 48 && seq[1] < 55 {
					if _, err = in.Read(seq2); err != nil {
						return
					}
					//fmt.Print(" >>", seq2) //!!! DEBUG

					//!!! doesn't works
					if seq[1] == 51 && seq2[0] == 126 { // Delete
						if ln.Delete() {
							goto _refresh
						}
					}
				}
			}
			if seq[0] == 79 {
				switch seq[1] {
				case 72: // Home
					goto _start
				case 70: // End
					goto _end
				}
			}

		case 21: // Ctrl+u, delete the whole line.
			goto _deleteLine

		case 11: // Ctrl+k, delete from current to end of line.
			ln.size = ln.cursor
			goto _refresh

		case 1: // Ctrl+a, go to the start of the line.
			goto _start

		case 5: // Ctrl+e, go to the end of the line.
			goto _end
		}
		continue

	_leftArrow:
		if ln.Left() {
			goto _refresh
		}
		continue

	_rightArrow:
		if ln.Right() {
			goto _refresh
		}
		continue

	_upDownArrow: // Up and down arrow: history
		// Up
		if seq[1] == 65 {
			anotherLine, err = ln.hist.Prev()
		// Down
		} else {
			anotherLine, err = ln.hist.Next()
		}
		if err != nil {
			continue
		}

		// Update the current history entry before to overwrite it with
		// the next one.
		//!!! it has to be removed before of to be saved the history
		if !isHistoryUsed {
			ln.hist.Add(ln.String())
		}
		isHistoryUsed = true

		ln.grow(len(anotherLine))
		ln.size = len(anotherLine)
		ln.cursor = len(anotherLine)
		copy(ln.data[0:], anotherLine)
		goto _refresh

	_start:
		ln.cursor = 0
		goto _refresh

	_end:
		ln.cursor = ln.size
		goto _refresh

	_deleteLine:
		ln.cursor, ln.size = 0, 0
		//goto _refresh

	_refresh:
		if err = ln.Refresh(); err != nil {
			return
		}
		continue
	}

	return
}

<|endoftext|>


<commit_before>package server

import (
	"fmt"

	"github.com/blang/semver"
	"github.com/getlantern/golog"
)

var log = golog.LoggerFor("autoupdate-server")

// Initiative type.
type Initiative string

const (
	INITIATIVE_NEVER  Initiative = "never"
	INITIATIVE_AUTO              = "auto"
	INITIATIVE_MANUAL            = "manual"
)

// PatchType represents the type of a binary patch, if any. Only bsdiff is supported
type PatchType string

const (
	PATCHTYPE_BSDIFF PatchType = "bsdiff"
	PATCHTYPE_NONE             = ""
)

// Params represent parameters sent by the go-update client.
type Params struct {
	// protocol version
	Version int `json:"version"`
	// identifier of the application to update
	//AppId string `json:"app_id"`

	// version of the application updating itself
	AppVersion string `json:"app_version"`
	// operating system of target platform
	OS string `json:"-"`
	// hardware architecture of target platform
	Arch string `json:"-"`
	// application-level user identifier
	//UserId string `json:"user_id"`
	// checksum of the binary to replace (used for returning diff patches)
	Checksum string `json:"checksum"`
	// release channel (empty string means 'stable')
	//Channel string `json:"-"`
	// tags for custom update channels
	Tags map[string]string `json:"tags"`
}

// Result represents the answer to be sent to the client.
type Result struct {
	// should the update be applied automatically/manually
	Initiative Initiative `json:"initiative"`
	// url where to download the updated application
	URL string `json:"url"`
	// a URL to a patch to apply
	PatchURL string `json:"patch_url"`
	// the patch format (only bsdiff supported at the moment)
	PatchType PatchType `json:"patch_type"`
	// version of the new application
	Version string `json:"version"`
	// expected checksum of the new application
	Checksum string `json:"checksum"`
	// signature for verifying update authenticity
	Signature string `json:"signature"`
}

// CheckForUpdate receives a *Params message and emits a *Result. If both res
// and err are nil it means no update is available.
func (g *ReleaseManager) CheckForUpdate(p *Params) (res *Result, err error) {

	// Keep for the future.
	if p.Version < 1 {
		p.Version = 1
	}

	// p must not be nil.
	if p == nil {
		return nil, fmt.Errorf("Expecting params")
	}

	if p.Tags != nil {
		// Compatibility with go-check.
		if p.Tags["os"] != "" {
			p.OS = p.Tags["os"]
		}
		if p.Tags["arch"] != "" {
			p.Arch = p.Tags["arch"]
		}
	}

	if p.OS == "" {
		return nil, fmt.Errorf("OS is required")
	}

	// The checksum is optional if the OS is Android
	// since we aren't doing binary diffs
	if p.OS != OS.Android && p.Checksum == "" {
		return nil, fmt.Errorf("Checksum must not be nil")
	}

	if p.Arch == "" {
		return nil, fmt.Errorf("Arch is required")
	}

	// If this binary has a known checksum, the p.AppVersion will be changed to
	// 2.0.0beta8+manoto regardless of the value that was sent.
	if hasManotoChecksum(p.Checksum) {
		p.AppVersion = manotoBeta8
	}

	appVersion, err := semver.Parse(p.AppVersion)
	if err != nil {
		return nil, fmt.Errorf("Bad version string: %v", err)
	}

	var update *Asset

	// This is a hack that allows Lantern 2.0.0beta8+manoto clients to upgrade to
	// Lantern 2.0.0+manoto
	//
	// See https://github.com/getlantern/lantern/issues/2868
	if appVersion.String() == manotoBeta8 {
		// Always return 2.0.0+manoto
		if update, err = g.lookupAssetWithVersion(p.OS, p.Arch, manotoBeta8Upgrade); err != nil {
			return nil, fmt.Errorf("No upgrade for %s/%s", p.OS, p.Arch)
		}
	}

	// Looking if there is a newer version for the os/arch.
	if update == nil {
		if update, err = g.getProductUpdate(p.OS, p.Arch); err != nil {
			return nil, fmt.Errorf("Could not lookup for updates: %s", err)
		}
	}

	// No update available.
	if update.v.LTE(appVersion) {
		return nil, ErrNoUpdateAvailable
	}

	// A newer version is available!

	// Looking for the asset thay matches the current app checksum.
	var current *Asset
	if current, err = g.lookupAssetWithChecksum(p.OS, p.Arch, p.Checksum); err != nil {
		// No such asset with the given checksum, nothing to compare. Making the
		// client download the full binary
		return &Result{
			Initiative: INITIATIVE_AUTO,
			URL:        update.URL,
			PatchType:  PATCHTYPE_NONE,
			Version:    update.v.String(),
			Checksum:   update.Checksum,
			Signature:  update.Signature,
		}, nil
	}

	// Generate a binary diff of the two assets.
	var patch *Patch
	if patch, err = generatePatch(current.URL, update.URL); err != nil {
		return nil, fmt.Errorf("Unable to generate patch: %q", err)
	}

	// Generate result.
	r := &Result{
		Initiative: INITIATIVE_AUTO,
		URL:        update.URL,
		PatchURL:   patch.File,
		PatchType:  PATCHTYPE_BSDIFF,
		Version:    update.v.String(),
		Checksum:   update.Checksum,
		Signature:  update.Signature,
	}

	return r, nil
}
<commit_msg>use previous format<commit_after>package server

import (
	"fmt"

	"github.com/blang/semver"
	"github.com/getlantern/golog"
)

var log = golog.LoggerFor("autoupdate-server")

// Initiative type.
type Initiative string

const (
	INITIATIVE_NEVER  Initiative = "never"
	INITIATIVE_AUTO              = "auto"
	INITIATIVE_MANUAL            = "manual"
)

// PatchType represents the type of a binary patch, if any. Only bsdiff is supported
type PatchType string

const (
	PATCHTYPE_BSDIFF PatchType = "bsdiff"
	PATCHTYPE_NONE             = ""
)

// Params represent parameters sent by the go-update client.
type Params struct {
	// protocol version
	Version int `json:"version"`
	// identifier of the application to update
	//AppId string `json:"app_id"`

	// version of the application updating itself
	AppVersion string `json:"app_version"`
	// operating system of target platform
	OS string `json:"-"`
	// hardware architecture of target platform
	Arch string `json:"-"`
	// application-level user identifier
	//UserId string `json:"user_id"`
	// checksum of the binary to replace (used for returning diff patches)
	Checksum string `json:"checksum"`
	// release channel (empty string means 'stable')
	//Channel string `json:"-"`
	// tags for custom update channels
	Tags map[string]string `json:"tags"`
}

// Result represents the answer to be sent to the client.
type Result struct {
	// should the update be applied automatically/manually
	Initiative Initiative `json:"initiative"`
	// url where to download the updated application
	URL string `json:"url"`
	// a URL to a patch to apply
	PatchURL string `json:"patch_url"`
	// the patch format (only bsdiff supported at the moment)
	PatchType PatchType `json:"patch_type"`
	// version of the new application
	Version string `json:"version"`
	// expected checksum of the new application
	Checksum string `json:"checksum"`
	// signature for verifying update authenticity
	Signature string `json:"signature"`
}

// CheckForUpdate receives a *Params message and emits a *Result. If both res
// and err are nil it means no update is available.
func (g *ReleaseManager) CheckForUpdate(p *Params) (res *Result, err error) {

	// Keep for the future.
	if p.Version < 1 {
		p.Version = 1
	}

	// p must not be nil.
	if p == nil {
		return nil, fmt.Errorf("Expecting params")
	}

	if p.Tags != nil {
		// Compatibility with go-check.
		if p.Tags["os"] != "" {
			p.OS = p.Tags["os"]
		}
		if p.Tags["arch"] != "" {
			p.Arch = p.Tags["arch"]
		}
	}

	if p.OS == "" {
		return nil, fmt.Errorf("OS is required")
	}

	// The checksum is optional if the OS is Android
	// since we aren't doing binary diffs
	if p.OS != OS.Android && p.Checksum == "" {
		return nil, fmt.Errorf("Checksum must not be nil")
	}

	if p.Arch == "" {
		return nil, fmt.Errorf("Arch is required")
	}

	// If this binary has a known checksum, the p.AppVersion will be changed to
	// 2.0.0beta8+manoto regardless of the value that was sent.
	if hasManotoChecksum(p.Checksum) {
		p.AppVersion = manotoBeta8
	}

	appVersion, err := semver.Parse(p.AppVersion)
	if err != nil {
		return nil, fmt.Errorf("Bad version string: %v", err)
	}

	var update *Asset

	// This is a hack that allows Lantern 2.0.0beta8+manoto clients to upgrade to
	// Lantern 2.0.0+manoto
	//
	// See https://github.com/getlantern/lantern/issues/2868
	if appVersion.String() == manotoBeta8 {
		// Always return 2.0.0+manoto
		if update, err = g.lookupAssetWithVersion(p.OS, p.Arch, manotoBeta8Upgrade); err != nil {
			return nil, fmt.Errorf("No upgrade for %s/%s", p.OS, p.Arch)
		}
	}

	// Looking if there is a newer version for the os/arch.
	if update == nil {
		if update, err = g.getProductUpdate(p.OS, p.Arch); err != nil {
			return nil, fmt.Errorf("Could not lookup for updates: %s", err)
		}
	}

	// No update available.
	if update.v.LTE(appVersion) {
		return nil, ErrNoUpdateAvailable
	}

	// A newer version is available!

	// Looking for the asset thay matches the current app checksum.
	var current *Asset
	if current, err = g.lookupAssetWithChecksum(p.OS, p.Arch, p.Checksum); err != nil {
		// No such asset with the given checksum, nothing to compare. Making the
		// client download the full binary
		r := &Result{
			Initiative: INITIATIVE_AUTO,
			URL:        update.URL,
			PatchType:  PATCHTYPE_NONE,
			Version:    update.v.String(),
			Checksum:   update.Checksum,
			Signature:  update.Signature,
		}

		return r, nil
	}

	// Generate a binary diff of the two assets.
	var patch *Patch
	if patch, err = generatePatch(current.URL, update.URL); err != nil {
		return nil, fmt.Errorf("Unable to generate patch: %q", err)
	}

	// Generate result.
	r := &Result{
		Initiative: INITIATIVE_AUTO,
		URL:        update.URL,
		PatchURL:   patch.File,
		PatchType:  PATCHTYPE_BSDIFF,
		Version:    update.v.String(),
		Checksum:   update.Checksum,
		Signature:  update.Signature,
	}

	return r, nil
}
<|endoftext|>


<commit_before>package server

import (
	"encoding/json"
	"errors"
	"net/http"
	"time"

	log "github.com/coreos/fleet/third_party/github.com/golang/glog"

	"github.com/coreos/fleet/agent"
	"github.com/coreos/fleet/config"
	"github.com/coreos/fleet/engine"
	"github.com/coreos/fleet/etcd"
	"github.com/coreos/fleet/event"
	"github.com/coreos/fleet/machine"
	"github.com/coreos/fleet/registry"
	"github.com/coreos/fleet/sign"
	"github.com/coreos/fleet/systemd"
	"github.com/coreos/fleet/unit"
	"github.com/coreos/fleet/version"
)

const (
	// machineStateRefreshInterval is the amount of time the server will
	// wait before each attempt to refresh the local machine state
	machineStateRefreshInterval = time.Minute
)

type Server struct {
	agent   *agent.Agent
	engine  *engine.Engine
	rStream *registry.EventStream
	sStream *systemd.EventStream
	eBus    *event.EventBus
	mach    *machine.CoreOSMachine

	stop chan bool
}

func New(cfg config.Config) (*Server, error) {
	mach, err := newMachineFromConfig(cfg)
	if err != nil {
		return nil, err
	}

	mgr, err := systemd.NewSystemdUnitManager(systemd.DefaultUnitsDirectory)
	if err != nil {
		return nil, err
	}

	eClient, err := etcd.NewClient(cfg.EtcdServers, http.Transport{})
	if err != nil {
		return nil, err
	}

	reg := registry.New(eClient, cfg.EtcdKeyPrefix)

	a, err := newAgentFromConfig(mach, reg, cfg, mgr)
	if err != nil {
		return nil, err
	}

	e := engine.New(reg, mach)

	sStream := systemd.NewEventStream(mgr)

	rStream, err := registry.NewEventStream(eClient, reg)
	if err != nil {
		return nil, err
	}

	aHandler := agent.NewEventHandler(a)
	eHandler := engine.NewEventHandler(e)

	eBus := event.NewEventBus()
	eBus.AddListener("engine", eHandler)
	eBus.AddListener("agent", aHandler)

	return &Server{a, e, rStream, sStream, eBus, mach, nil}, nil
}

func newMachineFromConfig(cfg config.Config) (*machine.CoreOSMachine, error) {
	state := machine.MachineState{
		PublicIP: cfg.PublicIP,
		Metadata: cfg.Metadata(),
		Version:  version.Version,
	}

	mach := machine.NewCoreOSMachine(state)
	mach.Refresh()

	if mach.State().ID == "" {
		return nil, errors.New("unable to determine local machine ID")
	}

	return mach, nil
}

func newAgentFromConfig(mach machine.Machine, reg registry.Registry, cfg config.Config, mgr unit.UnitManager) (*agent.Agent, error) {
	var verifier *sign.SignatureVerifier
	if cfg.VerifyUnits {
		var err error
		verifier, err = sign.NewSignatureVerifierFromAuthorizedKeysFile(cfg.AuthorizedKeysFile)
		if err != nil {
			log.Errorln("Failed to get any key from authorized key file in verify_units mode:", err)
			verifier = sign.NewSignatureVerifier()
		}
	}

	return agent.New(mgr, reg, mach, cfg.AgentTTL, verifier)
}

func (s *Server) Run() {
	idx := s.agent.Initialize()

	s.stop = make(chan bool)
	go s.mach.PeriodicRefresh(machineStateRefreshInterval, s.stop)
	go s.rStream.Stream(idx, s.eBus.Dispatch, s.stop)
	go s.sStream.Stream(s.eBus.Dispatch, s.stop)
	go s.agent.Heartbeat(s.stop)

	s.engine.CheckForWork()
}

func (s *Server) Stop() {
	close(s.stop)
}

func (s *Server) Purge() {
	s.agent.Purge()
}

func (s *Server) MarshalJSON() ([]byte, error) {
	return json.Marshal(struct{ Agent *agent.Agent }{Agent: s.agent})
}
<commit_msg>feat(API): Socket-activate API server<commit_after>package server

import (
	"encoding/json"
	"errors"
	"net/http"
	"time"

	"github.com/coreos/fleet/third_party/github.com/coreos/go-systemd/activation"
	log "github.com/coreos/fleet/third_party/github.com/golang/glog"

	"github.com/coreos/fleet/agent"
	"github.com/coreos/fleet/api"
	"github.com/coreos/fleet/config"
	"github.com/coreos/fleet/engine"
	"github.com/coreos/fleet/etcd"
	"github.com/coreos/fleet/event"
	"github.com/coreos/fleet/machine"
	"github.com/coreos/fleet/registry"
	"github.com/coreos/fleet/sign"
	"github.com/coreos/fleet/systemd"
	"github.com/coreos/fleet/unit"
	"github.com/coreos/fleet/version"
)

const (
	// machineStateRefreshInterval is the amount of time the server will
	// wait before each attempt to refresh the local machine state
	machineStateRefreshInterval = time.Minute
)

type Server struct {
	agent   *agent.Agent
	engine  *engine.Engine
	rStream *registry.EventStream
	sStream *systemd.EventStream
	eBus    *event.EventBus
	mach    *machine.CoreOSMachine

	stop chan bool
}

func New(cfg config.Config) (*Server, error) {
	mach, err := newMachineFromConfig(cfg)
	if err != nil {
		return nil, err
	}

	mgr, err := systemd.NewSystemdUnitManager(systemd.DefaultUnitsDirectory)
	if err != nil {
		return nil, err
	}

	eClient, err := etcd.NewClient(cfg.EtcdServers, http.Transport{})
	if err != nil {
		return nil, err
	}

	reg := registry.New(eClient, cfg.EtcdKeyPrefix)

	a, err := newAgentFromConfig(mach, reg, cfg, mgr)
	if err != nil {
		return nil, err
	}

	e := engine.New(reg, mach)

	sStream := systemd.NewEventStream(mgr)

	rStream, err := registry.NewEventStream(eClient, reg)
	if err != nil {
		return nil, err
	}

	aHandler := agent.NewEventHandler(a)
	eHandler := engine.NewEventHandler(e)

	eBus := event.NewEventBus()
	eBus.AddListener("engine", eHandler)
	eBus.AddListener("agent", aHandler)

	listeners, err := activation.Listeners(false)
	if err != nil {
		return nil, err
	}

	mux := api.NewServeMux(reg)
	for _, f := range listeners {
		go http.Serve(f, mux)
	}

	return &Server{a, e, rStream, sStream, eBus, mach, nil}, nil
}

func newMachineFromConfig(cfg config.Config) (*machine.CoreOSMachine, error) {
	state := machine.MachineState{
		PublicIP: cfg.PublicIP,
		Metadata: cfg.Metadata(),
		Version:  version.Version,
	}

	mach := machine.NewCoreOSMachine(state)
	mach.Refresh()

	if mach.State().ID == "" {
		return nil, errors.New("unable to determine local machine ID")
	}

	return mach, nil
}

func newAgentFromConfig(mach machine.Machine, reg registry.Registry, cfg config.Config, mgr unit.UnitManager) (*agent.Agent, error) {
	var verifier *sign.SignatureVerifier
	if cfg.VerifyUnits {
		var err error
		verifier, err = sign.NewSignatureVerifierFromAuthorizedKeysFile(cfg.AuthorizedKeysFile)
		if err != nil {
			log.Errorln("Failed to get any key from authorized key file in verify_units mode:", err)
			verifier = sign.NewSignatureVerifier()
		}
	}

	return agent.New(mgr, reg, mach, cfg.AgentTTL, verifier)
}

func (s *Server) Run() {
	idx := s.agent.Initialize()

	s.stop = make(chan bool)
	go s.mach.PeriodicRefresh(machineStateRefreshInterval, s.stop)
	go s.rStream.Stream(idx, s.eBus.Dispatch, s.stop)
	go s.sStream.Stream(s.eBus.Dispatch, s.stop)
	go s.agent.Heartbeat(s.stop)

	s.engine.CheckForWork()
}

func (s *Server) Stop() {
	close(s.stop)
}

func (s *Server) Purge() {
	s.agent.Purge()
}

func (s *Server) MarshalJSON() ([]byte, error) {
	return json.Marshal(struct{ Agent *agent.Agent }{Agent: s.agent})
}
<|endoftext|>


<commit_before>// An IMAP server.
package server

import (
	"crypto/tls"
	"errors"
	"io"
	"log"
	"net"

	"github.com/emersion/go-imap/common"
	"github.com/emersion/go-imap/backend"
	"github.com/emersion/go-imap/responses"
	"github.com/emersion/go-sasl"
)

// A command handler.
type Handler interface {
	common.Parser

	// Handle this command for a given connection.
	Handle(conn *Conn) error
}

// A function that creates handlers.
type HandlerFactory func() Handler

// A function that creates SASL servers.
type SaslServerFactory func(conn *Conn) sasl.Server

// An IMAP server.
type Server struct {
	listener net.Listener
	conns []*Conn

	caps map[string]common.ConnState
	commands map[string]HandlerFactory
	auths map[string]SaslServerFactory

	// This server's backend.
	Backend backend.Backend
	// Backend updates that will be sent to connected clients.
	Updates *backend.Updates
	// This server's TLS configuration.
	TLSConfig *tls.Config
	// Allow authentication over unencrypted connections.
	AllowInsecureAuth bool
	// Print all network activity to STDOUT.
	Debug bool
}

// Get this server's address.
func (s *Server) Addr() net.Addr {
	return s.listener.Addr()
}

func (s *Server) listen() error {
	defer s.listener.Close()

	for {
		c, err := s.listener.Accept()
		if err != nil {
			return err
		}

		conn := newConn(s, c)
		if s.Debug {
			conn.SetDebug(true)
		}

		go s.handleConn(conn)
	}
}

func (s *Server) handleConn(conn *Conn) error {
	s.conns = append(s.conns, conn)
	defer conn.Close()

	// Send greeting
	if err := conn.greet(); err != nil {
		return err
	}

	for {
		if conn.State == common.LogoutState {
			return nil
		}

		conn.Wait()

		fields, err := conn.ReadLine()
		if err == io.EOF || conn.State == common.LogoutState {
			return nil
		}
		if err != nil {
			log.Println("Error reading command:", err)
			return err
		}

		var res common.WriterTo

		cmd := &common.Command{}
		if err := cmd.Parse(fields); err != nil {
			res = &common.StatusResp{
				Tag: "*",
				Type: common.BAD,
				Info: err.Error(),
			}
		} else {
			var err error
			res, err = s.handleCommand(cmd, conn)
			if err != nil {
				res = &common.StatusResp{
					Tag: cmd.Tag,
					Type: common.BAD,
					Info: err.Error(),
				}
			}
		}

		if err := conn.WriteRes(res); err != nil {
			log.Println("Error writing response:", err)
			continue
		}
	}
}

func (s *Server) getCommandHandler(cmd *common.Command) (hdlr Handler, err error) {
	newHandler, ok := s.commands[cmd.Name]
	if !ok {
		err = errors.New("Unknown command")
		return
	}

	hdlr = newHandler()
	err = hdlr.Parse(cmd.Arguments)
	return
}

func (s *Server) handleCommand(cmd *common.Command, conn *Conn) (res common.WriterTo, err error) {
	hdlr, err := s.getCommandHandler(cmd)
	if err != nil {
		return
	}

	if err := hdlr.Handle(conn); err != nil {
		res = &common.StatusResp{
			Tag: cmd.Tag,
			Type: common.NO,
			Info: err.Error(),
		}
	} else {
		res = &common.StatusResp{
			Tag: cmd.Tag,
			Type: common.OK,
			Info: cmd.Name + " completed",
		}
	}

	return
}

func (s *Server) listenUpdates() (err error) {
	updater, ok := s.Backend.(backend.Updater)
	if !ok {
		return
	}
	s.Updates = updater.Updates()

	var update *backend.Update
	var res common.WriterTo
	for {
		select {
		case status := <-s.Updates.Statuses:
			update = &status.Update
			res = status.StatusResp
		case mailbox := <-s.Updates.Mailboxes:
			update = &mailbox.Update
			res = &responses.Select{Mailbox: mailbox.MailboxStatus}
		case message := <-s.Updates.Messages:
			update = &message.Update

			ch := make(chan *common.Message)
			go (func() {
				ch <- message.Message
				close(ch)
			})()

			res = &responses.Fetch{Messages: ch}
		case expunge := <-s.Updates.Expunges:
			update = &expunge.Update

			ch := make(chan uint32)
			go (func() {
				ch <- expunge.SeqNum
				close(ch)
			})()

			res = &responses.Expunge{SeqIds: ch}
		}

		// TODO: this doesn't work with responses using channels
		// Use a buffer instead, making sure the response is generated if at least
		// one client matches
		for _, conn := range s.conns {
			if update.Username != "" && (conn.User == nil || conn.User.Username() != update.Username) {
				continue
			}
			if update.Mailbox != "" && (conn.Mailbox == nil || conn.Mailbox.Name() != update.Mailbox) {
				continue
			}
			if conn.silent {
				// If silent is set, do not send message updates
				if _, ok := res.(*responses.Fetch); ok {
					continue
				}
			}

			if err := conn.WriteRes(res); err != nil {
				log.Println("WARN: error sending unilateral update:", err)
			}
		}
	}
}

func (s *Server) getCaps(currentState common.ConnState) (caps []string) {
	for name, state := range s.caps {
		if currentState & state != 0 {
			caps = append(caps, name)
		}
	}
	return
}

// Stops listening and closes all current connections.
func (s *Server) Close() error {
	if err := s.listener.Close(); err != nil {
		return err
	}

	for _, conn := range s.conns {
		conn.Close()
	}

	return nil
}

// Register a new capability that will be advertised by this server.
//
// This function should not be called directly, it must only be used by
// libraries implementing extensions of the IMAP protocol.
func (s *Server) RegisterCapability(name string, state common.ConnState) {
	s.caps[name] = state
}

// Register a new authentication mechanism for this server.
//
// This function should not be called directly, it must only be used by
// libraries implementing extensions of the IMAP protocol.
func (s *Server) RegisterAuth(name string, f SaslServerFactory) {
	s.auths[name] = f
}

// Register a new command for this server.
//
// This function should not be called directly, it must only be used by
// libraries implementing extensions of the IMAP protocol.
func (s *Server) RegisterCommand(name string, f HandlerFactory) {
	s.commands[name] = f
}

// Create a new IMAP server from an existing listener.
func NewServer(l net.Listener, bkd backend.Backend) *Server {
	s := &Server{
		listener: l,
		caps: map[string]common.ConnState{},
		Backend: bkd,
	}

	s.auths = map[string]SaslServerFactory{
		"PLAIN": func(conn *Conn) sasl.Server {
			return sasl.NewPlainServer(func(identity, username, password string) error {
				if identity != "" && identity != username {
					return errors.New("Identities not supported")
				}

				user, err := bkd.Login(username, password)
				if err != nil {
					return err
				}

				conn.State = common.AuthenticatedState
				conn.User = user
				return nil
			})
		},
	}

	s.commands = map[string]HandlerFactory{
		common.Noop: func() Handler { return &Noop{} },
		common.Capability: func() Handler { return &Capability{} },
		common.Logout: func() Handler { return &Logout{} },

		common.StartTLS: func() Handler { return &StartTLS{} },
		common.Login: func() Handler { return &Login{} },
		common.Authenticate: func() Handler { return &Authenticate{} },

		common.Select: func() Handler { return &Select{} },
		common.Examine: func() Handler {
			hdlr := &Select{}
			hdlr.ReadOnly = true
			return hdlr
		},
		common.Create: func() Handler { return &Create{} },
		common.Delete: func() Handler { return &Delete{} },
		common.Rename: func() Handler { return &Rename{} },
		common.Subscribe: func() Handler { return &Subscribe{} },
		common.Unsubscribe: func() Handler { return &Unsubscribe{} },
		common.List: func() Handler { return &List{} },
		common.Lsub: func() Handler {
			hdlr := &List{}
			hdlr.Subscribed = true
			return hdlr
		},
		common.Status: func() Handler { return &Status{} },
		common.Append: func() Handler { return &Append{} },

		common.Check: func() Handler { return &Check{} },
		common.Close: func() Handler { return &Close{} },
		common.Expunge: func() Handler { return &Expunge{} },
		common.Search: func() Handler { return &Search{} },
		common.Fetch: func() Handler { return &Fetch{} },
		common.Store: func() Handler { return &Store{} },
		common.Copy: func() Handler { return &Copy{} },
		common.Uid: func() Handler { return &Uid{} },
	}

	go s.listen()
	return s
}

func Listen(addr string, bkd backend.Backend) (s *Server, err error) {
	l, err := net.Listen("tcp", addr)
	if err != nil {
		return
	}

	s = NewServer(l, bkd)
	return
}

func ListenTLS(addr string, bkd backend.Backend, tlsConfig *tls.Config) (s *Server, err error) {
	l, err := tls.Listen("tcp", addr, tlsConfig)
	if err != nil {
		return
	}

	s = NewServer(l, bkd)
	return
}
<commit_msg>server: fixes sending unilateral message/expunge updates to more than one conn<commit_after>// An IMAP server.
package server

import (
	"bytes"
	"crypto/tls"
	"errors"
	"io"
	"log"
	"net"

	"github.com/emersion/go-imap/common"
	"github.com/emersion/go-imap/backend"
	"github.com/emersion/go-imap/responses"
	"github.com/emersion/go-sasl"
)

// A command handler.
type Handler interface {
	common.Parser

	// Handle this command for a given connection.
	Handle(conn *Conn) error
}

// A function that creates handlers.
type HandlerFactory func() Handler

// A function that creates SASL servers.
type SaslServerFactory func(conn *Conn) sasl.Server

// An IMAP server.
type Server struct {
	listener net.Listener
	conns []*Conn

	caps map[string]common.ConnState
	commands map[string]HandlerFactory
	auths map[string]SaslServerFactory

	// This server's backend.
	Backend backend.Backend
	// Backend updates that will be sent to connected clients.
	Updates *backend.Updates
	// This server's TLS configuration.
	TLSConfig *tls.Config
	// Allow authentication over unencrypted connections.
	AllowInsecureAuth bool
	// Print all network activity to STDOUT.
	Debug bool
}

// Get this server's address.
func (s *Server) Addr() net.Addr {
	return s.listener.Addr()
}

func (s *Server) listen() error {
	defer s.listener.Close()

	for {
		c, err := s.listener.Accept()
		if err != nil {
			return err
		}

		conn := newConn(s, c)
		if s.Debug {
			conn.SetDebug(true)
		}

		go s.handleConn(conn)
	}
}

func (s *Server) handleConn(conn *Conn) error {
	s.conns = append(s.conns, conn)
	defer conn.Close()

	// Send greeting
	if err := conn.greet(); err != nil {
		return err
	}

	for {
		if conn.State == common.LogoutState {
			return nil
		}

		conn.Wait()

		fields, err := conn.ReadLine()
		if err == io.EOF || conn.State == common.LogoutState {
			return nil
		}
		if err != nil {
			log.Println("Error reading command:", err)
			return err
		}

		var res common.WriterTo

		cmd := &common.Command{}
		if err := cmd.Parse(fields); err != nil {
			res = &common.StatusResp{
				Tag: "*",
				Type: common.BAD,
				Info: err.Error(),
			}
		} else {
			var err error
			res, err = s.handleCommand(cmd, conn)
			if err != nil {
				res = &common.StatusResp{
					Tag: cmd.Tag,
					Type: common.BAD,
					Info: err.Error(),
				}
			}
		}

		if err := conn.WriteRes(res); err != nil {
			log.Println("Error writing response:", err)
			continue
		}
	}
}

func (s *Server) getCommandHandler(cmd *common.Command) (hdlr Handler, err error) {
	newHandler, ok := s.commands[cmd.Name]
	if !ok {
		err = errors.New("Unknown command")
		return
	}

	hdlr = newHandler()
	err = hdlr.Parse(cmd.Arguments)
	return
}

func (s *Server) handleCommand(cmd *common.Command, conn *Conn) (res common.WriterTo, err error) {
	hdlr, err := s.getCommandHandler(cmd)
	if err != nil {
		return
	}

	if err := hdlr.Handle(conn); err != nil {
		res = &common.StatusResp{
			Tag: cmd.Tag,
			Type: common.NO,
			Info: err.Error(),
		}
	} else {
		res = &common.StatusResp{
			Tag: cmd.Tag,
			Type: common.OK,
			Info: cmd.Name + " completed",
		}
	}

	return
}

func (s *Server) listenUpdates() (err error) {
	updater, ok := s.Backend.(backend.Updater)
	if !ok {
		return
	}
	s.Updates = updater.Updates()

	var update *backend.Update
	var res common.WriterTo
	for {
		// TODO: do not generate response if nobody will receive it

		select {
		case status := <-s.Updates.Statuses:
			update = &status.Update
			res = status.StatusResp
		case mailbox := <-s.Updates.Mailboxes:
			update = &mailbox.Update
			res = &responses.Select{Mailbox: mailbox.MailboxStatus}
		case message := <-s.Updates.Messages:
			update = &message.Update

			ch := make(chan *common.Message)
			go (func() {
				ch <- message.Message
				close(ch)
			})()

			res = &responses.Fetch{Messages: ch}
		case expunge := <-s.Updates.Expunges:
			update = &expunge.Update

			ch := make(chan uint32)
			go (func() {
				ch <- expunge.SeqNum
				close(ch)
			})()

			res = &responses.Expunge{SeqIds: ch}
		}

		// Format response
		b := &bytes.Buffer{}
		w := common.NewWriter(b)
		if err := res.WriteTo(w); err != nil {
			log.Println("WARN: cannot format unlateral update:", err)
		}

		for _, conn := range s.conns {
			if update.Username != "" && (conn.User == nil || conn.User.Username() != update.Username) {
				continue
			}
			if update.Mailbox != "" && (conn.Mailbox == nil || conn.Mailbox.Name() != update.Mailbox) {
				continue
			}
			if conn.silent {
				// If silent is set, do not send message updates
				if _, ok := res.(*responses.Fetch); ok {
					continue
				}
			}

			if _, err := conn.Writer.Write(b.Bytes()); err != nil {
				log.Println("WARN: error sending unilateral update:", err)
				continue
			}
			conn.Flush()
		}
	}
}

func (s *Server) getCaps(currentState common.ConnState) (caps []string) {
	for name, state := range s.caps {
		if currentState & state != 0 {
			caps = append(caps, name)
		}
	}
	return
}

// Stops listening and closes all current connections.
func (s *Server) Close() error {
	if err := s.listener.Close(); err != nil {
		return err
	}

	for _, conn := range s.conns {
		conn.Close()
	}

	return nil
}

// Register a new capability that will be advertised by this server.
//
// This function should not be called directly, it must only be used by
// libraries implementing extensions of the IMAP protocol.
func (s *Server) RegisterCapability(name string, state common.ConnState) {
	s.caps[name] = state
}

// Register a new authentication mechanism for this server.
//
// This function should not be called directly, it must only be used by
// libraries implementing extensions of the IMAP protocol.
func (s *Server) RegisterAuth(name string, f SaslServerFactory) {
	s.auths[name] = f
}

// Register a new command for this server.
//
// This function should not be called directly, it must only be used by
// libraries implementing extensions of the IMAP protocol.
func (s *Server) RegisterCommand(name string, f HandlerFactory) {
	s.commands[name] = f
}

// Create a new IMAP server from an existing listener.
func NewServer(l net.Listener, bkd backend.Backend) *Server {
	s := &Server{
		listener: l,
		caps: map[string]common.ConnState{},
		Backend: bkd,
	}

	s.auths = map[string]SaslServerFactory{
		"PLAIN": func(conn *Conn) sasl.Server {
			return sasl.NewPlainServer(func(identity, username, password string) error {
				if identity != "" && identity != username {
					return errors.New("Identities not supported")
				}

				user, err := bkd.Login(username, password)
				if err != nil {
					return err
				}

				conn.State = common.AuthenticatedState
				conn.User = user
				return nil
			})
		},
	}

	s.commands = map[string]HandlerFactory{
		common.Noop: func() Handler { return &Noop{} },
		common.Capability: func() Handler { return &Capability{} },
		common.Logout: func() Handler { return &Logout{} },

		common.StartTLS: func() Handler { return &StartTLS{} },
		common.Login: func() Handler { return &Login{} },
		common.Authenticate: func() Handler { return &Authenticate{} },

		common.Select: func() Handler { return &Select{} },
		common.Examine: func() Handler {
			hdlr := &Select{}
			hdlr.ReadOnly = true
			return hdlr
		},
		common.Create: func() Handler { return &Create{} },
		common.Delete: func() Handler { return &Delete{} },
		common.Rename: func() Handler { return &Rename{} },
		common.Subscribe: func() Handler { return &Subscribe{} },
		common.Unsubscribe: func() Handler { return &Unsubscribe{} },
		common.List: func() Handler { return &List{} },
		common.Lsub: func() Handler {
			hdlr := &List{}
			hdlr.Subscribed = true
			return hdlr
		},
		common.Status: func() Handler { return &Status{} },
		common.Append: func() Handler { return &Append{} },

		common.Check: func() Handler { return &Check{} },
		common.Close: func() Handler { return &Close{} },
		common.Expunge: func() Handler { return &Expunge{} },
		common.Search: func() Handler { return &Search{} },
		common.Fetch: func() Handler { return &Fetch{} },
		common.Store: func() Handler { return &Store{} },
		common.Copy: func() Handler { return &Copy{} },
		common.Uid: func() Handler { return &Uid{} },
	}

	go s.listen()
	return s
}

func Listen(addr string, bkd backend.Backend) (s *Server, err error) {
	l, err := net.Listen("tcp", addr)
	if err != nil {
		return
	}

	s = NewServer(l, bkd)
	return
}

func ListenTLS(addr string, bkd backend.Backend, tlsConfig *tls.Config) (s *Server, err error) {
	l, err := tls.Listen("tcp", addr, tlsConfig)
	if err != nil {
		return
	}

	s = NewServer(l, bkd)
	return
}
<|endoftext|>


<commit_before>package server

import (
	"github.com/gilcrest/env"
	"github.com/gilcrest/errors"
	"github.com/rs/zerolog"
)

// Server struct contains the environment and additional methods
// for running our HTTP server
type Server struct {
	*env.Env
}

// NewServer is a constructor for the Server struct
// Sets up the struct and registers routes
func NewServer(name env.Name, lvl zerolog.Level) (*Server, error) {
	const op errors.Op = "server/NewServer"

	// call constructor for Env struct from env module
	env, err := env.NewEnv(name, lvl)
	if err != nil {
		return nil, errors.E(op, err)
	}

	// Use type embedding to make env.Env struct part of Server struct
	server := &Server{env}

	// routes registers handlers to the Server router
	err = server.routes()
	if err != nil {
		return nil, errors.E(op, err)
	}

	return server, nil
}
<commit_msg>comment update<commit_after>package server

import (
	"github.com/gilcrest/env"
	"github.com/gilcrest/errors"
	"github.com/rs/zerolog"
)

// Server struct contains the environment (env.Env) and additional methods
// for running our HTTP server
type Server struct {
	*env.Env
}

// NewServer is a constructor for the Server struct
// Sets up the struct and registers routes
func NewServer(name env.Name, lvl zerolog.Level) (*Server, error) {
	const op errors.Op = "server/NewServer"

	// call constructor for Env struct from env module
	env, err := env.NewEnv(name, lvl)
	if err != nil {
		return nil, errors.E(op, err)
	}

	// Use type embedding to make env.Env struct part of Server struct
	server := &Server{env}

	// routes registers handlers to the Server router
	err = server.routes()
	if err != nil {
		return nil, errors.E(op, err)
	}

	return server, nil
}
<|endoftext|>


<commit_before>// Package server implements the mog protocol.
package server

import (
	"bytes"
	"compress/gzip"
	crand "crypto/rand"
	"encoding/gob"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math"
	"math/big"
	"math/rand"
	"net/http"
	"net/url"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/boltdb/bolt"
	"github.com/mjibson/mog/codec"
	"github.com/mjibson/mog/models"
	"github.com/mjibson/mog/protocol"
	"github.com/pkg/browser"
)

func init() {
	i, err := crand.Int(crand.Reader, big.NewInt(math.MaxInt64))
	if err != nil {
		return
	}
	rand.Seed(i.Int64())
}

func printErr(e error) {
	log.Println(e)
	b := make([]byte, 4096)
	runtime.Stack(b, false)
	println(string(b))
}

func ListenAndServe(stateFile, addr, central string, devMode bool) error {
	server, err := New(stateFile, central)
	if err != nil {
		return err
	}
	if !devMode {
		host := addr
		if strings.HasPrefix(host, ":") {
			host = "localhost" + host
		}
		err := browser.OpenURL("http://" + host + "/")
		if err != nil {
			log.Println(err)
		}
	}
	return server.ListenAndServe(addr, devMode)
}

const (
	statePlay State = iota
	stateStop
	statePause
)

type State int

func (s State) String() string {
	switch s {
	case statePlay:
		return "play"
	case stateStop:
		return "stop"
	case statePause:
		return "pause"
	}
	return ""
}

type Playlist []SongID

type SongID codec.ID

func (s SongID) MarshalJSON() ([]byte, error) {
	sp := strings.SplitN(string(s), codec.IdSep, 3)
	if len(sp) != 3 {
		return json.Marshal("")
	}
	return json.Marshal(struct {
		Protocol string
		Key      string
		ID       string
		UID      string
	}{
		sp[0],
		sp[1],
		sp[2],
		string(s),
	})
}

func (s SongID) Protocol() string {
	return codec.ID(s).Top()
}

func (s SongID) Key() string {
	_, c := codec.ID(s).Pop()
	return c.Top()
}

func (s SongID) ID() codec.ID {
	_, c := codec.ID(s).Pop()
	_, c = c.Pop()
	return c
}

func (s SongID) Triple() (protocol, key string, id codec.ID) {
	protocol, id = codec.ID(s).Pop()
	key, id = id.Pop()
	return
}

type Server struct {
	Queue     Playlist
	Playlists map[string]Playlist

	Username string
	Token    string

	Repeat      bool
	Random      bool
	Protocols   map[string]map[string]protocol.Instance
	MinDuration time.Duration

	// Current song data.
	PlaylistIndex int
	songID        SongID
	song          codec.Song
	info          codec.SongInfo
	elapsed       time.Duration

	centralURL  string
	inprogress  map[codec.ID]bool
	ch          chan interface{}
	audioch     chan interface{}
	state       State
	db          *bolt.DB
	savePending bool
}

func (srv *Server) removeDeleted(p Playlist) Playlist {
	var r Playlist
	for _, id := range p {
		if !srv.hasSong(id) {
			continue
		}
		r = append(r, id)
	}
	return r
}

type PlaylistInfo []listItem

func (srv *Server) playlistInfo(p Playlist) PlaylistInfo {
	r := make(PlaylistInfo, len(p))
	for idx, id := range p {
		info, _ := srv.getSong(id)
		r[idx] = listItem{
			ID:   id,
			Info: info,
		}
	}
	return r
}

var dir = filepath.Join("server")

func New(stateFile, central string) (*Server, error) {
	srv := Server{
		ch:          make(chan interface{}),
		audioch:     make(chan interface{}),
		Protocols:   protocol.Map(),
		Playlists:   make(map[string]Playlist),
		MinDuration: time.Second * 30,
		centralURL:  central,
		inprogress:  make(map[codec.ID]bool),
	}
	db, err := bolt.Open(stateFile, 0600, nil)
	if err != nil {
		return nil, err
	}
	srv.db = db
	if err := srv.restore(); err != nil {
		log.Println(err)
	}
	log.Println("started from", stateFile)
	go srv.commands()
	go srv.audio()
	return &srv, nil
}

const (
	dbBucket = "bucket"
	dbServer = "server"
)

func (srv *Server) restore() error {
	decode := func(name string, dst interface{}) error {
		var data []byte
		err := srv.db.View(func(tx *bolt.Tx) error {
			b := tx.Bucket([]byte(dbBucket))
			if b == nil {
				return fmt.Errorf("unknown bucket: %v", dbBucket)
			}
			data = b.Get([]byte(name))
			return nil
		})
		if err != nil {
			return err
		}
		gr, err := gzip.NewReader(bytes.NewReader(data))
		if err != nil {
			return err
		}
		defer gr.Close()
		return gob.NewDecoder(gr).Decode(dst)
	}
	if err := decode(dbServer, srv); err != nil {
		return err
	}
	return nil
}

func (srv *Server) save() error {
	defer func() {
		srv.savePending = false
	}()
	store := map[string]interface{}{
		dbServer: srv,
	}
	tostore := make(map[string][]byte)
	for name, data := range store {
		f := new(bytes.Buffer)
		gz := gzip.NewWriter(f)
		enc := gob.NewEncoder(gz)
		if err := enc.Encode(data); err != nil {
			return err
		}
		if err := gz.Flush(); err != nil {
			return err
		}
		if err := gz.Close(); err != nil {
			return err
		}
		tostore[name] = f.Bytes()
	}
	err := srv.db.Update(func(tx *bolt.Tx) error {
		b, err := tx.CreateBucketIfNotExists([]byte(dbBucket))
		if err != nil {
			return err
		}
		for name, data := range tostore {
			if err := b.Put([]byte(name), data); err != nil {
				return err
			}
		}
		return nil
	})
	if err != nil {
		return err
	}
	log.Println("save to db complete")
	return nil
}

func (srv *Server) GetInstance(name, key string) (protocol.Instance, error) {
	prots, ok := srv.Protocols[name]
	if !ok {
		return nil, fmt.Errorf("unknown protocol: %s", name)
	}
	inst := prots[key]
	if inst == nil {
		return nil, fmt.Errorf("unknown key: %s", key)
	}
	return inst, nil
}

func (srv *Server) protocolRefresh(protocol, key string, list, doDelete bool) error {
	inst, err := srv.GetInstance(protocol, key)
	if err != nil {
		return err
	}
	f := inst.Refresh
	if list {
		f = inst.List
	}
	songs, err := f()
	if err != nil {
		return err
	}
	for k, v := range songs {
		if v.Time > 0 && v.Time < srv.MinDuration {
			delete(songs, k)
		}
	}
	if doDelete {
		srv.ch <- cmdRemoveDeleted{}
	}
	return err
}

type PlaylistChange [][]string

func (srv *Server) playlistChange(p Playlist, plc PlaylistChange, isq bool) (pl Playlist, cleared bool, err error) {
	m := make([]SongID, len(p))
	copy(m, p)
	for _, c := range plc {
		cmd := c[0]
		var arg string
		if len(c) > 1 {
			arg = c[1]
		}
		switch cmd {
		case "clear":
			cleared = true
			for i := range m {
				m[i] = ""
			}
		case "rem":
			i, err := strconv.Atoi(arg)
			if err != nil {
				return nil, false, err
			}
			if len(m) <= i {
				return nil, false, fmt.Errorf("unknown index: %v", i)
			}
			m[i] = ""
		case "add":
			m = append(m, SongID(arg))
		default:
			return nil, false, fmt.Errorf("unknown command: %v", cmd)
		}
	}
	for _, id := range m {
		if id != "" {
			pl = append(pl, id)
		}
	}
	return
}

type listItem struct {
	ID   SongID
	Info *codec.SongInfo
}

type Status struct {
	// Playback state
	State State
	// Song ID.
	Song     SongID
	SongInfo codec.SongInfo
	// Elapsed time of current song.
	Elapsed time.Duration
	// Duration of current song.
	Time       time.Duration
	Random     bool
	Repeat     bool
	Username   string
	Hostname   string
	CentralURL string
}

func (srv *Server) request(path string, body interface{}) (io.ReadCloser, error) {
	// TODO: srv.Token is subject to a race condition because this function is
	// called in go routines in the control loop, and srv.Token is set in the
	// main go routine.
	tv := url.Values{"token": []string{srv.Token}}.Encode()
	var br io.Reader
	if body != nil {
		b, err := json.Marshal(body)
		if err != nil {
			return nil, err
		}
		br = bytes.NewReader(b)
	}
	r, err := http.Post(srv.centralURL+path+"?"+tv, "application/json", br)
	if err != nil {
		return nil, err
	}
	if r.StatusCode != 200 {
		b, err := ioutil.ReadAll(r.Body)
		r.Body.Close()
		if err != nil {
			return nil, err
		}
		return nil, fmt.Errorf("%s: %v: %s", path, r.Status, b)
	}
	return r.Body, nil
}

// name and key may be empty to match all
func (srv *Server) putSource(protocol, key string) error {
	var ss []*models.Source
	for prot, m := range srv.Protocols {
		if prot != protocol && protocol != "" {
			continue
		}
		for name, p := range m {
			if name != key && key != "" {
				continue
			}
			buf := new(bytes.Buffer)
			gw := gzip.NewWriter(buf)
			if err := gob.NewEncoder(gw).Encode(p); err != nil {
				return err
			}
			if err := gw.Close(); err != nil {
				return err
			}
			ss = append(ss, &models.Source{
				Protocol: prot,
				Name:     name,
				Blob:     buf.Bytes(),
			})
		}
	}
	r, err := srv.request("/api/source/set", &ss)
	if err != nil {
		return fmt.Errorf("could not set sources: %v", err)
	}
	r.Close()
	return nil
}

func (srv *Server) getSong(id SongID) (*codec.SongInfo, error) {
	name, key, cid := id.Triple()
	p, ok := srv.Protocols[name]
	if !ok {
		return nil, fmt.Errorf("unknown protocol: %s", name)
	}
	inst, ok := p[key]
	if !ok {
		return nil, fmt.Errorf("unknown instance: %s", key)
	}
	return inst.Info(cid)
}

func (srv *Server) hasSong(id SongID) bool {
	info, _ := srv.getSong(id)
	return info != nil
}
<commit_msg>Sync data after refresh<commit_after>// Package server implements the mog protocol.
package server

import (
	"bytes"
	"compress/gzip"
	crand "crypto/rand"
	"encoding/gob"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math"
	"math/big"
	"math/rand"
	"net/http"
	"net/url"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/boltdb/bolt"
	"github.com/mjibson/mog/codec"
	"github.com/mjibson/mog/models"
	"github.com/mjibson/mog/protocol"
	"github.com/pkg/browser"
)

func init() {
	i, err := crand.Int(crand.Reader, big.NewInt(math.MaxInt64))
	if err != nil {
		return
	}
	rand.Seed(i.Int64())
}

func printErr(e error) {
	log.Println(e)
	b := make([]byte, 4096)
	runtime.Stack(b, false)
	println(string(b))
}

func ListenAndServe(stateFile, addr, central string, devMode bool) error {
	server, err := New(stateFile, central)
	if err != nil {
		return err
	}
	if !devMode {
		host := addr
		if strings.HasPrefix(host, ":") {
			host = "localhost" + host
		}
		err := browser.OpenURL("http://" + host + "/")
		if err != nil {
			log.Println(err)
		}
	}
	return server.ListenAndServe(addr, devMode)
}

const (
	statePlay State = iota
	stateStop
	statePause
)

type State int

func (s State) String() string {
	switch s {
	case statePlay:
		return "play"
	case stateStop:
		return "stop"
	case statePause:
		return "pause"
	}
	return ""
}

type Playlist []SongID

type SongID codec.ID

func (s SongID) MarshalJSON() ([]byte, error) {
	sp := strings.SplitN(string(s), codec.IdSep, 3)
	if len(sp) != 3 {
		return json.Marshal("")
	}
	return json.Marshal(struct {
		Protocol string
		Key      string
		ID       string
		UID      string
	}{
		sp[0],
		sp[1],
		sp[2],
		string(s),
	})
}

func (s SongID) Protocol() string {
	return codec.ID(s).Top()
}

func (s SongID) Key() string {
	_, c := codec.ID(s).Pop()
	return c.Top()
}

func (s SongID) ID() codec.ID {
	_, c := codec.ID(s).Pop()
	_, c = c.Pop()
	return c
}

func (s SongID) Triple() (protocol, key string, id codec.ID) {
	protocol, id = codec.ID(s).Pop()
	key, id = id.Pop()
	return
}

type Server struct {
	Queue     Playlist
	Playlists map[string]Playlist

	Username string
	Token    string

	Repeat      bool
	Random      bool
	Protocols   map[string]map[string]protocol.Instance
	MinDuration time.Duration

	// Current song data.
	PlaylistIndex int
	songID        SongID
	song          codec.Song
	info          codec.SongInfo
	elapsed       time.Duration

	centralURL  string
	inprogress  map[codec.ID]bool
	ch          chan interface{}
	audioch     chan interface{}
	state       State
	db          *bolt.DB
	savePending bool
}

func (srv *Server) removeDeleted(p Playlist) Playlist {
	var r Playlist
	for _, id := range p {
		if !srv.hasSong(id) {
			continue
		}
		r = append(r, id)
	}
	return r
}

type PlaylistInfo []listItem

func (srv *Server) playlistInfo(p Playlist) PlaylistInfo {
	r := make(PlaylistInfo, len(p))
	for idx, id := range p {
		info, _ := srv.getSong(id)
		r[idx] = listItem{
			ID:   id,
			Info: info,
		}
	}
	return r
}

var dir = filepath.Join("server")

func New(stateFile, central string) (*Server, error) {
	srv := Server{
		ch:          make(chan interface{}),
		audioch:     make(chan interface{}),
		Protocols:   protocol.Map(),
		Playlists:   make(map[string]Playlist),
		MinDuration: time.Second * 30,
		centralURL:  central,
		inprogress:  make(map[codec.ID]bool),
	}
	db, err := bolt.Open(stateFile, 0600, nil)
	if err != nil {
		return nil, err
	}
	srv.db = db
	if err := srv.restore(); err != nil {
		log.Println(err)
	}
	log.Println("started from", stateFile)
	go srv.commands()
	go srv.audio()
	return &srv, nil
}

const (
	dbBucket = "bucket"
	dbServer = "server"
)

func (srv *Server) restore() error {
	decode := func(name string, dst interface{}) error {
		var data []byte
		err := srv.db.View(func(tx *bolt.Tx) error {
			b := tx.Bucket([]byte(dbBucket))
			if b == nil {
				return fmt.Errorf("unknown bucket: %v", dbBucket)
			}
			data = b.Get([]byte(name))
			return nil
		})
		if err != nil {
			return err
		}
		gr, err := gzip.NewReader(bytes.NewReader(data))
		if err != nil {
			return err
		}
		defer gr.Close()
		return gob.NewDecoder(gr).Decode(dst)
	}
	if err := decode(dbServer, srv); err != nil {
		return err
	}
	return nil
}

func (srv *Server) save() error {
	defer func() {
		srv.savePending = false
	}()
	store := map[string]interface{}{
		dbServer: srv,
	}
	tostore := make(map[string][]byte)
	for name, data := range store {
		f := new(bytes.Buffer)
		gz := gzip.NewWriter(f)
		enc := gob.NewEncoder(gz)
		if err := enc.Encode(data); err != nil {
			return err
		}
		if err := gz.Flush(); err != nil {
			return err
		}
		if err := gz.Close(); err != nil {
			return err
		}
		tostore[name] = f.Bytes()
	}
	err := srv.db.Update(func(tx *bolt.Tx) error {
		b, err := tx.CreateBucketIfNotExists([]byte(dbBucket))
		if err != nil {
			return err
		}
		for name, data := range tostore {
			if err := b.Put([]byte(name), data); err != nil {
				return err
			}
		}
		return nil
	})
	if err != nil {
		return err
	}
	log.Println("save to db complete")
	return nil
}

func (srv *Server) GetInstance(name, key string) (protocol.Instance, error) {
	prots, ok := srv.Protocols[name]
	if !ok {
		return nil, fmt.Errorf("unknown protocol: %s", name)
	}
	inst := prots[key]
	if inst == nil {
		return nil, fmt.Errorf("unknown key: %s", key)
	}
	return inst, nil
}

func (srv *Server) protocolRefresh(protocol, key string, list, doDelete bool) error {
	inst, err := srv.GetInstance(protocol, key)
	if err != nil {
		return err
	}
	f := inst.Refresh
	if list {
		f = inst.List
	}
	songs, err := f()
	if err != nil {
		return err
	}
	for k, v := range songs {
		if v.Time > 0 && v.Time < srv.MinDuration {
			delete(songs, k)
		}
	}
	if doDelete {
		srv.ch <- cmdRemoveDeleted{}
	}
	if srv.Token != "" {
		go func() {
			if err := srv.putSource(protocol, key); err != nil {
				srv.ch <- cmdError(err)
			}
		}()
	}
	return err
}

type PlaylistChange [][]string

func (srv *Server) playlistChange(p Playlist, plc PlaylistChange, isq bool) (pl Playlist, cleared bool, err error) {
	m := make([]SongID, len(p))
	copy(m, p)
	for _, c := range plc {
		cmd := c[0]
		var arg string
		if len(c) > 1 {
			arg = c[1]
		}
		switch cmd {
		case "clear":
			cleared = true
			for i := range m {
				m[i] = ""
			}
		case "rem":
			i, err := strconv.Atoi(arg)
			if err != nil {
				return nil, false, err
			}
			if len(m) <= i {
				return nil, false, fmt.Errorf("unknown index: %v", i)
			}
			m[i] = ""
		case "add":
			m = append(m, SongID(arg))
		default:
			return nil, false, fmt.Errorf("unknown command: %v", cmd)
		}
	}
	for _, id := range m {
		if id != "" {
			pl = append(pl, id)
		}
	}
	return
}

type listItem struct {
	ID   SongID
	Info *codec.SongInfo
}

type Status struct {
	// Playback state
	State State
	// Song ID.
	Song     SongID
	SongInfo codec.SongInfo
	// Elapsed time of current song.
	Elapsed time.Duration
	// Duration of current song.
	Time       time.Duration
	Random     bool
	Repeat     bool
	Username   string
	Hostname   string
	CentralURL string
}

func (srv *Server) request(path string, body interface{}) (io.ReadCloser, error) {
	// TODO: srv.Token is subject to a race condition because this function is
	// called in go routines in the control loop, and srv.Token is set in the
	// main go routine.
	tv := url.Values{"token": []string{srv.Token}}.Encode()
	var br io.Reader
	if body != nil {
		b, err := json.Marshal(body)
		if err != nil {
			return nil, err
		}
		br = bytes.NewReader(b)
	}
	r, err := http.Post(srv.centralURL+path+"?"+tv, "application/json", br)
	if err != nil {
		return nil, err
	}
	if r.StatusCode != 200 {
		b, err := ioutil.ReadAll(r.Body)
		r.Body.Close()
		if err != nil {
			return nil, err
		}
		return nil, fmt.Errorf("%s: %v: %s", path, r.Status, b)
	}
	return r.Body, nil
}

// name and key may be empty to match all
func (srv *Server) putSource(protocol, key string) error {
	var ss []*models.Source
	for prot, m := range srv.Protocols {
		if prot != protocol && protocol != "" {
			continue
		}
		for name, p := range m {
			if name != key && key != "" {
				continue
			}
			buf := new(bytes.Buffer)
			gw := gzip.NewWriter(buf)
			if err := gob.NewEncoder(gw).Encode(p); err != nil {
				return err
			}
			if err := gw.Close(); err != nil {
				return err
			}
			ss = append(ss, &models.Source{
				Protocol: prot,
				Name:     name,
				Blob:     buf.Bytes(),
			})
		}
	}
	r, err := srv.request("/api/source/set", &ss)
	if err != nil {
		return fmt.Errorf("could not set sources: %v", err)
	}
	r.Close()
	return nil
}

func (srv *Server) getSong(id SongID) (*codec.SongInfo, error) {
	name, key, cid := id.Triple()
	p, ok := srv.Protocols[name]
	if !ok {
		return nil, fmt.Errorf("unknown protocol: %s", name)
	}
	inst, ok := p[key]
	if !ok {
		return nil, fmt.Errorf("unknown instance: %s", key)
	}
	return inst.Info(cid)
}

func (srv *Server) hasSong(id SongID) bool {
	info, _ := srv.getSong(id)
	return info != nil
}
<|endoftext|>


<commit_before>package local_test

import (
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"testing"

	"github.com/cheekybits/is"
	"github.com/graymeta/stow"
	"github.com/graymeta/stow/local"
)

func TestItemReader(t *testing.T) {
	is := is.New(t)
	testDir, teardown, err := setup()
	is.NoErr(err)
	defer teardown()

	cfg := stow.ConfigMap{"path": testDir}
	l, err := stow.Dial(local.Kind, cfg)
	is.NoErr(err)
	is.OK(l)
	containers, cursor, err := l.Containers("t", stow.CursorStart, 10)
	is.NoErr(err)
	is.OK(containers)
	is.Equal(cursor, "")
	three, err := l.Container(containers[0].ID())

	items, cursor, err := three.Items("", stow.CursorStart, 10)
	is.NoErr(err)
	is.Equal(cursor, "")
	item1 := items[0]

	rc, err := item1.Open()
	defer rc.Close()
	is.NoErr(err)
	b, err := ioutil.ReadAll(rc)
	is.NoErr(err)
	is.Equal("3.1", string(b))

}

func TestHardlink(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.SkipNow()
	}
	is := is.New(t)
	testDir, teardown, err := setup()
	is.NoErr(err)
	defer teardown()

	cfg := stow.ConfigMap{"path": testDir}
	l, err := stow.Dial(local.Kind, cfg)
	is.NoErr(err)
	is.OK(l)

	containers, cursor, err := l.Containers("z", stow.CursorStart, 10)
	is.NoErr(err)
	is.OK(containers)
	is.Equal(cursor, "")

	links, err := l.Container(containers[0].ID())
	is.NoErr(err)

	items, cursor, err := links.Items("", stow.CursorStart, 10)
	is.NoErr(err)
	is.Equal(cursor, "")

	for _, item := range items {
		if item.Name() == "hardlink" {
			meta, err := item.Metadata()
			is.NoErr(err)
			is.OK(meta)

			is.Equal(meta["is_dir"], false)
			is.True(meta["is_hardlink"])
			is.False(meta["is_symlink"])
			break
		}
	}
}

func TestSymLink(t *testing.T) {
	is := is.New(t)
	testDir, teardown, err := setup()
	is.NoErr(err)
	defer teardown()

	cfg := stow.ConfigMap{"path": testDir}
	l, err := stow.Dial(local.Kind, cfg)
	is.NoErr(err)
	is.OK(l)

	containers, cursor, err := l.Containers("z", stow.CursorStart, 10)
	is.NoErr(err)
	is.OK(containers)
	is.Equal(cursor, "")

	links, err := l.Container(containers[0].ID())
	is.NoErr(err)

	items, cursor, err := links.Items("", stow.CursorStart, 10)
	is.NoErr(err)
	is.Equal(cursor, "")

	for _, item := range items {
		if item.Name() == "symlink" {
			meta, err := item.Metadata()
			is.NoErr(err)
			is.OK(meta)

			is.Equal(meta["is_dir"], false)
			is.False(meta["is_hardlink"])
			is.True(meta["is_symlink"])

			linkStr, ok := meta["link"].(string)
			is.True(ok)
			is.OK(linkStr)

			is.True(strings.Contains(linkStr, "symtarget"))
			break
		}
	}
}

func TestItemFromURL(t *testing.T) {
	//A file-system path /a/b/c can be broken into (container, name)
	//either as (/a, b/c) or as (/a/b,c). This test imposes
	//the convention that the name is the last path component.
	is := is.New(t)
	testDir, teardown, err := setup()
	is.NoErr(err)
	defer teardown()
	os.MkdirAll(filepath.Join(testDir, "a", "b"), 0777)
	ioutil.WriteFile(filepath.Join(testDir, "a", "f2"), []byte("abc"), 0666)
	ioutil.WriteFile(filepath.Join(testDir, "a", "b", "f3"), []byte("abc"), 0666)

	cfg := stow.ConfigMap{"path": testDir}
	l, err := stow.Dial(local.Kind, cfg)
	is.NoErr(err)

	item2, err := l.ItemByURL(&url.URL{
		Scheme: "file",
		Path:   filepath.Join(testDir, "a", "f2"),
	})
	is.NoErr(err)
	is.Equal(item2.Name(), "f2")

	item3, err := l.ItemByURL(&url.URL{
		Scheme: "file",
		Path:   filepath.Join(testDir, "a", "b", "f3"),
	})
	is.NoErr(err)
	is.Equal(item3.Name(), "f3")
}
<commit_msg>local: fix dropped error<commit_after>package local_test

import (
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"testing"

	"github.com/cheekybits/is"
	"github.com/graymeta/stow"
	"github.com/graymeta/stow/local"
)

func TestItemReader(t *testing.T) {
	is := is.New(t)
	testDir, teardown, err := setup()
	is.NoErr(err)
	defer teardown()

	cfg := stow.ConfigMap{"path": testDir}
	l, err := stow.Dial(local.Kind, cfg)
	is.NoErr(err)
	is.OK(l)
	containers, cursor, err := l.Containers("t", stow.CursorStart, 10)
	is.NoErr(err)
	is.OK(containers)
	is.Equal(cursor, "")
	three, err := l.Container(containers[0].ID())
	is.NoErr(err)

	items, cursor, err := three.Items("", stow.CursorStart, 10)
	is.NoErr(err)
	is.Equal(cursor, "")
	item1 := items[0]

	rc, err := item1.Open()
	is.NoErr(err)
	defer rc.Close()
	b, err := ioutil.ReadAll(rc)
	is.NoErr(err)
	is.Equal("3.1", string(b))

}

func TestHardlink(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.SkipNow()
	}
	is := is.New(t)
	testDir, teardown, err := setup()
	is.NoErr(err)
	defer teardown()

	cfg := stow.ConfigMap{"path": testDir}
	l, err := stow.Dial(local.Kind, cfg)
	is.NoErr(err)
	is.OK(l)

	containers, cursor, err := l.Containers("z", stow.CursorStart, 10)
	is.NoErr(err)
	is.OK(containers)
	is.Equal(cursor, "")

	links, err := l.Container(containers[0].ID())
	is.NoErr(err)

	items, cursor, err := links.Items("", stow.CursorStart, 10)
	is.NoErr(err)
	is.Equal(cursor, "")

	for _, item := range items {
		if item.Name() == "hardlink" {
			meta, err := item.Metadata()
			is.NoErr(err)
			is.OK(meta)

			is.Equal(meta["is_dir"], false)
			is.True(meta["is_hardlink"])
			is.False(meta["is_symlink"])
			break
		}
	}
}

func TestSymLink(t *testing.T) {
	is := is.New(t)
	testDir, teardown, err := setup()
	is.NoErr(err)
	defer teardown()

	cfg := stow.ConfigMap{"path": testDir}
	l, err := stow.Dial(local.Kind, cfg)
	is.NoErr(err)
	is.OK(l)

	containers, cursor, err := l.Containers("z", stow.CursorStart, 10)
	is.NoErr(err)
	is.OK(containers)
	is.Equal(cursor, "")

	links, err := l.Container(containers[0].ID())
	is.NoErr(err)

	items, cursor, err := links.Items("", stow.CursorStart, 10)
	is.NoErr(err)
	is.Equal(cursor, "")

	for _, item := range items {
		if item.Name() == "symlink" {
			meta, err := item.Metadata()
			is.NoErr(err)
			is.OK(meta)

			is.Equal(meta["is_dir"], false)
			is.False(meta["is_hardlink"])
			is.True(meta["is_symlink"])

			linkStr, ok := meta["link"].(string)
			is.True(ok)
			is.OK(linkStr)

			is.True(strings.Contains(linkStr, "symtarget"))
			break
		}
	}
}

func TestItemFromURL(t *testing.T) {
	//A file-system path /a/b/c can be broken into (container, name)
	//either as (/a, b/c) or as (/a/b,c). This test imposes
	//the convention that the name is the last path component.
	is := is.New(t)
	testDir, teardown, err := setup()
	is.NoErr(err)
	defer teardown()
	os.MkdirAll(filepath.Join(testDir, "a", "b"), 0777)
	ioutil.WriteFile(filepath.Join(testDir, "a", "f2"), []byte("abc"), 0666)
	ioutil.WriteFile(filepath.Join(testDir, "a", "b", "f3"), []byte("abc"), 0666)

	cfg := stow.ConfigMap{"path": testDir}
	l, err := stow.Dial(local.Kind, cfg)
	is.NoErr(err)

	item2, err := l.ItemByURL(&url.URL{
		Scheme: "file",
		Path:   filepath.Join(testDir, "a", "f2"),
	})
	is.NoErr(err)
	is.Equal(item2.Name(), "f2")

	item3, err := l.ItemByURL(&url.URL{
		Scheme: "file",
		Path:   filepath.Join(testDir, "a", "b", "f3"),
	})
	is.NoErr(err)
	is.Equal(item3.Name(), "f3")
}
<|endoftext|>


<commit_before>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Daniel Theophanes (kardianos@gmail.com)

package server

import (
	"fmt"
	"io/ioutil"
	"net/url"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/elastic/gosigar"

	"github.com/cockroachdb/cockroach/base"
	"github.com/cockroachdb/cockroach/gossip/resolver"
	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/security"
	"github.com/cockroachdb/cockroach/storage"
	"github.com/cockroachdb/cockroach/storage/engine"
	"github.com/cockroachdb/cockroach/util/log"
	"github.com/cockroachdb/cockroach/util/stop"
)

// Context defaults.
const (
	defaultCGroupMemPath      = "/sys/fs/cgroup/memory/memory.limit_in_bytes"
	defaultAddr               = ":" + base.DefaultPort
	defaultMaxOffset          = 250 * time.Millisecond
	defaultCacheSize          = 512 << 20 // 512 MB
	defaultMemtableBudget     = 512 << 20 // 512 MB
	defaultScanInterval       = 10 * time.Minute
	defaultScanMaxIdleTime    = 5 * time.Second
	defaultMetricsFrequency   = 10 * time.Second
	defaultTimeUntilStoreDead = 5 * time.Minute
	defaultBalanceMode        = storage.BalanceModeUsage
)

// Context holds parameters needed to setup a server.
// Calling "cli".initFlags(ctx *Context) will initialize Context using
// command flags. Keep in sync with "cli/flags.go".
type Context struct {
	// Embed the base context.
	base.Context

	// Addr is the host:port to bind for HTTP/RPC traffic.
	Addr string

	// Stores is specified to enable durable key-value storage.
	Stores StoreSpecList

	// Attrs specifies a colon-separated list of node topography or machine
	// capabilities, used to match capabilities or location preferences specified
	// in zone configs.
	Attrs string

	// Maximum clock offset for the cluster.
	MaxOffset time.Duration

	// JoinUsing is a comma-separated list of node addresses that
	// act as bootstrap hosts for connecting to the gossip network.
	JoinUsing string

	// Enables linearizable behaviour of operations on this node by making sure
	// that no commit timestamp is reported back to the client until all other
	// node clocks have necessarily passed it.
	Linearizable bool

	// CacheSize is the amount of memory in bytes to use for caching data.
	// The value is split evenly between the stores if there are more than one.
	CacheSize uint64

	// MemtableBudget is the amount of memory in bytes to use for the memory
	// table. The value is split evenly between the stores if there are more than one.
	MemtableBudget uint64

	// BalanceMode determines how this node makes balancing decisions.
	BalanceMode storage.BalanceMode

	// Parsed values.

	// Engines is the storage instances specified by Stores.
	Engines []engine.Engine

	// NodeAttributes is the parsed representation of Attrs.
	NodeAttributes roachpb.Attributes

	// GossipBootstrapResolvers is a list of gossip resolvers used
	// to find bootstrap nodes for connecting to the gossip network.
	GossipBootstrapResolvers []resolver.Resolver

	// ScanInterval determines a duration during which each range should be
	// visited approximately once by the range scanner.
	ScanInterval time.Duration

	// ScanMaxIdleTime is the maximum time the scanner will be idle between ranges.
	// If enabled (> 0), the scanner may complete in less than ScanInterval for small
	// stores.
	ScanMaxIdleTime time.Duration

	// MetricsFrequency determines the frequency at which the server should
	// record internal metrics.
	MetricsFrequency time.Duration

	// TimeUntilStoreDead is the time after which if there is no new gossiped
	// information about a store, it is considered dead.
	TimeUntilStoreDead time.Duration
}

// GetTotalMemory returns either the total system memory or if possible the
// cgroups available memory.
func GetTotalMemory() (uint64, error) {
	mem := gosigar.Mem{}
	if err := mem.Get(); err != nil {
		return 0, err
	}
	totalMem := mem.Total
	var cgAvlMem uint64
	if runtime.GOOS == "linux" {
		var err error
		var buf []byte
		if buf, err = ioutil.ReadFile(defaultCGroupMemPath); err != nil {
			if log.V(1) {
				log.Infof("can't read available memory from cgroups (%s)", err)
			}
			return totalMem, nil
		}
		if cgAvlMem, err = strconv.ParseUint(strings.TrimSpace(string(buf)), 10, 64); err != nil {
			if log.V(1) {
				log.Infof("can't parse available memory from cgroups (%s)", err)
			}
			return totalMem, nil
		}
		return cgAvlMem, nil
	}
	return totalMem, nil
}

// NewContext returns a Context with default values.
func NewContext() *Context {
	ctx := &Context{}
	ctx.InitDefaults()
	return ctx
}

// InitDefaults sets up the default values for a Context.
//
// Note: This method should only perform simple initialization of fields
// because it is called very early in the lifetime of a cockroach process at
// which point we do not know if we are initializing a server or using the
// cli. Do not call any functions which could log or error. In fact, it is best
// if you don't call any other functions at all.
func (ctx *Context) InitDefaults() {
	ctx.Context.InitDefaults()
	ctx.Addr = defaultAddr
	ctx.MaxOffset = defaultMaxOffset
	ctx.CacheSize = defaultCacheSize
	ctx.MemtableBudget = defaultMemtableBudget
	ctx.ScanInterval = defaultScanInterval
	ctx.ScanMaxIdleTime = defaultScanMaxIdleTime
	ctx.MetricsFrequency = defaultMetricsFrequency
	ctx.TimeUntilStoreDead = defaultTimeUntilStoreDead
	ctx.BalanceMode = defaultBalanceMode
	ctx.Stores.Specs = append(ctx.Stores.Specs, StoreSpec{Path: "cockroach-data"})
}

// InitStores initializes ctx.Engines based on ctx.Stores.
func (ctx *Context) InitStores(stopper *stop.Stopper) error {
	// TODO(peter): The comments and docs say that CacheSize and MemtableBudget
	// are split evenly if there are multiple stores, but we aren't doing that
	// currently.
	for _, spec := range ctx.Stores.Specs {
		var sizeInBytes = spec.SizeInBytes
		if spec.InMemory {
			if spec.SizePercent > 0 {
				sysMem, err := GetTotalMemory()
				if err != nil {
					return fmt.Errorf("could not retrieve system memory")
				}
				sizeInBytes = int64(float64(sysMem) * spec.SizePercent / 100)
			}
			if sizeInBytes != 0 && sizeInBytes < minimumStoreSize {
				return fmt.Errorf("%f%% of memory is only %s bytes, which is below the minimum requirement of %s",
					spec.SizePercent, humanize.IBytes(uint64(sizeInBytes)), humanize.IBytes(uint64(minimumStoreSize)))
			}
			ctx.Engines = append(ctx.Engines, engine.NewInMem(spec.Attributes, uint64(sizeInBytes), stopper))
		} else {
			if spec.SizePercent > 0 {
				fileSystemUsage := gosigar.FileSystemUsage{}
				if err := fileSystemUsage.Get(spec.Path); err != nil {
					return err
				}
				sizeInBytes = int64(float64(fileSystemUsage.Total) * spec.SizePercent / 100)
			}
			if sizeInBytes != 0 && sizeInBytes < minimumStoreSize {
				return fmt.Errorf("%f%% of %s's total free space is only %s bytes, which is below the minimum requirement of %s",
					spec.SizePercent, spec.Path, humanize.IBytes(uint64(sizeInBytes)),
					humanize.IBytes(uint64(minimumStoreSize)))
			}
			ctx.Engines = append(ctx.Engines, engine.NewRocksDB(spec.Attributes, spec.Path, ctx.CacheSize,
				ctx.MemtableBudget, sizeInBytes, stopper))
		}
	}
	if len(ctx.Engines) == 1 {
		log.Infof("1 storage engine initialized")
	} else {
		log.Infof("%d storage engines initialized", len(ctx.Engines))
	}
	return nil
}

// InitNode parses node attributes and initializes the gossip bootstrap
// resolvers.
func (ctx *Context) InitNode() error {
	// Initialize attributes.
	ctx.NodeAttributes = parseAttributes(ctx.Attrs)

	// Get the gossip bootstrap resolvers.
	resolvers, err := ctx.parseGossipBootstrapResolvers()
	if err != nil {
		return err
	}
	if len(resolvers) > 0 {
		ctx.GossipBootstrapResolvers = resolvers
	}

	return nil
}

// AdminURL returns the URL for the admin UI.
func (ctx *Context) AdminURL() string {
	return fmt.Sprintf("%s://%s", ctx.HTTPRequestScheme(), ctx.Addr)
}

// PGURL returns the URL for the postgres endpoint.
func (ctx *Context) PGURL(user string) *url.URL {
	// Try to convert path to an absolute path. Failing to do so return path
	// unchanged.
	absPath := func(path string) string {
		r, err := filepath.Abs(path)
		if err != nil {
			return path
		}
		return r
	}

	options := url.Values{}
	if ctx.Insecure {
		options.Add("sslmode", "disable")
	} else {
		options.Add("sslmode", "verify-full")
		options.Add("sslcert", absPath(security.ClientCertPath(ctx.Certs, user)))
		options.Add("sslkey", absPath(security.ClientKeyPath(ctx.Certs, user)))
		options.Add("sslrootcert", absPath(security.CACertPath(ctx.Certs)))
	}
	return &url.URL{
		Scheme:   "postgresql",
		User:     url.User(user),
		Host:     ctx.Addr,
		RawQuery: options.Encode(),
	}
}

// parseGossipBootstrapResolvers parses a comma-separated list of
// gossip bootstrap resolvers.
func (ctx *Context) parseGossipBootstrapResolvers() ([]resolver.Resolver, error) {
	var bootstrapResolvers []resolver.Resolver
	addresses := strings.Split(ctx.JoinUsing, ",")
	for _, address := range addresses {
		if len(address) == 0 {
			continue
		}
		resolver, err := resolver.NewResolver(&ctx.Context, address)
		if err != nil {
			return nil, err
		}
		bootstrapResolvers = append(bootstrapResolvers, resolver)
	}

	return bootstrapResolvers, nil
}

// parseAttributes parses a colon-separated list of strings,
// filtering empty strings (i.e. "::" will yield no attributes.
// Returns the list of strings as Attributes.
func parseAttributes(attrsStr string) roachpb.Attributes {
	var filtered []string
	for _, attr := range strings.Split(attrsStr, ":") {
		if len(attr) != 0 {
			filtered = append(filtered, attr)
		}
	}
	return roachpb.Attributes{Attrs: filtered}
}
<commit_msg>Storage: Split rocksdb cache evenly between instances.<commit_after>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Daniel Theophanes (kardianos@gmail.com)

package server

import (
	"fmt"
	"io/ioutil"
	"net/url"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/elastic/gosigar"

	"github.com/cockroachdb/cockroach/base"
	"github.com/cockroachdb/cockroach/gossip/resolver"
	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/security"
	"github.com/cockroachdb/cockroach/storage"
	"github.com/cockroachdb/cockroach/storage/engine"
	"github.com/cockroachdb/cockroach/util/log"
	"github.com/cockroachdb/cockroach/util/stop"
)

// Context defaults.
const (
	defaultCGroupMemPath      = "/sys/fs/cgroup/memory/memory.limit_in_bytes"
	defaultAddr               = ":" + base.DefaultPort
	defaultMaxOffset          = 250 * time.Millisecond
	defaultCacheSize          = 512 << 20 // 512 MB
	defaultMemtableBudget     = 512 << 20 // 512 MB
	defaultScanInterval       = 10 * time.Minute
	defaultScanMaxIdleTime    = 5 * time.Second
	defaultMetricsFrequency   = 10 * time.Second
	defaultTimeUntilStoreDead = 5 * time.Minute
	defaultBalanceMode        = storage.BalanceModeUsage
)

// Context holds parameters needed to setup a server.
// Calling "cli".initFlags(ctx *Context) will initialize Context using
// command flags. Keep in sync with "cli/flags.go".
type Context struct {
	// Embed the base context.
	base.Context

	// Addr is the host:port to bind for HTTP/RPC traffic.
	Addr string

	// Stores is specified to enable durable key-value storage.
	Stores StoreSpecList

	// Attrs specifies a colon-separated list of node topography or machine
	// capabilities, used to match capabilities or location preferences specified
	// in zone configs.
	Attrs string

	// Maximum clock offset for the cluster.
	MaxOffset time.Duration

	// JoinUsing is a comma-separated list of node addresses that
	// act as bootstrap hosts for connecting to the gossip network.
	JoinUsing string

	// Enables linearizable behaviour of operations on this node by making sure
	// that no commit timestamp is reported back to the client until all other
	// node clocks have necessarily passed it.
	Linearizable bool

	// CacheSize is the amount of memory in bytes to use for caching data.
	// The value is split evenly between the stores if there are more than one.
	CacheSize uint64

	// MemtableBudget is the amount of memory in bytes to use for the memory
	// table. The value is split evenly between the stores if there are more than one.
	MemtableBudget uint64

	// BalanceMode determines how this node makes balancing decisions.
	BalanceMode storage.BalanceMode

	// Parsed values.

	// Engines is the storage instances specified by Stores.
	Engines []engine.Engine

	// NodeAttributes is the parsed representation of Attrs.
	NodeAttributes roachpb.Attributes

	// GossipBootstrapResolvers is a list of gossip resolvers used
	// to find bootstrap nodes for connecting to the gossip network.
	GossipBootstrapResolvers []resolver.Resolver

	// ScanInterval determines a duration during which each range should be
	// visited approximately once by the range scanner.
	ScanInterval time.Duration

	// ScanMaxIdleTime is the maximum time the scanner will be idle between ranges.
	// If enabled (> 0), the scanner may complete in less than ScanInterval for small
	// stores.
	ScanMaxIdleTime time.Duration

	// MetricsFrequency determines the frequency at which the server should
	// record internal metrics.
	MetricsFrequency time.Duration

	// TimeUntilStoreDead is the time after which if there is no new gossiped
	// information about a store, it is considered dead.
	TimeUntilStoreDead time.Duration
}

// GetTotalMemory returns either the total system memory or if possible the
// cgroups available memory.
func GetTotalMemory() (uint64, error) {
	mem := gosigar.Mem{}
	if err := mem.Get(); err != nil {
		return 0, err
	}
	totalMem := mem.Total
	var cgAvlMem uint64
	if runtime.GOOS == "linux" {
		var err error
		var buf []byte
		if buf, err = ioutil.ReadFile(defaultCGroupMemPath); err != nil {
			if log.V(1) {
				log.Infof("can't read available memory from cgroups (%s)", err)
			}
			return totalMem, nil
		}
		if cgAvlMem, err = strconv.ParseUint(strings.TrimSpace(string(buf)), 10, 64); err != nil {
			if log.V(1) {
				log.Infof("can't parse available memory from cgroups (%s)", err)
			}
			return totalMem, nil
		}
		return cgAvlMem, nil
	}
	return totalMem, nil
}

// NewContext returns a Context with default values.
func NewContext() *Context {
	ctx := &Context{}
	ctx.InitDefaults()
	return ctx
}

// InitDefaults sets up the default values for a Context.
//
// Note: This method should only perform simple initialization of fields
// because it is called very early in the lifetime of a cockroach process at
// which point we do not know if we are initializing a server or using the
// cli. Do not call any functions which could log or error. In fact, it is best
// if you don't call any other functions at all.
func (ctx *Context) InitDefaults() {
	ctx.Context.InitDefaults()
	ctx.Addr = defaultAddr
	ctx.MaxOffset = defaultMaxOffset
	ctx.CacheSize = defaultCacheSize
	ctx.MemtableBudget = defaultMemtableBudget
	ctx.ScanInterval = defaultScanInterval
	ctx.ScanMaxIdleTime = defaultScanMaxIdleTime
	ctx.MetricsFrequency = defaultMetricsFrequency
	ctx.TimeUntilStoreDead = defaultTimeUntilStoreDead
	ctx.BalanceMode = defaultBalanceMode
	ctx.Stores.Specs = append(ctx.Stores.Specs, StoreSpec{Path: "cockroach-data"})
}

// InitStores initializes ctx.Engines based on ctx.Stores.
func (ctx *Context) InitStores(stopper *stop.Stopper) error {
	// TODO(peter): The comments and docs say that CacheSize and MemtableBudget
	// are split evenly if there are multiple stores, but we aren't doing that
	// currently. See #4979 and #4980.
	for _, spec := range ctx.Stores.Specs {
		var sizeInBytes = spec.SizeInBytes
		if spec.InMemory {
			if spec.SizePercent > 0 {
				sysMem, err := GetTotalMemory()
				if err != nil {
					return fmt.Errorf("could not retrieve system memory")
				}
				sizeInBytes = int64(float64(sysMem) * spec.SizePercent / 100)
			}
			if sizeInBytes != 0 && sizeInBytes < minimumStoreSize {
				return fmt.Errorf("%f%% of memory is only %s bytes, which is below the minimum requirement of %s",
					spec.SizePercent, humanize.IBytes(uint64(sizeInBytes)), humanize.IBytes(uint64(minimumStoreSize)))
			}
			ctx.Engines = append(ctx.Engines, engine.NewInMem(spec.Attributes, uint64(sizeInBytes), stopper))
		} else {
			if spec.SizePercent > 0 {
				fileSystemUsage := gosigar.FileSystemUsage{}
				if err := fileSystemUsage.Get(spec.Path); err != nil {
					return err
				}
				sizeInBytes = int64(float64(fileSystemUsage.Total) * spec.SizePercent / 100)
			}
			if sizeInBytes != 0 && sizeInBytes < minimumStoreSize {
				return fmt.Errorf("%f%% of %s's total free space is only %s bytes, which is below the minimum requirement of %s",
					spec.SizePercent, spec.Path, humanize.IBytes(uint64(sizeInBytes)),
					humanize.IBytes(uint64(minimumStoreSize)))
			}
			ctx.Engines = append(ctx.Engines, engine.NewRocksDB(spec.Attributes, spec.Path,
				ctx.CacheSize/uint64(len(ctx.Stores.Specs)), ctx.MemtableBudget, sizeInBytes, stopper))
		}
	}
	if len(ctx.Engines) == 1 {
		log.Infof("1 storage engine initialized")
	} else {
		log.Infof("%d storage engines initialized", len(ctx.Engines))
	}
	return nil
}

// InitNode parses node attributes and initializes the gossip bootstrap
// resolvers.
func (ctx *Context) InitNode() error {
	// Initialize attributes.
	ctx.NodeAttributes = parseAttributes(ctx.Attrs)

	// Get the gossip bootstrap resolvers.
	resolvers, err := ctx.parseGossipBootstrapResolvers()
	if err != nil {
		return err
	}
	if len(resolvers) > 0 {
		ctx.GossipBootstrapResolvers = resolvers
	}

	return nil
}

// AdminURL returns the URL for the admin UI.
func (ctx *Context) AdminURL() string {
	return fmt.Sprintf("%s://%s", ctx.HTTPRequestScheme(), ctx.Addr)
}

// PGURL returns the URL for the postgres endpoint.
func (ctx *Context) PGURL(user string) *url.URL {
	// Try to convert path to an absolute path. Failing to do so return path
	// unchanged.
	absPath := func(path string) string {
		r, err := filepath.Abs(path)
		if err != nil {
			return path
		}
		return r
	}

	options := url.Values{}
	if ctx.Insecure {
		options.Add("sslmode", "disable")
	} else {
		options.Add("sslmode", "verify-full")
		options.Add("sslcert", absPath(security.ClientCertPath(ctx.Certs, user)))
		options.Add("sslkey", absPath(security.ClientKeyPath(ctx.Certs, user)))
		options.Add("sslrootcert", absPath(security.CACertPath(ctx.Certs)))
	}
	return &url.URL{
		Scheme:   "postgresql",
		User:     url.User(user),
		Host:     ctx.Addr,
		RawQuery: options.Encode(),
	}
}

// parseGossipBootstrapResolvers parses a comma-separated list of
// gossip bootstrap resolvers.
func (ctx *Context) parseGossipBootstrapResolvers() ([]resolver.Resolver, error) {
	var bootstrapResolvers []resolver.Resolver
	addresses := strings.Split(ctx.JoinUsing, ",")
	for _, address := range addresses {
		if len(address) == 0 {
			continue
		}
		resolver, err := resolver.NewResolver(&ctx.Context, address)
		if err != nil {
			return nil, err
		}
		bootstrapResolvers = append(bootstrapResolvers, resolver)
	}

	return bootstrapResolvers, nil
}

// parseAttributes parses a colon-separated list of strings,
// filtering empty strings (i.e. "::" will yield no attributes.
// Returns the list of strings as Attributes.
func parseAttributes(attrsStr string) roachpb.Attributes {
	var filtered []string
	for _, attr := range strings.Split(attrsStr, ":") {
		if len(attr) != 0 {
			filtered = append(filtered, attr)
		}
	}
	return roachpb.Attributes{Attrs: filtered}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/docopt/docopt-go"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
)

func main() {
	version := "arxiv-api 0.0.1"
	usage := `Call arXiv api. http://arxiv.org/help/api/user-manual

Usage:
  arxiv-api <query> [--id_list=<id_list>] [--start=<s>] [--max_results=<n>]
  arxiv-api -h | --help
  arxiv-api --version

Options:
  --id_list=<id_list>    Id list (comma-delimited)
  --start=<s>            Index of start [default: 0]
  --max_results=<n>      Number of results [default: 10]
  -h --help              Show this screen.
  --version              Show version.`

	query, id_list, start, max_results := parse(usage, version)
	resp := call(query, id_list, start, max_results)
	defer resp.Body.Close()
	execute(resp)
}

func parse(usage, version string) (query, id_list, start, max_results string) {
	arguments, _ := docopt.Parse(usage, os.Args[1:], true, version, false)

	var casted = make(map[string]string)
	for key, val := range arguments {
		casted[key], _ = val.(string)
	}

	return casted["<query>"], casted["--id_list"], casted["--start"], casted["--max_results"]
}

func call(query, id_list, start, max_results string) *http.Response {
	values := url.Values{}
	values.Add("search_query", query)
	values.Add("id_list", id_list)
	values.Add("start", start)
	values.Add("max_results", max_results)

	resp, err := http.Get("http://export.arxiv.org/api/query" + "?" + values.Encode())

	if err != nil {
		fmt.Print(err)
		os.Exit(1)
	}

	return resp
}

func execute(response *http.Response) {
	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println(string(body))
}
<commit_msg>fix packaging<commit_after>package arxiv-api

import (
	"fmt"
	"github.com/docopt/docopt-go"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
)

func main() {
	version := "arxiv-api 0.0.1"
	usage := `Call arXiv api. http://arxiv.org/help/api/user-manual

Usage:
  arxiv-api <query> [--id_list=<id_list>] [--start=<s>] [--max_results=<n>]
  arxiv-api -h | --help
  arxiv-api --version

Options:
  --id_list=<id_list>    Id list (comma-delimited)
  --start=<s>            Index of start [default: 0]
  --max_results=<n>      Number of results [default: 10]
  -h --help              Show this screen.
  --version              Show version.`

	query, id_list, start, max_results := parse(usage, version)
	resp := call(query, id_list, start, max_results)
	defer resp.Body.Close()
	execute(resp)
}

func parse(usage, version string) (query, id_list, start, max_results string) {
	arguments, _ := docopt.Parse(usage, os.Args[1:], true, version, false)

	var casted = make(map[string]string)
	for key, val := range arguments {
		casted[key], _ = val.(string)
	}

	return casted["<query>"], casted["--id_list"], casted["--start"], casted["--max_results"]
}

func call(query, id_list, start, max_results string) *http.Response {
	values := url.Values{}
	values.Add("search_query", query)
	values.Add("id_list", id_list)
	values.Add("start", start)
	values.Add("max_results", max_results)

	resp, err := http.Get("http://export.arxiv.org/api/query" + "?" + values.Encode())

	if err != nil {
		fmt.Print(err)
		os.Exit(1)
	}

	return resp
}

func execute(response *http.Response) {
	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println(string(body))
}
<|endoftext|>


<commit_before>package log

import (
	"os"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/segmentio/go-loggly"
)

// LogglyHook sends logs to loggly
type LogglyHook struct {
	client       *loggly.Client
	host         string
	FilteredKeys map[string]bool
}

// NewLogglyHook creates a new hook
func NewLogglyHook(token string) *LogglyHook {
	client := loggly.New(token, "horizon")
	host, err := os.Hostname()

	if err != nil {
		panic("couldn't get hostname")
	}

	return &LogglyHook{
		client: client,
		host:   host,
	}
}

func (hook *LogglyHook) Fire(entry *logrus.Entry) error {
	logglyMessage := loggly.Message{
		"timestamp": entry.Time.UTC().Format(time.RFC3339Nano),
		"level":     entry.Level.String(),
		"message":   entry.Message,
		"hostname":  hook.host,
	}

	for k, v := range entry.Data {
		//Filter out keys
		if _, ok := hook.FilteredKeys[k]; ok {
			continue
		}

		logglyMessage[k] = v
	}

	return hook.client.Send(logglyMessage)
}

func (hook *LogglyHook) Flush() {
	hook.client.Flush()
}

func (hook *LogglyHook) Levels() []logrus.Level {
	return []logrus.Level{
		logrus.PanicLevel,
		logrus.FatalLevel,
		logrus.ErrorLevel,
		logrus.WarnLevel,
		logrus.InfoLevel,
		logrus.DebugLevel,
	}
}
<commit_msg>Format extra data as a string in loggly hook, to avoid nesting<commit_after>package log

import (
	"fmt"
	"os"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/segmentio/go-loggly"
)

// LogglyHook sends logs to loggly
type LogglyHook struct {
	client       *loggly.Client
	host         string
	FilteredKeys map[string]bool
}

// NewLogglyHook creates a new hook
func NewLogglyHook(token string) *LogglyHook {
	client := loggly.New(token, "horizon")
	host, err := os.Hostname()

	if err != nil {
		panic("couldn't get hostname")
	}

	return &LogglyHook{
		client: client,
		host:   host,
	}
}

func (hook *LogglyHook) Fire(entry *logrus.Entry) error {
	logglyMessage := loggly.Message{
		"timestamp": entry.Time.UTC().Format(time.RFC3339Nano),
		"level":     entry.Level.String(),
		"message":   entry.Message,
		"hostname":  hook.host,
	}

	for k, v := range entry.Data {
		//Filter out keys
		if _, ok := hook.FilteredKeys[k]; ok {
			continue
		}

		logglyMessage[k] = fmt.Sprint(v)
	}

	return hook.client.Send(logglyMessage)
}

func (hook *LogglyHook) Flush() {
	hook.client.Flush()
}

func (hook *LogglyHook) Levels() []logrus.Level {
	return []logrus.Level{
		logrus.PanicLevel,
		logrus.FatalLevel,
		logrus.ErrorLevel,
		logrus.WarnLevel,
		logrus.InfoLevel,
		logrus.DebugLevel,
	}
}
<|endoftext|>