<commit_before>package main

import (
	"bytes"
	"io/ioutil"
	"os"
	"strings"
	"testing"
)

func expectBufferEquality(t *testing.T, name string, buffer *bytes.Buffer, expected string) {
	output := buffer.String()
	if output != expected {
		t.Errorf("incorrect %s:\n%s\n\nexpected %s:\n%s", name, output, name, expected)
		t.Log([]rune(output))
		t.Log([]rune(expected))
	}
}

func expectProcessMainResults(t *testing.T, input string, args []string, exitCode int, expectedOutput string, expectedError string) {
	inputReader := strings.NewReader(input)
	outputBuffer := new(bytes.Buffer)
	errorBuffer := new(bytes.Buffer)

	returnCode := processMain(args, inputReader, outputBuffer, errorBuffer)

	expectBufferEquality(t, "output", outputBuffer, expectedOutput)
	expectBufferEquality(t, "error", errorBuffer, expectedError)

	if returnCode != exitCode {
		t.Error("incorrect return code:", returnCode, "expected", exitCode)
	}
}

func TestProcessMainReadFromStdin(t *testing.T) {
	input := `
		[mytoml]
		a = 42`
	expectedOutput := `{
  "mytoml": {
    "a": 42
  }
}
`
	expectedError := ``
	expectedExitCode := 0

	expectProcessMainResults(t, input, []string{}, expectedExitCode, expectedOutput, expectedError)
}

func TestProcessMainReadFromFile(t *testing.T) {
	input := `
		[mytoml]
		a = 42`

	tmpfile, err := ioutil.TempFile("", "example.toml")
	if err != nil {
		t.Fatal(err)
	}
	if _, err := tmpfile.Write([]byte(input)); err != nil {
		t.Fatal(err)
	}

	defer os.Remove(tmpfile.Name())

	expectedOutput := `{
  "mytoml": {
    "a": 42
  }
}
`
	expectedError := ``
	expectedExitCode := 0

	expectProcessMainResults(t, ``, []string{tmpfile.Name()}, expectedExitCode, expectedOutput, expectedError)
}

func TestProcessMainReadFromMissingFile(t *testing.T) {
	expectedError := `open /this/file/does/not/exist: no such file or directory
`
	expectProcessMainResults(t, ``, []string{"/this/file/does/not/exist"}, -1, ``, expectedError)
}
<commit_msg>Fix path not found message on Windows (#227)<commit_after>package main

import (
	"bytes"
	"io/ioutil"
	"os"
	"runtime"
	"strings"
	"testing"
)

func expectBufferEquality(t *testing.T, name string, buffer *bytes.Buffer, expected string) {
	output := buffer.String()
	if output != expected {
		t.Errorf("incorrect %s:\n%s\n\nexpected %s:\n%s", name, output, name, expected)
		t.Log([]rune(output))
		t.Log([]rune(expected))
	}
}

func expectProcessMainResults(t *testing.T, input string, args []string, exitCode int, expectedOutput string, expectedError string) {
	inputReader := strings.NewReader(input)
	outputBuffer := new(bytes.Buffer)
	errorBuffer := new(bytes.Buffer)

	returnCode := processMain(args, inputReader, outputBuffer, errorBuffer)

	expectBufferEquality(t, "output", outputBuffer, expectedOutput)
	expectBufferEquality(t, "error", errorBuffer, expectedError)

	if returnCode != exitCode {
		t.Error("incorrect return code:", returnCode, "expected", exitCode)
	}
}

func TestProcessMainReadFromStdin(t *testing.T) {
	input := `
		[mytoml]
		a = 42`
	expectedOutput := `{
  "mytoml": {
    "a": 42
  }
}
`
	expectedError := ``
	expectedExitCode := 0

	expectProcessMainResults(t, input, []string{}, expectedExitCode, expectedOutput, expectedError)
}

func TestProcessMainReadFromFile(t *testing.T) {
	input := `
		[mytoml]
		a = 42`

	tmpfile, err := ioutil.TempFile("", "example.toml")
	if err != nil {
		t.Fatal(err)
	}
	if _, err := tmpfile.Write([]byte(input)); err != nil {
		t.Fatal(err)
	}

	defer os.Remove(tmpfile.Name())

	expectedOutput := `{
  "mytoml": {
    "a": 42
  }
}
`
	expectedError := ``
	expectedExitCode := 0

	expectProcessMainResults(t, ``, []string{tmpfile.Name()}, expectedExitCode, expectedOutput, expectedError)
}

func TestProcessMainReadFromMissingFile(t *testing.T) {
	var expectedError string
	if runtime.GOOS == "windows" {
		expectedError = `open /this/file/does/not/exist: The system cannot find the path specified.
`
	} else {
		expectedError = `open /this/file/does/not/exist: no such file or directory
`
	}

	expectProcessMainResults(t, ``, []string{"/this/file/does/not/exist"}, -1, ``, expectedError)
}
<|endoftext|>


<commit_before>// Copyright 2019 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// hdparm perform control operations on disks
// modeled after linux command of the same name.
// The hdparm command is a mess. It is modeled on the
// command [switches] verb model but many of the verbs
// are switches which can conflict. To add to the
// fun they decided a conf file in /etc/ would be
// a good idea; we have no plans to support that.
//
// We also have no plans to support ata12. It's 2019.
//
// Synopsis:
//     hdparm [not all the switches hdparm supports] [device ...]
package main

import (
	"flag"
	"fmt"
	"log"
	"os"

	"github.com/u-root/u-root/pkg/scuzz"
)

// Because all the verbs and options are switches, hence global,
// we can have all the functions take one Disk as a parameter and
// return an error. This simplifies other aspects of this program.
type op = func(scuzz.Disk) error

var (
	verbose = flag.Bool("v", false, "verbose log")
	debug   = func(string, ...interface{}) {}
	unlock  = flag.String("security-unlock", "", "Unlock the drive")
	master  = flag.Bool("user-master", false, "Unlock master (true) or user (false)")
	timeout = flag.Uint("timeout", 15000, "Timeout for operations")
	verbs   = map[string]op{
		"--security-unlock": unlockop,
	}
	verb op
)

// The hdparm switches can conflict. This function returns nil if there is no conflict, and a (hopefully)
// helpful error message otherwise. As a side effect it assigns verb.
func checkConflict() error {
	var v []string

	for _, a := range os.Args {
		if f, ok := verbs[a]; ok {
			v = append(v, a)
			verb = f
		}
	}

	if len(v) > 1 {
		return fmt.Errorf("%v verbs were invoked and only one is allowed", v)
	}
	if len(v) < 1 {
		return fmt.Errorf("no verbs were invoked and one of %v is required", verbs)
	}
	return nil
}

func unlockop(d scuzz.Disk) error {
	return scuzz.Unlock(d, *unlock, *timeout, *master)
}

func main() {
	flag.Parse()

	if err := checkConflict(); err != nil {
		log.Fatal(err)
	}

	if *verbose {
		debug = log.Printf
	}
	scuzz.Debug = debug

	for _, n := range flag.Args() {
		d, err := scuzz.NewSGDisk(n)
		if err != nil {
			log.Printf("%v: %v", n, err)
		}
		if err := verb(d); err != nil {
			log.Printf("%v: %v", n, err)
		}
	}

}
<commit_msg>hdparm: increase readability<commit_after>// Copyright 2019 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// hdparm performs control operations on disks.
//
// hdparm is modeled after linux command of the same name,
// using a command [switches] verb model, but many of the verbs
// are also switches which can conflict.
//
// Upstream hdparm also decided a conf file in /etc/ would be
// a good idea; we have no plans to support that.
//
// We also have no plans to support ata12. It's 2019.
//
// Synopsis:
//     hdparm [--security-unlock] [--user-master|--timeout] [device ...]
package main

import (
	"flag"
	"fmt"
	"log"
	"os"

	"github.com/u-root/u-root/pkg/scuzz"
)

// Because all the verbs and options are switches, hence global,
// we can have all the functions take one Disk as a parameter and
// return an error. This simplifies other aspects of this program.
type op func(scuzz.Disk) error

var (
	verbose = flag.Bool("v", false, "verbose log")
	debug   = func(string, ...interface{}) {}
	unlock  = flag.String("security-unlock", "", "Unlock the drive")
	master  = flag.Bool("user-master", false, "Unlock master (true) or user (false)")
	timeout = flag.Uint("timeout", 15000, "Timeout for operations")
	verbs   = map[string]op{
		"--security-unlock": unlockop,
	}
)

// The hdparm switches can conflict. This function returns nil if there is no conflict, and a (hopefully)
// helpful error message otherwise. As a side effect it assigns verb.
func checkVerbs() (op, error) {
	var v []string
	var verb op

	for _, a := range os.Args {
		if f, ok := verbs[a]; ok {
			v = append(v, a)
			verb = f
		}
	}

	if len(v) > 1 {
		return nil, fmt.Errorf("%v verbs were invoked and only one is allowed", v)
	}
	if len(v) < 1 {
		return nil, fmt.Errorf("no verbs were invoked and one of %v is required", verbs)
	}
	return verb, nil
}

func unlockop(d scuzz.Disk) error {
	return scuzz.Unlock(d, *unlock, *timeout, *master)
}

func main() {
	flag.Parse()

	verb, err := checkVerbs()
	if err != nil {
		log.Fatal(err)
	}

	if *verbose {
		debug = log.Printf
	}
	scuzz.Debug = debug

	for _, n := range flag.Args() {
		d, err := scuzz.NewSGDisk(n)
		if err != nil {
			log.Printf("%v: %v", n, err)
		}
		if err := verb(d); err != nil {
			log.Printf("%v: %v", n, err)
		}
	}

}
<|endoftext|>


<commit_before>package base

import (
	"strings"
	"sync/atomic"
)

// LogKey is a bitfield of log keys.
type LogKey uint32

// Values for log keys.
const (
	// KeyNone is shorthand for no log keys.
	KeyNone LogKey = 0

	// KeyAll is a wildcard for all log keys.
	KeyAll LogKey = 1 << iota

	KeyAccess
	KeyAttach
	KeyAuth
	KeyBucket
	KeyCache
	KeyChanges
	KeyChannelIndex
	KeyChannelStorage
	KeyCRUD
	KeyDCP
	KeyDIndex
	KeyEvents
	KeyFeed
	KeyGoCB
	KeyHeartbeat
	KeyHTTP
	KeyImport
	KeyIndex
	KeyMigrate
	KeyOIDC
	KeyQuery
	KeyReplicate
	KeySequences
	KeyShadow
	KeySync
	KeySyncMsg
	KeyWebSocket
	KeyWebSocketFrame
)

var (
	logKeyNames = map[LogKey]string{
		KeyNone:           "",
		KeyAll:            "*",
		KeyAccess:         "Access",
		KeyAttach:         "Attach",
		KeyAuth:           "Auth",
		KeyBucket:         "Bucket",
		KeyCache:          "Cache",
		KeyChanges:        "Changes",
		KeyChannelIndex:   "ChannelIndex",
		KeyChannelStorage: "ChannelStorage",
		KeyCRUD:           "CRUD",
		KeyDCP:            "DCP",
		KeyDIndex:         "DIndex",
		KeyEvents:         "Events",
		KeyFeed:           "Feed",
		KeyGoCB:           "gocb",
		KeyHeartbeat:      "Heartbeat",
		KeyHTTP:           "HTTP",
		KeyImport:         "Import",
		KeyIndex:          "Index",
		KeyMigrate:        "Migrate",
		KeyOIDC:           "OIDC",
		KeyQuery:          "Query",
		KeyReplicate:      "Replicate",
		KeySequences:      "Sequences",
		KeyShadow:         "Shadow",
		KeySync:           "Sync",
		KeySyncMsg:        "SyncMsg",
		KeyWebSocket:      "WS",
		KeyWebSocketFrame: "WS+", // backwards compatibility for WS++ logkey
	}

	// Inverse of the map above. Optimisation for string -> LogKey lookups in ToLogKey
	logKeyNamesInverse = inverselogKeyNames(logKeyNames)
)

// Enable will enable the given logKey in keyMask.
func (keyMask *LogKey) Enable(logKey LogKey) {
	val := atomic.LoadUint32((*uint32)(keyMask))
	atomic.StoreUint32((*uint32)(keyMask), val|uint32(logKey))
}

// Disable will disable the given logKey in keyMask.
func (keyMask *LogKey) Disable(logKey LogKey) {
	val := atomic.LoadUint32((*uint32)(keyMask))
	atomic.StoreUint32((*uint32)(keyMask), val & ^uint32(logKey))
}

// Enabled returns true if the given logKey is enabled in keyMask.
// Always returns true if KeyAll is enabled in keyMask.
func (keyMask *LogKey) Enabled(logKey LogKey) bool {
	return keyMask.enabled(logKey, true)
}

// Enabled returns true if the given logKey is enabled in keyMask.
func (keyMask *LogKey) EnabledExcludingWildcard(logKey LogKey) bool {
	return keyMask.enabled(logKey, false)
}

// enabled returns true if the given logKey is enabled in keyMask, with an optional wildcard check.
func (keyMask *LogKey) enabled(logKey LogKey, checkWildcards bool) bool {
	if keyMask == nil {
		return false
	}

	flag := atomic.LoadUint32((*uint32)(keyMask))

	// If KeyAll is set, return true for everything.
	if checkWildcards && flag&uint32(KeyAll) != 0 {
		return true
	}

	return flag&uint32(logKey) != 0
}

// LogKeyName returns the string representation of a single log key.
func LogKeyName(logKey LogKey) string {
	// No lock required to read concurrently, as long as nobody writes to logKeyNames.
	return logKeyNames[logKey]
}

// EnabledLogKeys returns a slice of enabled log key names.
func (keyMask *LogKey) EnabledLogKeys() []string {
	if keyMask == nil {
		return []string{}
	}
	var logKeys = make([]string, 0, len(logKeyNames))
	for i := 0; i < len(logKeyNames); i++ {
		logKey := LogKey(1) << uint32(i)
		if keyMask.enabled(logKey, false) {
			logKeys = append(logKeys, LogKeyName(logKey))
		}
	}
	return logKeys
}

// ToLogKey takes a slice of case-sensitive log key names and will return a LogKey bitfield.
func ToLogKey(keysStr []string) LogKey {
	var logKeys LogKey
	for _, name := range keysStr {
		// Ignore "+" in log keys (for backwards compatibility)
		name := strings.Replace(name, "+", "", -1)

		if logKey, ok := logKeyNamesInverse[name]; ok {
			logKeys.Enable(logKey)
		} else {
			Warnf(KeyAll, "Invalid log key: %v", name)
		}
	}
	return logKeys
}

func inverselogKeyNames(in map[LogKey]string) map[string]LogKey {
	var out = make(map[string]LogKey, len(in))
	for k, v := range in {
		out[v] = k
	}
	return out
}
<commit_msg>#3464 Override log level to debug if we see any + log key (#3467)<commit_after>package base

import (
	"strings"
	"sync/atomic"
)

// LogKey is a bitfield of log keys.
type LogKey uint32

// Values for log keys.
const (
	// KeyNone is shorthand for no log keys.
	KeyNone LogKey = 0

	// KeyAll is a wildcard for all log keys.
	KeyAll LogKey = 1 << iota

	KeyAccess
	KeyAttach
	KeyAuth
	KeyBucket
	KeyCache
	KeyChanges
	KeyChannelIndex
	KeyChannelStorage
	KeyCRUD
	KeyDCP
	KeyDIndex
	KeyEvents
	KeyFeed
	KeyGoCB
	KeyHeartbeat
	KeyHTTP
	KeyImport
	KeyIndex
	KeyMigrate
	KeyOIDC
	KeyQuery
	KeyReplicate
	KeySequences
	KeyShadow
	KeySync
	KeySyncMsg
	KeyWebSocket
	KeyWebSocketFrame
)

var (
	logKeyNames = map[LogKey]string{
		KeyNone:           "",
		KeyAll:            "*",
		KeyAccess:         "Access",
		KeyAttach:         "Attach",
		KeyAuth:           "Auth",
		KeyBucket:         "Bucket",
		KeyCache:          "Cache",
		KeyChanges:        "Changes",
		KeyChannelIndex:   "ChannelIndex",
		KeyChannelStorage: "ChannelStorage",
		KeyCRUD:           "CRUD",
		KeyDCP:            "DCP",
		KeyDIndex:         "DIndex",
		KeyEvents:         "Events",
		KeyFeed:           "Feed",
		KeyGoCB:           "gocb",
		KeyHeartbeat:      "Heartbeat",
		KeyHTTP:           "HTTP",
		KeyImport:         "Import",
		KeyIndex:          "Index",
		KeyMigrate:        "Migrate",
		KeyOIDC:           "OIDC",
		KeyQuery:          "Query",
		KeyReplicate:      "Replicate",
		KeySequences:      "Sequences",
		KeyShadow:         "Shadow",
		KeySync:           "Sync",
		KeySyncMsg:        "SyncMsg",
		KeyWebSocket:      "WS",
		KeyWebSocketFrame: "WS+", // backwards compatibility for WS++ logkey
	}

	// Inverse of the map above. Optimisation for string -> LogKey lookups in ToLogKey
	logKeyNamesInverse = inverselogKeyNames(logKeyNames)
)

// Enable will enable the given logKey in keyMask.
func (keyMask *LogKey) Enable(logKey LogKey) {
	val := atomic.LoadUint32((*uint32)(keyMask))
	atomic.StoreUint32((*uint32)(keyMask), val|uint32(logKey))
}

// Disable will disable the given logKey in keyMask.
func (keyMask *LogKey) Disable(logKey LogKey) {
	val := atomic.LoadUint32((*uint32)(keyMask))
	atomic.StoreUint32((*uint32)(keyMask), val & ^uint32(logKey))
}

// Enabled returns true if the given logKey is enabled in keyMask.
// Always returns true if KeyAll is enabled in keyMask.
func (keyMask *LogKey) Enabled(logKey LogKey) bool {
	return keyMask.enabled(logKey, true)
}

// Enabled returns true if the given logKey is enabled in keyMask.
func (keyMask *LogKey) EnabledExcludingWildcard(logKey LogKey) bool {
	return keyMask.enabled(logKey, false)
}

// enabled returns true if the given logKey is enabled in keyMask, with an optional wildcard check.
func (keyMask *LogKey) enabled(logKey LogKey, checkWildcards bool) bool {
	if keyMask == nil {
		return false
	}

	flag := atomic.LoadUint32((*uint32)(keyMask))

	// If KeyAll is set, return true for everything.
	if checkWildcards && flag&uint32(KeyAll) != 0 {
		return true
	}

	return flag&uint32(logKey) != 0
}

// LogKeyName returns the string representation of a single log key.
func LogKeyName(logKey LogKey) string {
	// No lock required to read concurrently, as long as nobody writes to logKeyNames.
	return logKeyNames[logKey]
}

// EnabledLogKeys returns a slice of enabled log key names.
func (keyMask *LogKey) EnabledLogKeys() []string {
	if keyMask == nil {
		return []string{}
	}
	var logKeys = make([]string, 0, len(logKeyNames))
	for i := 0; i < len(logKeyNames); i++ {
		logKey := LogKey(1) << uint32(i)
		if keyMask.enabled(logKey, false) {
			logKeys = append(logKeys, LogKeyName(logKey))
		}
	}
	return logKeys
}

// ToLogKey takes a slice of case-sensitive log key names and will return a LogKey bitfield.
func ToLogKey(keysStr []string) LogKey {
	var logKeys LogKey
	for _, name := range keysStr {

		// Strip "+" in log keys and warn (for backwards compatibility)
		if strings.HasSuffix(name, "+") {
			newName := strings.Replace(name, "+", "", -1)
			Warnf(KeyAll, "Deprecated log key: %q found. Changing to: %q.", name, newName)
			name = newName
		}

		if logKey, ok := logKeyNamesInverse[name]; ok {
			logKeys.Enable(logKey)
		} else {
			Warnf(KeyAll, "Invalid log key: %v", name)
		}
	}
	return logKeys
}

func inverselogKeyNames(in map[LogKey]string) map[string]LogKey {
	var out = make(map[string]LogKey, len(in))
	for k, v := range in {
		out[v] = k
	}
	return out
}
<|endoftext|>


<commit_before>// memory.go
//
// Copyright (c) 2016, Ayke van Laethem
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// Package memory implements the file tree interface (tree.Entry). All data is
// stored in memory. This implementation is mostly intended for testing
// purposes.
package memory

import (
	"bytes"
	"io"
	"time"

	"github.com/aykevl/dtsync/tree"
)

// Entry is one file or directory.
type Entry struct {
	fileType tree.Type
	modTime  time.Time
	name     string
	contents []byte
	children map[string]*Entry
}

// NewRoot creates a new in-memory filesystem root.
func NewRoot() *Entry {
	return &Entry{
		fileType: tree.TYPE_DIRECTORY,
		// do not set the modTime to ease comparing two trees
	}
}

// String returns a string for debugging purposes (pretty printing).
func (e *Entry) String() string {
	return "memory.Entry(" + e.name + ")"
}

// Type returns the file type (file, directory)
func (e *Entry) Type() tree.Type {
	return e.fileType
}

// Name returns the filename (not the path)
func (e *Entry) Name() string {
	return e.name
}

// Size returns the filesize for files, or the number of direct children for
// directories.
func (e *Entry) Size() int64 {
	switch e.fileType {
	case tree.TYPE_REGULAR:
		return int64(len(e.contents))
	case tree.TYPE_DIRECTORY:
		return int64(len(e.children))
	default:
		panic("unknown fileType")
	}
}

// ModTime returns the modification or creation time.
func (e *Entry) ModTime() time.Time {
	return e.modTime
}

// List returns a list of directory entries for directories. It returns an error
// when attempting to list something other than a directory.
func (e *Entry) List() ([]tree.Entry, error) {
	if e.fileType != tree.TYPE_DIRECTORY {
		return nil, tree.ErrNoDirectory
	}

	ret := make([]tree.Entry, 0, len(e.children))
	for _, entry := range e.children {
		ret = append(ret, entry)
	}
	tree.SortEntries(ret)
	return ret, nil
}

// CopyTo copies this file into the given parent, returning an error if the file
// already exists.
func (e *Entry) CopyTo(otherParent tree.Entry) (tree.Entry, error) {
	file, ok := otherParent.(tree.FileEntry)
	if !ok {
		return nil, tree.ErrNotImplemented
	}
	other, out, err := file.CreateFile(e.name, e.modTime)
	if err != nil {
		return nil, err
	}
	_, err = out.Write(e.contents)
	if err != nil {
		return nil, err
	}
	err = out.Close()
	if err != nil {
		return nil, err
	}
	return other, nil
}

// UpdateOver copies data and metadata to the given other file.
func (e *Entry) UpdateOver(other tree.Entry) error {
	file, ok := other.(tree.FileEntry)
	if !ok {
		return tree.ErrNotImplemented
	}
	out, err := file.UpdateFile(e.modTime)
	if err != nil {
		return err
	}

	_, err = out.Write(e.contents)
	if err != nil {
		return err
	}

	err = out.Close()
	if err != nil {
		return err
	}

	return nil
}

// Remove removes this file or directory tree, recursively.
func (e *Entry) Remove(child tree.Entry) error {
	if e.fileType != tree.TYPE_DIRECTORY {
		return tree.ErrNoDirectory
	}
	if e.children[child.Name()] != child {
		return tree.ErrNotFound
	}
	delete(e.children, child.Name())
	return child.(*Entry).removeSelf()
}

// removeSelf removes this file, but does not remove the entry from the parent
// map.
// TODO unnecessary?
func (e *Entry) removeSelf() error {
	if e.fileType == tree.TYPE_DIRECTORY {
		list, err := e.List()
		if err != nil {
			return err
		}
		for _, child := range list {
			child.(*Entry).removeSelf()
		}
	}
	return nil
}

// GetFile returns a file handle (io.ReadCloser) that can be used to read a
// status file.
func (e *Entry) GetFile(name string) (io.ReadCloser, error) {
	if entry, ok := e.children[name]; ok {
		return newReadCloseBuffer(entry.contents), nil
	} else {
		return nil, tree.ErrNotFound
	}
}

// SetFile returns a file handle (io.WriteCloser) to a newly created/replaced
// child file that can be used to save the replica state.
func (e *Entry) SetFile(name string) (io.WriteCloser, error) {
	if child, ok := e.children[name]; ok {
		return newFileCopier(func(buf *bytes.Buffer) {
			child.contents = buf.Bytes()
		}), nil
	} else {
		return newFileCopier(func(buf *bytes.Buffer) {
			e.AddRegular(name, buf.Bytes())
		}), nil
	}
}

// CreateDir creates a directory with the given name.
func (e *Entry) CreateDir(name string) (tree.Entry, error) {
	child := &Entry{
		fileType: tree.TYPE_DIRECTORY,
		name:     name,
	}
	err := e.addChild(child)
	if err != nil {
		return nil, err
	}
	return child, nil
}

// CreateFile is part of tree.FileEntry. It returns the created entry, a
// WriteCloser to write the data to, and possibly an error.
// The file's contents is stored when the returned WriteCloser is closed.
func (e *Entry) CreateFile(name string, modTime time.Time) (tree.Entry, io.WriteCloser, error) {
	child := &Entry{
		fileType: tree.TYPE_REGULAR,
		modTime:  modTime,
		name:     name,
	}
	err := e.addChild(child)
	if err != nil {
		return nil, nil, err
	}

	file := newFileCopier(func(buffer *bytes.Buffer) {
		child.contents = buffer.Bytes()
	})

	return child, file, nil
}

func (e *Entry) addChild(child *Entry) error {
	if e.fileType != tree.TYPE_DIRECTORY {
		return tree.ErrNoDirectory
	}
	if e.children == nil {
		e.children = make(map[string]*Entry)
	}
	if _, ok := e.children[child.Name()]; ok {
		return tree.ErrAlreadyExists
	}

	e.children[child.Name()] = child
	return nil
}

// UpdateFile is part of tree.FileEntry and implements replacing a file.
// When closing the returned WriteCloser, the file is actually replaced.
func (e *Entry) UpdateFile(modTime time.Time) (io.WriteCloser, error) {
	if e.fileType != tree.TYPE_REGULAR {
		return nil, tree.ErrNoRegular
	}
	file := newFileCopier(func(buffer *bytes.Buffer) {
		e.modTime = modTime
		e.contents = buffer.Bytes()
	})
	return file, nil
}

// AddRegular creates a new regular file.
// This function only exists for testing purposes.
func (e *Entry) AddRegular(name string, contents []byte) (*Entry, error) {
	if e.fileType != tree.TYPE_DIRECTORY {
		return nil, tree.ErrNoDirectory
	}
	if e.children == nil {
		e.children = make(map[string]*Entry)
	}
	if _, ok := e.children[name]; ok {
		return nil, tree.ErrAlreadyExists
	}
	child := &Entry{
		fileType: tree.TYPE_REGULAR,
		modTime:  time.Now(),
		name:     name,
		contents: contents,
	}
	e.children[name] = child
	return child, nil
}

// SetContents sets the internal contents of the file, for debugging.
func (e *Entry) SetContents(contents []byte) {
	e.modTime = time.Now()
	e.contents = contents
}

// Equal compares this entry with another entry, only returning true when this
// file and possible children (for directories) are exactly equal.
func (e *Entry) Equal(other *Entry) bool {
	if e.name != other.name || !e.modTime.Equal(other.modTime) || e.fileType != other.fileType {
		return false
	}
	switch e.fileType {
	case tree.TYPE_REGULAR:
		return bytes.Equal(e.contents, other.contents)
	case tree.TYPE_DIRECTORY:
		if len(e.children) != len(other.children) {
			return false
		}
		for name, child := range e.children {
			otherchild, ok := other.children[name]
			if !ok || !child.Equal(otherchild) {
				return false
			}
		}
		return true
	default:
		panic("unknown fileType")
	}
}
<commit_msg>tree/memory: use addChild for AddRegular<commit_after>// memory.go
//
// Copyright (c) 2016, Ayke van Laethem
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// Package memory implements the file tree interface (tree.Entry). All data is
// stored in memory. This implementation is mostly intended for testing
// purposes.
package memory

import (
	"bytes"
	"io"
	"time"

	"github.com/aykevl/dtsync/tree"
)

// Entry is one file or directory.
type Entry struct {
	fileType tree.Type
	modTime  time.Time
	name     string
	contents []byte
	children map[string]*Entry
}

// NewRoot creates a new in-memory filesystem root.
func NewRoot() *Entry {
	return &Entry{
		fileType: tree.TYPE_DIRECTORY,
		// do not set the modTime to ease comparing two trees
	}
}

// String returns a string for debugging purposes (pretty printing).
func (e *Entry) String() string {
	return "memory.Entry(" + e.name + ")"
}

// Type returns the file type (file, directory)
func (e *Entry) Type() tree.Type {
	return e.fileType
}

// Name returns the filename (not the path)
func (e *Entry) Name() string {
	return e.name
}

// Size returns the filesize for files, or the number of direct children for
// directories.
func (e *Entry) Size() int64 {
	switch e.fileType {
	case tree.TYPE_REGULAR:
		return int64(len(e.contents))
	case tree.TYPE_DIRECTORY:
		return int64(len(e.children))
	default:
		panic("unknown fileType")
	}
}

// ModTime returns the modification or creation time.
func (e *Entry) ModTime() time.Time {
	return e.modTime
}

// List returns a list of directory entries for directories. It returns an error
// when attempting to list something other than a directory.
func (e *Entry) List() ([]tree.Entry, error) {
	if e.fileType != tree.TYPE_DIRECTORY {
		return nil, tree.ErrNoDirectory
	}

	ret := make([]tree.Entry, 0, len(e.children))
	for _, entry := range e.children {
		ret = append(ret, entry)
	}
	tree.SortEntries(ret)
	return ret, nil
}

// CopyTo copies this file into the given parent, returning an error if the file
// already exists.
func (e *Entry) CopyTo(otherParent tree.Entry) (tree.Entry, error) {
	file, ok := otherParent.(tree.FileEntry)
	if !ok {
		return nil, tree.ErrNotImplemented
	}
	other, out, err := file.CreateFile(e.name, e.modTime)
	if err != nil {
		return nil, err
	}
	_, err = out.Write(e.contents)
	if err != nil {
		return nil, err
	}
	err = out.Close()
	if err != nil {
		return nil, err
	}
	return other, nil
}

// UpdateOver copies data and metadata to the given other file.
func (e *Entry) UpdateOver(other tree.Entry) error {
	file, ok := other.(tree.FileEntry)
	if !ok {
		return tree.ErrNotImplemented
	}
	out, err := file.UpdateFile(e.modTime)
	if err != nil {
		return err
	}

	_, err = out.Write(e.contents)
	if err != nil {
		return err
	}

	err = out.Close()
	if err != nil {
		return err
	}

	return nil
}

// Remove removes this file or directory tree, recursively.
func (e *Entry) Remove(child tree.Entry) error {
	if e.fileType != tree.TYPE_DIRECTORY {
		return tree.ErrNoDirectory
	}
	if e.children[child.Name()] != child {
		return tree.ErrNotFound
	}
	delete(e.children, child.Name())
	return child.(*Entry).removeSelf()
}

// removeSelf removes this file, but does not remove the entry from the parent
// map.
// TODO unnecessary?
func (e *Entry) removeSelf() error {
	if e.fileType == tree.TYPE_DIRECTORY {
		list, err := e.List()
		if err != nil {
			return err
		}
		for _, child := range list {
			child.(*Entry).removeSelf()
		}
	}
	return nil
}

// GetFile returns a file handle (io.ReadCloser) that can be used to read a
// status file.
func (e *Entry) GetFile(name string) (io.ReadCloser, error) {
	if entry, ok := e.children[name]; ok {
		return newReadCloseBuffer(entry.contents), nil
	} else {
		return nil, tree.ErrNotFound
	}
}

// SetFile returns a file handle (io.WriteCloser) to a newly created/replaced
// child file that can be used to save the replica state.
func (e *Entry) SetFile(name string) (io.WriteCloser, error) {
	if child, ok := e.children[name]; ok {
		return newFileCopier(func(buf *bytes.Buffer) {
			child.contents = buf.Bytes()
		}), nil
	} else {
		return newFileCopier(func(buf *bytes.Buffer) {
			e.AddRegular(name, buf.Bytes())
		}), nil
	}
}

// CreateDir creates a directory with the given name.
func (e *Entry) CreateDir(name string) (tree.Entry, error) {
	child := &Entry{
		fileType: tree.TYPE_DIRECTORY,
		name:     name,
	}
	err := e.addChild(child)
	if err != nil {
		return nil, err
	}
	return child, nil
}

// CreateFile is part of tree.FileEntry. It returns the created entry, a
// WriteCloser to write the data to, and possibly an error.
// The file's contents is stored when the returned WriteCloser is closed.
func (e *Entry) CreateFile(name string, modTime time.Time) (tree.Entry, io.WriteCloser, error) {
	child := &Entry{
		fileType: tree.TYPE_REGULAR,
		modTime:  modTime,
		name:     name,
	}
	err := e.addChild(child)
	if err != nil {
		return nil, nil, err
	}

	file := newFileCopier(func(buffer *bytes.Buffer) {
		child.contents = buffer.Bytes()
	})

	return child, file, nil
}

func (e *Entry) addChild(child *Entry) error {
	if e.fileType != tree.TYPE_DIRECTORY {
		return tree.ErrNoDirectory
	}
	if e.children == nil {
		e.children = make(map[string]*Entry)
	}
	if _, ok := e.children[child.Name()]; ok {
		return tree.ErrAlreadyExists
	}

	e.children[child.Name()] = child
	return nil
}

// UpdateFile is part of tree.FileEntry and implements replacing a file.
// When closing the returned WriteCloser, the file is actually replaced.
func (e *Entry) UpdateFile(modTime time.Time) (io.WriteCloser, error) {
	if e.fileType != tree.TYPE_REGULAR {
		return nil, tree.ErrNoRegular
	}
	file := newFileCopier(func(buffer *bytes.Buffer) {
		e.modTime = modTime
		e.contents = buffer.Bytes()
	})
	return file, nil
}

// AddRegular creates a new regular file.
// This function only exists for testing purposes.
func (e *Entry) AddRegular(name string, contents []byte) (*Entry, error) {
	child := &Entry{
		fileType: tree.TYPE_REGULAR,
		modTime:  time.Now(),
		name:     name,
		contents: contents,
	}
	err := e.addChild(child)
	if err != nil {
		return nil, err
	}
	return child, nil
}

// SetContents sets the internal contents of the file, for debugging.
func (e *Entry) SetContents(contents []byte) {
	e.modTime = time.Now()
	e.contents = contents
}

// Equal compares this entry with another entry, only returning true when this
// file and possible children (for directories) are exactly equal.
func (e *Entry) Equal(other *Entry) bool {
	if e.name != other.name || !e.modTime.Equal(other.modTime) || e.fileType != other.fileType {
		return false
	}
	switch e.fileType {
	case tree.TYPE_REGULAR:
		return bytes.Equal(e.contents, other.contents)
	case tree.TYPE_DIRECTORY:
		if len(e.children) != len(other.children) {
			return false
		}
		for name, child := range e.children {
			otherchild, ok := other.children[name]
			if !ok || !child.Equal(otherchild) {
				return false
			}
		}
		return true
	default:
		panic("unknown fileType")
	}
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"fmt"

	log "github.com/couchbaselabs/clog"
)

// A janitor maintains PIndexes and Feeds, creating, deleting &
// hooking them up as necessary to try to match to latest plans.

func (mgr *Manager) JanitorNOOP(msg string) {
	SyncWorkReq(mgr.janitorCh, WORK_NOOP, msg)
}

func (mgr *Manager) JanitorKick(msg string) {
	SyncWorkReq(mgr.janitorCh, WORK_KICK, msg)
}

func (mgr *Manager) JanitorLoop() {
	for m := range mgr.janitorCh {
		if m.op == WORK_KICK {
			err := mgr.JanitorOnce(m.msg)
			if err != nil {
				log.Printf("error: JanitorOnce, err: %v", err)
				// Keep looping as perhaps it's a transient issue.
				// TODO: Need a better way to track warnings from errors,
				// and which need a future, rescheduled janitor kick.
			}
		} else if m.op == WORK_NOOP {
			// NOOP.
		} else {
			log.Printf("error: unknown janitor op: %s, m: %#v", m.op, m)
		}
		if m.resCh != nil {
			close(m.resCh)
		}
	}
}

func (mgr *Manager) JanitorOnce(reason string) error {
	log.Printf("janitor awakes, reason: %s", reason)

	if mgr.cfg == nil { // Can occur during testing.
		return fmt.Errorf("janitor skipped due to nil cfg")
	}

	planPIndexes, _, err := CfgGetPlanPIndexes(mgr.cfg)
	if err != nil {
		return fmt.Errorf("janitor skipped on CfgGetPlanPIndexes err: %v", err)
	}
	if planPIndexes == nil {
		// Might happen if janitor wins a race.
		// TODO: Should we reschedule another future janitor kick?
		return fmt.Errorf("janitor skipped on nil planPIndexes")
	}

	currFeeds, currPIndexes := mgr.CurrentMaps()

	addPlanPIndexes, removePIndexes :=
		CalcPIndexesDelta(mgr.uuid, currPIndexes, planPIndexes)

	log.Printf("janitor pindexes to add:")
	for _, ppi := range addPlanPIndexes {
		log.Printf("  %+v", ppi)
	}
	log.Printf("janitor pindexes to remove:")
	for _, pi := range removePIndexes {
		log.Printf("  %+v", pi)
	}

	// First, teardown pindexes that need to be removed.
	for _, removePIndex := range removePIndexes {
		log.Printf("janitor removing pindex: %s", removePIndex.Name)
		err = mgr.stopPIndex(removePIndex)
		if err != nil {
			return fmt.Errorf("error: janitor removing pindex: %s, err: %v",
				removePIndex.Name, err)
		}
	}
	// Then, (re-)create pindexes that we're missing.
	for _, addPlanPIndex := range addPlanPIndexes {
		log.Printf("janitor adding pindex: %s", addPlanPIndex.Name)
		err = mgr.StartPIndex(addPlanPIndex)
		if err != nil {
			return fmt.Errorf("error: janitor adding pindex: %s, err: %v",
				addPlanPIndex.Name, err)
		}
	}

	currFeeds, currPIndexes = mgr.CurrentMaps()

	addFeeds, removeFeeds := CalcFeedsDelta(currFeeds, currPIndexes)

	log.Printf("janitor feeds to add:")
	for _, targetPIndexes := range addFeeds {
		if len(targetPIndexes) > 0 {
			log.Printf("  %s", FeedName(targetPIndexes[0]))
		}
	}
	log.Printf("janitor feeds to remove:")
	for _, removeFeed := range removeFeeds {
		log.Printf("  %s", removeFeed.Name())
	}

	// First, teardown feeds that need to be removed.
	for _, removeFeed := range removeFeeds {
		err = mgr.stopFeed(removeFeed)
		if err != nil {
			return fmt.Errorf("error: janitor removing feed name: %s, err: %v",
				removeFeed.Name, err)
		}
	}
	// Then, (re-)create feeds that we're missing.
	for addFeedName, targetPIndexes := range addFeeds {
		err = mgr.startFeed(targetPIndexes)
		if err != nil {
			return fmt.Errorf("error: janitor adding feed, addFeedName: %s, err: %v",
				addFeedName, err)
		}
	}

	return nil
}

// --------------------------------------------------------

// Functionally determine the delta of which pindexes need creation
// and which should be shut down on our local node (mgrUUID).
func CalcPIndexesDelta(mgrUUID string,
	currPIndexes map[string]*PIndex,
	wantedPlanPIndexes *PlanPIndexes) (
	addPlanPIndexes []*PlanPIndex,
	removePIndexes []*PIndex) {
	// Allocate our return arrays.
	addPlanPIndexes = make([]*PlanPIndex, 0)
	removePIndexes = make([]*PIndex, 0)

	// Just for fast transient lookups.
	mapWantedPlanPIndex := make(map[string]*PlanPIndex)
	mapRemovePIndex := make(map[string]*PIndex)

	// For each wanted plan pindex, if a pindex does not exist or is
	// different, then schedule to add.
	for _, wantedPlanPIndex := range wantedPlanPIndexes.PlanPIndexes {
	nodeUUIDs:
		for nodeUUID, nodeState := range wantedPlanPIndex.NodeUUIDs {
			if nodeUUID != mgrUUID || nodeState != "active" {
				continue nodeUUIDs
			}

			mapWantedPlanPIndex[wantedPlanPIndex.Name] = wantedPlanPIndex

			currPIndex, exists := currPIndexes[wantedPlanPIndex.Name]
			if !exists {
				addPlanPIndexes = append(addPlanPIndexes, wantedPlanPIndex)
			} else if PIndexMatchesPlan(currPIndex, wantedPlanPIndex) == false {
				addPlanPIndexes = append(addPlanPIndexes, wantedPlanPIndex)
				removePIndexes = append(removePIndexes, currPIndex)
				mapRemovePIndex[currPIndex.Name] = currPIndex
			}

			break nodeUUIDs
		}
	}

	// For each existing pindex, if not part of wanted plan pindex,
	// then schedule for removal.
	for _, currPIndex := range currPIndexes {
		if _, exists := mapWantedPlanPIndex[currPIndex.Name]; !exists {
			if _, exists = mapRemovePIndex[currPIndex.Name]; !exists {
				removePIndexes = append(removePIndexes, currPIndex)
				mapRemovePIndex[currPIndex.Name] = currPIndex
			}
		}
	}

	return addPlanPIndexes, removePIndexes
}

func PIndexMatchesPlan(pindex *PIndex, planPIndex *PlanPIndex) bool {
	same := pindex.Name == planPIndex.Name &&
		pindex.IndexName == planPIndex.IndexName &&
		pindex.IndexUUID == planPIndex.IndexUUID &&
		pindex.IndexMapping == planPIndex.IndexMapping &&
		pindex.SourceType == planPIndex.SourceType &&
		pindex.SourceName == planPIndex.SourceName &&
		pindex.SourceUUID == planPIndex.SourceUUID &&
		pindex.SourcePartitions == planPIndex.SourcePartitions
	if !same {
		log.Printf("PIndexMatchesPlan false, pindex: %#v, planPIndex: %#v",
			pindex, planPIndex)
	}
	return same
}

// --------------------------------------------------------

// Functionally determine the delta of which feeds need creation and
// which should be shut down.
func CalcFeedsDelta(currFeeds map[string]Feed, pindexes map[string]*PIndex) (
	addFeeds [][]*PIndex, removeFeeds []Feed) {
	// Allocate result holders.
	addFeeds = make([][]*PIndex, 0)
	removeFeeds = make([]Feed, 0)

	// Group the pindexes by their feed names.
	groupedPIndexes := make(map[string][]*PIndex)
	for _, pindex := range pindexes {
		feedName := FeedName(pindex)
		arr, exists := groupedPIndexes[feedName]
		if !exists {
			arr = make([]*PIndex, 0)
		}
		groupedPIndexes[feedName] = append(arr, pindex)
	}

	for feedName, feedPIndexes := range groupedPIndexes {
		if _, exists := currFeeds[feedName]; !exists {
			addFeeds = append(addFeeds, feedPIndexes)
		}
	}

	for currFeedName, currFeed := range currFeeds {
		if _, exists := groupedPIndexes[currFeedName]; !exists {
			removeFeeds = append(removeFeeds, currFeed)
		}
	}

	return addFeeds, removeFeeds
}

func FeedName(pindex *PIndex) string {
	// TODO: Different feed types might have different name formulas
	// depending on whether they have a "single cluster" abstraction
	// or work on a "node by node" basis (in where stream destinations
	// need to be part of the name encoding).
	//
	// NOTE: We're depending on the IndexName/IndexUUID to
	// functionally "cover" the SourceType/SourceName/SourceUUID, so
	// we don't need to encode the source parts into the feed name.
	return pindex.IndexName + "_" + pindex.IndexUUID
}

// --------------------------------------------------------

func (mgr *Manager) StartPIndex(planPIndex *PlanPIndex) error {
	pindex, err := NewPIndex(planPIndex.Name, NewUUID(),
		planPIndex.IndexType,
		planPIndex.IndexName,
		planPIndex.IndexUUID,
		planPIndex.IndexMapping,
		planPIndex.SourceType,
		planPIndex.SourceName,
		planPIndex.SourceUUID,
		planPIndex.SourcePartitions,
		mgr.PIndexPath(planPIndex.Name))
	if err != nil {
		return fmt.Errorf("error: NewPIndex, name: %s, err: %v",
			planPIndex.Name, err)
	}

	if err = mgr.registerPIndex(pindex); err != nil {
		close(pindex.Stream)
		return err
	}

	return nil
}

func (mgr *Manager) stopPIndex(pindex *PIndex) error {
	feeds, _ := mgr.CurrentMaps()
	for _, feed := range feeds {
		for _, stream := range feed.Streams() {
			if stream == pindex.Stream {
				if err := mgr.stopFeed(feed); err != nil {
					panic(fmt.Sprintf("error: could not stop feed, err: %v", err))
				}
			}
		}
	}

	pindexUnreg := mgr.unregisterPIndex(pindex.Name)
	if pindexUnreg != nil && pindexUnreg != pindex {
		panic("unregistered pindex isn't the one we're stopping")
	}

	close(pindex.Stream)

	return nil
}

// --------------------------------------------------------

func (mgr *Manager) startFeed(pindexes []*PIndex) error {
	if len(pindexes) <= 0 {
		return nil
	}

	pindexFirst := pindexes[0]
	feedName := FeedName(pindexFirst)

	streams := make(map[string]Stream)
	for _, pindex := range pindexes {
		if f := FeedName(pindex); f != feedName {
			panic(fmt.Sprintf("error: unexpected feedName: %s != %s", f, feedName))
		}

		streams[pindex.SourcePartitions] = pindex.Stream
	}

	// TODO: Make this more extensible one day for more SourceType possibilities.
	if pindexFirst.SourceType == "couchbase" {
		// TODO: Should default to DCP feed or projector feed one day.
		return mgr.startTAPFeed(feedName,
			pindexFirst.IndexName, pindexFirst.IndexUUID,
			pindexFirst.SourceName, pindexFirst.SourceUUID, streams)
	}

	return fmt.Errorf("error: startFeed() got unknown source type: %s",
		pindexFirst.SourceType)
}

func (mgr *Manager) stopFeed(feed Feed) error {
	feedUnreg := mgr.unregisterFeed(feed.Name())
	if feedUnreg != nil && feedUnreg != feed {
		panic("error: unregistered feed isn't the one we're closing")
	}

	// NOTE: We're depending on feed to synchronously close, so we
	// know it'll no longer be sending to any of its streams anymore.
	//
	return feed.Close()
}

// --------------------------------------------------------

func (mgr *Manager) startTAPFeed(feedName, indexName, indexUUID,
	bucketName, bucketUUID string, streams map[string]Stream) error {
	feed, err := NewTAPFeed(feedName, mgr.server, "default",
		bucketName, bucketUUID, streams)
	if err != nil {
		return fmt.Errorf("error: could not prepare TAP stream to server: %s,"+
			" bucketName: %s, indexName: %s, err: %v",
			mgr.server, bucketName, indexName, err)
		// TODO: need a way to collect these errors so REST api
		// can show them to user ("hey, perhaps you deleted a bucket
		// and should delete these related full-text indexes?
		// or the couchbase cluster is just down.");
		// perhaps as specialized clog writer?
		// TODO: cleanup on error?
	}

	if err = feed.Start(); err != nil {
		// TODO: need way to track dead cows (non-beef)
		// TODO: cleanup?
		return fmt.Errorf("error: could not start feed, server: %s, err: %v",
			mgr.server, err)
	}

	if err = mgr.registerFeed(feed); err != nil {
		// TODO: cleanup?
		return err
	}

	return nil
}
<commit_msg>made startPIndex non-public<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"fmt"

	log "github.com/couchbaselabs/clog"
)

// A janitor maintains PIndexes and Feeds, creating, deleting &
// hooking them up as necessary to try to match to latest plans.

func (mgr *Manager) JanitorNOOP(msg string) {
	SyncWorkReq(mgr.janitorCh, WORK_NOOP, msg)
}

func (mgr *Manager) JanitorKick(msg string) {
	SyncWorkReq(mgr.janitorCh, WORK_KICK, msg)
}

func (mgr *Manager) JanitorLoop() {
	for m := range mgr.janitorCh {
		if m.op == WORK_KICK {
			err := mgr.JanitorOnce(m.msg)
			if err != nil {
				log.Printf("error: JanitorOnce, err: %v", err)
				// Keep looping as perhaps it's a transient issue.
				// TODO: Need a better way to track warnings from errors,
				// and which need a future, rescheduled janitor kick.
			}
		} else if m.op == WORK_NOOP {
			// NOOP.
		} else {
			log.Printf("error: unknown janitor op: %s, m: %#v", m.op, m)
		}
		if m.resCh != nil {
			close(m.resCh)
		}
	}
}

func (mgr *Manager) JanitorOnce(reason string) error {
	log.Printf("janitor awakes, reason: %s", reason)

	if mgr.cfg == nil { // Can occur during testing.
		return fmt.Errorf("janitor skipped due to nil cfg")
	}

	planPIndexes, _, err := CfgGetPlanPIndexes(mgr.cfg)
	if err != nil {
		return fmt.Errorf("janitor skipped on CfgGetPlanPIndexes err: %v", err)
	}
	if planPIndexes == nil {
		// Might happen if janitor wins a race.
		// TODO: Should we reschedule another future janitor kick?
		return fmt.Errorf("janitor skipped on nil planPIndexes")
	}

	currFeeds, currPIndexes := mgr.CurrentMaps()

	addPlanPIndexes, removePIndexes :=
		CalcPIndexesDelta(mgr.uuid, currPIndexes, planPIndexes)

	log.Printf("janitor pindexes to add:")
	for _, ppi := range addPlanPIndexes {
		log.Printf("  %+v", ppi)
	}
	log.Printf("janitor pindexes to remove:")
	for _, pi := range removePIndexes {
		log.Printf("  %+v", pi)
	}

	// First, teardown pindexes that need to be removed.
	for _, removePIndex := range removePIndexes {
		log.Printf("janitor removing pindex: %s", removePIndex.Name)
		err = mgr.stopPIndex(removePIndex)
		if err != nil {
			return fmt.Errorf("error: janitor removing pindex: %s, err: %v",
				removePIndex.Name, err)
		}
	}
	// Then, (re-)create pindexes that we're missing.
	for _, addPlanPIndex := range addPlanPIndexes {
		log.Printf("janitor adding pindex: %s", addPlanPIndex.Name)
		err = mgr.startPIndex(addPlanPIndex)
		if err != nil {
			return fmt.Errorf("error: janitor adding pindex: %s, err: %v",
				addPlanPIndex.Name, err)
		}
	}

	currFeeds, currPIndexes = mgr.CurrentMaps()

	addFeeds, removeFeeds := CalcFeedsDelta(currFeeds, currPIndexes)

	log.Printf("janitor feeds to add:")
	for _, targetPIndexes := range addFeeds {
		if len(targetPIndexes) > 0 {
			log.Printf("  %s", FeedName(targetPIndexes[0]))
		}
	}
	log.Printf("janitor feeds to remove:")
	for _, removeFeed := range removeFeeds {
		log.Printf("  %s", removeFeed.Name())
	}

	// First, teardown feeds that need to be removed.
	for _, removeFeed := range removeFeeds {
		err = mgr.stopFeed(removeFeed)
		if err != nil {
			return fmt.Errorf("error: janitor removing feed name: %s, err: %v",
				removeFeed.Name, err)
		}
	}
	// Then, (re-)create feeds that we're missing.
	for addFeedName, targetPIndexes := range addFeeds {
		err = mgr.startFeed(targetPIndexes)
		if err != nil {
			return fmt.Errorf("error: janitor adding feed, addFeedName: %s, err: %v",
				addFeedName, err)
		}
	}

	return nil
}

// --------------------------------------------------------

// Functionally determine the delta of which pindexes need creation
// and which should be shut down on our local node (mgrUUID).
func CalcPIndexesDelta(mgrUUID string,
	currPIndexes map[string]*PIndex,
	wantedPlanPIndexes *PlanPIndexes) (
	addPlanPIndexes []*PlanPIndex,
	removePIndexes []*PIndex) {
	// Allocate our return arrays.
	addPlanPIndexes = make([]*PlanPIndex, 0)
	removePIndexes = make([]*PIndex, 0)

	// Just for fast transient lookups.
	mapWantedPlanPIndex := make(map[string]*PlanPIndex)
	mapRemovePIndex := make(map[string]*PIndex)

	// For each wanted plan pindex, if a pindex does not exist or is
	// different, then schedule to add.
	for _, wantedPlanPIndex := range wantedPlanPIndexes.PlanPIndexes {
	nodeUUIDs:
		for nodeUUID, nodeState := range wantedPlanPIndex.NodeUUIDs {
			if nodeUUID != mgrUUID || nodeState != "active" {
				continue nodeUUIDs
			}

			mapWantedPlanPIndex[wantedPlanPIndex.Name] = wantedPlanPIndex

			currPIndex, exists := currPIndexes[wantedPlanPIndex.Name]
			if !exists {
				addPlanPIndexes = append(addPlanPIndexes, wantedPlanPIndex)
			} else if PIndexMatchesPlan(currPIndex, wantedPlanPIndex) == false {
				addPlanPIndexes = append(addPlanPIndexes, wantedPlanPIndex)
				removePIndexes = append(removePIndexes, currPIndex)
				mapRemovePIndex[currPIndex.Name] = currPIndex
			}

			break nodeUUIDs
		}
	}

	// For each existing pindex, if not part of wanted plan pindex,
	// then schedule for removal.
	for _, currPIndex := range currPIndexes {
		if _, exists := mapWantedPlanPIndex[currPIndex.Name]; !exists {
			if _, exists = mapRemovePIndex[currPIndex.Name]; !exists {
				removePIndexes = append(removePIndexes, currPIndex)
				mapRemovePIndex[currPIndex.Name] = currPIndex
			}
		}
	}

	return addPlanPIndexes, removePIndexes
}

func PIndexMatchesPlan(pindex *PIndex, planPIndex *PlanPIndex) bool {
	same := pindex.Name == planPIndex.Name &&
		pindex.IndexName == planPIndex.IndexName &&
		pindex.IndexUUID == planPIndex.IndexUUID &&
		pindex.IndexMapping == planPIndex.IndexMapping &&
		pindex.SourceType == planPIndex.SourceType &&
		pindex.SourceName == planPIndex.SourceName &&
		pindex.SourceUUID == planPIndex.SourceUUID &&
		pindex.SourcePartitions == planPIndex.SourcePartitions
	if !same {
		log.Printf("PIndexMatchesPlan false, pindex: %#v, planPIndex: %#v",
			pindex, planPIndex)
	}
	return same
}

// --------------------------------------------------------

// Functionally determine the delta of which feeds need creation and
// which should be shut down.
func CalcFeedsDelta(currFeeds map[string]Feed, pindexes map[string]*PIndex) (
	addFeeds [][]*PIndex, removeFeeds []Feed) {
	// Allocate result holders.
	addFeeds = make([][]*PIndex, 0)
	removeFeeds = make([]Feed, 0)

	// Group the pindexes by their feed names.
	groupedPIndexes := make(map[string][]*PIndex)
	for _, pindex := range pindexes {
		feedName := FeedName(pindex)
		arr, exists := groupedPIndexes[feedName]
		if !exists {
			arr = make([]*PIndex, 0)
		}
		groupedPIndexes[feedName] = append(arr, pindex)
	}

	for feedName, feedPIndexes := range groupedPIndexes {
		if _, exists := currFeeds[feedName]; !exists {
			addFeeds = append(addFeeds, feedPIndexes)
		}
	}

	for currFeedName, currFeed := range currFeeds {
		if _, exists := groupedPIndexes[currFeedName]; !exists {
			removeFeeds = append(removeFeeds, currFeed)
		}
	}

	return addFeeds, removeFeeds
}

func FeedName(pindex *PIndex) string {
	// TODO: Different feed types might have different name formulas
	// depending on whether they have a "single cluster" abstraction
	// or work on a "node by node" basis (in where stream destinations
	// need to be part of the name encoding).
	//
	// NOTE: We're depending on the IndexName/IndexUUID to
	// functionally "cover" the SourceType/SourceName/SourceUUID, so
	// we don't need to encode the source parts into the feed name.
	return pindex.IndexName + "_" + pindex.IndexUUID
}

// --------------------------------------------------------

func (mgr *Manager) startPIndex(planPIndex *PlanPIndex) error {
	pindex, err := NewPIndex(planPIndex.Name, NewUUID(),
		planPIndex.IndexType,
		planPIndex.IndexName,
		planPIndex.IndexUUID,
		planPIndex.IndexMapping,
		planPIndex.SourceType,
		planPIndex.SourceName,
		planPIndex.SourceUUID,
		planPIndex.SourcePartitions,
		mgr.PIndexPath(planPIndex.Name))
	if err != nil {
		return fmt.Errorf("error: NewPIndex, name: %s, err: %v",
			planPIndex.Name, err)
	}

	if err = mgr.registerPIndex(pindex); err != nil {
		close(pindex.Stream)
		return err
	}

	return nil
}

func (mgr *Manager) stopPIndex(pindex *PIndex) error {
	feeds, _ := mgr.CurrentMaps()
	for _, feed := range feeds {
		for _, stream := range feed.Streams() {
			if stream == pindex.Stream {
				if err := mgr.stopFeed(feed); err != nil {
					panic(fmt.Sprintf("error: could not stop feed, err: %v", err))
				}
			}
		}
	}

	pindexUnreg := mgr.unregisterPIndex(pindex.Name)
	if pindexUnreg != nil && pindexUnreg != pindex {
		panic("unregistered pindex isn't the one we're stopping")
	}

	close(pindex.Stream)

	return nil
}

// --------------------------------------------------------

func (mgr *Manager) startFeed(pindexes []*PIndex) error {
	if len(pindexes) <= 0 {
		return nil
	}

	pindexFirst := pindexes[0]
	feedName := FeedName(pindexFirst)

	streams := make(map[string]Stream)
	for _, pindex := range pindexes {
		if f := FeedName(pindex); f != feedName {
			panic(fmt.Sprintf("error: unexpected feedName: %s != %s", f, feedName))
		}

		streams[pindex.SourcePartitions] = pindex.Stream
	}

	// TODO: Make this more extensible one day for more SourceType possibilities.
	if pindexFirst.SourceType == "couchbase" {
		// TODO: Should default to DCP feed or projector feed one day.
		return mgr.startTAPFeed(feedName,
			pindexFirst.IndexName, pindexFirst.IndexUUID,
			pindexFirst.SourceName, pindexFirst.SourceUUID, streams)
	}

	return fmt.Errorf("error: startFeed() got unknown source type: %s",
		pindexFirst.SourceType)
}

func (mgr *Manager) stopFeed(feed Feed) error {
	feedUnreg := mgr.unregisterFeed(feed.Name())
	if feedUnreg != nil && feedUnreg != feed {
		panic("error: unregistered feed isn't the one we're closing")
	}

	// NOTE: We're depending on feed to synchronously close, so we
	// know it'll no longer be sending to any of its streams anymore.
	//
	return feed.Close()
}

// --------------------------------------------------------

func (mgr *Manager) startTAPFeed(feedName, indexName, indexUUID,
	bucketName, bucketUUID string, streams map[string]Stream) error {
	feed, err := NewTAPFeed(feedName, mgr.server, "default",
		bucketName, bucketUUID, streams)
	if err != nil {
		return fmt.Errorf("error: could not prepare TAP stream to server: %s,"+
			" bucketName: %s, indexName: %s, err: %v",
			mgr.server, bucketName, indexName, err)
		// TODO: need a way to collect these errors so REST api
		// can show them to user ("hey, perhaps you deleted a bucket
		// and should delete these related full-text indexes?
		// or the couchbase cluster is just down.");
		// perhaps as specialized clog writer?
		// TODO: cleanup on error?
	}

	if err = feed.Start(); err != nil {
		// TODO: need way to track dead cows (non-beef)
		// TODO: cleanup?
		return fmt.Errorf("error: could not start feed, server: %s, err: %v",
			mgr.server, err)
	}

	if err = mgr.registerFeed(feed); err != nil {
		// TODO: cleanup?
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package beat

import (
	"flag"
	"fmt"
	"os"
	"time"

	"github.com/elastic/libbeat/beat"
	"github.com/elastic/libbeat/cfgfile"
	"github.com/elastic/libbeat/common"
	"github.com/elastic/libbeat/logp"
	"github.com/elastic/libbeat/publisher"

	cfg "github.com/elastic/filebeat/config"
	. "github.com/elastic/filebeat/crawler"
	. "github.com/elastic/filebeat/input"
)

// TODO: Cleanup if possible
var exitStat = struct {
	ok, usageError, faulted int
}{
	ok:         0,
	usageError: 1,
	faulted:    2,
}

var configDirPath string

// Init config path flag
func init() {
	flag.StringVar(&configDirPath, "configDir", "", "path to additional filebeat configuration directory with .yml files")
}

// Beater object. Contains all objects needed to run the beat
type Filebeat struct {
	FbConfig *cfg.Config
	// Channel from harvesters to spooler
	SpoolChan     chan *FileEvent
	publisherChan chan []*FileEvent
	RegistrarChan chan []*FileEvent
	Spooler       *Spooler
}

// Config setups up the filebeat configuration by fetch all additional config files
func (fb *Filebeat) Config(b *beat.Beat) error {

	// Load Base config
	err := cfgfile.Read(&fb.FbConfig, "")

	if err != nil {
		return fmt.Errorf("Error reading config file: %v", err)
	}

	// This is optiona
	if configDirPath != "" {
		logp.Info("Additional config files are fetched from:", configDirPath)
		fb.FbConfig.FetchConfigs(configDirPath)
	}

	return nil
}

func (fb *Filebeat) Setup(b *beat.Beat) error {
	return nil
}

func (fb *Filebeat) Run(b *beat.Beat) error {

	defer func() {
		p := recover()
		if p == nil {
			return
		}

		fmt.Printf("recovered panic: %v", p)
		os.Exit(exitStat.faulted)
	}()

	// Init channels
	fb.SpoolChan = make(chan *FileEvent, 16)
	fb.publisherChan = make(chan []*FileEvent, 1)
	fb.RegistrarChan = make(chan []*FileEvent, 1)

	// Setup registrar to persist state
	registrar := NewRegistrar(fb.FbConfig.Filebeat.RegistryFile)

	crawl := &Crawler{
		Registrar: registrar,
	}

	// Load the previous log file locations now, for use in prospector
	registrar.LoadState()
	crawl.Start(fb.FbConfig.Filebeat.Prospectors, fb.SpoolChan)

	// Init and Start spooler: Harvesters dump events into the spooler.
	spooler := NewSpooler(fb)
	err := spooler.Config()

	if err != nil {
		logp.Err("Could not init spooler: %v", err)
		return err
	}

	fb.Spooler = spooler

	// TODO: Check if spooler shouldn't start earlier?
	go spooler.Start()

	// Publishes event to output
	go Publish(b, fb)

	// registrar records last acknowledged positions in all files.
	registrar.WriteState(fb.RegistrarChan)

	return nil
}

func (fb *Filebeat) Cleanup(b *beat.Beat) error {
	return nil
}

func (fb *Filebeat) Stop() {

	// Stop harvesters
	// Stop prospectors
	// Flush what is in spooler
	// Write state

	fb.Spooler.Stop()

	// FIXME: Improve to first write state and then close channels
	close(fb.SpoolChan)
	close(fb.publisherChan)
	close(fb.RegistrarChan)
}

func Publish(beat *beat.Beat, fb *Filebeat) {

	// Receives events from spool during flush
	for events := range fb.publisherChan {

		eventsSlice := make([]common.MapStr, len(events))

		logp.Debug("filebeat", "Send events to output")
		for _, event := range events {
			bEvent := common.MapStr{
				"timestamp": common.Time(time.Now()),
				"source":    event.Source,
				"offset":    event.Offset,
				"line":      event.Line,
				"text":      event.Text,
				"fields":    event.Fields,
				"fileinfo":  event.Fileinfo,
				"type":      "log",
			}

			eventsSlice = append(eventsSlice, bEvent)
		}

		publishEvents(beat.Events, eventsSlice)

		logp.Debug("filebeat", "Events sent: %d", len(events))

		// Tell the registrar that we've successfully sent these events
		fb.RegistrarChan <- events
	}
}

func publishEvents(client publisher.Client, events []common.MapStr) {

	// Sends event to beat (outputs).
	// Wait/Repeat until all events are published
	for {
		ok := client.PublishEvents(events, publisher.Confirm)
		if ok {
			break
		}
	}
}
<commit_msg>Fix slice allocation<commit_after>package beat

import (
	"flag"
	"fmt"
	"os"
	"time"

	"github.com/elastic/libbeat/beat"
	"github.com/elastic/libbeat/cfgfile"
	"github.com/elastic/libbeat/common"
	"github.com/elastic/libbeat/logp"
	"github.com/elastic/libbeat/publisher"

	cfg "github.com/elastic/filebeat/config"
	. "github.com/elastic/filebeat/crawler"
	. "github.com/elastic/filebeat/input"
)

// TODO: Cleanup if possible
var exitStat = struct {
	ok, usageError, faulted int
}{
	ok:         0,
	usageError: 1,
	faulted:    2,
}

var configDirPath string

// Init config path flag
func init() {
	flag.StringVar(&configDirPath, "configDir", "", "path to additional filebeat configuration directory with .yml files")
}

// Beater object. Contains all objects needed to run the beat
type Filebeat struct {
	FbConfig *cfg.Config
	// Channel from harvesters to spooler
	SpoolChan     chan *FileEvent
	publisherChan chan []*FileEvent
	RegistrarChan chan []*FileEvent
	Spooler       *Spooler
}

// Config setups up the filebeat configuration by fetch all additional config files
func (fb *Filebeat) Config(b *beat.Beat) error {

	// Load Base config
	err := cfgfile.Read(&fb.FbConfig, "")

	if err != nil {
		return fmt.Errorf("Error reading config file: %v", err)
	}

	// This is optiona
	if configDirPath != "" {
		logp.Info("Additional config files are fetched from:", configDirPath)
		fb.FbConfig.FetchConfigs(configDirPath)
	}

	return nil
}

func (fb *Filebeat) Setup(b *beat.Beat) error {
	return nil
}

func (fb *Filebeat) Run(b *beat.Beat) error {

	defer func() {
		p := recover()
		if p == nil {
			return
		}

		fmt.Printf("recovered panic: %v", p)
		os.Exit(exitStat.faulted)
	}()

	// Init channels
	fb.SpoolChan = make(chan *FileEvent, 16)
	fb.publisherChan = make(chan []*FileEvent, 1)
	fb.RegistrarChan = make(chan []*FileEvent, 1)

	// Setup registrar to persist state
	registrar := NewRegistrar(fb.FbConfig.Filebeat.RegistryFile)

	crawl := &Crawler{
		Registrar: registrar,
	}

	// Load the previous log file locations now, for use in prospector
	registrar.LoadState()
	crawl.Start(fb.FbConfig.Filebeat.Prospectors, fb.SpoolChan)

	// Init and Start spooler: Harvesters dump events into the spooler.
	spooler := NewSpooler(fb)
	err := spooler.Config()

	if err != nil {
		logp.Err("Could not init spooler: %v", err)
		return err
	}

	fb.Spooler = spooler

	// TODO: Check if spooler shouldn't start earlier?
	go spooler.Start()

	// Publishes event to output
	go Publish(b, fb)

	// registrar records last acknowledged positions in all files.
	registrar.WriteState(fb.RegistrarChan)

	return nil
}

func (fb *Filebeat) Cleanup(b *beat.Beat) error {
	return nil
}

func (fb *Filebeat) Stop() {

	// Stop harvesters
	// Stop prospectors
	// Flush what is in spooler
	// Write state

	fb.Spooler.Stop()

	// FIXME: Improve to first write state and then close channels
	close(fb.SpoolChan)
	close(fb.publisherChan)
	close(fb.RegistrarChan)
}

func Publish(beat *beat.Beat, fb *Filebeat) {

	// Receives events from spool during flush
	for events := range fb.publisherChan {

		pubEvents := make([]common.MapStr, 0, len(events))

		logp.Debug("filebeat", "Send events to output")
		for _, event := range events {
			bEvent := common.MapStr{
				"timestamp": common.Time(time.Now()),
				"source":    event.Source,
				"offset":    event.Offset,
				"line":      event.Line,
				"text":      event.Text,
				"fields":    event.Fields,
				"fileinfo":  event.Fileinfo,
				"type":      "log",
			}

			pubEvents = append(pubEvents, bEvent)
		}

		publishEvents(beat.Events, pubEvents)

		logp.Debug("filebeat", "Events sent: %d", len(events))

		// Tell the registrar that we've successfully sent these events
		fb.RegistrarChan <- events
	}
}

func publishEvents(client publisher.Client, events []common.MapStr) {

	// Sends event to beat (outputs).
	// Wait/Repeat until all events are published
	for {
		ok := client.PublishEvents(events, publisher.Confirm)
		if ok {
			break
		}
	}
}
<|endoftext|>


<commit_before>package validator

import (
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestEnumValidator(t *testing.T) {

	enums := []string{"hoge", "fuga"}

	Convey("When creating a Enum validator", t, func() {
		v := Enum{enums}
		Convey("give \"hoge\" paramater", func() {
			So(v.Validate("hoge"), ShouldBeTrue)
		})
		Convey("give \"fuga\" paramater", func() {
			So(v.Validate("fuga"), ShouldBeTrue)
		})
		Convey("give \"foo\" paramater", func() {
			So(v.Validate("foo"), ShouldBeFalse)
		})
		Convey("give \"hogefuga\" paramater", func() {
			So(v.Validate("hogefuga"), ShouldBeFalse)
		})
	})

	Convey("When creating a EnumIfNotEmpty validator", t, func() {
		v := EnumIfNotEmpty{enums}
		Convey("give \"hoge\" paramater", func() {
			So(v.Validate("hoge"), ShouldBeTrue)
		})
		Convey("give \"fuga\" paramater", func() {
			So(v.Validate("fuga"), ShouldBeTrue)
		})
		Convey("give \"foo\" paramater", func() {
			So(v.Validate("foo"), ShouldBeFalse)
		})
		Convey("give \"hogefuga\" paramater", func() {
			So(v.Validate("hogefuga"), ShouldBeFalse)
		})
	})

}
<commit_msg>Add test case<commit_after>package validator

import (
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestEnumValidator(t *testing.T) {

	enums := []string{"hoge", "fuga"}

	Convey("When creating a Enum validator", t, func() {
		v := Enum{enums}
		Convey("give \"hoge\" paramater", func() {
			So(v.Validate("hoge"), ShouldBeTrue)
		})
		Convey("give \"fuga\" paramater", func() {
			So(v.Validate("fuga"), ShouldBeTrue)
		})
		Convey("give \"foo\" paramater", func() {
			So(v.Validate("foo"), ShouldBeFalse)
		})
		Convey("give \"hogefuga\" paramater", func() {
			So(v.Validate("hogefuga"), ShouldBeFalse)
		})
	})

	Convey("When creating a EnumIfNotEmpty validator", t, func() {
		v := EnumIfNotEmpty{enums}
		Convey("give \"hoge\" paramater", func() {
			So(v.Validate("hoge"), ShouldBeTrue)
		})
		Convey("give \"fuga\" paramater", func() {
			So(v.Validate("fuga"), ShouldBeTrue)
		})
		Convey("give \"foo\" paramater", func() {
			So(v.Validate("foo"), ShouldBeFalse)
		})
		Convey("give \"hogefuga\" paramater", func() {
			So(v.Validate("hogefuga"), ShouldBeFalse)
		})
		Convey("give a empty paramater", func() {
			So(v.Validate(""), ShouldBeTrue)
		})
	})

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math"
)

var mod10plus7 = int(math.Pow(10, 9) + 7)

func hashIt(x, y int) string {
	return fmt.Sprintf("%d.%d", x, y)
}

// if I know the sum from 0,10 the sum from 3,7 is inside of that index, so I need
// to figure it out how to take this difference

func sumInRange(nums []int, queries [][]int) int {
	var (
		hashSet = map[string]int{}
		sum     int
	)

	for x := 0; x < len(queries); x++ {
		var hashed string = hashIt(queries[x][0], queries[x][1])
		if v, ok := hashSet[hashed]; ok {
			sum = sum + v
			continue
		}

		var innerSum int
		for i := queries[x][0]; i <= queries[x][1]; i++ {
			innerSum = innerSum + nums[i]
		}

		hashSet[hashed] = innerSum

		sum = sum + innerSum
	}

	return (sum + mod10plus7) % mod10plus7
}

func main() {
	fmt.Println(fmt.Sprintf("Expected output to be 10: %v", sumInRange([]int{3, 0, -2, 6, -3, 2}, [][]int{[]int{0, 2}, []int{2, 5}, []int{0, 5}})))
}
<commit_msg>finish sum in range with interesting technique<commit_after>package main

import (
	"fmt"
	"math"
)

var (
	mod10plus7 = int(math.Pow(10, 9) + 7)
)

func sumInRange(nums []int, queries [][]int) int {
	var (
		prefix = make([]int, len(nums)+1)
		sum    int
	)

	for x := 0; x < len(nums); x++ {
		sum = sum + nums[x]
		prefix[x+1] = sum
	}

	sum = 0
	for _, query := range queries {
		sum = sum + (prefix[query[1]+1] - prefix[query[0]])
	}

	return (sum + mod10plus7) % mod10plus7
}

func main() {
	fmt.Println(fmt.Sprintf("Expected output to be 10: %v", sumInRange([]int{3, 0, -2, 6, -3, 2}, [][]int{[]int{0, 2}, []int{2, 5}, []int{0, 5}})))
}
<|endoftext|>


<commit_before>package video

import "github.com/32bitkid/bitreader"

const (
	_ uint32 = iota
	PictureStructure_TopField
	PictureStructure_BottomField
	PictureStructure_FramePicture
)

type PictureCodingExtension struct {
	f_code                     [2][2]uint32
	intra_dc_precision         uint32 // 2 uimsbf
	picture_structure          uint32 // 2 uimsbf
	top_field_first            bool   // 1 uimsbf
	frame_pred_frame_dct       uint32 // 1 uimsbf
	concealment_motion_vectors bool   // 1 uimsbf
	q_scale_type               uint32 // 1 uimsbf
	intra_vlc_format           uint32 // 1 uimsbf
	alternate_scan             uint32 // 1 uimsbf
	repeat_first_field         bool   // 1 uimsbf
	chroma_420_type            bool   // 1 uimsbf
	progressive_frame          bool   // 1 uimsbf
	composite_display_flag     bool   // 1 uimsbf

	v_axis            bool   // 1 uimsbf
	field_sequence    uint32 // 3 uimsbf
	sub_carrier       bool   // 1 uimsbf
	burst_amplitude   uint32 // 7 uimsbf
	sub_carrier_phase uint32 // 8 uimsbf
}

func picture_coding_extension(br bitreader.BitReader) (*PictureCodingExtension, error) {

	err := start_code_check(br, ExtensionStartCode)
	if err != nil {
		return nil, err
	}

	err = extension_code_check(br, PictureCodingExtensionID)
	if err != nil {
		return nil, err
	}

	pce := PictureCodingExtension{}

	pce.f_code[0][0], err = br.Read32(4)
	if err != nil {
		return nil, err
	}
	pce.f_code[0][1], err = br.Read32(4)
	if err != nil {
		return nil, err
	}
	pce.f_code[1][0], err = br.Read32(4)
	if err != nil {
		return nil, err
	}
	pce.f_code[1][1], err = br.Read32(4)
	if err != nil {
		return nil, err
	}

	pce.intra_dc_precision, err = br.Read32(2)
	if err != nil {
		return nil, err
	}

	pce.picture_structure, err = br.Read32(2)
	if err != nil {
		return nil, err
	}

	pce.top_field_first, err = br.ReadBit()
	if err != nil {
		return nil, err
	}

	pce.frame_pred_frame_dct, err = br.Read32(1)
	if err != nil {
		return nil, err
	}

	pce.concealment_motion_vectors, err = br.ReadBit()
	if err != nil {
		return nil, err
	}

	pce.q_scale_type, err = br.Read32(1)
	if err != nil {
		return nil, err
	}

	pce.intra_vlc_format, err = br.Read32(1)
	if err != nil {
		return nil, err
	}

	pce.alternate_scan, err = br.Read32(1)
	if err != nil {
		return nil, err
	}

	pce.repeat_first_field, err = br.ReadBit()
	if err != nil {
		return nil, err
	}

	pce.chroma_420_type, err = br.ReadBit()
	if err != nil {
		return nil, err
	}

	pce.progressive_frame, err = br.ReadBit()
	if err != nil {
		return nil, err
	}

	pce.composite_display_flag, err = br.ReadBit()
	if err != nil {
		return nil, err
	}

	if pce.composite_display_flag {
		pce.v_axis, err = br.ReadBit()
		if err != nil {
			return nil, err
		}
		pce.field_sequence, err = br.Read32(3)
		if err != nil {
			return nil, err
		}
		pce.sub_carrier, err = br.ReadBit()
		if err != nil {
			return nil, err
		}
		pce.burst_amplitude, err = br.Read32(7)
		if err != nil {
			return nil, err
		}
		pce.sub_carrier_phase, err = br.Read32(8)
		if err != nil {
			return nil, err
		}
	}

	return &pce, next_start_code(br)
}
<commit_msg>typing FCode<commit_after>package video

import "github.com/32bitkid/bitreader"

const (
	_ uint32 = iota
	PictureStructure_TopField
	PictureStructure_BottomField
	PictureStructure_FramePicture
)

type FCode [2][2]uint32

type PictureCodingExtension struct {
	f_code                     FCode
	intra_dc_precision         uint32 // 2 uimsbf
	picture_structure          uint32 // 2 uimsbf
	top_field_first            bool   // 1 uimsbf
	frame_pred_frame_dct       uint32 // 1 uimsbf
	concealment_motion_vectors bool   // 1 uimsbf
	q_scale_type               uint32 // 1 uimsbf
	intra_vlc_format           uint32 // 1 uimsbf
	alternate_scan             uint32 // 1 uimsbf
	repeat_first_field         bool   // 1 uimsbf
	chroma_420_type            bool   // 1 uimsbf
	progressive_frame          bool   // 1 uimsbf
	composite_display_flag     bool   // 1 uimsbf

	v_axis            bool   // 1 uimsbf
	field_sequence    uint32 // 3 uimsbf
	sub_carrier       bool   // 1 uimsbf
	burst_amplitude   uint32 // 7 uimsbf
	sub_carrier_phase uint32 // 8 uimsbf
}

func picture_coding_extension(br bitreader.BitReader) (*PictureCodingExtension, error) {

	err := start_code_check(br, ExtensionStartCode)
	if err != nil {
		return nil, err
	}

	err = extension_code_check(br, PictureCodingExtensionID)
	if err != nil {
		return nil, err
	}

	pce := PictureCodingExtension{}

	pce.f_code[0][0], err = br.Read32(4)
	if err != nil {
		return nil, err
	}
	pce.f_code[0][1], err = br.Read32(4)
	if err != nil {
		return nil, err
	}
	pce.f_code[1][0], err = br.Read32(4)
	if err != nil {
		return nil, err
	}
	pce.f_code[1][1], err = br.Read32(4)
	if err != nil {
		return nil, err
	}

	pce.intra_dc_precision, err = br.Read32(2)
	if err != nil {
		return nil, err
	}

	pce.picture_structure, err = br.Read32(2)
	if err != nil {
		return nil, err
	}

	pce.top_field_first, err = br.ReadBit()
	if err != nil {
		return nil, err
	}

	pce.frame_pred_frame_dct, err = br.Read32(1)
	if err != nil {
		return nil, err
	}

	pce.concealment_motion_vectors, err = br.ReadBit()
	if err != nil {
		return nil, err
	}

	pce.q_scale_type, err = br.Read32(1)
	if err != nil {
		return nil, err
	}

	pce.intra_vlc_format, err = br.Read32(1)
	if err != nil {
		return nil, err
	}

	pce.alternate_scan, err = br.Read32(1)
	if err != nil {
		return nil, err
	}

	pce.repeat_first_field, err = br.ReadBit()
	if err != nil {
		return nil, err
	}

	pce.chroma_420_type, err = br.ReadBit()
	if err != nil {
		return nil, err
	}

	pce.progressive_frame, err = br.ReadBit()
	if err != nil {
		return nil, err
	}

	pce.composite_display_flag, err = br.ReadBit()
	if err != nil {
		return nil, err
	}

	if pce.composite_display_flag {
		pce.v_axis, err = br.ReadBit()
		if err != nil {
			return nil, err
		}
		pce.field_sequence, err = br.Read32(3)
		if err != nil {
			return nil, err
		}
		pce.sub_carrier, err = br.ReadBit()
		if err != nil {
			return nil, err
		}
		pce.burst_amplitude, err = br.Read32(7)
		if err != nil {
			return nil, err
		}
		pce.sub_carrier_phase, err = br.Read32(8)
		if err != nil {
			return nil, err
		}
	}

	return &pce, next_start_code(br)
}
<|endoftext|>


<commit_before>package commands

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/github/git-lfs/lfs"
	"github.com/github/git-lfs/vendor/_nuts/github.com/spf13/cobra"
)

var (
	trackCmd = &cobra.Command{
		Use: "track",
		Run: trackCommand,
	}
)

func trackCommand(cmd *cobra.Command, args []string) {
	if lfs.LocalGitDir == "" {
		Print("Not a git repository.")
		os.Exit(128)
	}

	if lfs.LocalWorkingDir == "" {
		Print("This operation must be run in a work tree.")
		os.Exit(128)
	}

	lfs.InstallHooks(false)
	knownPaths := findPaths()

	if len(args) == 0 {
		Print("Listing tracked paths")
		for _, t := range knownPaths {
			Print("    %s (%s)", t.Path, t.Source)
		}
		return
	}

	addTrailingLinebreak := needsTrailingLinebreak(".gitattributes")
	attributesFile, err := os.OpenFile(".gitattributes", os.O_RDWR|os.O_APPEND|os.O_CREATE, 0660)
	if err != nil {
		Print("Error opening .gitattributes file")
		return
	}
	defer attributesFile.Close()

	if addTrailingLinebreak {
		if _, err := attributesFile.WriteString("\n"); err != nil {
			Print("Error writing to .gitattributes")
		}
	}

	wd, _ := os.Getwd()
	relpath, err := filepath.Rel(lfs.LocalWorkingDir, wd)
	if err != nil {
		Exit("Current directory %q outside of git working directory %q.", wd, lfs.LocalWorkingDir)
	}

ArgsLoop:
	for _, pattern := range args {
		for _, known := range knownPaths {
			if known.Path == filepath.Join(relpath, pattern) {
				Print("%s already supported", pattern)
				continue ArgsLoop
			}
		}

		encodedArg := strings.Replace(pattern, " ", "[[:space:]]", -1)
		_, err := attributesFile.WriteString(fmt.Sprintf("%s filter=lfs diff=lfs merge=lfs -text\n", encodedArg))
		if err != nil {
			Print("Error adding path %s", pattern)
			continue
		}
		Print("Tracking %s", pattern)
	}
}

type mediaPath struct {
	Path   string
	Source string
}

func findPaths() []mediaPath {
	paths := make([]mediaPath, 0)

	for _, path := range findAttributeFiles() {
		attributes, err := os.Open(path)
		if err != nil {
			continue
		}

		scanner := bufio.NewScanner(attributes)

		for scanner.Scan() {
			line := scanner.Text()
			if strings.Contains(line, "filter=lfs") {
				fields := strings.Fields(line)
				relfile, _ := filepath.Rel(lfs.LocalWorkingDir, path)
				pattern := fields[0]
				if reldir := filepath.Dir(relfile); len(reldir) > 0 {
					pattern = filepath.Join(reldir, pattern)
				}

				paths = append(paths, mediaPath{Path: pattern, Source: relfile})
			}
		}
	}

	return paths
}

func findAttributeFiles() []string {
	paths := make([]string, 0)

	repoAttributes := filepath.Join(lfs.LocalGitDir, "info", "attributes")
	if info, err := os.Stat(repoAttributes); err == nil && !info.IsDir() {
		paths = append(paths, repoAttributes)
	}

	filepath.Walk(lfs.LocalWorkingDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if !info.IsDir() && (filepath.Base(path) == ".gitattributes") {
			paths = append(paths, path)
		}
		return nil
	})

	return paths
}

func needsTrailingLinebreak(filename string) bool {
	file, err := os.Open(filename)
	if err != nil {
		return false
	}
	defer file.Close()

	buf := make([]byte, 16384)
	bytesRead := 0
	for {
		n, err := file.Read(buf)
		if err == io.EOF {
			break
		} else if err != nil {
			return false
		}
		bytesRead = n
	}

	return !strings.HasSuffix(string(buf[0:bytesRead]), "\n")
}

func init() {
	RootCmd.AddCommand(trackCmd)
}
<commit_msg>'git lfs track' now automatically touches matching files already in git<commit_after>package commands

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/github/git-lfs/git"

	"github.com/github/git-lfs/lfs"
	"github.com/github/git-lfs/vendor/_nuts/github.com/spf13/cobra"
)

var (
	trackCmd = &cobra.Command{
		Use: "track",
		Run: trackCommand,
	}
)

func trackCommand(cmd *cobra.Command, args []string) {
	if lfs.LocalGitDir == "" {
		Print("Not a git repository.")
		os.Exit(128)
	}

	if lfs.LocalWorkingDir == "" {
		Print("This operation must be run in a work tree.")
		os.Exit(128)
	}

	lfs.InstallHooks(false)
	knownPaths := findPaths()

	if len(args) == 0 {
		Print("Listing tracked paths")
		for _, t := range knownPaths {
			Print("    %s (%s)", t.Path, t.Source)
		}
		return
	}

	addTrailingLinebreak := needsTrailingLinebreak(".gitattributes")
	attributesFile, err := os.OpenFile(".gitattributes", os.O_RDWR|os.O_APPEND|os.O_CREATE, 0660)
	if err != nil {
		Print("Error opening .gitattributes file")
		return
	}
	defer attributesFile.Close()

	if addTrailingLinebreak {
		if _, err := attributesFile.WriteString("\n"); err != nil {
			Print("Error writing to .gitattributes")
		}
	}

	wd, _ := os.Getwd()
	relpath, err := filepath.Rel(lfs.LocalWorkingDir, wd)
	if err != nil {
		Exit("Current directory %q outside of git working directory %q.", wd, lfs.LocalWorkingDir)
	}

ArgsLoop:
	for _, pattern := range args {
		for _, known := range knownPaths {
			if known.Path == filepath.Join(relpath, pattern) {
				Print("%s already supported", pattern)
				continue ArgsLoop
			}
		}

		encodedArg := strings.Replace(pattern, " ", "[[:space:]]", -1)
		_, err := attributesFile.WriteString(fmt.Sprintf("%s filter=lfs diff=lfs merge=lfs -text\n", encodedArg))
		if err != nil {
			Print("Error adding path %s", pattern)
			continue
		}
		Print("Tracking %s", pattern)

		// Make sure any existing git tracked files have their timestamp updated
		// so they will now show as modifed
		// note this is relative to current dir which is how we write .gitattributes
		// deliberately not done in parallel as a chan because we'll be marking modified
		gittracked, err := git.GetTrackedFiles(pattern)
		if err != nil {
			LoggedError(err, "Error getting git tracked files")
			continue
		}
		now := time.Now()
		for _, f := range gittracked {
			err := os.Chtimes(f, now, now)
			if err != nil {
				LoggedError(err, "Error marking %q modified", f)
				continue
			}
		}

	}
}

type mediaPath struct {
	Path   string
	Source string
}

func findPaths() []mediaPath {
	paths := make([]mediaPath, 0)

	for _, path := range findAttributeFiles() {
		attributes, err := os.Open(path)
		if err != nil {
			continue
		}

		scanner := bufio.NewScanner(attributes)

		for scanner.Scan() {
			line := scanner.Text()
			if strings.Contains(line, "filter=lfs") {
				fields := strings.Fields(line)
				relfile, _ := filepath.Rel(lfs.LocalWorkingDir, path)
				pattern := fields[0]
				if reldir := filepath.Dir(relfile); len(reldir) > 0 {
					pattern = filepath.Join(reldir, pattern)
				}

				paths = append(paths, mediaPath{Path: pattern, Source: relfile})
			}
		}
	}

	return paths
}

func findAttributeFiles() []string {
	paths := make([]string, 0)

	repoAttributes := filepath.Join(lfs.LocalGitDir, "info", "attributes")
	if info, err := os.Stat(repoAttributes); err == nil && !info.IsDir() {
		paths = append(paths, repoAttributes)
	}

	filepath.Walk(lfs.LocalWorkingDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if !info.IsDir() && (filepath.Base(path) == ".gitattributes") {
			paths = append(paths, path)
		}
		return nil
	})

	return paths
}

func needsTrailingLinebreak(filename string) bool {
	file, err := os.Open(filename)
	if err != nil {
		return false
	}
	defer file.Close()

	buf := make([]byte, 16384)
	bytesRead := 0
	for {
		n, err := file.Read(buf)
		if err == io.EOF {
			break
		} else if err != nil {
			return false
		}
		bytesRead = n
	}

	return !strings.HasSuffix(string(buf[0:bytesRead]), "\n")
}

func init() {
	RootCmd.AddCommand(trackCmd)
}
<|endoftext|>


<commit_before>package commands

import (
	"errors"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/aisk/logp"
	"github.com/leancloud/go-upload"
	"github.com/leancloud/lean-cli/api"
	"github.com/leancloud/lean-cli/api/regions"
	"github.com/leancloud/lean-cli/apps"
	"github.com/leancloud/lean-cli/runtimes"
	"github.com/leancloud/lean-cli/utils"
	"github.com/leancloud/lean-cli/version"
	"github.com/urfave/cli"
)

func deployAction(c *cli.Context) error {
	version.PrintCurrentVersion()
	isDeployFromGit := c.Bool("g")
	isDeployFromJavaWar := c.Bool("war")
	ignoreFilePath := c.String("leanignore")
	noDepsCache := c.Bool("no-cache")
	overwriteFuncs := c.Bool("overwrite-functions")
	message := c.String("message")
	keepFile := c.Bool("keep-deploy-file")
	revision := c.String("revision")
	prodString := c.String("prod")
	isDirect := c.Bool("direct")

	var prod int

	appID, err := apps.GetCurrentAppID(".")
	if err != nil {
		return err
	}

	groupName, err := apps.GetCurrentGroup(".")
	if err != nil {
		return err
	}

	logp.Info("Retrieving app info ...")

	region, err := apps.GetAppRegion(appID)
	if err != nil {
		return err
	}

	appInfo, err := api.GetAppInfo(appID)
	if err != nil {
		return err
	}

	groupInfo, err := api.GetGroup(appID, groupName)
	if err != nil {
		return err
	}

	if prodString == "" {
		if groupInfo.Staging.Deployable {
			prod = 0
		} else {
			prod = 1
		}
	} else {
		prod, err = strconv.Atoi(prodString)
		if err != nil {
			return err
		}
	}
	if prod == 0 && !groupInfo.Staging.Deployable {
		return cli.NewExitError("Deployment failed: no staging instance", 1)
	} else if prod == 1 && !groupInfo.Production.Deployable {
		return cli.NewExitError("Deployment failed: no production instance", 1)
	}

	if prod == 1 {
		logp.Infof("Preparing to deploy %s(%s) to region: %s group: %s production\r\n", appInfo.AppName, appID, region, groupName)
	} else {
		logp.Infof("Preparing to deploy %s(%s) to region: %s group: %s staging\r\n", appInfo.AppName, appID, region, groupName)
	}

	opts := &api.DeployOptions{
		NoDepsCache:    noDepsCache,
		OverwriteFuncs: overwriteFuncs,
		Options:        c.String("options"),
	}

	if isDeployFromGit {
		err = deployFromGit(appID, groupName, prod, revision, opts)
		if err != nil {
			return err
		}
	} else {
		opts.Message = getCommentMessage(message)
		opts.DirectUpload = isDirect
		err = deployFromLocal(appID, groupName, prod, isDeployFromJavaWar, ignoreFilePath, keepFile, opts)
		if err != nil {
			return err
		}
	}
	return nil
}

func packageProject(repoPath, ignoreFilePath string) (string, error) {
	fileDir, err := ioutil.TempDir("", "leanengine")
	if err != nil {
		return "", err
	}

	archiveFile := filepath.Join(fileDir, "leanengine.zip")

	runtime, err := runtimes.DetectRuntime(repoPath)
	if err == runtimes.ErrRuntimeNotFound {
		logp.Warn("Failed to recognize project type. Please inspect the directory structure if the deployment failed.")
	} else if err != nil {
		return "", err
	}

	if err := runtime.ArchiveUploadFiles(archiveFile, ignoreFilePath); err != nil {
		return "", err
	}

	return archiveFile, nil
}

func uploadProject(appID string, region regions.Region, repoPath string, ignoreFilePath string) (*upload.File, error) {
	archiveFile, err := packageProject(repoPath, ignoreFilePath)
	if err != nil {
		return nil, err
	}

	return api.UploadToRepoStorage(region, archiveFile)
}

func packageWar(repoPath string) (string, error) {
	var warPath string
	files, err := ioutil.ReadDir(filepath.Join(repoPath, "target"))
	if err != nil {
		return "", err
	}
	for _, file := range files {
		if strings.HasSuffix(file.Name(), ".war") && !file.IsDir() {
			warPath = filepath.Join(repoPath, "target", file.Name())
		}
	}
	if warPath == "" {
		return "", errors.New("cannot find .war file in ./target")
	}

	logp.Info("Found .war file:", warPath)

	fileDir, err := ioutil.TempDir("", "leanengine")
	if err != nil {
		return "", err
	}
	archivePath := filepath.Join(fileDir, "ROOT.war.zip")

	file := []struct{ Name, Path string }{{
		Name: "ROOT.war",
		Path: warPath,
	}}
	if err = utils.ArchiveFiles(archivePath, file); err != nil {
		return "", err
	}

	return archivePath, nil
}

func uploadWar(appID string, region regions.Region, repoPath string) (*upload.File, error) {
	archivePath, err := packageWar(repoPath)
	if err != nil {
		return nil, err
	}
	return api.UploadToRepoStorage(region, archivePath)
}

func deployFromLocal(appID string, group string, prod int, isDeployFromJavaWar bool, ignoreFilePath string, keepFile bool, opts *api.DeployOptions) error {
	region, err := apps.GetAppRegion(appID)
	if err != nil {
		return err
	}

	var file *upload.File
	var archiveFilePath string
	if isDeployFromJavaWar {
		if opts.DirectUpload {
			archiveFilePath, err = packageWar(".")
		} else {
			file, err = uploadWar(appID, region, ".")
		}
		if err != nil {
			return err
		}
	} else {
		if opts.DirectUpload {
			archiveFilePath, err = packageProject(".", ignoreFilePath)
		} else {
			file, err = uploadProject(appID, region, ".", ignoreFilePath)
		}
		if err != nil {
			return err
		}
	}

	if !opts.DirectUpload {
		if !keepFile {
			defer func() {
				logp.Info("Deleting temporary files")
				err := api.DeleteFromRepoStorage(region, file.ObjectID)
				if err != nil {
					logp.Error(err)
				}
			}()
		}
	}

	var eventTok string
	if opts.DirectUpload {
		eventTok, err = api.DeployAppFromFile(appID, group, prod, archiveFilePath, opts)
	} else {
		eventTok, err = api.DeployAppFromFile(appID, group, prod, file.URL, opts)
	}
	if err != nil {
		return err
	}
	ok, err := api.PollEvents(appID, eventTok)
	if err != nil {
		return err
	}
	if !ok {
		return cli.NewExitError("Deployment failed", 1)
	}
	return nil
}

func deployFromGit(appID string, group string, prod int, revision string, opts *api.DeployOptions) error {
	eventTok, err := api.DeployAppFromGit(appID, group, prod, revision, opts)
	if err != nil {
		return err
	}
	ok, err := api.PollEvents(appID, eventTok)
	if err != nil {
		return err
	}
	if !ok {
		return cli.NewExitError("Deployment failed", 1)
	}
	return nil
}

func getCommentMessage(message string) string {
	if message == "" {
		_, err := exec.LookPath("git")

		if err == nil {
			if _, err := os.Stat("./.git"); !os.IsNotExist(err) {
				messageBuf, err := exec.Command("git", "log", "-1", "--no-color", "--pretty=%B").CombinedOutput()
				messageStr := string(messageBuf)

				if err != nil {
					logp.Error("failed to load git message: ", err)
				} else {
					message = "WIP on: " + strings.TrimSpace(messageStr)
				}
			}
		}
	}

	if message == "" {
		message = "Creating from the CLI"
	}

	return message
}
<commit_msg>upload additional leanengine.yaml and system.properties files with ROOT.war<commit_after>package commands

import (
	"errors"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/aisk/logp"
	"github.com/leancloud/go-upload"
	"github.com/leancloud/lean-cli/api"
	"github.com/leancloud/lean-cli/api/regions"
	"github.com/leancloud/lean-cli/apps"
	"github.com/leancloud/lean-cli/runtimes"
	"github.com/leancloud/lean-cli/utils"
	"github.com/leancloud/lean-cli/version"
	"github.com/urfave/cli"
)

func deployAction(c *cli.Context) error {
	version.PrintCurrentVersion()
	isDeployFromGit := c.Bool("g")
	isDeployFromJavaWar := c.Bool("war")
	ignoreFilePath := c.String("leanignore")
	noDepsCache := c.Bool("no-cache")
	overwriteFuncs := c.Bool("overwrite-functions")
	message := c.String("message")
	keepFile := c.Bool("keep-deploy-file")
	revision := c.String("revision")
	prodString := c.String("prod")
	isDirect := c.Bool("direct")

	var prod int

	appID, err := apps.GetCurrentAppID(".")
	if err != nil {
		return err
	}

	groupName, err := apps.GetCurrentGroup(".")
	if err != nil {
		return err
	}

	logp.Info("Retrieving app info ...")

	region, err := apps.GetAppRegion(appID)
	if err != nil {
		return err
	}

	appInfo, err := api.GetAppInfo(appID)
	if err != nil {
		return err
	}

	groupInfo, err := api.GetGroup(appID, groupName)
	if err != nil {
		return err
	}

	if prodString == "" {
		if groupInfo.Staging.Deployable {
			prod = 0
		} else {
			prod = 1
		}
	} else {
		prod, err = strconv.Atoi(prodString)
		if err != nil {
			return err
		}
	}
	if prod == 0 && !groupInfo.Staging.Deployable {
		return cli.NewExitError("Deployment failed: no staging instance", 1)
	} else if prod == 1 && !groupInfo.Production.Deployable {
		return cli.NewExitError("Deployment failed: no production instance", 1)
	}

	if prod == 1 {
		logp.Infof("Preparing to deploy %s(%s) to region: %s group: %s production\r\n", appInfo.AppName, appID, region, groupName)
	} else {
		logp.Infof("Preparing to deploy %s(%s) to region: %s group: %s staging\r\n", appInfo.AppName, appID, region, groupName)
	}

	opts := &api.DeployOptions{
		NoDepsCache:    noDepsCache,
		OverwriteFuncs: overwriteFuncs,
		Options:        c.String("options"),
	}

	if isDeployFromGit {
		err = deployFromGit(appID, groupName, prod, revision, opts)
		if err != nil {
			return err
		}
	} else {
		opts.Message = getCommentMessage(message)
		opts.DirectUpload = isDirect
		err = deployFromLocal(appID, groupName, prod, isDeployFromJavaWar, ignoreFilePath, keepFile, opts)
		if err != nil {
			return err
		}
	}
	return nil
}

func packageProject(repoPath, ignoreFilePath string) (string, error) {
	fileDir, err := ioutil.TempDir("", "leanengine")
	if err != nil {
		return "", err
	}

	archiveFile := filepath.Join(fileDir, "leanengine.zip")

	runtime, err := runtimes.DetectRuntime(repoPath)
	if err == runtimes.ErrRuntimeNotFound {
		logp.Warn("Failed to recognize project type. Please inspect the directory structure if the deployment failed.")
	} else if err != nil {
		return "", err
	}

	if err := runtime.ArchiveUploadFiles(archiveFile, ignoreFilePath); err != nil {
		return "", err
	}

	return archiveFile, nil
}

func uploadProject(appID string, region regions.Region, repoPath string, ignoreFilePath string) (*upload.File, error) {
	archiveFile, err := packageProject(repoPath, ignoreFilePath)
	if err != nil {
		return nil, err
	}

	return api.UploadToRepoStorage(region, archiveFile)
}

func packageWar(repoPath string) (string, error) {
	var warPath string
	files, err := ioutil.ReadDir(filepath.Join(repoPath, "target"))
	if err != nil {
		return "", err
	}
	for _, file := range files {
		if strings.HasSuffix(file.Name(), ".war") && !file.IsDir() {
			warPath = filepath.Join(repoPath, "target", file.Name())
		}
	}
	if warPath == "" {
		return "", errors.New("cannot find .war file in ./target")
	}

	logp.Info("Found .war file:", warPath)

	file := []struct{ Name, Path string }{{
		Name: "ROOT.war",
		Path: warPath,
	}}

	for _, filename := range []string{"leanengine.yaml", "system.properties"} {
		path := filepath.Join(repoPath, filename)
		if utils.IsFileExists(path) {
			file = append(file, struct{ Name, Path string }{
				Name: filename,
				Path: path,
			})
		}
	}

	fileDir, err := ioutil.TempDir("", "leanengine")
	if err != nil {
		return "", err
	}
	archivePath := filepath.Join(fileDir, "ROOT.war.zip")
	if err = utils.ArchiveFiles(archivePath, file); err != nil {
		return "", err
	}

	return archivePath, nil
}

func uploadWar(appID string, region regions.Region, repoPath string) (*upload.File, error) {
	archivePath, err := packageWar(repoPath)
	if err != nil {
		return nil, err
	}
	return api.UploadToRepoStorage(region, archivePath)
}

func deployFromLocal(appID string, group string, prod int, isDeployFromJavaWar bool, ignoreFilePath string, keepFile bool, opts *api.DeployOptions) error {
	region, err := apps.GetAppRegion(appID)
	if err != nil {
		return err
	}

	var file *upload.File
	var archiveFilePath string
	if isDeployFromJavaWar {
		if opts.DirectUpload {
			archiveFilePath, err = packageWar(".")
		} else {
			file, err = uploadWar(appID, region, ".")
		}
		if err != nil {
			return err
		}
	} else {
		if opts.DirectUpload {
			archiveFilePath, err = packageProject(".", ignoreFilePath)
		} else {
			file, err = uploadProject(appID, region, ".", ignoreFilePath)
		}
		if err != nil {
			return err
		}
	}

	if !opts.DirectUpload {
		if !keepFile {
			defer func() {
				logp.Info("Deleting temporary files")
				err := api.DeleteFromRepoStorage(region, file.ObjectID)
				if err != nil {
					logp.Error(err)
				}
			}()
		}
	}

	var eventTok string
	if opts.DirectUpload {
		eventTok, err = api.DeployAppFromFile(appID, group, prod, archiveFilePath, opts)
	} else {
		eventTok, err = api.DeployAppFromFile(appID, group, prod, file.URL, opts)
	}
	if err != nil {
		return err
	}
	ok, err := api.PollEvents(appID, eventTok)
	if err != nil {
		return err
	}
	if !ok {
		return cli.NewExitError("Deployment failed", 1)
	}
	return nil
}

func deployFromGit(appID string, group string, prod int, revision string, opts *api.DeployOptions) error {
	eventTok, err := api.DeployAppFromGit(appID, group, prod, revision, opts)
	if err != nil {
		return err
	}
	ok, err := api.PollEvents(appID, eventTok)
	if err != nil {
		return err
	}
	if !ok {
		return cli.NewExitError("Deployment failed", 1)
	}
	return nil
}

func getCommentMessage(message string) string {
	if message == "" {
		_, err := exec.LookPath("git")

		if err == nil {
			if _, err := os.Stat("./.git"); !os.IsNotExist(err) {
				messageBuf, err := exec.Command("git", "log", "-1", "--no-color", "--pretty=%B").CombinedOutput()
				messageStr := string(messageBuf)

				if err != nil {
					logp.Error("failed to load git message: ", err)
				} else {
					message = "WIP on: " + strings.TrimSpace(messageStr)
				}
			}
		}
	}

	if message == "" {
		message = "Creating from the CLI"
	}

	return message
}
<|endoftext|>