<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"reflect"
	"sync"
	"time"

	log "github.com/couchbaselabs/clog"
)

type Manager struct {
	startTime time.Time
	version   string // Our software VERSION.
	cfg       Cfg
	uuid      string // Unique to every Manager process instance.
	tags      []string
	bindAddr  string
	dataDir   string
	server    string // The datasource that cbft will index.
	m         sync.Mutex
	feeds     map[string]Feed    // Key is Feed.Name().
	pindexes  map[string]*PIndex // Key is PIndex.Name().
	plannerCh chan *WorkReq      // Used to kick the planner that there's more work.
	janitorCh chan *WorkReq      // Used to kick the janitor that there's more work.
	meh       ManagerEventHandlers
}

type ManagerEventHandlers interface {
	OnRegisterPIndex(pindex *PIndex)
	OnUnregisterPIndex(pindex *PIndex)
}

func NewManager(version string, cfg Cfg, uuid string, tags []string,
	bindAddr, dataDir string, server string, meh ManagerEventHandlers) *Manager {
	return &Manager{
		startTime: time.Now(),
		version:   version,
		cfg:       cfg,
		uuid:      uuid,
		bindAddr:  bindAddr,
		dataDir:   dataDir,
		server:    server,
		feeds:     make(map[string]Feed),
		pindexes:  make(map[string]*PIndex),
		plannerCh: make(chan *WorkReq),
		janitorCh: make(chan *WorkReq),
		meh:       meh,
	}
}

func (mgr *Manager) Start(registerAsWanted bool) error {
	// Save our nodeDef (with our UUID) into the Cfg.
	if err := mgr.SaveNodeDef(NODE_DEFS_KNOWN); err != nil {
		return err
	}
	if registerAsWanted {
		if err := mgr.SaveNodeDef(NODE_DEFS_WANTED); err != nil {
			return err
		}
	}

	if err := mgr.LoadDataDir(); err != nil {
		return err
	}

	go mgr.PlannerLoop()
	mgr.PlannerKick("start")

	go mgr.JanitorLoop()
	mgr.JanitorKick("start")

	return nil
}

// Walk the data dir and register pindexes.
func (mgr *Manager) LoadDataDir() error {
	log.Printf("loading dataDir...")

	dirEntries, err := ioutil.ReadDir(mgr.dataDir)
	if err != nil {
		return fmt.Errorf("error: could not read dataDir: %s, err: %v",
			mgr.dataDir, err)
	}

	for _, dirInfo := range dirEntries {
		path := mgr.dataDir + string(os.PathSeparator) + dirInfo.Name()
		_, ok := mgr.ParsePIndexPath(path)
		if !ok {
			continue // Skip the entry that doesn't match the naming pattern.
		}

		log.Printf("  opening pindex: %s", path)
		pindex, err := OpenPIndex(path)
		if err != nil {
			log.Printf("error: could not open pindex: %s, err: %v",
				path, err)
			continue
		}

		mgr.registerPIndex(pindex)
	}

	log.Printf("loading dataDir... done")
	return nil
}

func (mgr *Manager) SaveNodeDef(kind string) error {
	if mgr.cfg == nil {
		return nil // Occurs during testing.
	}

	for {
		nodeDefs, cas, err := CfgGetNodeDefs(mgr.cfg, kind)
		if err != nil {
			return err
		}
		if nodeDefs == nil {
			nodeDefs = NewNodeDefs(mgr.version)
		}
		nodeDef, exists := nodeDefs.NodeDefs[mgr.bindAddr]
		if exists {
			// TODO: need a way to force overwrite other node's UUID?
			if nodeDef.UUID != mgr.uuid {
				return fmt.Errorf("some other node is running at our bindAddr: %v,"+
					" with different uuid: %s, than our uuid: %s",
					mgr.bindAddr, nodeDef.UUID, mgr.uuid)
			}
			if nodeDef.ImplVersion == mgr.version &&
				reflect.DeepEqual(nodeDef.Tags, mgr.tags) {
				return nil // No changes, so leave the existing nodeDef.
			}
		}

		nodeDef = &NodeDef{
			HostPort:    mgr.bindAddr, // TODO: need FQDN:port instead of ":8095".
			UUID:        mgr.uuid,
			ImplVersion: mgr.version,
			Tags:        mgr.tags,
		}

		nodeDefs.UUID = NewUUID()
		nodeDefs.NodeDefs[mgr.bindAddr] = nodeDef
		nodeDefs.ImplVersion = mgr.version // TODO: ImplVersion bump?

		_, err = CfgSetNodeDefs(mgr.cfg, kind, nodeDefs, cas)
		if err != nil {
			if _, ok := err.(*CfgCASError); ok {
				// Retry if it was a CAS mismatch, as perhaps
				// multiple nodes are all racing to register themselves,
				// such as in a full datacenter power restart.
				continue
			}
			return err
		}
		break
	}
	return nil
}

func (mgr *Manager) registerFeed(feed Feed) error {
	mgr.m.Lock()
	defer mgr.m.Unlock()

	if _, exists := mgr.feeds[feed.Name()]; exists {
		return fmt.Errorf("error: registered feed already exists, name: %s",
			feed.Name())
	}
	mgr.feeds[feed.Name()] = feed
	return nil
}

func (mgr *Manager) unregisterFeed(name string) Feed {
	mgr.m.Lock()
	defer mgr.m.Unlock()

	rv, ok := mgr.feeds[name]
	if ok {
		delete(mgr.feeds, name)
		return rv
	}
	return nil
}

func (mgr *Manager) registerPIndex(pindex *PIndex) error {
	mgr.m.Lock()
	defer mgr.m.Unlock()

	if _, exists := mgr.pindexes[pindex.Name]; exists {
		return fmt.Errorf("error: registered pindex already exists, name: %s",
			pindex.Name)
	}
	mgr.pindexes[pindex.Name] = pindex
	if mgr.meh != nil {
		mgr.meh.OnRegisterPIndex(pindex)
	}
	return nil
}

func (mgr *Manager) unregisterPIndex(name string) *PIndex {
	mgr.m.Lock()
	defer mgr.m.Unlock()

	pindex, ok := mgr.pindexes[name]
	if ok {
		delete(mgr.pindexes, name)
		if mgr.meh != nil {
			mgr.meh.OnUnregisterPIndex(pindex)
		}
		return pindex
	}
	return nil
}

// Returns a snapshot copy of the current feeds and pindexes.
func (mgr *Manager) CurrentMaps() (map[string]Feed, map[string]*PIndex) {
	feeds := make(map[string]Feed)
	pindexes := make(map[string]*PIndex)

	mgr.m.Lock()
	defer mgr.m.Unlock()

	for k, v := range mgr.feeds {
		feeds[k] = v
	}
	for k, v := range mgr.pindexes {
		pindexes[k] = v
	}
	return feeds, pindexes
}

func (mgr *Manager) PIndexPath(pindexName string) string {
	return PIndexPath(mgr.dataDir, pindexName)
}

func (mgr *Manager) ParsePIndexPath(pindexPath string) (string, bool) {
	return ParsePIndexPath(mgr.dataDir, pindexPath)
}

func (mgr *Manager) DataDir() string {
	return mgr.dataDir
}
<commit_msg>check tags for pindex/planner/janitor-local cases<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"reflect"
	"sync"
	"time"

	log "github.com/couchbaselabs/clog"
)

type Manager struct {
	startTime time.Time
	version   string // Our software VERSION.
	cfg       Cfg
	uuid      string // Unique to every Manager process instance.
	tags      []string
	bindAddr  string
	dataDir   string
	server    string // The datasource that cbft will index.
	m         sync.Mutex
	feeds     map[string]Feed    // Key is Feed.Name().
	pindexes  map[string]*PIndex // Key is PIndex.Name().
	plannerCh chan *WorkReq      // Used to kick the planner that there's more work.
	janitorCh chan *WorkReq      // Used to kick the janitor that there's more work.
	meh       ManagerEventHandlers
}

type ManagerEventHandlers interface {
	OnRegisterPIndex(pindex *PIndex)
	OnUnregisterPIndex(pindex *PIndex)
}

func NewManager(version string, cfg Cfg, uuid string, tags []string,
	bindAddr, dataDir string, server string, meh ManagerEventHandlers) *Manager {
	return &Manager{
		startTime: time.Now(),
		version:   version,
		cfg:       cfg,
		uuid:      uuid,
		bindAddr:  bindAddr,
		dataDir:   dataDir,
		server:    server,
		feeds:     make(map[string]Feed),
		pindexes:  make(map[string]*PIndex),
		plannerCh: make(chan *WorkReq),
		janitorCh: make(chan *WorkReq),
		meh:       meh,
	}
}

func (mgr *Manager) Start(registerAsWanted bool) error {
	// Save our nodeDef (with our UUID) into the Cfg.
	if err := mgr.SaveNodeDef(NODE_DEFS_KNOWN); err != nil {
		return err
	}
	if registerAsWanted {
		if err := mgr.SaveNodeDef(NODE_DEFS_WANTED); err != nil {
			return err
		}
	}

	tags := map[string]bool{}
	for _, tag := range mgr.tags {
		tags[tag] = true
	}

	if mgr.tags == nil || tags["pindex"] {
		if err := mgr.LoadDataDir(); err != nil {
			return err
		}
	}

	if mgr.tags == nil || tags["planner"] {
		go mgr.PlannerLoop()
		mgr.PlannerKick("start")
	}

	if mgr.tags == nil || (tags["pindex"] && tags["janitor-local"]) {
		go mgr.JanitorLoop()
		mgr.JanitorKick("start")
	}

	return nil
}

// Walk the data dir and register pindexes.
func (mgr *Manager) LoadDataDir() error {
	log.Printf("loading dataDir...")

	dirEntries, err := ioutil.ReadDir(mgr.dataDir)
	if err != nil {
		return fmt.Errorf("error: could not read dataDir: %s, err: %v",
			mgr.dataDir, err)
	}

	for _, dirInfo := range dirEntries {
		path := mgr.dataDir + string(os.PathSeparator) + dirInfo.Name()
		_, ok := mgr.ParsePIndexPath(path)
		if !ok {
			continue // Skip the entry that doesn't match the naming pattern.
		}

		log.Printf("  opening pindex: %s", path)
		pindex, err := OpenPIndex(path)
		if err != nil {
			log.Printf("error: could not open pindex: %s, err: %v",
				path, err)
			continue
		}

		mgr.registerPIndex(pindex)
	}

	log.Printf("loading dataDir... done")
	return nil
}

func (mgr *Manager) SaveNodeDef(kind string) error {
	if mgr.cfg == nil {
		return nil // Occurs during testing.
	}

	for {
		nodeDefs, cas, err := CfgGetNodeDefs(mgr.cfg, kind)
		if err != nil {
			return err
		}
		if nodeDefs == nil {
			nodeDefs = NewNodeDefs(mgr.version)
		}
		nodeDef, exists := nodeDefs.NodeDefs[mgr.bindAddr]
		if exists {
			// TODO: need a way to force overwrite other node's UUID?
			if nodeDef.UUID != mgr.uuid {
				return fmt.Errorf("some other node is running at our bindAddr: %v,"+
					" with different uuid: %s, than our uuid: %s",
					mgr.bindAddr, nodeDef.UUID, mgr.uuid)
			}
			if nodeDef.ImplVersion == mgr.version &&
				reflect.DeepEqual(nodeDef.Tags, mgr.tags) {
				return nil // No changes, so leave the existing nodeDef.
			}
		}

		nodeDef = &NodeDef{
			HostPort:    mgr.bindAddr, // TODO: need FQDN:port instead of ":8095".
			UUID:        mgr.uuid,
			ImplVersion: mgr.version,
			Tags:        mgr.tags,
		}

		nodeDefs.UUID = NewUUID()
		nodeDefs.NodeDefs[mgr.bindAddr] = nodeDef
		nodeDefs.ImplVersion = mgr.version // TODO: ImplVersion bump?

		_, err = CfgSetNodeDefs(mgr.cfg, kind, nodeDefs, cas)
		if err != nil {
			if _, ok := err.(*CfgCASError); ok {
				// Retry if it was a CAS mismatch, as perhaps
				// multiple nodes are all racing to register themselves,
				// such as in a full datacenter power restart.
				continue
			}
			return err
		}
		break
	}
	return nil
}

func (mgr *Manager) registerFeed(feed Feed) error {
	mgr.m.Lock()
	defer mgr.m.Unlock()

	if _, exists := mgr.feeds[feed.Name()]; exists {
		return fmt.Errorf("error: registered feed already exists, name: %s",
			feed.Name())
	}
	mgr.feeds[feed.Name()] = feed
	return nil
}

func (mgr *Manager) unregisterFeed(name string) Feed {
	mgr.m.Lock()
	defer mgr.m.Unlock()

	rv, ok := mgr.feeds[name]
	if ok {
		delete(mgr.feeds, name)
		return rv
	}
	return nil
}

func (mgr *Manager) registerPIndex(pindex *PIndex) error {
	mgr.m.Lock()
	defer mgr.m.Unlock()

	if _, exists := mgr.pindexes[pindex.Name]; exists {
		return fmt.Errorf("error: registered pindex already exists, name: %s",
			pindex.Name)
	}
	mgr.pindexes[pindex.Name] = pindex
	if mgr.meh != nil {
		mgr.meh.OnRegisterPIndex(pindex)
	}
	return nil
}

func (mgr *Manager) unregisterPIndex(name string) *PIndex {
	mgr.m.Lock()
	defer mgr.m.Unlock()

	pindex, ok := mgr.pindexes[name]
	if ok {
		delete(mgr.pindexes, name)
		if mgr.meh != nil {
			mgr.meh.OnUnregisterPIndex(pindex)
		}
		return pindex
	}
	return nil
}

// Returns a snapshot copy of the current feeds and pindexes.
func (mgr *Manager) CurrentMaps() (map[string]Feed, map[string]*PIndex) {
	feeds := make(map[string]Feed)
	pindexes := make(map[string]*PIndex)

	mgr.m.Lock()
	defer mgr.m.Unlock()

	for k, v := range mgr.feeds {
		feeds[k] = v
	}
	for k, v := range mgr.pindexes {
		pindexes[k] = v
	}
	return feeds, pindexes
}

func (mgr *Manager) PIndexPath(pindexName string) string {
	return PIndexPath(mgr.dataDir, pindexName)
}

func (mgr *Manager) ParsePIndexPath(pindexPath string) (string, bool) {
	return ParsePIndexPath(mgr.dataDir, pindexPath)
}

func (mgr *Manager) DataDir() string {
	return mgr.dataDir
}
<|endoftext|>


<commit_before>package manikyr

import (
	"errors"
	"os"
	"path"

	"github.com/go-fsnotify/fsnotify"
	"github.com/disintegration/imaging"
)

var (
	ErrRootNotWatched 	= errors.New("root is not watched")
	ErrRootWatched 		= errors.New("root is already watched")
	ErrSubdirNotWatched	= errors.New("subdir is not watched")
	ErrSubdirWatched	= errors.New("subdir is already watched")

	NearestNeighbor		= imaging.NearestNeighbor
	Box			= imaging.Box
	Linear			= imaging.Linear
	Hermite			= imaging.Hermite
	MitchellNetravali	= imaging.MitchellNetravali
	CatmullRom		= imaging.CatmullRom
	BSpline			= imaging.BSpline
	Gaussian		= imaging.Gaussian
	Bartlett		= imaging.Bartlett
	Lanczos			= imaging.Lanczos
	Hann			= imaging.Hann
	Hamming			= imaging.Hamming
	Blackman		= imaging.Blackman
	Welch			= imaging.Welch
	Cosine			= imaging.Cosine
)

type Manikyr struct {
	roots			map[string]*fsnotify.Watcher
	subdirs			map[string][]string
	errChans		map[string]chan error
	doneChans		map[string]chan bool
	thumbDirPerms		os.FileMode
	thumbWidth		int
	thumbHeight		int
	thumbAlgo		imaging.ResampleFilter
	ThumbDirGetter		func(string) string
	ThumbNameGetter		func(string) string
	ShouldCreateThumb	func(string, string) bool
	ShouldWatchSubdir	func(string, string) bool
}
func (m *Manikyr) Roots() []string {
	keys := make([]string, len(m.roots))
	i := 0
	for k := range m.roots {
		keys[i] = k
		i = i + 1
	}
	return keys
}
func (m *Manikyr) AddRoot(root string, errChan chan error) error {
	if _, ok := m.roots[root]; ok {
		return ErrRootWatched
	}

	w, err := fsnotify.NewWatcher()
	if err != nil {
		return err
	}

	doneChan := make(chan bool)

	m.roots[root] = w
	m.errChans[root] = errChan
	m.doneChans[root] = doneChan
	go m.watch(root, errChan, doneChan)

	return m.roots[root].Add(root)
}
func (m *Manikyr) RemoveRoot(root string) error {
	if _, ok := m.roots[root]; !ok {
		return ErrRootNotWatched
	}
	m.roots[root].Close()
	m.doneChans[root] <-true

	delete(m.roots, root)
	delete(m.errChans, root)
	delete(m.doneChans, root)
	delete(m.subdirs, root)
	return nil
}
func (m *Manikyr) watch(root string, errChan chan error, doneChan chan bool) {
	w, ok := m.roots[root]
	if !ok {
		errChan <-ErrRootNotWatched
		return
	}

	defer w.Close()
	for {
		select {
		case evt := <- w.Events:
			if evt.Op == fsnotify.Create {
				// If a file was created

				// Get some info about the file
				info, err := os.Stat(evt.Name)
				if os.IsNotExist(err) {
					errChan <-err
					continue
				}

				switch mode := info.Mode(); {
				case mode.IsDir():
					if m.ShouldWatchSubdir(root, evt.Name) {
						w.Add(evt.Name)
					}
				case mode.IsRegular():
					if m.ShouldCreateThumb(root, evt.Name) {
						go m.createThumb(evt.Name, errChan)
					}
				}
			} else {
				// Something else happened to the file
				_, err := os.Stat(evt.Name)
				if os.IsNotExist(err) {
					// Try to delete thumb.
					// Error is useless here because the file could
					// have been a directory or a non-image file.
					m.removeThumb(evt.Name)
				} else if err != nil {
					errChan <-err
					continue
				}
			}
		case err := <- w.Errors:
			errChan <-err
		case done := <- doneChan:
			break
		}
	}
}
func (m *Manikyr) AddSubdir(root, subdir string) error {
	if _, ok := m.roots[root]; !ok {
		return ErrRootNotWatched
	}
	for i := range m.subdirs[root] {
		if m.subdirs[root][i] == subdir {
			return ErrSubdirWatched
		}
	}

	err := m.roots[root].Add(subdir)
	if err != nil {
		return err
	}
	m.subdirs[root] = append(m.subdirs[root], subdir)
	return nil
}
func (m *Manikyr) RemoveSubdir(root, subdir string) error {
	if _, ok := m.roots[root]; !ok {
		return ErrRootNotWatched
	}

	for i := range m.subdirs[root] {
		if m.subdirs[root][i] == subdir {
			m.subdirs[root] = append(m.subdirs[root][:i], m.subdirs[root][i+1:]...) // Keep indexes <i || >i
			return m.roots[root].Remove(subdir)
		}
	}
	return ErrSubdirNotWatched
}
func (m *Manikyr) Subdirs(root string) ([]string, error) {
	subdirs := make([]string, len(m.subdirs[root]))

	if _, ok := m.roots[root]; !ok {
		return subdirs, ErrRootNotWatched
	}

	for i := range m.subdirs[root] {
		subdirs[i] = m.subdirs[root][i]
	}
	return subdirs, nil
}
func (m *Manikyr) removeThumb(parentFile string) error {
	thumbPath := path.Join(m.ThumbDirGetter(parentFile), m.ThumbNameGetter(parentFile))
	return os.Remove(thumbPath)
}
func (m *Manikyr) createThumb(parentFile string, errChan chan error) {
	img, err := openImageWhenReady(parentFile)
	if err != nil {
		errChan <-err
		return
	}

	localThumbs := m.ThumbDirGetter(parentFile)
	_, err = os.Stat(localThumbs)
	// If thumbDir does not exist...
	if os.IsNotExist(err) {
		// ..create it
		err := os.Mkdir(localThumbs, m.thumbDirPerms)
		if err != nil {
			errChan <-err
			return
		}
	} else if err != nil {
		errChan <-err
		return
	}

	// Save the thumbnail
	thumb := imaging.Thumbnail(img, m.thumbWidth, m.thumbHeight, m.thumbAlgo)
	thumbPath := path.Join(localThumbs, m.ThumbNameGetter(parentFile))
	if err = imaging.Save(thumb, thumbPath); err != nil {
		errChan <-err
	}
}
func (m *Manikyr) ThumbSize() (int, int) {
	return m.thumbWidth, m.thumbHeight
}
func (m *Manikyr) SetThumbSize(w, h int) {
	// Dimensions must be positive

	if w < 1 {
		w = 1
	}
	m.thumbWidth = w

	if h < 1 {
		h = 1
	}
	m.thumbHeight = h
}
func (m *Manikyr) ThumbDirFileMode() uint32 {
	return uint32(m.thumbDirPerms)
}
func (m *Manikyr) SetThumbDirFileMode(fm uint32) {
	m.thumbDirPerms = os.FileMode(fm)
}
func (m *Manikyr) ThumbAlgorithm() imaging.ResampleFilter {
	return m.thumbAlgo
}
func (m *Manikyr) SetThumbAlgorithm(filter imaging.ResampleFilter) {
	m.thumbAlgo = filter
}

func New() *Manikyr {
	// Sensible defaults
	return &Manikyr{
		roots:			make(map[string]*fsnotify.Watcher),
		subdirs:		make(map[string][]string),
		errChans:		make(map[string]chan error),
		doneChans:		make(map[string]chan bool),
		thumbWidth: 		100,
		thumbHeight: 		100,
		thumbAlgo:		NearestNeighbor,
		thumbDirPerms:		0777,
		ThumbDirGetter: func(parentFile string) string {
			return path.Join(path.Dir(parentFile), ".thumbs")
		},
		ThumbNameGetter: func(parentFile string) string {
			return path.Base(parentFile)
		},
		ShouldCreateThumb: func(root, parentFile string) bool {
			ok, _ := NthSubdir(root, parentFile, 1)
			if ok {
				return true
			}
			return false
		},
		ShouldWatchSubdir: func(root, parentFile string) bool {
			ok, _ := NthSubdir(root, parentFile, 0)
			if ok && parentFile[0] != '.' {
				return true
			}
			return false
		},
	}
}
<commit_msg>Fix Manikyr default thumbDir<commit_after>package manikyr

import (
	"errors"
	"os"
	"path"

	"github.com/go-fsnotify/fsnotify"
	"github.com/disintegration/imaging"
)

var (
	ErrRootNotWatched 	= errors.New("root is not watched")
	ErrRootWatched 		= errors.New("root is already watched")
	ErrSubdirNotWatched	= errors.New("subdir is not watched")
	ErrSubdirWatched	= errors.New("subdir is already watched")

	NearestNeighbor		= imaging.NearestNeighbor
	Box			= imaging.Box
	Linear			= imaging.Linear
	Hermite			= imaging.Hermite
	MitchellNetravali	= imaging.MitchellNetravali
	CatmullRom		= imaging.CatmullRom
	BSpline			= imaging.BSpline
	Gaussian		= imaging.Gaussian
	Bartlett		= imaging.Bartlett
	Lanczos			= imaging.Lanczos
	Hann			= imaging.Hann
	Hamming			= imaging.Hamming
	Blackman		= imaging.Blackman
	Welch			= imaging.Welch
	Cosine			= imaging.Cosine
)

type Manikyr struct {
	roots			map[string]*fsnotify.Watcher
	subdirs			map[string][]string
	errChans		map[string]chan error
	doneChans		map[string]chan bool
	thumbDirPerms		os.FileMode
	thumbWidth		int
	thumbHeight		int
	thumbAlgo		imaging.ResampleFilter
	ThumbDirGetter		func(string) string
	ThumbNameGetter		func(string) string
	ShouldCreateThumb	func(string, string) bool
	ShouldWatchSubdir	func(string, string) bool
}
func (m *Manikyr) Roots() []string {
	keys := make([]string, len(m.roots))
	i := 0
	for k := range m.roots {
		keys[i] = k
		i = i + 1
	}
	return keys
}
func (m *Manikyr) AddRoot(root string, errChan chan error) error {
	if _, ok := m.roots[root]; ok {
		return ErrRootWatched
	}

	w, err := fsnotify.NewWatcher()
	if err != nil {
		return err
	}

	doneChan := make(chan bool)

	m.roots[root] = w
	m.errChans[root] = errChan
	m.doneChans[root] = doneChan
	go m.watch(root, errChan, doneChan)

	return m.roots[root].Add(root)
}
func (m *Manikyr) RemoveRoot(root string) error {
	if _, ok := m.roots[root]; !ok {
		return ErrRootNotWatched
	}
	m.roots[root].Close()
	m.doneChans[root] <-true

	delete(m.roots, root)
	delete(m.errChans, root)
	delete(m.doneChans, root)
	delete(m.subdirs, root)
	return nil
}
func (m *Manikyr) watch(root string, errChan chan error, doneChan chan bool) {
	w, ok := m.roots[root]
	if !ok {
		errChan <-ErrRootNotWatched
		return
	}

	defer w.Close()
	for {
		select {
		case evt := <- w.Events:
			if evt.Op == fsnotify.Create {
				// If a file was created

				// Get some info about the file
				info, err := os.Stat(evt.Name)
				if os.IsNotExist(err) {
					errChan <-err
					continue
				}

				switch mode := info.Mode(); {
				case mode.IsDir():
					if m.ShouldWatchSubdir(root, evt.Name) {
						w.Add(evt.Name)
					}
				case mode.IsRegular():
					if m.ShouldCreateThumb(root, evt.Name) {
						go m.createThumb(evt.Name, errChan)
					}
				}
			} else {
				// Something else happened to the file
				_, err := os.Stat(evt.Name)
				if os.IsNotExist(err) {
					// Try to delete thumb.
					// Error is useless here because the file could
					// have been a directory or a non-image file.
					m.removeThumb(evt.Name)
				} else if err != nil {
					errChan <-err
					continue
				}
			}
		case err := <- w.Errors:
			errChan <-err
		case done := <- doneChan:
			break
		}
	}
}
func (m *Manikyr) AddSubdir(root, subdir string) error {
	if _, ok := m.roots[root]; !ok {
		return ErrRootNotWatched
	}
	for i := range m.subdirs[root] {
		if m.subdirs[root][i] == subdir {
			return ErrSubdirWatched
		}
	}

	err := m.roots[root].Add(subdir)
	if err != nil {
		return err
	}
	m.subdirs[root] = append(m.subdirs[root], subdir)
	return nil
}
func (m *Manikyr) RemoveSubdir(root, subdir string) error {
	if _, ok := m.roots[root]; !ok {
		return ErrRootNotWatched
	}

	for i := range m.subdirs[root] {
		if m.subdirs[root][i] == subdir {
			m.subdirs[root] = append(m.subdirs[root][:i], m.subdirs[root][i+1:]...) // Keep indexes <i || >i
			return m.roots[root].Remove(subdir)
		}
	}
	return ErrSubdirNotWatched
}
func (m *Manikyr) Subdirs(root string) ([]string, error) {
	subdirs := make([]string, len(m.subdirs[root]))

	if _, ok := m.roots[root]; !ok {
		return subdirs, ErrRootNotWatched
	}

	for i := range m.subdirs[root] {
		subdirs[i] = m.subdirs[root][i]
	}
	return subdirs, nil
}
func (m *Manikyr) removeThumb(parentFile string) error {
	thumbPath := path.Join(m.ThumbDirGetter(parentFile), m.ThumbNameGetter(parentFile))
	return os.Remove(thumbPath)
}
func (m *Manikyr) createThumb(parentFile string, errChan chan error) {
	img, err := openImageWhenReady(parentFile)
	if err != nil {
		errChan <-err
		return
	}

	localThumbs := m.ThumbDirGetter(parentFile)
	_, err = os.Stat(localThumbs)
	// If thumbDir does not exist...
	if os.IsNotExist(err) {
		// ..create it
		err := os.Mkdir(localThumbs, m.thumbDirPerms)
		if err != nil {
			errChan <-err
			return
		}
	} else if err != nil {
		errChan <-err
		return
	}

	// Save the thumbnail
	thumb := imaging.Thumbnail(img, m.thumbWidth, m.thumbHeight, m.thumbAlgo)
	thumbPath := path.Join(localThumbs, m.ThumbNameGetter(parentFile))
	if err = imaging.Save(thumb, thumbPath); err != nil {
		errChan <-err
	}
}
func (m *Manikyr) ThumbSize() (int, int) {
	return m.thumbWidth, m.thumbHeight
}
func (m *Manikyr) SetThumbSize(w, h int) {
	// Dimensions must be positive

	if w < 1 {
		w = 1
	}
	m.thumbWidth = w

	if h < 1 {
		h = 1
	}
	m.thumbHeight = h
}
func (m *Manikyr) ThumbDirFileMode() uint32 {
	return uint32(m.thumbDirPerms)
}
func (m *Manikyr) SetThumbDirFileMode(fm uint32) {
	m.thumbDirPerms = os.FileMode(fm)
}
func (m *Manikyr) ThumbAlgorithm() imaging.ResampleFilter {
	return m.thumbAlgo
}
func (m *Manikyr) SetThumbAlgorithm(filter imaging.ResampleFilter) {
	m.thumbAlgo = filter
}

func New() *Manikyr {
	// Sensible defaults
	return &Manikyr{
		roots:			make(map[string]*fsnotify.Watcher),
		subdirs:		make(map[string][]string),
		errChans:		make(map[string]chan error),
		doneChans:		make(map[string]chan bool),
		thumbWidth: 		100,
		thumbHeight: 		100,
		thumbAlgo:		NearestNeighbor,
		thumbDirPerms:		0777,
		ThumbDirGetter: func(parentFile string) string {
			return os.TempDir()
		},
		ThumbNameGetter: func(parentFile string) string {
			return path.Base(parentFile)
		},
		ShouldCreateThumb: func(root, parentFile string) bool {
			ok, _ := NthSubdir(root, parentFile, 1)
			if ok {
				return true
			}
			return false
		},
		ShouldWatchSubdir: func(root, parentFile string) bool {
			ok, _ := NthSubdir(root, parentFile, 0)
			if ok && parentFile[0] != '.' {
				return true
			}
			return false
		},
	}
}
<|endoftext|>


<commit_before>package models

import (
	// Go Builtin Packages
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	// Google Appengine Packages
	"appengine"
	"appengine/datastore"
	"appengine/user"

	// Google OAuth2/Drive Packages
	"golang.org/x/oauth2"
	"google.golang.org/api/drive/v3"
	go_ae "google.golang.org/appengine"
	"google.golang.org/appengine/memcache"

	// Local Packages
	"pastebin/utils"
)

type OAuthToken struct {
	UserID  string       `datastore:"user_id"`
	Token   oauth2.Token `datastore:"token,noindex"`
	BatchID string       `datastore:"batch_id,noindex"`
}

func SaveOAuthToken(c appengine.Context, t *oauth2.Token) error {
	user_id := user.Current(c).ID
	token := new(OAuthToken)
	token.UserID = user_id
	token.Token = *t
	token.BatchID = fmt.Sprintf("%s_%s", user_id, time.Now().Format(time.RFC3339Nano))

	key := datastore.NewKey(c, "OAuthToken", user_id, 0, nil)
	if _, err := datastore.Put(c, key, token); err != nil {
		return err
	}
	return nil
}

func DeleteOAuthToken(c appengine.Context, batch_id string, user_id string) error {
	if berr := NewBatchCleanQ(c, batch_id); berr != nil {
		return berr
	}

	key := datastore.NewKey(c, "OAuthToken", user_id, 0, nil)
	err := datastore.Delete(c, key)
	return err
}

func CheckOAuthToken(c appengine.Context) (bool, error) {
	if usr := user.Current(c); usr != nil {
		if count, err := datastore.NewQuery("OAuthToken").Filter("user_id =", usr.ID).KeysOnly().Limit(1).Count(c); err != nil {
			return false, err
		} else if count > 0 {
			return true, nil
		}
	}
	return false, nil
}

func GetOAuthToken(c appengine.Context, user_id string) (*OAuthToken, *datastore.Key, error) {
	key := datastore.NewKey(c, "OAuthToken", user_id, 0, nil)
	token := new(OAuthToken)
	err := datastore.Get(c, key, token)
	return token, key, err
}

func GetOAuthClient(c appengine.Context, r *http.Request, user_id string) (*http.Client, string, error) {
	if token, key, err := GetOAuthToken(c, user_id); err == nil {
		ctx := go_ae.NewContext(r)
		config, cerr := utils.OAuthConfigDance(c)
		if cerr != nil {
			return nil, "", cerr
		}

		t_source := config.TokenSource(ctx, &token.Token)
		client := oauth2.NewClient(ctx, t_source)
		n_token, terr := t_source.Token()
		if terr != nil { // terrrr!!
			return nil, "", terr
		}
		token.Token = *n_token

		if _, derr := datastore.Put(c, key, token); derr != nil {
			return nil, "", derr
		}

		return client, token.BatchID, nil
	} else {
		return nil, "", err
	}

	c.Errorf("Oops, it's an error to arrive here just to return a nil client O_o")
	return nil, "", nil
}

func makePastebinFolder(c appengine.Context, client *http.Client) (string, error) {
	if service, aerr := drive.New(client); aerr != nil {
		return "", aerr
	} else {
		fl_call := service.Files.List().Fields("files(id)").PageSize(1).Spaces("drive")
		fl_call = fl_call.Q("name='Pastebin!!' and trashed=false")
		response, err := fl_call.Do()

		if err != nil {
			c.Errorf("Meep! We had an error when trying to do the FilesListCall call!")
			c.Errorf(err.Error())
			return "", err

		} else {
			if len(response.Files) > 0 { // teh Pastebin!! folder is there!
				pbdir_id := response.Files[0].Id
				return pbdir_id, nil
			} else { // teh Pastebin!! folder is NOT there!
				pbdir := new(drive.File)
				pbdir.Name = "Pastebin!!"
				pbdir.MimeType = "application/vnd.google-apps.folder"
				fc_call := service.Files.Create(pbdir).Fields("id")
				d_response, berr := fc_call.Do()
				if berr != nil {
					return "", berr
				} else {
					pbdir_id := d_response.Id
					return pbdir_id, nil
				}
			}
		}
	}
	return "", nil
}

func (p *Paste) saveToDrive(c appengine.Context, r *http.Request, paste_id string) error {
	client, batch_id, cerr := GetOAuthClient(c, r, p.UserID)
	if cerr != nil {
		return cerr
	}

	p.BatchID = batch_id
	if service, err := drive.New(client); err != nil {
		return err
	} else {
		p_content := new(drive.File)

		if len(p.Title) > 0 {
			p_content.Name = p.Title
		} else {
			p_content.Name = paste_id
		}

		if p.Format == "html" {
			p_content.Name = p_content.Name + ".html"
		} else {
			p_content.Name = p_content.Name + ".txt"
		}

		// Here be metadata
		appProperties := make(map[string]string)
		appProperties["PasteID"] = paste_id
		appProperties["Title"] = p.Title
		appProperties["Tags"] = strings.Join(p.Tags, ",")
		appProperties["Format"] = p.Format
		appProperties["Date"] = p.Date.Format(time.RFC3339Nano)
		appProperties["Zlib"] = fmt.Sprintf("%v", p.Zlib)
		appProperties["BatchID"] = p.BatchID

		p_content.AppProperties = appProperties
		if pbdir_id, aerr := makePastebinFolder(c, client); aerr == nil {
			p_content.Parents = []string{pbdir_id}
		} else {
			return parseAPIError(c, r, aerr, p, false)
		}

		buffer := bytes.NewReader([]byte(p.uContent))
		fc_call := service.Files.Create(p_content).Fields("id")
		fc_call = fc_call.Media(buffer)
		response, err := fc_call.Do()

		if err != nil {
			c.Errorf(err.Error())
			return parseAPIError(c, r, err, p, false)
		} else {
			c.Infof("Received Google Drive File ID -> " + response.Id)
			p.GDriveID = response.Id

			// Set the thing in memcache for immediate retrieval
			mc_item := &memcache.Item{
				Key:   response.Id,
				Value: []byte(p.uContent),
			}

			ctx := go_ae.NewContext(r)
			memcache.Add(ctx, mc_item)
		}
	}

	return nil
}

func (p *Paste) loadFromDrive(c appengine.Context, r *http.Request) error {
	ctx := go_ae.NewContext(r)
	if item, err := memcache.Get(ctx, p.GDriveID); err == nil {
		p.Content = item.Value
		return nil
	} else if err == memcache.ErrCacheMiss {
		client, _, cerr := GetOAuthClient(c, r, p.UserID)
		if cerr != nil {
			return cerr
		}

		if service, err := drive.New(client); err != nil {
			return err
		} else {
			fg_call := service.Files.Get(p.GDriveID)
			// We should probably issue a Do() call first to find out whether the file was trashed before
			// downloading it. Oorrrr... we could just tell the users to delete it from trash as well! xD
			response, err := fg_call.Download()
			if err != nil {
				c.Errorf(err.Error())
				return parseAPIError(c, r, err, p, false)
			} else {
				if response.StatusCode == 200 {
					if p_content, err := ioutil.ReadAll(response.Body); err == nil {
						p.Content = p_content

						// Set the thing in memcache for immediate retrieval
						mc_item := &memcache.Item{
							Key:   p.GDriveID,
							Value: p_content,
						}

						ctx := go_ae.NewContext(r)
						memcache.Add(ctx, mc_item)

					} else {
						return err
					}

				}
			}
		}
	} else {
		return err
	}

	return nil
}

func (p *Paste) deleteFromDrive(c appengine.Context, r *http.Request) error {
	client, _, cerr := GetOAuthClient(c, r, p.UserID)
	if cerr != nil {
		return cerr
	}

	if service, err := drive.New(client); err != nil {
		return err
	} else {
		fd_call := service.Files.Delete(p.GDriveID)
		err := fd_call.Do()
		if err != nil {
			c.Errorf(err.Error())
			return parseAPIError(c, r, err, p, true)
		}
	}

	return nil
}
<commit_msg>We should CLING to that refresh token like a sloth on a tree!@<commit_after>package models

import (
	// Go Builtin Packages
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	// Google Appengine Packages
	"appengine"
	"appengine/datastore"
	"appengine/user"

	// Google OAuth2/Drive Packages
	"golang.org/x/oauth2"
	"google.golang.org/api/drive/v3"
	go_ae "google.golang.org/appengine"
	"google.golang.org/appengine/memcache"

	// Local Packages
	"pastebin/utils"
)

type OAuthToken struct {
	UserID  string       `datastore:"user_id"`
	Token   oauth2.Token `datastore:"token,noindex"`
	BatchID string       `datastore:"batch_id,noindex"`
}

func SaveOAuthToken(c appengine.Context, t *oauth2.Token) error {
	user_id := user.Current(c).ID
	token := new(OAuthToken)
	token.UserID = user_id
	token.Token = *t
	token.BatchID = fmt.Sprintf("%s_%s", user_id, time.Now().Format(time.RFC3339Nano))

	key := datastore.NewKey(c, "OAuthToken", user_id, 0, nil)
	if _, err := datastore.Put(c, key, token); err != nil {
		return err
	}
	return nil
}

func DeleteOAuthToken(c appengine.Context, batch_id string, user_id string) error {
	if berr := NewBatchCleanQ(c, batch_id); berr != nil {
		return berr
	}

	key := datastore.NewKey(c, "OAuthToken", user_id, 0, nil)
	err := datastore.Delete(c, key)
	return err
}

func CheckOAuthToken(c appengine.Context) (bool, error) {
	if usr := user.Current(c); usr != nil {
		if count, err := datastore.NewQuery("OAuthToken").Filter("user_id =", usr.ID).KeysOnly().Limit(1).Count(c); err != nil {
			return false, err
		} else if count > 0 {
			return true, nil
		}
	}
	return false, nil
}

func GetOAuthToken(c appengine.Context, user_id string) (*OAuthToken, *datastore.Key, error) {
	key := datastore.NewKey(c, "OAuthToken", user_id, 0, nil)
	token := new(OAuthToken)
	err := datastore.Get(c, key, token)
	return token, key, err
}

func GetOAuthClient(c appengine.Context, r *http.Request, user_id string) (*http.Client, string, error) {
	if token, key, err := GetOAuthToken(c, user_id); err == nil {
		ctx := go_ae.NewContext(r)
		config, cerr := utils.OAuthConfigDance(c)
		if cerr != nil {
			return nil, "", cerr
		}

		t_source := config.TokenSource(ctx, &token.Token)
		client := oauth2.NewClient(ctx, t_source)
		n_token, terr := t_source.Token()
		if terr != nil { // terrrr!!
			return nil, "", terr
		}
		token.Token.AccessToken = n_token.AccessToken
		token.Token.Expiry = n_token.Expiry

		if _, derr := datastore.Put(c, key, token); derr != nil {
			return nil, "", derr
		}

		return client, token.BatchID, nil
	} else {
		return nil, "", err
	}

	c.Errorf("Oops, it's an error to arrive here just to return a nil client O_o")
	return nil, "", nil
}

func makePastebinFolder(c appengine.Context, client *http.Client) (string, error) {
	if service, aerr := drive.New(client); aerr != nil {
		return "", aerr
	} else {
		fl_call := service.Files.List().Fields("files(id)").PageSize(1).Spaces("drive")
		fl_call = fl_call.Q("name='Pastebin!!' and trashed=false")
		response, err := fl_call.Do()

		if err != nil {
			c.Errorf("Meep! We had an error when trying to do the FilesListCall call!")
			c.Errorf(err.Error())
			return "", err

		} else {
			if len(response.Files) > 0 { // teh Pastebin!! folder is there!
				pbdir_id := response.Files[0].Id
				return pbdir_id, nil
			} else { // teh Pastebin!! folder is NOT there!
				pbdir := new(drive.File)
				pbdir.Name = "Pastebin!!"
				pbdir.MimeType = "application/vnd.google-apps.folder"
				fc_call := service.Files.Create(pbdir).Fields("id")
				d_response, berr := fc_call.Do()
				if berr != nil {
					return "", berr
				} else {
					pbdir_id := d_response.Id
					return pbdir_id, nil
				}
			}
		}
	}
	return "", nil
}

func (p *Paste) saveToDrive(c appengine.Context, r *http.Request, paste_id string) error {
	client, batch_id, cerr := GetOAuthClient(c, r, p.UserID)
	if cerr != nil {
		return cerr
	}

	p.BatchID = batch_id
	if service, err := drive.New(client); err != nil {
		return err
	} else {
		p_content := new(drive.File)

		if len(p.Title) > 0 {
			p_content.Name = p.Title
		} else {
			p_content.Name = paste_id
		}

		if p.Format == "html" {
			p_content.Name = p_content.Name + ".html"
		} else {
			p_content.Name = p_content.Name + ".txt"
		}

		// Here be metadata
		appProperties := make(map[string]string)
		appProperties["PasteID"] = paste_id
		appProperties["Title"] = p.Title
		appProperties["Tags"] = strings.Join(p.Tags, ",")
		appProperties["Format"] = p.Format
		appProperties["Date"] = p.Date.Format(time.RFC3339Nano)
		appProperties["Zlib"] = fmt.Sprintf("%v", p.Zlib)
		appProperties["BatchID"] = p.BatchID

		p_content.AppProperties = appProperties
		if pbdir_id, aerr := makePastebinFolder(c, client); aerr == nil {
			p_content.Parents = []string{pbdir_id}
		} else {
			return parseAPIError(c, r, aerr, p, false)
		}

		buffer := bytes.NewReader([]byte(p.uContent))
		fc_call := service.Files.Create(p_content).Fields("id")
		fc_call = fc_call.Media(buffer)
		response, err := fc_call.Do()

		if err != nil {
			c.Errorf(err.Error())
			return parseAPIError(c, r, err, p, false)
		} else {
			c.Infof("Received Google Drive File ID -> " + response.Id)
			p.GDriveID = response.Id

			// Set the thing in memcache for immediate retrieval
			mc_item := &memcache.Item{
				Key:   response.Id,
				Value: []byte(p.uContent),
			}

			ctx := go_ae.NewContext(r)
			memcache.Add(ctx, mc_item)
		}
	}

	return nil
}

func (p *Paste) loadFromDrive(c appengine.Context, r *http.Request) error {
	ctx := go_ae.NewContext(r)
	if item, err := memcache.Get(ctx, p.GDriveID); err == nil {
		p.Content = item.Value
		return nil
	} else if err == memcache.ErrCacheMiss {
		client, _, cerr := GetOAuthClient(c, r, p.UserID)
		if cerr != nil {
			return cerr
		}

		if service, err := drive.New(client); err != nil {
			return err
		} else {
			fg_call := service.Files.Get(p.GDriveID)
			// We should probably issue a Do() call first to find out whether the file was trashed before
			// downloading it. Oorrrr... we could just tell the users to delete it from trash as well! xD
			response, err := fg_call.Download()
			if err != nil {
				c.Errorf(err.Error())
				return parseAPIError(c, r, err, p, false)
			} else {
				if response.StatusCode == 200 {
					if p_content, err := ioutil.ReadAll(response.Body); err == nil {
						p.Content = p_content

						// Set the thing in memcache for immediate retrieval
						mc_item := &memcache.Item{
							Key:   p.GDriveID,
							Value: p_content,
						}

						ctx := go_ae.NewContext(r)
						memcache.Add(ctx, mc_item)

					} else {
						return err
					}

				}
			}
		}
	} else {
		return err
	}

	return nil
}

func (p *Paste) deleteFromDrive(c appengine.Context, r *http.Request) error {
	client, _, cerr := GetOAuthClient(c, r, p.UserID)
	if cerr != nil {
		return cerr
	}

	if service, err := drive.New(client); err != nil {
		return err
	} else {
		fd_call := service.Files.Delete(p.GDriveID)
		err := fd_call.Do()
		if err != nil {
			c.Errorf(err.Error())
			return parseAPIError(c, r, err, p, true)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package drivers

import (
	"fmt"
	"sync"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/db/cluster"
	"github.com/lxc/lxd/lxd/db/warningtype"
	"github.com/lxc/lxd/lxd/device"
	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/lxd/revert"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
)

// Instance driver definitions.
var instanceDrivers = map[string]func() instance.Instance{
	"lxc":  func() instance.Instance { return &lxc{} },
	"qemu": func() instance.Instance { return &qemu{} },
}

// DriverStatus definition.
type DriverStatus struct {
	Info      instance.Info
	Warning   *cluster.Warning
	Supported bool
}

// Supported instance drivers cache variables.
var driverStatusesMu sync.Mutex
var driverStatuses map[instancetype.Type]*DriverStatus

func init() {
	// Expose load to the instance package, to avoid circular imports.
	instance.Load = load

	// Expose validDevices to the instance package, to avoid circular imports.
	instance.ValidDevices = validDevices

	// Expose create to the instance package, to avoid circular imports.
	instance.Create = create
}

// load creates the underlying instance type struct and returns it as an Instance.
func load(s *state.State, args db.InstanceArgs, p api.Project) (instance.Instance, error) {
	var inst instance.Instance
	var err error

	if args.Type == instancetype.Container {
		inst, err = lxcLoad(s, args, p)
	} else if args.Type == instancetype.VM {
		inst, err = qemuLoad(s, args, p)
	} else {
		return nil, fmt.Errorf("Invalid instance type for instance %s", args.Name)
	}

	if err != nil {
		return nil, err
	}

	return inst, nil
}

// validDevices validate instance device configs.
func validDevices(state *state.State, p api.Project, instanceType instancetype.Type, devices deviceConfig.Devices, expanded bool) error {
	// Empty device list
	if devices == nil {
		return nil
	}

	instConf := &common{
		dbType:       instanceType,
		localDevices: devices.Clone(),
		project:      p,
	}

	// In non-expanded validation expensive checks should be avoided.
	if expanded {
		// The devices being validated are already expanded, so just use the same
		// devices clone as we used for the main devices config.
		instConf.expandedDevices = instConf.localDevices
	}

	// Check each device individually using the device package.
	// Use instConf.localDevices so that the cloned config is passed into the driver, so it cannot modify it.
	for name, config := range instConf.localDevices {
		// Enforce a maximum name length of 64 characters (safe maximum allowing use for sockets and other filesystem use).
		if len(name) > 64 {
			return fmt.Errorf("The maximum device name length is 64 characters")
		}

		err := device.Validate(instConf, state, name, config)
		if err != nil {
			return fmt.Errorf("Device validation failed for %q: %w", name, err)
		}
	}

	// Check we have a root disk if in expanded validation mode.
	if expanded {
		_, _, err := shared.GetRootDiskDevice(devices.CloneNative())
		if err != nil {
			return fmt.Errorf("Failed detecting root disk device: %w", err)
		}
	}

	return nil
}

func create(s *state.State, args db.InstanceArgs, p api.Project) (instance.Instance, revert.Hook, error) {
	if args.Type == instancetype.Container {
		return lxcCreate(s, args, p)
	} else if args.Type == instancetype.VM {
		return qemuCreate(s, args, p)
	}

	return nil, nil, fmt.Errorf("Instance type invalid")
}

// DriverStatuses returns a map of DriverStatus structs for all instance type drivers.
// The first time this function is called each of the instance drivers will be probed for support and the result
// will be cached internally to make subsequent calls faster.
func DriverStatuses() map[instancetype.Type]*DriverStatus {
	driverStatusesMu.Lock()
	defer driverStatusesMu.Unlock()

	if driverStatuses != nil {
		return driverStatuses
	}

	driverStatuses = make(map[instancetype.Type]*DriverStatus, len(instanceDrivers))

	for _, instanceDriver := range instanceDrivers {
		driverStatus := &DriverStatus{}

		driverInfo := instanceDriver().Info()
		driverStatus.Info = driverInfo
		driverStatus.Supported = true

		if driverInfo.Error != nil || driverInfo.Version == "" {
			logger.Warn("Instance type not operational", logger.Ctx{"type": driverInfo.Type, "driver": driverInfo.Name, "err": driverInfo.Error})

			driverStatus.Supported = false
			driverStatus.Warning = &cluster.Warning{
				TypeCode:    warningtype.InstanceTypeNotOperational,
				LastMessage: fmt.Sprintf("%v", driverInfo.Error),
			}
		}

		driverStatuses[driverInfo.Type] = driverStatus
	}

	return driverStatuses
}
<commit_msg>lxd/instance/drivers/load: Updates validDevices to accept local and expanded config<commit_after>package drivers

import (
	"errors"
	"fmt"
	"sync"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/db/cluster"
	"github.com/lxc/lxd/lxd/db/warningtype"
	"github.com/lxc/lxd/lxd/device"
	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/lxd/revert"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
)

// Instance driver definitions.
var instanceDrivers = map[string]func() instance.Instance{
	"lxc":  func() instance.Instance { return &lxc{} },
	"qemu": func() instance.Instance { return &qemu{} },
}

// DriverStatus definition.
type DriverStatus struct {
	Info      instance.Info
	Warning   *cluster.Warning
	Supported bool
}

// Supported instance drivers cache variables.
var driverStatusesMu sync.Mutex
var driverStatuses map[instancetype.Type]*DriverStatus

func init() {
	// Expose load to the instance package, to avoid circular imports.
	instance.Load = load

	// Expose validDevices to the instance package, to avoid circular imports.
	instance.ValidDevices = validDevices

	// Expose create to the instance package, to avoid circular imports.
	instance.Create = create
}

// load creates the underlying instance type struct and returns it as an Instance.
func load(s *state.State, args db.InstanceArgs, p api.Project) (instance.Instance, error) {
	var inst instance.Instance
	var err error

	if args.Type == instancetype.Container {
		inst, err = lxcLoad(s, args, p)
	} else if args.Type == instancetype.VM {
		inst, err = qemuLoad(s, args, p)
	} else {
		return nil, fmt.Errorf("Invalid instance type for instance %s", args.Name)
	}

	if err != nil {
		return nil, err
	}

	return inst, nil
}

// validDevices validate instance device configs.
func validDevices(state *state.State, p api.Project, instanceType instancetype.Type, localDevices deviceConfig.Devices, expandedDevices deviceConfig.Devices) error {
	instConf := &common{
		dbType:          instanceType,
		localDevices:    localDevices.Clone(),
		expandedDevices: expandedDevices.Clone(),
		project:         p,
	}

	var checkedDevices []string

	checkDevices := func(devices deviceConfig.Devices, expanded bool) error {
		// Check each device individually using the device package.
		for deviceName, deviceConfig := range devices {
			if expanded && shared.StringInSlice(deviceName, checkedDevices) {
				continue // Don't check the device twice if present in both local and expanded.
			}

			// Enforce a maximum name length of 64 characters.
			// This is a safe maximum allowing use for sockets and other filesystem use.
			if len(deviceName) > 64 {
				return fmt.Errorf("The maximum device name length is 64 characters")
			}

			err := device.Validate(instConf, state, deviceName, deviceConfig)
			if err != nil {
				if expanded && errors.Is(err, device.ErrUnsupportedDevType) {
					// Skip unsupported devices in expanded config.
					// This allows mixed instance type profiles to be used where some devices
					// are only supported with specific instance types.
					continue
				}

				return fmt.Errorf("Device validation failed for %q: %w", deviceName, err)
			}

			checkedDevices = append(checkedDevices, deviceName)
		}

		return nil
	}

	// Check each local device individually using the device package.
	// Use the cloned config from instConf.localDevices so that the driver cannot modify it.
	err := checkDevices(instConf.localDevices, false)
	if err != nil {
		return err
	}

	if len(expandedDevices) > 0 {
		// Check we have a root disk if in expanded validation mode.
		_, _, err := shared.GetRootDiskDevice(expandedDevices.CloneNative())
		if err != nil {
			return fmt.Errorf("Failed detecting root disk device: %w", err)
		}

		// Check each expanded device individually using the device package.
		// Use the cloned config from instConf.expandedDevices so that the driver cannot modify it.
		err = checkDevices(instConf.expandedDevices, true)
		if err != nil {
			return err
		}
	}

	return nil
}

func create(s *state.State, args db.InstanceArgs, p api.Project) (instance.Instance, revert.Hook, error) {
	if args.Type == instancetype.Container {
		return lxcCreate(s, args, p)
	} else if args.Type == instancetype.VM {
		return qemuCreate(s, args, p)
	}

	return nil, nil, fmt.Errorf("Instance type invalid")
}

// DriverStatuses returns a map of DriverStatus structs for all instance type drivers.
// The first time this function is called each of the instance drivers will be probed for support and the result
// will be cached internally to make subsequent calls faster.
func DriverStatuses() map[instancetype.Type]*DriverStatus {
	driverStatusesMu.Lock()
	defer driverStatusesMu.Unlock()

	if driverStatuses != nil {
		return driverStatuses
	}

	driverStatuses = make(map[instancetype.Type]*DriverStatus, len(instanceDrivers))

	for _, instanceDriver := range instanceDrivers {
		driverStatus := &DriverStatus{}

		driverInfo := instanceDriver().Info()
		driverStatus.Info = driverInfo
		driverStatus.Supported = true

		if driverInfo.Error != nil || driverInfo.Version == "" {
			logger.Warn("Instance type not operational", logger.Ctx{"type": driverInfo.Type, "driver": driverInfo.Name, "err": driverInfo.Error})

			driverStatus.Supported = false
			driverStatus.Warning = &cluster.Warning{
				TypeCode:    warningtype.InstanceTypeNotOperational,
				LastMessage: fmt.Sprintf("%v", driverInfo.Error),
			}
		}

		driverStatuses[driverInfo.Type] = driverStatus
	}

	return driverStatuses
}
<|endoftext|>


<commit_before>package sarama

import "time"

type ProduceResponseBlock struct {
	Err       KError
	Offset    int64
	Timestamp time.Time // only provided if Version >= 2
}

func (b *ProduceResponseBlock) decode(pd packetDecoder, version int16) (err error) {
	tmp, err := pd.getInt16()
	if err != nil {
		return err
	}
	b.Err = KError(tmp)

	b.Offset, err = pd.getInt64()
	if err != nil {
		return err
	}

	if version >= 2 {
		if millis, err := pd.getInt64(); err != nil {
			return err
		} else {
			b.Timestamp = time.Unix(millis/1000, (millis%1000)*int64(time.Millisecond))
		}
	}

	return nil
}

type ProduceResponse struct {
	Blocks       map[string]map[int32]*ProduceResponseBlock
	Version      int16
	ThrottleTime time.Duration // only provided if Version >= 1
}

func (r *ProduceResponse) decode(pd packetDecoder, version int16) (err error) {
	r.Version = version

	numTopics, err := pd.getArrayLength()
	if err != nil {
		return err
	}

	r.Blocks = make(map[string]map[int32]*ProduceResponseBlock, numTopics)
	for i := 0; i < numTopics; i++ {
		name, err := pd.getString()
		if err != nil {
			return err
		}

		numBlocks, err := pd.getArrayLength()
		if err != nil {
			return err
		}

		r.Blocks[name] = make(map[int32]*ProduceResponseBlock, numBlocks)

		for j := 0; j < numBlocks; j++ {
			id, err := pd.getInt32()
			if err != nil {
				return err
			}

			block := new(ProduceResponseBlock)
			err = block.decode(pd, version)
			if err != nil {
				return err
			}
			r.Blocks[name][id] = block
		}
	}

	if r.Version >= 1 {
		if millis, err := pd.getInt32(); err != nil {
			return err
		} else {
			r.ThrottleTime = time.Duration(millis) * time.Millisecond
		}
	}

	return nil
}

func (r *ProduceResponse) encode(pe packetEncoder) error {
	err := pe.putArrayLength(len(r.Blocks))
	if err != nil {
		return err
	}
	for topic, partitions := range r.Blocks {
		err = pe.putString(topic)
		if err != nil {
			return err
		}
		err = pe.putArrayLength(len(partitions))
		if err != nil {
			return err
		}
		for id, prb := range partitions {
			pe.putInt32(id)
			pe.putInt16(int16(prb.Err))
			pe.putInt64(prb.Offset)
		}
	}
	if r.Version >= 1 {
		pe.putInt32(int32(r.ThrottleTime / time.Millisecond))
	}
	return nil
}

func (r *ProduceResponse) key() int16 {
	return 0
}

func (r *ProduceResponse) version() int16 {
	return r.Version
}

func (r *ProduceResponse) requiredVersion() KafkaVersion {
	switch r.Version {
	case 1:
		return V0_9_0_0
	case 2:
		return V0_10_0_0
	default:
		return minVersion
	}
}

func (r *ProduceResponse) GetBlock(topic string, partition int32) *ProduceResponseBlock {
	if r.Blocks == nil {
		return nil
	}

	if r.Blocks[topic] == nil {
		return nil
	}

	return r.Blocks[topic][partition]
}

// Testing API

func (r *ProduceResponse) AddTopicPartition(topic string, partition int32, err KError) {
	if r.Blocks == nil {
		r.Blocks = make(map[string]map[int32]*ProduceResponseBlock)
	}
	byTopic, ok := r.Blocks[topic]
	if !ok {
		byTopic = make(map[int32]*ProduceResponseBlock)
		r.Blocks[topic] = byTopic
	}
	byTopic[partition] = &ProduceResponseBlock{Err: err}
}
<commit_msg>Don't try and parse a timestamp of -1<commit_after>package sarama

import "time"

type ProduceResponseBlock struct {
	Err    KError
	Offset int64
	// only provided if Version >= 2 and the broker is configured with `LogAppendTime`
	Timestamp time.Time
}

func (b *ProduceResponseBlock) decode(pd packetDecoder, version int16) (err error) {
	tmp, err := pd.getInt16()
	if err != nil {
		return err
	}
	b.Err = KError(tmp)

	b.Offset, err = pd.getInt64()
	if err != nil {
		return err
	}

	if version >= 2 {
		if millis, err := pd.getInt64(); err != nil {
			return err
		} else if millis != -1 {
			b.Timestamp = time.Unix(millis/1000, (millis%1000)*int64(time.Millisecond))
		}
	}

	return nil
}

type ProduceResponse struct {
	Blocks       map[string]map[int32]*ProduceResponseBlock
	Version      int16
	ThrottleTime time.Duration // only provided if Version >= 1
}

func (r *ProduceResponse) decode(pd packetDecoder, version int16) (err error) {
	r.Version = version

	numTopics, err := pd.getArrayLength()
	if err != nil {
		return err
	}

	r.Blocks = make(map[string]map[int32]*ProduceResponseBlock, numTopics)
	for i := 0; i < numTopics; i++ {
		name, err := pd.getString()
		if err != nil {
			return err
		}

		numBlocks, err := pd.getArrayLength()
		if err != nil {
			return err
		}

		r.Blocks[name] = make(map[int32]*ProduceResponseBlock, numBlocks)

		for j := 0; j < numBlocks; j++ {
			id, err := pd.getInt32()
			if err != nil {
				return err
			}

			block := new(ProduceResponseBlock)
			err = block.decode(pd, version)
			if err != nil {
				return err
			}
			r.Blocks[name][id] = block
		}
	}

	if r.Version >= 1 {
		if millis, err := pd.getInt32(); err != nil {
			return err
		} else {
			r.ThrottleTime = time.Duration(millis) * time.Millisecond
		}
	}

	return nil
}

func (r *ProduceResponse) encode(pe packetEncoder) error {
	err := pe.putArrayLength(len(r.Blocks))
	if err != nil {
		return err
	}
	for topic, partitions := range r.Blocks {
		err = pe.putString(topic)
		if err != nil {
			return err
		}
		err = pe.putArrayLength(len(partitions))
		if err != nil {
			return err
		}
		for id, prb := range partitions {
			pe.putInt32(id)
			pe.putInt16(int16(prb.Err))
			pe.putInt64(prb.Offset)
		}
	}
	if r.Version >= 1 {
		pe.putInt32(int32(r.ThrottleTime / time.Millisecond))
	}
	return nil
}

func (r *ProduceResponse) key() int16 {
	return 0
}

func (r *ProduceResponse) version() int16 {
	return r.Version
}

func (r *ProduceResponse) requiredVersion() KafkaVersion {
	switch r.Version {
	case 1:
		return V0_9_0_0
	case 2:
		return V0_10_0_0
	default:
		return minVersion
	}
}

func (r *ProduceResponse) GetBlock(topic string, partition int32) *ProduceResponseBlock {
	if r.Blocks == nil {
		return nil
	}

	if r.Blocks[topic] == nil {
		return nil
	}

	return r.Blocks[topic][partition]
}

// Testing API

func (r *ProduceResponse) AddTopicPartition(topic string, partition int32, err KError) {
	if r.Blocks == nil {
		r.Blocks = make(map[string]map[int32]*ProduceResponseBlock)
	}
	byTopic, ok := r.Blocks[topic]
	if !ok {
		byTopic = make(map[int32]*ProduceResponseBlock)
		r.Blocks[topic] = byTopic
	}
	byTopic[partition] = &ProduceResponseBlock{Err: err}
}
<|endoftext|>


<commit_before>package hpack

import "fmt"

type Header struct {
	Name, Value string
}

func (h Header) size() uint32 {
	return uint32(len(h.Name + h.Value))
}

func (t *Table) FindHeader(h Header) (match bool, index int) {
	//here should be optimized
	preName := ""
	for i, header := range *STATIC_TABLE {
		if header == h {
			return true, i
		} else if header.Name == h.Name && index == 0 {
			preName = header.Name
			index = i
			match = false
		} else if index != 0 && preName != header.Name {
			return match, index
		}
	}

	ring := t.head
	for i := 0; i < int(t.info.currentEntryNum); i++ {
		if ring.header == h {
			return true, i + int(STATIC_TABLE_NUM)
		} else if ring.header.Name == h.Name && index == 0 {
			match = false
			index = i + int(STATIC_TABLE_NUM)
		}
		ring = ring.Next
	}
	if index > 0 {
		return match, index
	} else {
		return false, -1
	}
}

func (t *Table) GetHeader(index uint32) Header {
	fmt.Println(index, uint32(STATIC_TABLE_NUM+byte(t.info.currentEntryNum)))
	if 0 < index && index < uint32(STATIC_TABLE_NUM) {
		return (*STATIC_TABLE)[index]
	} else if uint32(STATIC_TABLE_NUM) <= index && index <= uint32(STATIC_TABLE_NUM+byte(t.info.currentEntryNum)) {
		return t.getFromHeaderTable(index) //from Header Table
	} else {
		panic("error")
	}
}

type Table struct {
	head, tail *RingTable
	info       Info
}

type RingTable struct {
	header    Header
	Next, Pre *RingTable
}

type Info struct {
	currentEntrySize uint32
	currentEntryNum  uint32
	headerTableSize  uint32
}

func InitTable() (table Table) {
	var ringTable RingTable
	table.head = &ringTable //*RingTalbe = &RingTable{Header{"", ""}, ringTable, nil}
	table.tail = &ringTable //*RingTable = &RingTable{Header{"", ""}, nil, ringTable}

	table.info.currentEntryNum = 0
	table.info.currentEntrySize = 0
	table.info.headerTableSize = 4096
	return
}

func (t *Table) getFromHeaderTable(index uint32) Header {
	index -= uint32(STATIC_TABLE_NUM)
	ring := t.head.Next
	for i := uint32(0); i < index; i++ {
		ring = ring.Next
	}
	return ring.header
}

var nilElem *RingTable

func (t *Table) delLast() {
	t.info.currentEntrySize -= t.tail.Pre.header.size()
	t.tail.Pre = t.tail.Pre.Pre
	t.info.currentEntryNum--
}

func (t *Table) insertFirst(header Header) {
	//here should be refactored
	elem := RingTable{header, t.head.Next, nil}
	if t.info.currentEntryNum >= 1 {
		t.head.Next.Pre = &elem
	}
	t.head.Next = &elem

	if t.info.currentEntryNum == 1 {
		t.tail.Pre = &elem
	}
	t.info.currentEntrySize += header.size()
	t.info.currentEntryNum++
}

func (t *Table) AddHeader(header Header) {
	for t.info.currentEntrySize+header.size() > t.info.headerTableSize {
		t.delLast()
	}
	t.insertFirst(header)
}

func (t *Table) SetMaxHeaderTableSize(size uint32) {
	t.info.headerTableSize = size
}

var STATIC_TABLE *[]Header = &[]Header{
	{"", ""},
	{":authority", ""},
	{":method", "GET"},
	{":method", "POST"},
	{":path", "/"},
	{":path", "/index.html"},
	{":scheme", "http"},
	{":scheme", "https"},
	{":status", "200"},
	{":status", "204"},
	{":status", "206"},
	{":status", "304"},
	{":status", "400"},
	{":status", "404"},
	{":status", "500"},
	{"accept-charset", ""},
	{"accept-encoding", "gzip, deflate"},
	{"accept-language", ""},
	{"accept-ranges", ""},
	{"accept", ""},
	{"access-control-allow-origin", ""},
	{"age", ""},
	{"allow", ""},
	{"authorization", ""},
	{"cache-control", ""},
	{"content-disposition", ""},
	{"content-encoding", ""},
	{"content-language", ""},
	{"content-length", ""},
	{"content-location", ""},
	{"content-range", ""},
	{"content-type", ""},
	{"cookie", ""},
	{"date", ""},
	{"etag", ""},
	{"expect", ""},
	{"expires", ""},
	{"from", ""},
	{"host", ""},
	{"if-match", ""},
	{"if-modified-since", ""},
	{"if-none-match", ""},
	{"if-range", ""},
	{"if-unmodified-since", ""},
	{"last-modified", ""},
	{"link", ""},
	{"location", ""},
	{"max-forwards", ""},
	{"proxy-authenticate", ""},
	{"proxy-authorization", ""},
	{"range", ""},
	{"referer", ""},
	{"refresh", ""},
	{"retry-after", ""},
	{"server", ""},
	{"set-cookie", ""},
	{"strict-transport-security", ""},
	{"transfer-encoding", ""},
	{"user-agent", ""},
	{"vary", ""},
	{"via", ""},
	{"www-authenticate", ""},
}
var STATIC_TABLE_NUM byte = byte(len(*STATIC_TABLE))
<commit_msg>remove Info type<commit_after>package hpack

import "fmt"

type Header struct {
	Name, Value string
}

func (h Header) size() uint32 {
	return uint32(len(h.Name + h.Value))
}

func (t *Table) FindHeader(h Header) (match bool, index int) {
	//here should be optimized
	preName := ""
	for i, header := range *STATIC_TABLE {
		if header == h {
			return true, i
		} else if header.Name == h.Name && index == 0 {
			preName = header.Name
			index = i
			match = false
		} else if index != 0 && preName != header.Name {
			return match, index
		}
	}

	ring := t.head
	for i := 0; i < int(t.currentEntryNum); i++ {
		if ring.header == h {
			return true, i + int(STATIC_TABLE_NUM)
		} else if ring.header.Name == h.Name && index == 0 {
			match = false
			index = i + int(STATIC_TABLE_NUM)
		}
		ring = ring.Next
	}
	if index > 0 {
		return match, index
	} else {
		return false, -1
	}
}

func (t *Table) GetHeader(index uint32) Header {
	fmt.Println(index, uint32(STATIC_TABLE_NUM+byte(t.currentEntryNum)))
	if 0 < index && index < uint32(STATIC_TABLE_NUM) {
		return (*STATIC_TABLE)[index]
	} else if uint32(STATIC_TABLE_NUM) <= index && index <= uint32(STATIC_TABLE_NUM+byte(t.currentEntryNum)) {
		return t.getFromHeaderTable(index) //from Header Table
	} else {
		panic("error")
	}
}

type Table struct {
	head, tail       *RingTable
	currentEntrySize uint32
	currentEntryNum  uint32
	headerTableSize  uint32
}

type RingTable struct {
	header    Header
	Next, Pre *RingTable
}

func InitTable() (t Table) {
	var ringTable RingTable
	t.head = &ringTable //*RingTalbe = &RingTable{Header{"", ""}, ringTable, nil}
	t.tail = &ringTable //*RingTable = &RingTable{Header{"", ""}, nil, ringTable}

	t.currentEntryNum = 0
	t.currentEntrySize = 0
	t.headerTableSize = 4096
	return
}

func (t *Table) getFromHeaderTable(index uint32) Header {
	index -= uint32(STATIC_TABLE_NUM)
	ring := t.head.Next
	for i := uint32(0); i < index; i++ {
		ring = ring.Next
	}
	return ring.header
}

var nilElem *RingTable

func (t *Table) delLast() {
	t.currentEntrySize -= t.tail.Pre.header.size()
	t.tail.Pre = t.tail.Pre.Pre
	t.currentEntryNum--
}

func (t *Table) insertFirst(header Header) {
	//here should be refactored
	elem := RingTable{header, t.head.Next, nil}
	if t.currentEntryNum >= 1 {
		t.head.Next.Pre = &elem
	}
	t.head.Next = &elem

	if t.currentEntryNum == 1 {
		t.tail.Pre = &elem
	}
	t.currentEntryNum++
	t.currentEntrySize += header.size()
}

func (t *Table) AddHeader(header Header) {
	for t.currentEntrySize+header.size() > t.headerTableSize {
		t.delLast()
	}
	t.insertFirst(header)
}

func (t *Table) SetMaxHeaderTableSize(size uint32) {
	t.headerTableSize = size
}

var STATIC_TABLE *[]Header = &[]Header{
	{"", ""},
	{":authority", ""},
	{":method", "GET"},
	{":method", "POST"},
	{":path", "/"},
	{":path", "/index.html"},
	{":scheme", "http"},
	{":scheme", "https"},
	{":status", "200"},
	{":status", "204"},
	{":status", "206"},
	{":status", "304"},
	{":status", "400"},
	{":status", "404"},
	{":status", "500"},
	{"accept-charset", ""},
	{"accept-encoding", "gzip, deflate"},
	{"accept-language", ""},
	{"accept-ranges", ""},
	{"accept", ""},
	{"access-control-allow-origin", ""},
	{"age", ""},
	{"allow", ""},
	{"authorization", ""},
	{"cache-control", ""},
	{"content-disposition", ""},
	{"content-encoding", ""},
	{"content-language", ""},
	{"content-length", ""},
	{"content-location", ""},
	{"content-range", ""},
	{"content-type", ""},
	{"cookie", ""},
	{"date", ""},
	{"etag", ""},
	{"expect", ""},
	{"expires", ""},
	{"from", ""},
	{"host", ""},
	{"if-match", ""},
	{"if-modified-since", ""},
	{"if-none-match", ""},
	{"if-range", ""},
	{"if-unmodified-since", ""},
	{"last-modified", ""},
	{"link", ""},
	{"location", ""},
	{"max-forwards", ""},
	{"proxy-authenticate", ""},
	{"proxy-authorization", ""},
	{"range", ""},
	{"referer", ""},
	{"refresh", ""},
	{"retry-after", ""},
	{"server", ""},
	{"set-cookie", ""},
	{"strict-transport-security", ""},
	{"transfer-encoding", ""},
	{"user-agent", ""},
	{"vary", ""},
	{"via", ""},
	{"www-authenticate", ""},
}
var STATIC_TABLE_NUM byte = byte(len(*STATIC_TABLE))
<|endoftext|>


<commit_before>/*
Copyright 2013 TamĂĄs GulĂĄcsi

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package oracall

import (
	"fmt"
	"strings"
)

// Log is discarded by default.
var Log = func(...interface{}) error { return nil }

const (
	MarkNull = "â" // 0x2400 = nul
	//MarkValid  = "ćťż" // 0x6eff = fill; full, satisfied
	MarkValid = "Valid" // 0x6eff = fill; full, satisfied
	//MarkHidden = "ĺż"     // 0x533f = hide
	MarkHidden = "_hidden"

	DefaultMaxVARCHARLength = 32767
	DefaultMaxCHARLength    = 10
)

type Function struct {
	Package, name string
	Returns       *Argument
	Args          []Argument
	Documentation string
}

func (f Function) Name() string {
	if f.Package == "" {
		return strings.ToLower(f.name)
	}
	return UnoCap(f.Package) + "." + strings.ToLower(f.name)
}

func (f Function) String() string {
	args := make([]string, len(f.Args))
	for i := range args {
		args[i] = f.Args[i].String()
	}
	s := f.Name() + "(" + strings.Join(args, ", ") + ")"
	if f.Documentation == "" {
		return s
	}
	return s + "\n" + f.Documentation
}

func (f Function) HasCursorOut() bool {
	if f.Returns != nil &&
		f.Returns.IsOutput() && f.Returns.Type == "REF CURSOR" {
		return true
	}
	for _, arg := range f.Args {
		if arg.IsOutput() && arg.Type == "REF CURSOR" {
			return true
		}
	}
	return false
}

const (
	DIR_IN  = 1
	DIR_OUT = 2

	FLAVOR_SIMPLE = 0
	FLAVOR_RECORD = 1
	FLAVOR_TABLE  = 2
)

type Argument struct {
	RecordOf []NamedArgument //this argument is a record (map) of this type
	PlsType
	Name           string
	Type, TypeName string
	AbsType        string
	Charset        string
	goTypeName     string
	TableOf        *Argument // this argument is a table (array) of this type
	Charlength     uint
	Flavor         uint8
	Direction      uint8
	Precision      uint8
	Scale          uint8
}
type NamedArgument struct {
	Name string
	Argument
}

func (a Argument) String() string {
	typ := a.Type
	switch a.Flavor {
	case FLAVOR_RECORD:
		typ = fmt.Sprintf("%s{%s}", a.PlsType, a.RecordOf)
	case FLAVOR_TABLE:
		typ = fmt.Sprintf("%s[%s]", a.PlsType, a.TableOf)
	}
	dir := ""
	switch a.Direction {
	case DIR_IN:
		dir = "IN"
	case DIR_OUT:
		dir = "OUT"
	default:
		dir = "INOUT"
	}
	return a.Name + " " + dir + " " + typ
}

func (a Argument) IsInput() bool {
	return a.Direction&DIR_IN > 0
}
func (a Argument) IsOutput() bool {
	return a.Direction&DIR_OUT > 0
}

func NewArgument(name, dataType, plsType, typeName, dirName string, dir uint8,
	charset string, precision, scale uint8, charlength uint) Argument {

	name = strings.ToLower(name)
	if typeName == "..@" {
		typeName = ""
	}
	if typeName != "" && typeName[len(typeName)-1] == '@' {
		typeName = typeName[:len(typeName)-1]
	}

	if dirName != "" {
		switch dirName {
		case "IN/OUT":
			dir = DIR_IN | DIR_OUT
		case "OUT":
			dir = DIR_OUT
		default:
			dir = DIR_IN
		}
	}
	if dir < DIR_IN {
		dir = DIR_IN
	}

	arg := Argument{Name: name, Type: dataType, PlsType: NewPlsType(plsType),
		TypeName: typeName, Direction: dir,
		Precision: precision, Scale: scale, Charlength: charlength,
		Charset: charset}
	switch arg.Type {
	case "PL/SQL RECORD":
		arg.Flavor = FLAVOR_RECORD
		arg.RecordOf = make([]NamedArgument, 1)
	case "TABLE", "PL/SQL TABLE", "REF CURSOR":
		arg.Flavor = FLAVOR_TABLE
	}

	switch arg.Type {
	case "CHAR", "NCHAR", "VARCHAR", "NVARCHAR", "VARCHAR2", "NVARCHAR2":
		if arg.Charlength <= 0 {
			if strings.Contains(arg.Type, "VAR") {
				arg.Charlength = DefaultMaxVARCHARLength
			} else {
				arg.Charlength = DefaultMaxCHARLength
			}
		}
		arg.AbsType = fmt.Sprintf("%s(%d)", arg.Type, arg.Charlength)
	case "NUMBER":
		if arg.Scale > 0 {
			arg.AbsType = fmt.Sprintf("NUMBER(%d, %d)", arg.Precision, arg.Scale)
		} else if arg.Precision > 0 {
			arg.AbsType = fmt.Sprintf("NUMBER(%d)", arg.Precision)
		} else {
			arg.AbsType = "NUMBER"
		}
	case "PLS_INTEGER", "BINARY_INTEGER":
		arg.AbsType = "INTEGER(10)"
	default:
		arg.AbsType = arg.Type
	}
	return arg
}

func UnoCap(text string) string {
	i := strings.Index(text, "_")
	if i == 0 {
		return capitalize(text)
	}
	return strings.ToUpper(text[:i]) + "_" + strings.ToLower(text[i+1:])
}
<commit_msg>maligned<commit_after>/*
Copyright 2013 TamĂĄs GulĂĄcsi

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package oracall

import (
	"fmt"
	"strings"
)

// Log is discarded by default.
var Log = func(...interface{}) error { return nil }

const (
	MarkNull = "â" // 0x2400 = nul
	//MarkValid  = "ćťż" // 0x6eff = fill; full, satisfied
	MarkValid = "Valid" // 0x6eff = fill; full, satisfied
	//MarkHidden = "ĺż"     // 0x533f = hide
	MarkHidden = "_hidden"

	DefaultMaxVARCHARLength = 32767
	DefaultMaxCHARLength    = 10
)

type Function struct {
	Package, name string
	Returns       *Argument
	Args          []Argument
	Documentation string
}

func (f Function) Name() string {
	if f.Package == "" {
		return strings.ToLower(f.name)
	}
	return UnoCap(f.Package) + "." + strings.ToLower(f.name)
}

func (f Function) String() string {
	args := make([]string, len(f.Args))
	for i := range args {
		args[i] = f.Args[i].String()
	}
	s := f.Name() + "(" + strings.Join(args, ", ") + ")"
	if f.Documentation == "" {
		return s
	}
	return s + "\n" + f.Documentation
}

func (f Function) HasCursorOut() bool {
	if f.Returns != nil &&
		f.Returns.IsOutput() && f.Returns.Type == "REF CURSOR" {
		return true
	}
	for _, arg := range f.Args {
		if arg.IsOutput() && arg.Type == "REF CURSOR" {
			return true
		}
	}
	return false
}

const (
	DIR_IN  = 1
	DIR_OUT = 2

	FLAVOR_SIMPLE = 0
	FLAVOR_RECORD = 1
	FLAVOR_TABLE  = 2
)

type Argument struct {
	RecordOf []NamedArgument //this argument is a record (map) of this type
	Name     string
	//Level                   uint8
	//Position                uint8
	Type, TypeName string
	AbsType        string
	Charset        string
	Charlength     uint
	TableOf        *Argument // this argument is a table (array) of this type
	goTypeName     string
	PlsType
	Flavor    uint8
	Direction uint8
	Precision uint8
	Scale     uint8
}
type NamedArgument struct {
	Name string
	Argument
}

func (a Argument) String() string {
	typ := a.Type
	switch a.Flavor {
	case FLAVOR_RECORD:
		typ = fmt.Sprintf("%s{%s}", a.PlsType, a.RecordOf)
	case FLAVOR_TABLE:
		typ = fmt.Sprintf("%s[%s]", a.PlsType, a.TableOf)
	}
	dir := ""
	switch a.Direction {
	case DIR_IN:
		dir = "IN"
	case DIR_OUT:
		dir = "OUT"
	default:
		dir = "INOUT"
	}
	return a.Name + " " + dir + " " + typ
}

func (a Argument) IsInput() bool {
	return a.Direction&DIR_IN > 0
}
func (a Argument) IsOutput() bool {
	return a.Direction&DIR_OUT > 0
}

func NewArgument(name, dataType, plsType, typeName, dirName string, dir uint8,
	charset string, precision, scale uint8, charlength uint) Argument {

	name = strings.ToLower(name)
	if typeName == "..@" {
		typeName = ""
	}
	if typeName != "" && typeName[len(typeName)-1] == '@' {
		typeName = typeName[:len(typeName)-1]
	}

	if dirName != "" {
		switch dirName {
		case "IN/OUT":
			dir = DIR_IN | DIR_OUT
		case "OUT":
			dir = DIR_OUT
		default:
			dir = DIR_IN
		}
	}
	if dir < DIR_IN {
		dir = DIR_IN
	}

	arg := Argument{Name: name, Type: dataType, PlsType: NewPlsType(plsType),
		TypeName: typeName, Direction: dir,
		Precision: precision, Scale: scale, Charlength: charlength,
		Charset: charset}
	switch arg.Type {
	case "PL/SQL RECORD":
		arg.Flavor = FLAVOR_RECORD
		arg.RecordOf = make([]NamedArgument, 1)
	case "TABLE", "PL/SQL TABLE", "REF CURSOR":
		arg.Flavor = FLAVOR_TABLE
	}

	switch arg.Type {
	case "CHAR", "NCHAR", "VARCHAR", "NVARCHAR", "VARCHAR2", "NVARCHAR2":
		if arg.Charlength <= 0 {
			if strings.Contains(arg.Type, "VAR") {
				arg.Charlength = DefaultMaxVARCHARLength
			} else {
				arg.Charlength = DefaultMaxCHARLength
			}
		}
		arg.AbsType = fmt.Sprintf("%s(%d)", arg.Type, arg.Charlength)
	case "NUMBER":
		if arg.Scale > 0 {
			arg.AbsType = fmt.Sprintf("NUMBER(%d, %d)", arg.Precision, arg.Scale)
		} else if arg.Precision > 0 {
			arg.AbsType = fmt.Sprintf("NUMBER(%d)", arg.Precision)
		} else {
			arg.AbsType = "NUMBER"
		}
	case "PLS_INTEGER", "BINARY_INTEGER":
		arg.AbsType = "INTEGER(10)"
	default:
		arg.AbsType = arg.Type
	}
	return arg
}

func UnoCap(text string) string {
	i := strings.Index(text, "_")
	if i == 0 {
		return capitalize(text)
	}
	return strings.ToUpper(text[:i]) + "_" + strings.ToLower(text[i+1:])
}
<|endoftext|>


<commit_before>package warden

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"sync"
	"time"
)

const (
	HTTPCACHEDIR       = "/opt/warden"
	AgentConfigFile    = "/etc/warden/warden-agent.json"
	GuardianConfigFile = "/etc/warden/warden-guardian.json"
)

var (
	DefaultZKConnectionTime    = time.Second * 5
	agentSettings              = AgentConfig{}
	guardianSettings           = GuardianConfig{}
	DefaultHealthCheckInterval = 5
	DefaultHealthCheckRetry    = 3
	ConfigLock                 = new(sync.RWMutex)
)

type AgentConfig struct {
	ZKHost       []string `json:"zkhost"`
	ZKAuth       string   `json:"zkauth"`
	AppHome      string   `json:"apphome"`
	DockerSocket string   `json:"dockersocket"`
}

type GuardianConfig struct {
	ZKHost       []string `json:"zkhost"`
	ZKAuth       string   `json:"zkauth"`
	AppHome      string   `json:"apphome"`
	Applications []string `json:"applications"`
}

func loadAgentConfig(config *AgentConfig, configfile string) error {
	configBody, readErr := ioutil.ReadFile(configfile)
	if readErr != nil {
		return readErr
	}
	ConfigLock.Lock()
	defer ConfigLock.Unlock()
	err := json.Unmarshal(configBody, config)
	if err != nil {
		return err
	}

	log.Printf("%+v", config)
	return nil
}

func loadGuardianConfig(config *GuardianConfig, configfile string) error {
	configBody, readErr := ioutil.ReadFile(configfile)
	if readErr != nil {
		return readErr
	}
	ConfigLock.Lock()
	defer ConfigLock.Unlock()
	err := json.Unmarshal(configBody, config)
	if err != nil {
		return err
	}

	log.Printf("%+v", config)
	return nil
}

func init() {
	if _, err := os.Stat(AgentConfigFile); err == nil {
		loadAgentConfig(&agentSettings, AgentConfigFile)
	}
	if _, err := os.Stat(GuardianConfigFile); err == nil {
		loadGuardianConfig(&guardianSettings, GuardianConfigFile)
	}
}

func AgentSettings() *AgentConfig {
	return &agentSettings
}

func ReloadAgentSettings() error {
	err := loadAgentConfig(&agentSettings, AgentConfigFile)
	return err
}

func ReloadGuardianSettings() error {
	err := loadGuardianConfig(&guardianSettings, GuardianConfigFile)
	return err
}

func GuardianSettings() *GuardianConfig {
	return &guardianSettings
}
<commit_msg>make zk session timeout up to 30s by default<commit_after>package warden

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"sync"
	"time"
)

const (
	HTTPCACHEDIR       = "/opt/warden"
	AgentConfigFile    = "/etc/warden/warden-agent.json"
	GuardianConfigFile = "/etc/warden/warden-guardian.json"
)

var (
	DefaultZKConnectionTime    = time.Second * 30
	agentSettings              = AgentConfig{}
	guardianSettings           = GuardianConfig{}
	DefaultHealthCheckInterval = 5
	DefaultHealthCheckRetry    = 3
	ConfigLock                 = new(sync.RWMutex)
)

type AgentConfig struct {
	ZKHost       []string `json:"zkhost"`
	ZKAuth       string   `json:"zkauth"`
	AppHome      string   `json:"apphome"`
	DockerSocket string   `json:"dockersocket"`
}

type GuardianConfig struct {
	ZKHost       []string `json:"zkhost"`
	ZKAuth       string   `json:"zkauth"`
	AppHome      string   `json:"apphome"`
	Applications []string `json:"applications"`
}

func loadAgentConfig(config *AgentConfig, configfile string) error {
	configBody, readErr := ioutil.ReadFile(configfile)
	if readErr != nil {
		return readErr
	}
	ConfigLock.Lock()
	defer ConfigLock.Unlock()
	err := json.Unmarshal(configBody, config)
	if err != nil {
		return err
	}

	log.Printf("%+v", config)
	return nil
}

func loadGuardianConfig(config *GuardianConfig, configfile string) error {
	configBody, readErr := ioutil.ReadFile(configfile)
	if readErr != nil {
		return readErr
	}
	ConfigLock.Lock()
	defer ConfigLock.Unlock()
	err := json.Unmarshal(configBody, config)
	if err != nil {
		return err
	}

	log.Printf("%+v", config)
	return nil
}

func init() {
	if _, err := os.Stat(AgentConfigFile); err == nil {
		loadAgentConfig(&agentSettings, AgentConfigFile)
	}
	if _, err := os.Stat(GuardianConfigFile); err == nil {
		loadGuardianConfig(&guardianSettings, GuardianConfigFile)
	}
}

func AgentSettings() *AgentConfig {
	return &agentSettings
}

func ReloadAgentSettings() error {
	err := loadAgentConfig(&agentSettings, AgentConfigFile)
	return err
}

func ReloadGuardianSettings() error {
	err := loadGuardianConfig(&guardianSettings, GuardianConfigFile)
	return err
}

func GuardianSettings() *GuardianConfig {
	return &guardianSettings
}
<|endoftext|>


<commit_before>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Radu Berinde (radu@cockroachlabs.com)
//
// The parallel_test adds an orchestration layer on top of the logic_test code
// with the capability of running multiple test data files in parallel.
//
// Each test lives in a separate subdir under testdata/paralleltest. Each test
// dir contains a "main" file along with a set of files in logic test format.
//
// The format of the main file is very simple (for now). Each line is of the
// form:
//     run <file1> <file2> ..
// This will run the given logic test files in parallel and wait for them to
// complete.  The same file can be specified multiple times.

package sql_test

import (
	"database/sql"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/cockroachdb/cockroach/security"
	"github.com/cockroachdb/cockroach/server"
	"github.com/cockroachdb/cockroach/testutils/sqlutils"
	"github.com/cockroachdb/cockroach/util/leaktest"
	"github.com/cockroachdb/cockroach/util/log"
)

var (
	paralleltestdata = flag.String("partestdata", "partestdata/[^.]*", "test data glob")
)

type testDB struct {
	db      *sql.DB
	cleanup func()
}

type parallelTest struct {
	*testing.T
	srv     *testServer
	clients []testDB
}

func (t *parallelTest) close() {
	for i := len(t.clients) - 1; i >= 0; i-- {
		t.clients[i].cleanup()
		t.clients[i].db.Close()
	}
	t.clients = nil
	if t.srv != nil {
		cleanupTestServer(t.srv)
		t.srv = nil
	}
}

func (t *parallelTest) addClient(createDB bool) {
	pgURL, cleanupFunc := sqlutils.PGUrl(t.T, &t.srv.TestServer, security.RootUser, "TestParallel")
	db, err := sql.Open("postgres", pgURL.String())
	if err != nil {
		t.Fatal(err)
	}

	if createDB {
		if _, err := db.Exec("CREATE DATABASE test;"); err != nil {
			t.Fatal(err)
		}
	}
	if _, err := db.Exec("SET DATABASE = test;"); err != nil {
		t.Fatal(err)
	}

	t.clients = append(t.clients, testDB{db: db, cleanup: cleanupFunc})
}

func (t *parallelTest) processTestFile(path string, db *sql.DB, ch chan bool) {
	if ch != nil {
		defer func() { ch <- true }()
	}

	// Set up a dummy logicTest structure to use that code.
	l := &logicTest{T: t.T, srv: t.srv, db: db, user: security.RootUser}
	l.processTestFile(path)
}

func (t *parallelTest) run(dir string) {
	defer t.close()
	t.setup()

	// Add the main client and set up the database.
	t.addClient(true)

	// Open the main faile.
	fmt.Printf("Running test %s\n", dir)
	mainFile := filepath.Join(dir, "main")
	file, err := os.Open(mainFile)
	if err != nil {
		t.Fatal(err)
	}
	defer file.Close()
	s := newLineScanner(file)
	for s.Scan() {
		fields := strings.Fields(s.Text())
		if len(fields) == 0 {
			continue
		}
		cmd := fields[0]
		if strings.HasPrefix(cmd, "#") {
			// Skip comment lines.
			continue
		}
		switch cmd {
		case "run":
			testFiles := fields[1:]
			for len(t.clients) < len(testFiles) {
				t.addClient(false)
			}
			if testing.Verbose() || log.V(1) {
				fmt.Printf("%s:%d: running %s\n", mainFile, s.line, strings.Join(testFiles, ","))
			}
			ch := make(chan bool)
			for i, f := range testFiles {
				go t.processTestFile(filepath.Join(dir, f), t.clients[i].db, ch)
			}
			// Wait for all clients to complete.
			for range testFiles {
				<-ch
			}
		default:
			t.Fatalf("%s:%d: unknown command: %s", mainFile, s.line, cmd)
		}
	}
}

func (t *parallelTest) setup() {
	ctx := server.NewTestContext()
	ctx.MaxOffset = logicMaxOffset
	ctx.TestingKnobs.ExecutorTestingKnobs.WaitForGossipUpdate = true
	ctx.TestingKnobs.ExecutorTestingKnobs.CheckStmtStringChange = true
	t.srv = setupTestServerWithContext(t.T, ctx)
}

func TestParallel(t *testing.T) {
	defer leaktest.AfterTest(t)()

	glob := string(*paralleltestdata)
	paths, err := filepath.Glob(glob)
	if err != nil {
		t.Fatal(err)
	}
	if len(paths) == 0 {
		t.Fatalf("No testfiles found (glob: %s)", glob)
	}
	total := 0
	for _, p := range paths {
		pt := parallelTest{T: t}
		pt.run(p)
		total++
	}
	fmt.Printf("%d parallel tests passed\n", total)
}
<commit_msg>sql: fix TestParallel.<commit_after>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Radu Berinde (radu@cockroachlabs.com)
//
// The parallel_test adds an orchestration layer on top of the logic_test code
// with the capability of running multiple test data files in parallel.
//
// Each test lives in a separate subdir under testdata/paralleltest. Each test
// dir contains a "main" file along with a set of files in logic test format.
//
// The format of the main file is very simple (for now). Each line is of the
// form:
//     run <file1> <file2> ..
// This will run the given logic test files in parallel and wait for them to
// complete.  The same file can be specified multiple times.

package sql_test

import (
	"database/sql"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/cockroachdb/cockroach/security"
	"github.com/cockroachdb/cockroach/server"
	"github.com/cockroachdb/cockroach/testutils/sqlutils"
	"github.com/cockroachdb/cockroach/util/leaktest"
	"github.com/cockroachdb/cockroach/util/log"
)

var (
	paralleltestdata = flag.String("partestdata", "partestdata/[^.]*", "test data glob")
)

type testDB struct {
	db      *sql.DB
	cleanup func()
}

type parallelTest struct {
	*testing.T
	srv     *testServer
	clients []testDB
}

func (t *parallelTest) close() {
	for i := len(t.clients) - 1; i >= 0; i-- {
		t.clients[i].cleanup()
		t.clients[i].db.Close()
	}
	t.clients = nil
	if t.srv != nil {
		cleanupTestServer(t.srv)
		t.srv = nil
	}
}

func (t *parallelTest) addClient(createDB bool) {
	pgURL, cleanupFunc := sqlutils.PGUrl(t.T, &t.srv.TestServer, security.RootUser, "TestParallel")
	db, err := sql.Open("postgres", pgURL.String())
	if err != nil {
		t.Fatal(err)
	}

	if createDB {
		if _, err := db.Exec("CREATE DATABASE test;"); err != nil {
			t.Fatal(err)
		}
	}
	if _, err := db.Exec("SET DATABASE = test;"); err != nil {
		t.Fatal(err)
	}

	t.clients = append(t.clients, testDB{db: db, cleanup: cleanupFunc})
}

func (t *parallelTest) processTestFile(path string, db *sql.DB, ch chan bool) {
	if ch != nil {
		defer func() { ch <- true }()
	}

	// Set up a dummy logicTest structure to use that code.
	l := &logicTest{T: t.T, srv: t.srv, db: db, user: security.RootUser}
	if err := l.processTestFile(path); err != nil {
		t.Error(err)
	}
}

func (t *parallelTest) run(dir string) {
	defer t.close()
	t.setup()

	// Add the main client and set up the database.
	t.addClient(true)

	// Open the main faile.
	fmt.Printf("Running test %s\n", dir)
	mainFile := filepath.Join(dir, "main")
	file, err := os.Open(mainFile)
	if err != nil {
		t.Fatal(err)
	}
	defer file.Close()
	s := newLineScanner(file)
	for s.Scan() {
		fields := strings.Fields(s.Text())
		if len(fields) == 0 {
			continue
		}
		cmd := fields[0]
		if strings.HasPrefix(cmd, "#") {
			// Skip comment lines.
			continue
		}
		switch cmd {
		case "run":
			testFiles := fields[1:]
			for len(t.clients) < len(testFiles) {
				t.addClient(false)
			}
			if testing.Verbose() || log.V(1) {
				fmt.Printf("%s:%d: running %s\n", mainFile, s.line, strings.Join(testFiles, ","))
			}
			ch := make(chan bool)
			for i, f := range testFiles {
				go t.processTestFile(filepath.Join(dir, f), t.clients[i].db, ch)
			}
			// Wait for all clients to complete.
			for range testFiles {
				<-ch
			}
		default:
			t.Fatalf("%s:%d: unknown command: %s", mainFile, s.line, cmd)
		}
	}
}

func (t *parallelTest) setup() {
	ctx := server.NewTestContext()
	ctx.MaxOffset = logicMaxOffset
	ctx.TestingKnobs.ExecutorTestingKnobs.WaitForGossipUpdate = true
	ctx.TestingKnobs.ExecutorTestingKnobs.CheckStmtStringChange = true
	t.srv = setupTestServerWithContext(t.T, ctx)
}

func TestParallel(t *testing.T) {
	defer leaktest.AfterTest(t)()

	glob := string(*paralleltestdata)
	paths, err := filepath.Glob(glob)
	if err != nil {
		t.Fatal(err)
	}
	if len(paths) == 0 {
		t.Fatalf("No testfiles found (glob: %s)", glob)
	}
	total := 0
	for _, p := range paths {
		pt := parallelTest{T: t}
		pt.run(p)
		total++
	}
	fmt.Printf("%d parallel tests passed\n", total)
}
<|endoftext|>


<commit_before>package logic

import (
	"encoding/json"
	"fmt"

	log "github.com/cihub/seelog"
	"github.com/stampzilla/stampzilla-go/protocol"
	serverprotocol "github.com/stampzilla/stampzilla-go/stampzilla-server/protocol"
)

type RuleAction interface {
	RunCommand()
}
type ruleAction struct {
	Command *protocol.Command `json:"command"`
	Uuid    string            `json:"uuid"`
	nodes   *serverprotocol.Nodes
}

func NewRuleAction(cmd *protocol.Command, uuid string) RuleAction {
	return &ruleAction{Command: cmd, Uuid: uuid}
}
func (ra *ruleAction) RunCommand() {
	fmt.Println("Running command", ra.Command)
	if ra.nodes == nil {
		fmt.Println("ra.nodes is nil!")
		return
	}
	node := ra.nodes.Search(ra.Uuid)
	if node != nil {
		jsonToSend, err := json.Marshal(&ra.Command)
		if err != nil {
			log.Error(err)
			return
		}
		node.Conn().Write(jsonToSend)
	}
}
<commit_msg>more logging<commit_after>package logic

import (
	"encoding/json"
	"fmt"

	log "github.com/cihub/seelog"
	"github.com/stampzilla/stampzilla-go/protocol"
	serverprotocol "github.com/stampzilla/stampzilla-go/stampzilla-server/protocol"
)

type RuleAction interface {
	RunCommand()
}
type ruleAction struct {
	Command *protocol.Command `json:"command"`
	Uuid    string            `json:"uuid"`
	nodes   *serverprotocol.Nodes
}

func NewRuleAction(cmd *protocol.Command, uuid string) RuleAction {
	return &ruleAction{Command: cmd, Uuid: uuid}
}
func (ra *ruleAction) RunCommand() {
	fmt.Println("Running command", ra.Command,"to",ra.Uuid)
	if ra.nodes == nil {
		fmt.Println("ra.nodes is nil!")
		return
	}
	node := ra.nodes.Search(ra.Uuid)
	if node != nil {
		jsonToSend, err := json.Marshal(&ra.Command)
		if err != nil {
			log.Error(err)
			return
		}
		node.Conn().Write(jsonToSend)
	}
}
<|endoftext|>