<commit_before>package config

import (
	"fmt"
	"regexp"
	"strings"
	"sync"

	"github.com/evanw/esbuild/internal/ast"
	"github.com/evanw/esbuild/internal/compat"
	"github.com/evanw/esbuild/internal/js_ast"
	"github.com/evanw/esbuild/internal/logger"
)

type JSXOptions struct {
	Factory  JSXExpr
	Fragment JSXExpr
	Parse    bool
	Preserve bool
}

type JSXExpr struct {
	Constant js_ast.E
	Parts    []string
}

type TSOptions struct {
	Parse               bool
	NoAmbiguousLessThan bool
}

type Platform uint8

const (
	PlatformBrowser Platform = iota
	PlatformNode
	PlatformNeutral
)

type StrictOptions struct {
	// Loose:  "class Foo { foo = 1 }" => "class Foo { constructor() { this.foo = 1; } }"
	// Strict: "class Foo { foo = 1 }" => "class Foo { constructor() { __publicField(this, 'foo', 1); } }"
	//
	// The disadvantage of strictness here is code bloat and performance. The
	// advantage is following the class field specification accurately. For
	// example, loose mode will incorrectly trigger setter methods while strict
	// mode won't.
	ClassFields bool
}

type SourceMap uint8

const (
	SourceMapNone SourceMap = iota
	SourceMapInline
	SourceMapLinkedWithComment
	SourceMapExternalWithoutComment
	SourceMapInlineAndExternal
)

type LegalComments uint8

const (
	LegalCommentsInline LegalComments = iota
	LegalCommentsNone
	LegalCommentsEndOfFile
	LegalCommentsLinkedWithComment
	LegalCommentsExternalWithoutComment
)

func (lc LegalComments) HasExternalFile() bool {
	return lc == LegalCommentsLinkedWithComment || lc == LegalCommentsExternalWithoutComment
}

type Loader uint8

const (
	LoaderNone Loader = iota
	LoaderJS
	LoaderJSX
	LoaderTS
	LoaderTSNoAmbiguousLessThan // Used with ".mts" and ".cts"
	LoaderTSX
	LoaderJSON
	LoaderText
	LoaderBase64
	LoaderDataURL
	LoaderFile
	LoaderBinary
	LoaderCSS
	LoaderDefault
)

func (loader Loader) IsTypeScript() bool {
	switch loader {
	case LoaderTS, LoaderTSNoAmbiguousLessThan, LoaderTSX:
		return true
	default:
		return false
	}
}

func (loader Loader) CanHaveSourceMap() bool {
	switch loader {
	case LoaderJS, LoaderJSX, LoaderTS, LoaderTSNoAmbiguousLessThan, LoaderTSX, LoaderCSS:
		return true
	default:
		return false
	}
}

type Format uint8

const (
	// This is used when not bundling. It means to preserve whatever form the
	// import or export was originally in. ES6 syntax stays ES6 syntax and
	// CommonJS syntax stays CommonJS syntax.
	FormatPreserve Format = iota

	// IIFE stands for immediately-invoked function expression. That looks like
	// this:
	//
	//   (() => {
	//     ... bundled code ...
	//   })();
	//
	// If the optional GlobalName is configured, then we'll write out this:
	//
	//   let globalName = (() => {
	//     ... bundled code ...
	//     return exports;
	//   })();
	//
	FormatIIFE

	// The CommonJS format looks like this:
	//
	//   ... bundled code ...
	//   module.exports = exports;
	//
	FormatCommonJS

	// The ES module format looks like this:
	//
	//   ... bundled code ...
	//   export {...};
	//
	FormatESModule
)

func (f Format) KeepES6ImportExportSyntax() bool {
	return f == FormatPreserve || f == FormatESModule
}

func (f Format) String() string {
	switch f {
	case FormatIIFE:
		return "iife"
	case FormatCommonJS:
		return "cjs"
	case FormatESModule:
		return "esm"
	}
	return ""
}

type StdinInfo struct {
	Contents      string
	SourceFile    string
	AbsResolveDir string
	Loader        Loader
}

type WildcardPattern struct {
	Prefix string
	Suffix string
}

type ExternalModules struct {
	NodeModules map[string]bool
	AbsPaths    map[string]bool
	Patterns    []WildcardPattern
}

type Mode uint8

const (
	ModePassThrough Mode = iota
	ModeConvertFormat
	ModeBundle
)

type MaybeBool uint8

const (
	Unspecified MaybeBool = iota
	True
	False
)

type Options struct {
	ModuleTypeData js_ast.ModuleTypeData
	Defines        *ProcessedDefines
	TSTarget       *TSTarget

	// This is the original information that was used to generate the
	// unsupported feature sets above. It's used for error messages.
	OriginalTargetEnv string

	ExtensionOrder  []string
	MainFields      []string
	Conditions      []string
	AbsNodePaths    []string // The "NODE_PATH" variable from Node.js
	ExternalModules ExternalModules

	AbsOutputFile      string
	AbsOutputDir       string
	AbsOutputBase      string
	OutputExtensionJS  string
	OutputExtensionCSS string
	GlobalName         []string
	TsConfigOverride   string
	ExtensionToLoader  map[string]Loader

	PublicPath      string
	InjectAbsPaths  []string
	InjectedDefines []InjectedDefine
	InjectedFiles   []InjectedFile

	JSBanner  string
	JSFooter  string
	CSSBanner string
	CSSFooter string

	EntryPathTemplate []PathTemplate
	ChunkPathTemplate []PathTemplate
	AssetPathTemplate []PathTemplate

	Plugins    []Plugin
	SourceRoot string
	Stdin      *StdinInfo
	JSX        JSXOptions

	UnsupportedJSFeatures  compat.JSFeature
	UnsupportedCSSFeatures compat.CSSFeature

	TS                TSOptions
	Mode              Mode
	PreserveSymlinks  bool
	RemoveWhitespace  bool
	MinifyIdentifiers bool
	MangleSyntax      bool
	ProfilerNames     bool
	CodeSplitting     bool
	WatchMode         bool
	AllowOverwrite    bool
	LegalComments     LegalComments

	// If true, make sure to generate a single file that can be written to stdout
	WriteToStdout bool

	OmitRuntimeForTests     bool
	UnusedImportsTS         UnusedImportsTS
	UseDefineForClassFields MaybeBool
	ASCIIOnly               bool
	KeepNames               bool
	IgnoreDCEAnnotations    bool
	TreeShaking             bool
	DropDebugger            bool
	Platform                Platform
	TargetFromAPI           TargetFromAPI
	OutputFormat            Format
	NeedsMetafile           bool
	SourceMap               SourceMap
	ExcludeSourcesContent   bool
}

type TargetFromAPI uint8

const (
	// In this state, the "target" field in "tsconfig.json" is respected
	TargetWasUnconfigured TargetFromAPI = iota

	// In this state, the "target" field in "tsconfig.json" is overridden
	TargetWasConfigured

	// In this state, "useDefineForClassFields" is true unless overridden
	TargetWasConfiguredIncludingESNext
)

type UnusedImportsTS uint8

const (
	// "import { unused } from 'foo'" => "" (TypeScript's default behavior)
	UnusedImportsRemoveStmt UnusedImportsTS = iota

	// "import { unused } from 'foo'" => "import 'foo'" ("importsNotUsedAsValues" != "remove")
	UnusedImportsKeepStmtRemoveValues

	// "import { unused } from 'foo'" => "import { unused } from 'foo'" ("preserveValueImports" == true)
	UnusedImportsKeepValues
)

func UnusedImportsFromTsconfigValues(preserveImportsNotUsedAsValues bool, preserveValueImports bool) UnusedImportsTS {
	if preserveValueImports {
		return UnusedImportsKeepValues
	}
	if preserveImportsNotUsedAsValues {
		return UnusedImportsKeepStmtRemoveValues
	}
	return UnusedImportsRemoveStmt
}

type TSTarget struct {
	Target                string
	Source                logger.Source
	Range                 logger.Range
	UnsupportedJSFeatures compat.JSFeature
}

type PathPlaceholder uint8

const (
	NoPlaceholder PathPlaceholder = iota

	// The relative path from the original parent directory to the configured
	// "outbase" directory, or to the lowest common ancestor directory
	DirPlaceholder

	// The original name of the file, or the manual chunk name, or the name of
	// the type of output file ("entry" or "chunk" or "asset")
	NamePlaceholder

	// A hash of the contents of this file, and the contents and output paths of
	// all dependencies (except for their hash placeholders)
	HashPlaceholder

	// The original extension of the file, or the name of the output file
	// (e.g. "css", "svg", "png")
	ExtPlaceholder
)

type PathTemplate struct {
	Data        string
	Placeholder PathPlaceholder
}

type PathPlaceholders struct {
	Dir  *string
	Name *string
	Hash *string
	Ext  *string
}

func (placeholders PathPlaceholders) Get(placeholder PathPlaceholder) *string {
	switch placeholder {
	case DirPlaceholder:
		return placeholders.Dir
	case NamePlaceholder:
		return placeholders.Name
	case HashPlaceholder:
		return placeholders.Hash
	case ExtPlaceholder:
		return placeholders.Ext
	}
	return nil
}

func TemplateToString(template []PathTemplate) string {
	if len(template) == 1 && template[0].Placeholder == NoPlaceholder {
		// Avoid allocations in this case
		return template[0].Data
	}
	sb := strings.Builder{}
	for _, part := range template {
		sb.WriteString(part.Data)
		switch part.Placeholder {
		case DirPlaceholder:
			sb.WriteString("[dir]")
		case NamePlaceholder:
			sb.WriteString("[name]")
		case HashPlaceholder:
			sb.WriteString("[hash]")
		case ExtPlaceholder:
			sb.WriteString("[ext]")
		}
	}
	return sb.String()
}

func HasPlaceholder(template []PathTemplate, placeholder PathPlaceholder) bool {
	for _, part := range template {
		if part.Placeholder == placeholder {
			return true
		}
	}
	return false
}

func SubstituteTemplate(template []PathTemplate, placeholders PathPlaceholders) []PathTemplate {
	// Don't allocate if no substitution is possible and the template is already minimal
	shouldSubstitute := false
	for i, part := range template {
		if placeholders.Get(part.Placeholder) != nil || (part.Placeholder == NoPlaceholder && i+1 < len(template)) {
			shouldSubstitute = true
			break
		}
	}
	if !shouldSubstitute {
		return template
	}

	// Otherwise, substitute and merge as appropriate
	result := make([]PathTemplate, 0, len(template))
	for _, part := range template {
		if sub := placeholders.Get(part.Placeholder); sub != nil {
			part.Data += *sub
			part.Placeholder = NoPlaceholder
		}
		if last := len(result) - 1; last >= 0 && result[last].Placeholder == NoPlaceholder {
			last := &result[last]
			last.Data += part.Data
			last.Placeholder = part.Placeholder
		} else {
			result = append(result, part)
		}
	}
	return result
}

func ShouldCallRuntimeRequire(mode Mode, outputFormat Format) bool {
	return mode == ModeBundle && outputFormat != FormatCommonJS
}

type InjectedDefine struct {
	Data   js_ast.E
	Name   string
	Source logger.Source
}

type InjectedFile struct {
	Exports    []InjectableExport
	DefineName string
	Source     logger.Source
}

type InjectableExport struct {
	Alias string
	Loc   logger.Loc
}

var filterMutex sync.Mutex
var filterCache map[string]*regexp.Regexp

func compileFilter(filter string) (result *regexp.Regexp) {
	if filter == "" {
		// Must provide a filter
		return nil
	}
	ok := false

	// Cache hit?
	(func() {
		filterMutex.Lock()
		defer filterMutex.Unlock()
		if filterCache != nil {
			result, ok = filterCache[filter]
		}
	})()
	if ok {
		return
	}

	// Cache miss
	result, err := regexp.Compile(filter)
	if err != nil {
		return nil
	}

	// Cache for next time
	filterMutex.Lock()
	defer filterMutex.Unlock()
	if filterCache == nil {
		filterCache = make(map[string]*regexp.Regexp)
	}
	filterCache[filter] = result
	return
}

func CompileFilterForPlugin(pluginName string, kind string, filter string) (*regexp.Regexp, error) {
	if filter == "" {
		return nil, fmt.Errorf("[%s] %q is missing a filter", pluginName, kind)
	}

	result := compileFilter(filter)
	if result == nil {
		return nil, fmt.Errorf("[%s] %q filter is not a valid Go regular expression: %q", pluginName, kind, filter)
	}

	return result, nil
}

func PluginAppliesToPath(path logger.Path, filter *regexp.Regexp, namespace string) bool {
	return (namespace == "" || path.Namespace == namespace) && filter.MatchString(path.Text)
}

////////////////////////////////////////////////////////////////////////////////
// Plugin API

type Plugin struct {
	Name      string
	OnStart   []OnStart
	OnResolve []OnResolve
	OnLoad    []OnLoad
}

type OnStart struct {
	Callback func() OnStartResult
	Name     string
}

type OnStartResult struct {
	ThrownError error
	Msgs        []logger.Msg
}

type OnResolve struct {
	Filter    *regexp.Regexp
	Callback  func(OnResolveArgs) OnResolveResult
	Name      string
	Namespace string
}

type OnResolveArgs struct {
	Path       string
	ResolveDir string
	PluginData interface{}
	Importer   logger.Path
	Kind       ast.ImportKind
}

type OnResolveResult struct {
	PluginName string

	Msgs        []logger.Msg
	ThrownError error

	AbsWatchFiles []string
	AbsWatchDirs  []string

	PluginData       interface{}
	Path             logger.Path
	External         bool
	IsSideEffectFree bool
}

type OnLoad struct {
	Filter    *regexp.Regexp
	Callback  func(OnLoadArgs) OnLoadResult
	Name      string
	Namespace string
}

type OnLoadArgs struct {
	PluginData interface{}
	Path       logger.Path
}

type OnLoadResult struct {
	PluginName string

	Contents      *string
	AbsResolveDir string
	PluginData    interface{}

	Msgs        []logger.Msg
	ThrownError error

	AbsWatchFiles []string
	AbsWatchDirs  []string

	Loader Loader
}
<commit_msg>remove an unused type<commit_after>package config

import (
	"fmt"
	"regexp"
	"strings"
	"sync"

	"github.com/evanw/esbuild/internal/ast"
	"github.com/evanw/esbuild/internal/compat"
	"github.com/evanw/esbuild/internal/js_ast"
	"github.com/evanw/esbuild/internal/logger"
)

type JSXOptions struct {
	Factory  JSXExpr
	Fragment JSXExpr
	Parse    bool
	Preserve bool
}

type JSXExpr struct {
	Constant js_ast.E
	Parts    []string
}

type TSOptions struct {
	Parse               bool
	NoAmbiguousLessThan bool
}

type Platform uint8

const (
	PlatformBrowser Platform = iota
	PlatformNode
	PlatformNeutral
)

type SourceMap uint8

const (
	SourceMapNone SourceMap = iota
	SourceMapInline
	SourceMapLinkedWithComment
	SourceMapExternalWithoutComment
	SourceMapInlineAndExternal
)

type LegalComments uint8

const (
	LegalCommentsInline LegalComments = iota
	LegalCommentsNone
	LegalCommentsEndOfFile
	LegalCommentsLinkedWithComment
	LegalCommentsExternalWithoutComment
)

func (lc LegalComments) HasExternalFile() bool {
	return lc == LegalCommentsLinkedWithComment || lc == LegalCommentsExternalWithoutComment
}

type Loader uint8

const (
	LoaderNone Loader = iota
	LoaderJS
	LoaderJSX
	LoaderTS
	LoaderTSNoAmbiguousLessThan // Used with ".mts" and ".cts"
	LoaderTSX
	LoaderJSON
	LoaderText
	LoaderBase64
	LoaderDataURL
	LoaderFile
	LoaderBinary
	LoaderCSS
	LoaderDefault
)

func (loader Loader) IsTypeScript() bool {
	switch loader {
	case LoaderTS, LoaderTSNoAmbiguousLessThan, LoaderTSX:
		return true
	default:
		return false
	}
}

func (loader Loader) CanHaveSourceMap() bool {
	switch loader {
	case LoaderJS, LoaderJSX, LoaderTS, LoaderTSNoAmbiguousLessThan, LoaderTSX, LoaderCSS:
		return true
	default:
		return false
	}
}

type Format uint8

const (
	// This is used when not bundling. It means to preserve whatever form the
	// import or export was originally in. ES6 syntax stays ES6 syntax and
	// CommonJS syntax stays CommonJS syntax.
	FormatPreserve Format = iota

	// IIFE stands for immediately-invoked function expression. That looks like
	// this:
	//
	//   (() => {
	//     ... bundled code ...
	//   })();
	//
	// If the optional GlobalName is configured, then we'll write out this:
	//
	//   let globalName = (() => {
	//     ... bundled code ...
	//     return exports;
	//   })();
	//
	FormatIIFE

	// The CommonJS format looks like this:
	//
	//   ... bundled code ...
	//   module.exports = exports;
	//
	FormatCommonJS

	// The ES module format looks like this:
	//
	//   ... bundled code ...
	//   export {...};
	//
	FormatESModule
)

func (f Format) KeepES6ImportExportSyntax() bool {
	return f == FormatPreserve || f == FormatESModule
}

func (f Format) String() string {
	switch f {
	case FormatIIFE:
		return "iife"
	case FormatCommonJS:
		return "cjs"
	case FormatESModule:
		return "esm"
	}
	return ""
}

type StdinInfo struct {
	Contents      string
	SourceFile    string
	AbsResolveDir string
	Loader        Loader
}

type WildcardPattern struct {
	Prefix string
	Suffix string
}

type ExternalModules struct {
	NodeModules map[string]bool
	AbsPaths    map[string]bool
	Patterns    []WildcardPattern
}

type Mode uint8

const (
	ModePassThrough Mode = iota
	ModeConvertFormat
	ModeBundle
)

type MaybeBool uint8

const (
	Unspecified MaybeBool = iota
	True
	False
)

type Options struct {
	ModuleTypeData js_ast.ModuleTypeData
	Defines        *ProcessedDefines
	TSTarget       *TSTarget

	// This is the original information that was used to generate the
	// unsupported feature sets above. It's used for error messages.
	OriginalTargetEnv string

	ExtensionOrder  []string
	MainFields      []string
	Conditions      []string
	AbsNodePaths    []string // The "NODE_PATH" variable from Node.js
	ExternalModules ExternalModules

	AbsOutputFile      string
	AbsOutputDir       string
	AbsOutputBase      string
	OutputExtensionJS  string
	OutputExtensionCSS string
	GlobalName         []string
	TsConfigOverride   string
	ExtensionToLoader  map[string]Loader

	PublicPath      string
	InjectAbsPaths  []string
	InjectedDefines []InjectedDefine
	InjectedFiles   []InjectedFile

	JSBanner  string
	JSFooter  string
	CSSBanner string
	CSSFooter string

	EntryPathTemplate []PathTemplate
	ChunkPathTemplate []PathTemplate
	AssetPathTemplate []PathTemplate

	Plugins    []Plugin
	SourceRoot string
	Stdin      *StdinInfo
	JSX        JSXOptions

	UnsupportedJSFeatures  compat.JSFeature
	UnsupportedCSSFeatures compat.CSSFeature

	TS                TSOptions
	Mode              Mode
	PreserveSymlinks  bool
	RemoveWhitespace  bool
	MinifyIdentifiers bool
	MangleSyntax      bool
	ProfilerNames     bool
	CodeSplitting     bool
	WatchMode         bool
	AllowOverwrite    bool
	LegalComments     LegalComments

	// If true, make sure to generate a single file that can be written to stdout
	WriteToStdout bool

	OmitRuntimeForTests     bool
	UnusedImportsTS         UnusedImportsTS
	UseDefineForClassFields MaybeBool
	ASCIIOnly               bool
	KeepNames               bool
	IgnoreDCEAnnotations    bool
	TreeShaking             bool
	DropDebugger            bool
	Platform                Platform
	TargetFromAPI           TargetFromAPI
	OutputFormat            Format
	NeedsMetafile           bool
	SourceMap               SourceMap
	ExcludeSourcesContent   bool
}

type TargetFromAPI uint8

const (
	// In this state, the "target" field in "tsconfig.json" is respected
	TargetWasUnconfigured TargetFromAPI = iota

	// In this state, the "target" field in "tsconfig.json" is overridden
	TargetWasConfigured

	// In this state, "useDefineForClassFields" is true unless overridden
	TargetWasConfiguredIncludingESNext
)

type UnusedImportsTS uint8

const (
	// "import { unused } from 'foo'" => "" (TypeScript's default behavior)
	UnusedImportsRemoveStmt UnusedImportsTS = iota

	// "import { unused } from 'foo'" => "import 'foo'" ("importsNotUsedAsValues" != "remove")
	UnusedImportsKeepStmtRemoveValues

	// "import { unused } from 'foo'" => "import { unused } from 'foo'" ("preserveValueImports" == true)
	UnusedImportsKeepValues
)

func UnusedImportsFromTsconfigValues(preserveImportsNotUsedAsValues bool, preserveValueImports bool) UnusedImportsTS {
	if preserveValueImports {
		return UnusedImportsKeepValues
	}
	if preserveImportsNotUsedAsValues {
		return UnusedImportsKeepStmtRemoveValues
	}
	return UnusedImportsRemoveStmt
}

type TSTarget struct {
	Target                string
	Source                logger.Source
	Range                 logger.Range
	UnsupportedJSFeatures compat.JSFeature
}

type PathPlaceholder uint8

const (
	NoPlaceholder PathPlaceholder = iota

	// The relative path from the original parent directory to the configured
	// "outbase" directory, or to the lowest common ancestor directory
	DirPlaceholder

	// The original name of the file, or the manual chunk name, or the name of
	// the type of output file ("entry" or "chunk" or "asset")
	NamePlaceholder

	// A hash of the contents of this file, and the contents and output paths of
	// all dependencies (except for their hash placeholders)
	HashPlaceholder

	// The original extension of the file, or the name of the output file
	// (e.g. "css", "svg", "png")
	ExtPlaceholder
)

type PathTemplate struct {
	Data        string
	Placeholder PathPlaceholder
}

type PathPlaceholders struct {
	Dir  *string
	Name *string
	Hash *string
	Ext  *string
}

func (placeholders PathPlaceholders) Get(placeholder PathPlaceholder) *string {
	switch placeholder {
	case DirPlaceholder:
		return placeholders.Dir
	case NamePlaceholder:
		return placeholders.Name
	case HashPlaceholder:
		return placeholders.Hash
	case ExtPlaceholder:
		return placeholders.Ext
	}
	return nil
}

func TemplateToString(template []PathTemplate) string {
	if len(template) == 1 && template[0].Placeholder == NoPlaceholder {
		// Avoid allocations in this case
		return template[0].Data
	}
	sb := strings.Builder{}
	for _, part := range template {
		sb.WriteString(part.Data)
		switch part.Placeholder {
		case DirPlaceholder:
			sb.WriteString("[dir]")
		case NamePlaceholder:
			sb.WriteString("[name]")
		case HashPlaceholder:
			sb.WriteString("[hash]")
		case ExtPlaceholder:
			sb.WriteString("[ext]")
		}
	}
	return sb.String()
}

func HasPlaceholder(template []PathTemplate, placeholder PathPlaceholder) bool {
	for _, part := range template {
		if part.Placeholder == placeholder {
			return true
		}
	}
	return false
}

func SubstituteTemplate(template []PathTemplate, placeholders PathPlaceholders) []PathTemplate {
	// Don't allocate if no substitution is possible and the template is already minimal
	shouldSubstitute := false
	for i, part := range template {
		if placeholders.Get(part.Placeholder) != nil || (part.Placeholder == NoPlaceholder && i+1 < len(template)) {
			shouldSubstitute = true
			break
		}
	}
	if !shouldSubstitute {
		return template
	}

	// Otherwise, substitute and merge as appropriate
	result := make([]PathTemplate, 0, len(template))
	for _, part := range template {
		if sub := placeholders.Get(part.Placeholder); sub != nil {
			part.Data += *sub
			part.Placeholder = NoPlaceholder
		}
		if last := len(result) - 1; last >= 0 && result[last].Placeholder == NoPlaceholder {
			last := &result[last]
			last.Data += part.Data
			last.Placeholder = part.Placeholder
		} else {
			result = append(result, part)
		}
	}
	return result
}

func ShouldCallRuntimeRequire(mode Mode, outputFormat Format) bool {
	return mode == ModeBundle && outputFormat != FormatCommonJS
}

type InjectedDefine struct {
	Data   js_ast.E
	Name   string
	Source logger.Source
}

type InjectedFile struct {
	Exports    []InjectableExport
	DefineName string
	Source     logger.Source
}

type InjectableExport struct {
	Alias string
	Loc   logger.Loc
}

var filterMutex sync.Mutex
var filterCache map[string]*regexp.Regexp

func compileFilter(filter string) (result *regexp.Regexp) {
	if filter == "" {
		// Must provide a filter
		return nil
	}
	ok := false

	// Cache hit?
	(func() {
		filterMutex.Lock()
		defer filterMutex.Unlock()
		if filterCache != nil {
			result, ok = filterCache[filter]
		}
	})()
	if ok {
		return
	}

	// Cache miss
	result, err := regexp.Compile(filter)
	if err != nil {
		return nil
	}

	// Cache for next time
	filterMutex.Lock()
	defer filterMutex.Unlock()
	if filterCache == nil {
		filterCache = make(map[string]*regexp.Regexp)
	}
	filterCache[filter] = result
	return
}

func CompileFilterForPlugin(pluginName string, kind string, filter string) (*regexp.Regexp, error) {
	if filter == "" {
		return nil, fmt.Errorf("[%s] %q is missing a filter", pluginName, kind)
	}

	result := compileFilter(filter)
	if result == nil {
		return nil, fmt.Errorf("[%s] %q filter is not a valid Go regular expression: %q", pluginName, kind, filter)
	}

	return result, nil
}

func PluginAppliesToPath(path logger.Path, filter *regexp.Regexp, namespace string) bool {
	return (namespace == "" || path.Namespace == namespace) && filter.MatchString(path.Text)
}

////////////////////////////////////////////////////////////////////////////////
// Plugin API

type Plugin struct {
	Name      string
	OnStart   []OnStart
	OnResolve []OnResolve
	OnLoad    []OnLoad
}

type OnStart struct {
	Callback func() OnStartResult
	Name     string
}

type OnStartResult struct {
	ThrownError error
	Msgs        []logger.Msg
}

type OnResolve struct {
	Filter    *regexp.Regexp
	Callback  func(OnResolveArgs) OnResolveResult
	Name      string
	Namespace string
}

type OnResolveArgs struct {
	Path       string
	ResolveDir string
	PluginData interface{}
	Importer   logger.Path
	Kind       ast.ImportKind
}

type OnResolveResult struct {
	PluginName string

	Msgs        []logger.Msg
	ThrownError error

	AbsWatchFiles []string
	AbsWatchDirs  []string

	PluginData       interface{}
	Path             logger.Path
	External         bool
	IsSideEffectFree bool
}

type OnLoad struct {
	Filter    *regexp.Regexp
	Callback  func(OnLoadArgs) OnLoadResult
	Name      string
	Namespace string
}

type OnLoadArgs struct {
	PluginData interface{}
	Path       logger.Path
}

type OnLoadResult struct {
	PluginName string

	Contents      *string
	AbsResolveDir string
	PluginData    interface{}

	Msgs        []logger.Msg
	ThrownError error

	AbsWatchFiles []string
	AbsWatchDirs  []string

	Loader Loader
}
<|endoftext|>


<commit_before>package config

import (
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"strings"

	"github.com/alexflint/go-filemutex"
	conf "github.com/micro/go-micro/v2/config"
	"github.com/micro/go-micro/v2/config/source/file"
	"github.com/micro/go-micro/v2/logger"
	"github.com/micro/go-micro/v2/util/log"
)

// FileName for global micro config
const FileName = ".micro"

// config is a singleton which is required to ensure
// each function call doesn't load the .micro file
// from disk
var config = newConfig()

// Get a value from the .micro file
func Get(path ...string) (string, error) {
	m := mutex()
	m.Lock()
	defer m.Unlock()

	tk := config.Get(path...).String("")
	return strings.TrimSpace(tk), nil
}

// Set a value in the .micro file
func Set(value string, path ...string) error {
	m := mutex()
	m.Lock()
	defer m.Unlock()

	// get the filepath
	fp, err := filePath()
	if err != nil {
		return err
	}

	// set the value
	config.Set(value, path...)

	// write to the file
	return ioutil.WriteFile(fp, config.Bytes(), 0644)
}

func filePath() (string, error) {
	usr, err := user.Current()
	if err != nil {
		return "", err
	}
	return filepath.Join(usr.HomeDir, FileName), nil
}

// newConfig returns a loaded config
func newConfig() conf.Config {
	m := mutex()
	m.Lock()
	defer m.Unlock()

	// get the filepath
	fp, err := filePath()
	if err != nil {
		log.Error(err)
		return conf.DefaultConfig
	}

	// write the file if it does not exist
	if _, err := os.Stat(fp); os.IsNotExist(err) {
		ioutil.WriteFile(fp, []byte{}, 0644)
	} else if err != nil {
		log.Error(err)
		return conf.DefaultConfig
	}

	// create a new config
	c, err := conf.NewConfig(
		conf.WithSource(
			file.NewSource(
				file.WithPath(fp),
			),
		),
	)
	if err != nil {
		log.Error(err)
		return conf.DefaultConfig
	}

	// load the config
	if err := c.Load(); err != nil {
		log.Error(err)
		return conf.DefaultConfig
	}

	// return the conf
	return c
}

func mutex() *filemutex.FileMutex {
	// detemine lock filepath
	fp, _ := filePath()
	lockFile := fp + ".lock"

	// check if file exists
	var _, err = os.Stat(lockFile)

	// create file if not exists
	if os.IsNotExist(err) {
		var file, err = os.Create(lockFile)
		if err != nil {
			logger.Fatalf("Error creating ~/.micro.lock: %v", err)
		}
		file.Close()
	}

	// create the mutex
	m, err := filemutex.New(lockFile)
	if err != nil {
		logger.Fatalf("Error locking ~/.micro: %v", err)
	}

	return m
}
<commit_msg>internal/config: stop locking on newConfig()<commit_after>package config

import (
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"strings"

	"github.com/alexflint/go-filemutex"
	conf "github.com/micro/go-micro/v2/config"
	"github.com/micro/go-micro/v2/config/source/file"
	"github.com/micro/go-micro/v2/logger"
	"github.com/micro/go-micro/v2/util/log"
)

// FileName for global micro config
const FileName = ".micro"

// config is a singleton which is required to ensure
// each function call doesn't load the .micro file
// from disk
var config = newConfig()

// Get a value from the .micro file
func Get(path ...string) (string, error) {
	m := mutex()
	m.Lock()
	defer m.Unlock()

	tk := config.Get(path...).String("")
	return strings.TrimSpace(tk), nil
}

// Set a value in the .micro file
func Set(value string, path ...string) error {
	m := mutex()
	m.Lock()
	defer m.Unlock()

	// get the filepath
	fp, err := filePath()
	if err != nil {
		return err
	}

	// set the value
	config.Set(value, path...)

	// write to the file
	return ioutil.WriteFile(fp, config.Bytes(), 0644)
}

func filePath() (string, error) {
	usr, err := user.Current()
	if err != nil {
		return "", err
	}
	return filepath.Join(usr.HomeDir, FileName), nil
}

// newConfig returns a loaded config
func newConfig() conf.Config {
	// get the filepath
	fp, err := filePath()
	if err != nil {
		log.Error(err)
		return conf.DefaultConfig
	}

	// write the file if it does not exist
	if _, err := os.Stat(fp); os.IsNotExist(err) {
		ioutil.WriteFile(fp, []byte{}, 0644)
	} else if err != nil {
		log.Error(err)
		return conf.DefaultConfig
	}

	// create a new config
	c, err := conf.NewConfig(
		conf.WithSource(
			file.NewSource(
				file.WithPath(fp),
			),
		),
	)
	if err != nil {
		log.Error(err)
		return conf.DefaultConfig
	}

	// load the config
	if err := c.Load(); err != nil {
		log.Error(err)
		return conf.DefaultConfig
	}

	// return the conf
	return c
}

func mutex() *filemutex.FileMutex {
	// detemine lock filepath
	fp, _ := filePath()
	lockFile := fp + ".lock"

	// check if file exists
	var _, err = os.Stat(lockFile)

	// create file if not exists
	if os.IsNotExist(err) {
		var file, err = os.Create(lockFile)
		if err != nil {
			logger.Fatalf("Error creating ~/.micro.lock: %v", err)
		}
		file.Close()
	}

	// create the mutex
	m, err := filemutex.New(lockFile)
	if err != nil {
		logger.Fatalf("Error locking ~/.micro: %v", err)
	}

	return m
}
<|endoftext|>


<commit_before>package internal

import (
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"testing"
)

func TestInstallingHealthChecker(t *testing.T) {
	try := func(desc string, mux *http.ServeMux, wantCode int, wantBody string) {
		installHealthChecker(mux)
		srv := httptest.NewServer(mux)
		defer srv.Close()

		resp, err := http.Get(srv.URL + "/_ah/health")
		if err != nil {
			t.Errorf("%s: http.Get: %v", desc, err)
			return
		}
		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			t.Errorf("%s: reading body: %v", err)
			return
		}

		if resp.StatusCode != wantCode {
			t.Errorf("%s: got HTTP %d, want %d", resp.StatusCode, wantCode)
			return
		}
		if wantBody != "" && string(body) != wantBody {
			t.Errorf("%s: got HTTP body %q, want %q", body, wantBody)
			return
		}
	}

	// If there's no handlers, or only a root handler, a health checker should be installed.
	try("empty mux", http.NewServeMux(), 200, "ok")
	mux := http.NewServeMux()
	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		io.WriteString(w, "root handler")
	})
	try("mux with root handler", mux, 200, "ok")

	// If there's a custom health check handler, one should not be installed.
	mux = http.NewServeMux()
	mux.HandleFunc("/_ah/health", func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(418)
		io.WriteString(w, "I'm short and stout!")
	})
	try("mux with custom health checker", mux, 418, "I'm short and stout!")
}
<commit_msg>Fix t.Errorf calls in internal_test.<commit_after>package internal

import (
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"testing"
)

func TestInstallingHealthChecker(t *testing.T) {
	try := func(desc string, mux *http.ServeMux, wantCode int, wantBody string) {
		installHealthChecker(mux)
		srv := httptest.NewServer(mux)
		defer srv.Close()

		resp, err := http.Get(srv.URL + "/_ah/health")
		if err != nil {
			t.Errorf("%s: http.Get: %v", desc, err)
			return
		}
		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			t.Errorf("%s: reading body: %v", desc, err)
			return
		}

		if resp.StatusCode != wantCode {
			t.Errorf("%s: got HTTP %d, want %d", desc, resp.StatusCode, wantCode)
			return
		}
		if wantBody != "" && string(body) != wantBody {
			t.Errorf("%s: got HTTP body %q, want %q", desc, body, wantBody)
			return
		}
	}

	// If there's no handlers, or only a root handler, a health checker should be installed.
	try("empty mux", http.NewServeMux(), 200, "ok")
	mux := http.NewServeMux()
	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		io.WriteString(w, "root handler")
	})
	try("mux with root handler", mux, 200, "ok")

	// If there's a custom health check handler, one should not be installed.
	mux = http.NewServeMux()
	mux.HandleFunc("/_ah/health", func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(418)
		io.WriteString(w, "I'm short and stout!")
	})
	try("mux with custom health checker", mux, 418, "I'm short and stout!")
}
<|endoftext|>


<commit_before>// SPDX-License-Identifier: MIT

// Package locale 提供了一个本地化翻译服务。
package locale

import (
	"golang.org/x/text/language"
	"golang.org/x/text/language/display"
	"golang.org/x/text/message"
)

var (
	// 当前使用的语言标签
	//
	// 保证有个初始化的值，部分包的测试功能依赖此变量
	localeTag     = language.MustParse("zh-Hans")
	localePrinter = message.NewPrinter(localeTag)
	displayNames  = map[language.Tag]string{}
)

// Err 为错误信息提供本地化的缓存机制
type Err struct {
	Key    message.Reference
	Values []interface{}
}

func (err *Err) Error() string {
	return Sprintf(err.Key, err.Values...)
}

func setMessages(tag language.Tag, messages map[string]string) {
	for key, val := range messages {
		if err := message.SetString(tag, key, val); err != nil {
			panic(err)
		}
	}
	displayNames[tag] = display.Self.Name(tag)
}

// SetLanguageTag 切换本地化环境
func SetLanguageTag(tag language.Tag) bool {
	if _, found := displayNames[tag]; !found {
		return false
	}

	localeTag = tag
	localePrinter = message.NewPrinter(localeTag)
	return true
}

// LanguageTag 获取当前的本地化 ID
func LanguageTag() language.Tag {
	return localeTag
}

// DisplayNames 所有支持语言的列表
func DisplayNames() map[language.Tag]string {
	ret := make(map[language.Tag]string, len(displayNames))
	for k, v := range displayNames {
		ret[k] = v
	}

	return ret
}

// Sprintf 类似 fmt.Sprintf，与特定的本地化绑定。
func Sprintf(key message.Reference, v ...interface{}) string {
	return localePrinter.Sprintf(key, v...)
}

// NewError 类似 fmt.NewError，与特定的本地化绑定。
func NewError(key message.Reference, v ...interface{}) error {
	return &Err{Key: key, Values: v}
}

// Translate 功能与 Sprintf 类似，但是可以指定本地化 ID 值。
func Translate(localeID string, key message.Reference, v ...interface{}) string {
	return message.NewPrinter(language.MustParse(localeID)).Sprintf(key, v...)
}
<commit_msg>feat(internal/locale): 添加 Locale 用以缓存普通的本地化对象<commit_after>// SPDX-License-Identifier: MIT

// Package locale 提供了一个本地化翻译服务。
package locale

import (
	"golang.org/x/text/language"
	"golang.org/x/text/language/display"
	"golang.org/x/text/message"
)

var (
	// 当前使用的语言标签
	//
	// 保证有个初始化的值，部分包的测试功能依赖此变量
	localeTag     = language.MustParse("zh-Hans")
	localePrinter = message.NewPrinter(localeTag)
	displayNames  = map[language.Tag]string{}
)

type Locale struct {
	Key    message.Reference
	Values []interface{}
}

// Err 为错误信息提供本地化的缓存机制
type Err Locale

func (l *Locale) String() string {
	return Sprintf(l.Key, l.Values...)
}

func (err *Err) Error() string {
	return (*Locale)(err).String()
}

func setMessages(tag language.Tag, messages map[string]string) {
	for key, val := range messages {
		if err := message.SetString(tag, key, val); err != nil {
			panic(err)
		}
	}
	displayNames[tag] = display.Self.Name(tag)
}

// SetLanguageTag 切换本地化环境
func SetLanguageTag(tag language.Tag) bool {
	if _, found := displayNames[tag]; !found {
		return false
	}

	localeTag = tag
	localePrinter = message.NewPrinter(localeTag)
	return true
}

// LanguageTag 获取当前的本地化 ID
func LanguageTag() language.Tag {
	return localeTag
}

// DisplayNames 所有支持语言的列表
func DisplayNames() map[language.Tag]string {
	ret := make(map[language.Tag]string, len(displayNames))
	for k, v := range displayNames {
		ret[k] = v
	}

	return ret
}

// Sprintf 类似 fmt.Sprintf，与特定的本地化绑定。
func Sprintf(key message.Reference, v ...interface{}) string {
	return localePrinter.Sprintf(key, v...)
}

// NewError 类似 fmt.NewError，与特定的本地化绑定。
func NewError(key message.Reference, v ...interface{}) error {
	return &Err{Key: key, Values: v}
}

// Translate 功能与 Sprintf 类似，但是可以指定本地化 ID 值。
func Translate(localeID string, key message.Reference, v ...interface{}) string {
	return message.NewPrinter(language.MustParse(localeID)).Sprintf(key, v...)
}
<|endoftext|>


<commit_before>// Copyright 2015 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build integration

package docker

import (
	"bytes"
	"testing"
	"time"

	dockerclient "github.com/fsouza/go-dockerclient"
	. "gopkg.in/check.v1"
)

func TestDocker(t *testing.T) { TestingT(t) }

type DockerSuite struct {
	dc     *dockerclient.Client
	docker *DockerClient
	regid  string
}

var _ = Suite(&DockerSuite{})

func (s *DockerSuite) SetUpSuite(c *C) {
	var err error
	s.dc, err = dockerclient.NewClient(DefaultSocket)
	if err != nil {
		c.Fatalf("Could not connect to docker client: %s", err)
	}
	s.docker = &DockerClient{dc: s.dc}
	if ctr, err := s.dc.InspectContainer("regtestserver"); err == nil {
		s.dc.KillContainer(dockerclient.KillContainerOptions{ID: ctr.ID})
		opts := dockerclient.RemoveContainerOptions{
			ID:            ctr.ID,
			RemoveVolumes: true,
			Force:         true,
		}
		s.dc.RemoveContainer(opts)
	} else {
		opts := dockerclient.PullImageOptions{
			Repository: "registry",
			Tag:        "2.1.1",
		}
		auth := dockerclient.AuthConfiguration{}
		s.dc.PullImage(opts, auth)
	}
	// Start the docker registry
	opts := dockerclient.CreateContainerOptions{Name: "regtestserver"}
	opts.Config = &dockerclient.Config{Image: "registry:2.1.1"}
	opts.HostConfig = &dockerclient.HostConfig{
		PortBindings: map[dockerclient.Port][]dockerclient.PortBinding{
			"5000/tcp": []dockerclient.PortBinding{
				{HostIP: "localhost", HostPort: "5000"},
			},
		},
	}
	ctr, err := s.dc.CreateContainer(opts)
	if err != nil {
		c.Fatalf("Could not initialize docker registry: %s", err)
	}
	s.regid = ctr.ID
	if err := s.dc.StartContainer(ctr.ID, nil); err != nil {
		c.Fatalf("Could not start docker registry: %s", err)
	}
}

func (s *DockerSuite) TearDownSuite(c *C) {
	// Shut down the docker registry
	s.dc.StopContainer(s.regid, 10)
	opts := dockerclient.RemoveContainerOptions{
		ID:            s.regid,
		RemoveVolumes: true,
		Force:         true,
	}
	s.dc.RemoveContainer(opts)
}

func (s *DockerSuite) SetUpTest(c *C) {
	opts := dockerclient.PullImageOptions{Repository: "busybox", Tag: "latest"}
	auth := dockerclient.AuthConfiguration{}
	if err := s.dc.PullImage(opts, auth); err != nil {
		c.Fatalf("Could not pull test image: %s", err)
	}
}

func (s *DockerSuite) TestFindImage(c *C) {
	_, err := s.docker.FindImage("fakebox")
	c.Assert(err, Equals, dockerclient.ErrNoSuchImage)
	expected, err := s.dc.InspectImage("busybox")
	c.Assert(err, IsNil)
	actual, err := s.docker.FindImage("busybox")
	c.Assert(err, IsNil)
	c.Assert(actual, DeepEquals, expected)
}

func (s *DockerSuite) TestSaveImages(c *C) {
	buffer := bytes.NewBufferString("")
	err := s.docker.SaveImages([]string{"busybox"}, buffer)
	c.Assert(err, IsNil)
	err = s.dc.RemoveImage("busybox")
	c.Assert(err, IsNil)
	err = s.dc.LoadImage(dockerclient.LoadImageOptions{InputStream: buffer})
	c.Assert(err, IsNil)
	_, err = s.dc.InspectImage("busybox")
	c.Assert(err, IsNil)
}

func (s *DockerSuite) TestLoadImage(c *C) {
	buffer := bytes.NewBufferString("")
	err := s.dc.ExportImages(dockerclient.ExportImagesOptions{Names: []string{"busybox"}, OutputStream: buffer})
	c.Assert(err, IsNil)
	err = s.dc.RemoveImage("busybox")
	c.Assert(err, IsNil)
	err = s.docker.LoadImage(buffer)
	c.Assert(err, IsNil)
	_, err = s.dc.InspectImage("busybox")
	c.Assert(err, IsNil)
}

func (s *DockerSuite) TestPushImage(c *C) {
	defer s.dc.RemoveImage("localhost:5000/busybox:push")
	err := s.docker.PushImage("localhost:5000/busybox:push")
	c.Assert(err, NotNil)
	err = s.dc.TagImage("busybox", dockerclient.TagImageOptions{Repo: "localhost:5000/busybox", Tag: "push"})
	c.Assert(err, IsNil)
	err = s.docker.PushImage("localhost:5000/busybox:push")
	c.Assert(err, IsNil)
	err = s.dc.RemoveImage("localhost:5000/busybox:push")
	c.Assert(err, IsNil)
	opts := dockerclient.PullImageOptions{
		Repository: "localhost:5000/busybox",
		Registry:   "localhost:5000",
		Tag:        "push",
	}
	auth := dockerclient.AuthConfiguration{}
	err = s.dc.PullImage(opts, auth)
	c.Assert(err, IsNil)
	_, err = s.dc.InspectImage("localhost:5000/busybox:push")
	c.Assert(err, IsNil)
}

func (s *DockerSuite) TestPullImage(c *C) {
	defer s.dc.RemoveImage("localhost:5000/busybox:pull")
	err := s.docker.PullImage("localhost:5000/busybox:pull")
	c.Assert(err, NotNil)
	err = s.dc.TagImage("busybox", dockerclient.TagImageOptions{Repo: "localhost:5000/busybox", Tag: "pull"})
	c.Assert(err, IsNil)
	opts := dockerclient.PushImageOptions{
		Name:     "localhost:5000/busybox",
		Tag:      "pull",
		Registry: "localhost:5000",
	}
	auth := dockerclient.AuthConfiguration{}
	err = s.dc.PushImage(opts, auth)
	c.Assert(err, IsNil)
	err = s.dc.RemoveImage("localhost:5000/busybox:pull")
	c.Assert(err, IsNil)
	err = s.docker.PullImage("localhost:5000/busybox:pull")
	c.Assert(err, IsNil)
	_, err = s.dc.InspectImage("localhost:5000/busybox:pull")
	c.Assert(err, IsNil)
}

func (s *DockerSuite) TestTagImage(c *C) {
	defer s.dc.RemoveImage("localhost:5000/busybox:tag")
	err := s.docker.TagImage("fakebox", "localhost:5000/busybox:tag")
	c.Assert(err, NotNil)
	err = s.docker.TagImage("busybox", "localhost:5000/busybox:tag")
	c.Assert(err, IsNil)
	_, err = s.dc.InspectImage("localhost:5000/busybox:tag")
	c.Assert(err, IsNil)
}

func (s *DockerSuite) TestRemoveImage(c *C) {
	defer s.dc.RemoveImage("localhost:5000/busybox:remove")
	err := s.docker.RemoveImage("localhost:5000/busybox:remove")
	c.Assert(err, NotNil)
	err = s.dc.TagImage("busybox", dockerclient.TagImageOptions{Repo: "localhost:5000/busybox", Tag: "remove"})
	c.Assert(err, IsNil)
	err = s.docker.RemoveImage("localhost:5000/busybox:remove")
	c.Assert(err, IsNil)
	_, err = s.dc.InspectImage("localhost:5000/busybox:remove")
	c.Assert(err, NotNil)
}

func (s *DockerSuite) TestFindContainer(c *C) {
	_, err := s.docker.FindContainer("fakecontainer")
	c.Assert(err, NotNil)
	opts := dockerclient.CreateContainerOptions{}
	opts.Config = &dockerclient.Config{Image: "busybox"}
	expected, err := s.dc.CreateContainer(opts)
	c.Assert(err, IsNil)
	defer s.dc.RemoveContainer(dockerclient.RemoveContainerOptions{ID: expected.ID, RemoveVolumes: true, Force: true})
	actual, err := s.docker.FindContainer(expected.ID)
	c.Assert(err, IsNil)
	c.Assert(actual.ID, DeepEquals, expected.ID)
}

func (s *DockerSuite) TestCommitContainer(c *C) {
	defer s.dc.RemoveImage("localhost:5000/busybox:commit")
	opts := dockerclient.CreateContainerOptions{}
	opts.Config = &dockerclient.Config{Image: "busybox"}
	ctr, err := s.dc.CreateContainer(opts)
	c.Assert(err, IsNil)
	defer s.dc.RemoveContainer(dockerclient.RemoveContainerOptions{ID: ctr.ID, RemoveVolumes: true, Force: true})
	expected, err := s.docker.CommitContainer(ctr.ID, "localhost:5000/busybox:commit")
	c.Assert(err, IsNil)
	actual, err := s.dc.InspectImage("localhost:5000/busybox:commit")
	c.Assert(err, IsNil)
	c.Assert(actual.ID, DeepEquals, expected.ID)
}

func (s *DockerSuite) TestGetImageHash(c *C) {
	_, err := s.docker.GetImageHash("fakebox")
	c.Assert(err, Equals, dockerclient.ErrNoSuchImage)
	hash1, err1 := s.docker.GetImageHash("busybox")
	hash2, err2 := s.docker.GetImageHash("registry:2.1.1")
	c.Assert(err1, IsNil)
	c.Assert(err2, IsNil)
	c.Assert(hash1, Not(Equals), "")
	c.Assert(hash2, Not(Equals), "")
	c.Assert(hash1, Not(Equals), hash2)
}

func (s *DockerSuite) TestGetContainerStats(c *C) {
	stats, err := s.docker.GetContainerStats("fakecontainer", 30*time.Second)
	c.Assert(err, NotNil)
	c.Assert(stats, IsNil)
	opts := dockerclient.CreateContainerOptions{}
	opts.Config = &dockerclient.Config{Image: "busybox"}
	container, err := s.dc.CreateContainer(opts)
	c.Assert(err, IsNil)
	defer s.dc.RemoveContainer(dockerclient.RemoveContainerOptions{ID: container.ID, RemoveVolumes: true, Force: true})
	stats, err = s.docker.GetContainerStats(container.ID, 30*time.Second)
	c.Assert(err, IsNil)
	c.Assert(stats, NotNil)
}
<commit_msg>integration test for FindImageByHash<commit_after>// Copyright 2015 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build integration

package docker

import (
	"bytes"
	"testing"
	"time"

	dockerclient "github.com/fsouza/go-dockerclient"
	. "gopkg.in/check.v1"
)

func TestDocker(t *testing.T) { TestingT(t) }

type DockerSuite struct {
	dc     *dockerclient.Client
	docker *DockerClient
	regid  string
}

var _ = Suite(&DockerSuite{})

func (s *DockerSuite) SetUpSuite(c *C) {
	var err error
	s.dc, err = dockerclient.NewClient(DefaultSocket)
	if err != nil {
		c.Fatalf("Could not connect to docker client: %s", err)
	}
	s.docker = &DockerClient{dc: s.dc}
	if ctr, err := s.dc.InspectContainer("regtestserver"); err == nil {
		s.dc.KillContainer(dockerclient.KillContainerOptions{ID: ctr.ID})
		opts := dockerclient.RemoveContainerOptions{
			ID:            ctr.ID,
			RemoveVolumes: true,
			Force:         true,
		}
		s.dc.RemoveContainer(opts)
	} else {
		opts := dockerclient.PullImageOptions{
			Repository: "registry",
			Tag:        "2.1.1",
		}
		auth := dockerclient.AuthConfiguration{}
		s.dc.PullImage(opts, auth)
	}
	// Start the docker registry
	opts := dockerclient.CreateContainerOptions{Name: "regtestserver"}
	opts.Config = &dockerclient.Config{Image: "registry:2.1.1"}
	opts.HostConfig = &dockerclient.HostConfig{
		PortBindings: map[dockerclient.Port][]dockerclient.PortBinding{
			"5000/tcp": []dockerclient.PortBinding{
				{HostIP: "localhost", HostPort: "5000"},
			},
		},
	}
	ctr, err := s.dc.CreateContainer(opts)
	if err != nil {
		c.Fatalf("Could not initialize docker registry: %s", err)
	}
	s.regid = ctr.ID
	if err := s.dc.StartContainer(ctr.ID, nil); err != nil {
		c.Fatalf("Could not start docker registry: %s", err)
	}
}

func (s *DockerSuite) TearDownSuite(c *C) {
	// Shut down the docker registry
	s.dc.StopContainer(s.regid, 10)
	opts := dockerclient.RemoveContainerOptions{
		ID:            s.regid,
		RemoveVolumes: true,
		Force:         true,
	}
	s.dc.RemoveContainer(opts)
}

func (s *DockerSuite) SetUpTest(c *C) {
	opts := dockerclient.PullImageOptions{Repository: "busybox", Tag: "latest"}
	auth := dockerclient.AuthConfiguration{}
	if err := s.dc.PullImage(opts, auth); err != nil {
		c.Fatalf("Could not pull test image: %s", err)
	}
}

func (s *DockerSuite) TestFindImage(c *C) {
	_, err := s.docker.FindImage("fakebox")
	c.Assert(err, Equals, dockerclient.ErrNoSuchImage)
	expected, err := s.dc.InspectImage("busybox")
	c.Assert(err, IsNil)
	actual, err := s.docker.FindImage("busybox")
	c.Assert(err, IsNil)
	c.Assert(actual, DeepEquals, expected)
}

func (s *DockerSuite) TestSaveImages(c *C) {
	buffer := bytes.NewBufferString("")
	err := s.docker.SaveImages([]string{"busybox"}, buffer)
	c.Assert(err, IsNil)
	err = s.dc.RemoveImage("busybox")
	c.Assert(err, IsNil)
	err = s.dc.LoadImage(dockerclient.LoadImageOptions{InputStream: buffer})
	c.Assert(err, IsNil)
	_, err = s.dc.InspectImage("busybox")
	c.Assert(err, IsNil)
}

func (s *DockerSuite) TestLoadImage(c *C) {
	buffer := bytes.NewBufferString("")
	err := s.dc.ExportImages(dockerclient.ExportImagesOptions{Names: []string{"busybox"}, OutputStream: buffer})
	c.Assert(err, IsNil)
	err = s.dc.RemoveImage("busybox")
	c.Assert(err, IsNil)
	err = s.docker.LoadImage(buffer)
	c.Assert(err, IsNil)
	_, err = s.dc.InspectImage("busybox")
	c.Assert(err, IsNil)
}

func (s *DockerSuite) TestPushImage(c *C) {
	defer s.dc.RemoveImage("localhost:5000/busybox:push")
	err := s.docker.PushImage("localhost:5000/busybox:push")
	c.Assert(err, NotNil)
	err = s.dc.TagImage("busybox", dockerclient.TagImageOptions{Repo: "localhost:5000/busybox", Tag: "push"})
	c.Assert(err, IsNil)
	err = s.docker.PushImage("localhost:5000/busybox:push")
	c.Assert(err, IsNil)
	err = s.dc.RemoveImage("localhost:5000/busybox:push")
	c.Assert(err, IsNil)
	opts := dockerclient.PullImageOptions{
		Repository: "localhost:5000/busybox",
		Registry:   "localhost:5000",
		Tag:        "push",
	}
	auth := dockerclient.AuthConfiguration{}
	err = s.dc.PullImage(opts, auth)
	c.Assert(err, IsNil)
	_, err = s.dc.InspectImage("localhost:5000/busybox:push")
	c.Assert(err, IsNil)
}

func (s *DockerSuite) TestPullImage(c *C) {
	defer s.dc.RemoveImage("localhost:5000/busybox:pull")
	err := s.docker.PullImage("localhost:5000/busybox:pull")
	c.Assert(err, NotNil)
	err = s.dc.TagImage("busybox", dockerclient.TagImageOptions{Repo: "localhost:5000/busybox", Tag: "pull"})
	c.Assert(err, IsNil)
	opts := dockerclient.PushImageOptions{
		Name:     "localhost:5000/busybox",
		Tag:      "pull",
		Registry: "localhost:5000",
	}
	auth := dockerclient.AuthConfiguration{}
	err = s.dc.PushImage(opts, auth)
	c.Assert(err, IsNil)
	err = s.dc.RemoveImage("localhost:5000/busybox:pull")
	c.Assert(err, IsNil)
	err = s.docker.PullImage("localhost:5000/busybox:pull")
	c.Assert(err, IsNil)
	_, err = s.dc.InspectImage("localhost:5000/busybox:pull")
	c.Assert(err, IsNil)
}

func (s *DockerSuite) TestTagImage(c *C) {
	defer s.dc.RemoveImage("localhost:5000/busybox:tag")
	err := s.docker.TagImage("fakebox", "localhost:5000/busybox:tag")
	c.Assert(err, NotNil)
	err = s.docker.TagImage("busybox", "localhost:5000/busybox:tag")
	c.Assert(err, IsNil)
	_, err = s.dc.InspectImage("localhost:5000/busybox:tag")
	c.Assert(err, IsNil)
}

func (s *DockerSuite) TestRemoveImage(c *C) {
	defer s.dc.RemoveImage("localhost:5000/busybox:remove")
	err := s.docker.RemoveImage("localhost:5000/busybox:remove")
	c.Assert(err, NotNil)
	err = s.dc.TagImage("busybox", dockerclient.TagImageOptions{Repo: "localhost:5000/busybox", Tag: "remove"})
	c.Assert(err, IsNil)
	err = s.docker.RemoveImage("localhost:5000/busybox:remove")
	c.Assert(err, IsNil)
	_, err = s.dc.InspectImage("localhost:5000/busybox:remove")
	c.Assert(err, NotNil)
}

func (s *DockerSuite) TestFindContainer(c *C) {
	_, err := s.docker.FindContainer("fakecontainer")
	c.Assert(err, NotNil)
	opts := dockerclient.CreateContainerOptions{}
	opts.Config = &dockerclient.Config{Image: "busybox"}
	expected, err := s.dc.CreateContainer(opts)
	c.Assert(err, IsNil)
	defer s.dc.RemoveContainer(dockerclient.RemoveContainerOptions{ID: expected.ID, RemoveVolumes: true, Force: true})
	actual, err := s.docker.FindContainer(expected.ID)
	c.Assert(err, IsNil)
	c.Assert(actual.ID, DeepEquals, expected.ID)
}

func (s *DockerSuite) TestCommitContainer(c *C) {
	defer s.dc.RemoveImage("localhost:5000/busybox:commit")
	opts := dockerclient.CreateContainerOptions{}
	opts.Config = &dockerclient.Config{Image: "busybox"}
	ctr, err := s.dc.CreateContainer(opts)
	c.Assert(err, IsNil)
	defer s.dc.RemoveContainer(dockerclient.RemoveContainerOptions{ID: ctr.ID, RemoveVolumes: true, Force: true})
	expected, err := s.docker.CommitContainer(ctr.ID, "localhost:5000/busybox:commit")
	c.Assert(err, IsNil)
	actual, err := s.dc.InspectImage("localhost:5000/busybox:commit")
	c.Assert(err, IsNil)
	c.Assert(actual.ID, DeepEquals, expected.ID)
}

func (s *DockerSuite) TestGetImageHash(c *C) {
	_, err := s.docker.GetImageHash("fakebox")
	c.Assert(err, Equals, dockerclient.ErrNoSuchImage)
	hash1, err1 := s.docker.GetImageHash("busybox")
	hash2, err2 := s.docker.GetImageHash("registry:2.1.1")
	c.Assert(err1, IsNil)
	c.Assert(err2, IsNil)
	c.Assert(hash1, Not(Equals), "")
	c.Assert(hash2, Not(Equals), "")
	c.Assert(hash1, Not(Equals), hash2)
}

func (s *DockerSuite) TestGetContainerStats(c *C) {
	stats, err := s.docker.GetContainerStats("fakecontainer", 30*time.Second)
	c.Assert(err, NotNil)
	c.Assert(stats, IsNil)
	opts := dockerclient.CreateContainerOptions{}
	opts.Config = &dockerclient.Config{Image: "busybox"}
	container, err := s.dc.CreateContainer(opts)
	c.Assert(err, IsNil)
	defer s.dc.RemoveContainer(dockerclient.RemoveContainerOptions{ID: container.ID, RemoveVolumes: true, Force: true})
	stats, err = s.docker.GetContainerStats(container.ID, 30*time.Second)
	c.Assert(err, IsNil)
	c.Assert(stats, NotNil)
}

func (s *DockerSuite) TestFindImageByHash(c *C) {
	_, err := s.docker.FindImageByHash("non_existant_hash")
	c.Assert(err, Equals, dockerclient.ErrNoSuchImage)
	hash, err := s.docker.GetImageHash("busybox")
	c.Assert(err, IsNil)
	expected, err := s.docker.FindImage("busybox")
	c.Assert(err, IsNil)
	img, err := s.docker.FindImageByHash(hash)
	c.Assert(err, IsNil)
	c.Assert(img, DeepEquals, expected)
}
<|endoftext|>


<commit_before>// Copyright 2016 The Linux Foundation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package specs

import "fmt"

const (
	// VersionMajor is for an API incompatible changes
	VersionMajor = 1
	// VersionMinor is for functionality in a backwards-compatible manner
	VersionMinor = 1
	// VersionPatch is for backwards-compatible bug fixes
	VersionPatch = 0

	// VersionDev indicates development branch. Releases will be empty string.
	VersionDev = "-rc2"
)

// Version is the specification version that the package types support.
var Version = fmt.Sprintf("%d.%d.%d%s", VersionMajor, VersionMinor, VersionPatch, VersionDev)
<commit_msg>version: bump main back to -dev<commit_after>// Copyright 2016 The Linux Foundation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package specs

import "fmt"

const (
	// VersionMajor is for an API incompatible changes
	VersionMajor = 1
	// VersionMinor is for functionality in a backwards-compatible manner
	VersionMinor = 1
	// VersionPatch is for backwards-compatible bug fixes
	VersionPatch = 0

	// VersionDev indicates development branch. Releases will be empty string.
	VersionDev = "-dev"
)

// Version is the specification version that the package types support.
var Version = fmt.Sprintf("%d.%d.%d%s", VersionMajor, VersionMinor, VersionPatch, VersionDev)
<|endoftext|>


<commit_before>package core

import (
	"errors"
	"fmt"
	"log"
	"pilosa/config"
	"pilosa/db"
	"pilosa/util"
	"sort"
	"strconv"
	"strings"
	"sync"

	"github.com/coreos/go-etcd/etcd"
	"github.com/davecgh/go-spew/spew"
)

type TopologyMapper struct {
	service   *Service
	namespace string
}

func (self *TopologyMapper) Run() {
	log.Println(self.namespace + "/db")
	db_path := self.namespace + "/db"
	resp, err := self.service.Etcd.Get(db_path, false, true)
	if err != nil {
		ee, ok := err.(*etcd.EtcdError)
		if ok && ee.ErrorCode == 100 { // node does not exist
			resp, err = self.service.Etcd.CreateDir(db_path, 0)
			if err != nil {
				log.Fatal(err)
			}
		} else {
			log.Fatal(err)
		}
	}
	for _, node := range flatten(resp.Node) {
		err := self.handlenode(node)
		if err != nil {
			log.Println(err)
		}
	}
	receiver := make(chan *etcd.Response)
	stop := make(chan bool)
	go func() {
		// TODO: add some terminating measure
		// TODO: use modindex to make sure watch catches everything
		for {
			ns := self.namespace + "/db"
			log.Println(" ETCD watcher:", ns)
			resp, err = self.service.Etcd.Watch(ns, 0, true, receiver, stop)
			log.Println("TopologyMapper ETCD watcher", resp, err)
		}
	}()
	go func() {
		for resp = range receiver {
			switch resp.Action {
			case "set":
				self.handlenode(resp.Node)
			case "delete":
				self.remove_fragment(resp.Node)
			}
			// TODO: handle deletes
		}
	}()
}

func NewTopologyMapper(service *Service, namespace string) *TopologyMapper {
	return &TopologyMapper{service, namespace}
}

type Pair struct {
	Key   string
	Value int
}

// A slice of Pairs that implements sort.Interface to sort by Value.
type PairList []Pair

func (p PairList) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p PairList) Len() int           { return len(p) }
func (p PairList) Less(i, j int) bool { return p[i].Value < p[j].Value }

func getLightestProcess(m map[string]int) (Pair, error) {

	l := len(m)
	if l == 0 {
		return Pair{}, errors.New("No Processes")
	}
	processlist := make(PairList, l)
	i := 0
	for k, v := range m {
		processlist[i] = Pair{k, v}
	}
	sort.Sort(processlist)

	return processlist[0], nil
}

func (self *TopologyMapper) MakeFragments(db string, slice_int int) error {
	frames_to_create := config.GetStringArrayDefault("supported_frames", []string{"b.n", "l.n", "t.t", "d"})
	for _, frame := range frames_to_create {
		err := self.AllocateFragment(db, frame, slice_int)
		if err != nil {
			log.Println(err)
		}
	}
	return nil

}

func (self *TopologyMapper) AllocateFragment(db, frame string, slice_int int) error {
	//get Lock to create the fragment
	ttl := uint64(300) //secs to hold the lock
	lock_key := fmt.Sprintf("%s/lock/%s-%s-%d", self.namespace, db, frame, slice_int)
	response, err := self.service.Etcd.RawCreate(lock_key, "0", ttl)

	if err == nil {
		switch response.StatusCode {
		case 201: //key created
			//figure out least loaded process..possibly check max process
			//to create the node, just write off the items to etcd and the watch should spawn
			//be nice if something would notify perhaps queue
			m := make(map[string]int)
			for _, dbs := range self.service.Cluster.GetDatabases() {
				for _, fsi := range dbs.GetFramSliceIntersects() {
					for _, fragment := range fsi.GetFragments() {
						process := fragment.GetProcess().Id().String()
						i := m[process]
						i++
						m[process] = i

					}

				}

			}
			p, err := getLightestProcess(m)
			if err != nil {
				return err
			}

			// PUT -d "value=5cb315c3-6e1d-4218-89b7-943d1dba985b" http://etcd0:4001/v2/keys/pilosa/0/db/29/frame/d/slice/5/fragment/a2b632fc4001b817/proces
			//so i need db, frame, slice , fragment_id
			fuid := util.SUUID_to_Hex(util.Id())
			fragment_key := fmt.Sprintf("%s/db/%s/frame/%s/slice/%d/fragment/%s/process", self.namespace, db, frame, slice_int, fuid)
			process_guid := p.Key
			// need to check value to see how many we have left
			_, err = self.service.Etcd.Set(fragment_key, process_guid, 0)
			log.Printf("Fragment sent to etcd: %s(%s)", fragment_key, process_guid)
		case 400: //key already present
			return errors.New("Fragment creation already in process:" + lock_key)
		default:
			return errors.New(fmt.Sprintf("Unknown Etcd status:%d", response.StatusCode))

		}

	}
	return err

}

func (self *TopologyMapper) handlenode(node *etcd.Node) error {
	key := node.Key[len(self.namespace)+1:]
	bits := strings.Split(key, "/")
	var database *db.Database
	var frame *db.Frame
	var fragment *db.Fragment
	var fragment_id util.SUUID
	var slice *db.Slice
	var slice_int int
	var process_uuid util.GUID
	var process *db.Process
	var err error

	if len(bits) <= 1 || bits[0] != "db" {
		return nil
	}
	if len(bits) > 1 {
		database = self.service.Cluster.GetOrCreateDatabase(bits[1])
	}
	if len(bits) > 2 {
		if bits[2] != "frame" {
			return errors.New("no frame")
		}
	}
	if len(bits) > 3 {
		frame = database.GetOrCreateFrame(bits[3])
	}
	if len(bits) > 4 {
		if bits[4] != "slice" {
			return errors.New("no slice")
		}
	}
	if len(bits) > 5 {
		slice_int, err = strconv.Atoi(bits[5])
		if err != nil {
			return err
		}
		slice = database.GetOrCreateSlice(slice_int)
	}
	if len(bits) > 6 {
		if bits[6] != "fragment" {
			return errors.New("no fragment")
		}
	}
	if len(bits) > 7 {
		fragment_id = util.Hex_to_SUUID(bits[7])
		fragment = database.GetOrCreateFragment(frame, slice, fragment_id)
	}

	if len(bits) > 8 {
		if bits[8] != "process" {
			return errors.New("no process")
		}
		process_uuid, err = util.ParseGUID(node.Value)
		if err != nil {
			return err
		}
		process = db.NewProcess(&process_uuid)
		fragment.SetProcess(process)

		if self.service.Id.String() == process_uuid.String() {
			self.service.Index.AddFragment(bits[1], bits[3], slice_int, fragment_id)
		}

	}
	return err
}
func (self *TopologyMapper) remove_fragment(node *etcd.Node) error {
	log.Println(" hot remove_fragment (Not Supported yet):", node)
	/*
		key := node.Key[len(self.namespace)+1:]
		bits := strings.Split(key, "/")
		process_uuid, err = util.ParseGUID(node.Value)
		if self.service.Id.String() == process_uuid.String() {
			fragment_id = util.Hex_to_SUUID(bits[7])
			self.service.Index.RemoveFragment(fragment_id)
		}
	*/
	return nil
}

func flatten(node *etcd.Node) []*etcd.Node {
	nodes := []*etcd.Node{node}
	for i := 0; i < len(node.Nodes); i++ {
		nodes = append(nodes, flatten(node.Nodes[i])...)
	}
	return nodes
}

type Node struct {
	id        *util.GUID
	ip        string
	port_tcp  int
	port_http int
}

type ProcessMap struct {
	nodes map[util.GUID]*db.Process
	mutex sync.Mutex
}

func NewProcessMap() *ProcessMap {
	p := ProcessMap{}
	p.nodes = make(map[util.GUID]*db.Process)
	return &p
}

func (self *ProcessMap) AddProcess(process *db.Process) {
	self.mutex.Lock()
	defer self.mutex.Unlock()
	self.nodes[process.Id()] = process
}

func (self *ProcessMap) GetProcess(id *util.GUID) (*db.Process, error) {
	self.mutex.Lock()
	defer self.mutex.Unlock()
	process, ok := self.nodes[*id]
	if !ok {
		return nil, errors.New("No such process")
	}
	return process, nil
}

func (self *ProcessMap) GetOrAddProcess(id *util.GUID) *db.Process {
	process, err := self.GetProcess(id)
	if err != nil {
		process = db.NewProcess(id)
		self.AddProcess(process)
	}
	return process
}

func (self *ProcessMap) GetHost(id *util.GUID) (string, error) {
	self.mutex.Lock()
	defer self.mutex.Unlock()
	process, ok := self.nodes[*id]
	if !ok {
		return "", errors.New("Process does not exist")
	}
	return process.Host(), nil
}

func (self *ProcessMap) GetPortTcp(id *util.GUID) (int, error) {
	self.mutex.Lock()
	defer self.mutex.Unlock()
	process, ok := self.nodes[*id]
	if !ok {
		return 0, errors.New("Process does not exist")
	}
	return process.PortTcp(), nil
}

func (self *ProcessMap) GetPortHttp(id *util.GUID) (int, error) {
	self.mutex.Lock()
	defer self.mutex.Unlock()
	process, ok := self.nodes[*id]
	if !ok {
		return 0, errors.New("Process does not exist")
	}
	return process.PortHttp(), nil
}

func (self *ProcessMap) GetMetadata() map[string]map[string]interface{} {
	self.mutex.Lock()
	defer self.mutex.Unlock()
	out := make(map[string]map[string]interface{})
	for id, process := range self.nodes {
		pdata := make(map[string]interface{})
		pdata["host"] = process.Host()
		pdata["port_tcp"] = process.PortTcp()
		pdata["port_http"] = process.PortHttp()
		out[id.String()] = pdata
	}
	return out
}

type ProcessMapper struct {
	service   *Service
	receiver  chan etcd.Response
	commands  chan ProcessMapperCommand
	namespace string
}

func NewProcessMapper(service *Service, namespace string) *ProcessMapper {
	return &ProcessMapper{
		service:   service,
		receiver:  make(chan etcd.Response),
		commands:  make(chan ProcessMapperCommand),
		namespace: namespace,
	}
}

type ProcessMapperCommand struct {
	key string
}

func getKey(input string) string {
	bits := strings.Split(input, "/")
	return bits[len(bits)-1]
}

func (self *ProcessMapper) getnode(u *util.GUID) *Node {
	return new(Node)
}

func crash_on_error(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func (self *ProcessMapper) handlenode(node *etcd.Node) error {
	var err error
	var process *db.Process
	key := node.Key[len(self.namespace)+1:]
	bits := strings.Split(key, "/")
	if len(bits) <= 1 || bits[0] != "process" {
		return nil
	}
	if len(bits) >= 2 {
		id_string := bits[1]
		id, err := util.ParseGUID(id_string)
		if err != nil {
			return errors.New("Invalid GUID: " + id_string)
		}
		process = self.service.ProcessMap.GetOrAddProcess(&id)
	}
	if len(bits) >= 3 {
		switch bits[2] {
		case "port_tcp":
			port_tcp, _ := strconv.Atoi(node.Value)
			process.SetPortTcp(port_tcp)
		case "port_http":
			port_http, _ := strconv.Atoi(node.Value)
			process.SetPortHttp(port_http)
		case "host":
			host := node.Value
			process.SetHost(host)
		}
	}

	return err
}

func (self *ProcessMapper) Run() {
	id_string := self.service.Id.String()
	path := self.namespace + "/process"
	self_path := path + "/" + id_string

	log.Println("Writing configuration to etcd...")
	log.Println(self_path)

	var err error
	_, err = self.service.Etcd.Set(self_path+"/port_tcp", strconv.Itoa(config.GetInt("port_tcp")), 0)
	crash_on_error(err)
	_, err = self.service.Etcd.Set(self_path+"/port_http", strconv.Itoa(config.GetInt("port_http")), 0)
	crash_on_error(err)
	_, err = self.service.Etcd.Set(self_path+"/host", config.GetString("host"), 0)
	crash_on_error(err)

	response, err := self.service.Etcd.Get(path, false, true)
	for _, node := range flatten(response.Node) {
		err := self.handlenode(node)
		if err != nil {
			spew.Dump(node)
			log.Println(err)
		}
	}

	receiver := make(chan *etcd.Response)
	stop := make(chan bool)
	go func() {
		// TODO: error check and restart watcher
		// TODO: use modindex to make sure watch catches everything
		_, _ = self.service.Etcd.Watch(path, 0, true, receiver, stop)
	}()

	go func() {
		for response = range receiver {
			switch response.Action {
			case "set":
				self.handlenode(response.Node)
			}
			// TODO: handle deletes
		}
	}()
}
<commit_msg>more verbose logging<commit_after>package core

import (
	"errors"
	"fmt"
	"log"
	"pilosa/config"
	"pilosa/db"
	"pilosa/util"
	"sort"
	"strconv"
	"strings"
	"sync"

	"github.com/coreos/go-etcd/etcd"
	"github.com/davecgh/go-spew/spew"
)

type TopologyMapper struct {
	service   *Service
	namespace string
}

func (self *TopologyMapper) Run() {
	log.Println(self.namespace + "/db")
	db_path := self.namespace + "/db"
	resp, err := self.service.Etcd.Get(db_path, false, true)
	if err != nil {
		ee, ok := err.(*etcd.EtcdError)
		if ok && ee.ErrorCode == 100 { // node does not exist
			resp, err = self.service.Etcd.CreateDir(db_path, 0)
			if err != nil {
				log.Fatal(err)
			}
		} else {
			log.Fatal(err)
		}
	}
	for _, node := range flatten(resp.Node) {
		err := self.handlenode(node)
		if err != nil {
			log.Println(err)
		}
	}
	receiver := make(chan *etcd.Response)
	stop := make(chan bool)
	go func() {
		// TODO: add some terminating measure
		// TODO: use modindex to make sure watch catches everything
		for {
			ns := self.namespace + "/db"
			log.Println(" ETCD watcher:", ns)
			resp, err = self.service.Etcd.Watch(ns, 0, true, receiver, stop)
			log.Println("TopologyMapper ETCD watcher", resp, err)
		}
	}()
	go func() {
		for resp = range receiver {
			switch resp.Action {
			case "set":
				self.handlenode(resp.Node)
			case "delete":
				self.remove_fragment(resp.Node)
			}
			// TODO: handle deletes
		}
	}()
}

func NewTopologyMapper(service *Service, namespace string) *TopologyMapper {
	return &TopologyMapper{service, namespace}
}

type Pair struct {
	Key   string
	Value int
}

// A slice of Pairs that implements sort.Interface to sort by Value.
type PairList []Pair

func (p PairList) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p PairList) Len() int           { return len(p) }
func (p PairList) Less(i, j int) bool { return p[i].Value < p[j].Value }

func getLightestProcess(m map[string]int) (Pair, error) {

	l := len(m)
	if l == 0 {
		return Pair{}, errors.New("No Processes")
	}
	processlist := make(PairList, l)
	i := 0
	for k, v := range m {
		processlist[i] = Pair{k, v}
	}
	sort.Sort(processlist)

	return processlist[0], nil
}

func (self *TopologyMapper) MakeFragments(db string, slice_int int) error {
	frames_to_create := config.GetStringArrayDefault("supported_frames", []string{"b.n", "l.n", "t.t", "d"})
	for _, frame := range frames_to_create {
		err := self.AllocateFragment(db, frame, slice_int)
		if err != nil {
			log.Println(err)
		}
	}
	return nil

}

func (self *TopologyMapper) AllocateFragment(db, frame string, slice_int int) error {
	//get Lock to create the fragment
	ttl := uint64(300) //secs to hold the lock
	lock_key := fmt.Sprintf("%s/lock/%s-%s-%d", self.namespace, db, frame, slice_int)
	response, err := self.service.Etcd.RawCreate(lock_key, "0", ttl)

	if err == nil {
		switch response.StatusCode {
		case 201: //key created
			//figure out least loaded process..possibly check max process
			//to create the node, just write off the items to etcd and the watch should spawn
			//be nice if something would notify perhaps queue
			m := make(map[string]int)
			for _, dbs := range self.service.Cluster.GetDatabases() {
				for _, fsi := range dbs.GetFramSliceIntersects() {
					for _, fragment := range fsi.GetFragments() {
						process := fragment.GetProcess().Id().String()
						i := m[process]
						i++
						m[process] = i

					}

				}

			}
			p, err := getLightestProcess(m)
			if err != nil {
				return err
			}

			// PUT -d "value=5cb315c3-6e1d-4218-89b7-943d1dba985b" http://etcd0:4001/v2/keys/pilosa/0/db/29/frame/d/slice/5/fragment/a2b632fc4001b817/proces
			//so i need db, frame, slice , fragment_id
			fuid := util.SUUID_to_Hex(util.Id())
			fragment_key := fmt.Sprintf("%s/db/%s/frame/%s/slice/%d/fragment/%s/process", self.namespace, db, frame, slice_int, fuid)
			process_guid := p.Key
			// need to check value to see how many we have left
			_, err = self.service.Etcd.Set(fragment_key, process_guid, 0)
			log.Printf("Fragment sent to etcd: %s(%s)", fragment_key, process_guid)
		case 400: //key already present
			return errors.New("Fragment creation already in process:" + lock_key)
		default:
			log.Println(spew.Sdump(response))
			return errors.New(fmt.Sprintf("Unknown Etcd status:%d", response.StatusCode))

		}

	}
	return err

}

func (self *TopologyMapper) handlenode(node *etcd.Node) error {
	key := node.Key[len(self.namespace)+1:]
	bits := strings.Split(key, "/")
	var database *db.Database
	var frame *db.Frame
	var fragment *db.Fragment
	var fragment_id util.SUUID
	var slice *db.Slice
	var slice_int int
	var process_uuid util.GUID
	var process *db.Process
	var err error

	if len(bits) <= 1 || bits[0] != "db" {
		return nil
	}
	if len(bits) > 1 {
		database = self.service.Cluster.GetOrCreateDatabase(bits[1])
	}
	if len(bits) > 2 {
		if bits[2] != "frame" {
			return errors.New("no frame")
		}
	}
	if len(bits) > 3 {
		frame = database.GetOrCreateFrame(bits[3])
	}
	if len(bits) > 4 {
		if bits[4] != "slice" {
			return errors.New("no slice")
		}
	}
	if len(bits) > 5 {
		slice_int, err = strconv.Atoi(bits[5])
		if err != nil {
			return err
		}
		slice = database.GetOrCreateSlice(slice_int)
	}
	if len(bits) > 6 {
		if bits[6] != "fragment" {
			return errors.New("no fragment")
		}
	}
	if len(bits) > 7 {
		fragment_id = util.Hex_to_SUUID(bits[7])
		fragment = database.GetOrCreateFragment(frame, slice, fragment_id)
	}

	if len(bits) > 8 {
		if bits[8] != "process" {
			return errors.New("no process")
		}
		process_uuid, err = util.ParseGUID(node.Value)
		if err != nil {
			return err
		}
		process = db.NewProcess(&process_uuid)
		fragment.SetProcess(process)

		if self.service.Id.String() == process_uuid.String() {
			self.service.Index.AddFragment(bits[1], bits[3], slice_int, fragment_id)
		}

	}
	return err
}
func (self *TopologyMapper) remove_fragment(node *etcd.Node) error {
	log.Println(" hot remove_fragment (Not Supported yet):", node)
	/*
		key := node.Key[len(self.namespace)+1:]
		bits := strings.Split(key, "/")
		process_uuid, err = util.ParseGUID(node.Value)
		if self.service.Id.String() == process_uuid.String() {
			fragment_id = util.Hex_to_SUUID(bits[7])
			self.service.Index.RemoveFragment(fragment_id)
		}
	*/
	return nil
}

func flatten(node *etcd.Node) []*etcd.Node {
	nodes := []*etcd.Node{node}
	for i := 0; i < len(node.Nodes); i++ {
		nodes = append(nodes, flatten(node.Nodes[i])...)
	}
	return nodes
}

type Node struct {
	id        *util.GUID
	ip        string
	port_tcp  int
	port_http int
}

type ProcessMap struct {
	nodes map[util.GUID]*db.Process
	mutex sync.Mutex
}

func NewProcessMap() *ProcessMap {
	p := ProcessMap{}
	p.nodes = make(map[util.GUID]*db.Process)
	return &p
}

func (self *ProcessMap) AddProcess(process *db.Process) {
	self.mutex.Lock()
	defer self.mutex.Unlock()
	self.nodes[process.Id()] = process
}

func (self *ProcessMap) GetProcess(id *util.GUID) (*db.Process, error) {
	self.mutex.Lock()
	defer self.mutex.Unlock()
	process, ok := self.nodes[*id]
	if !ok {
		return nil, errors.New("No such process")
	}
	return process, nil
}

func (self *ProcessMap) GetOrAddProcess(id *util.GUID) *db.Process {
	process, err := self.GetProcess(id)
	if err != nil {
		process = db.NewProcess(id)
		self.AddProcess(process)
	}
	return process
}

func (self *ProcessMap) GetHost(id *util.GUID) (string, error) {
	self.mutex.Lock()
	defer self.mutex.Unlock()
	process, ok := self.nodes[*id]
	if !ok {
		return "", errors.New("Process does not exist")
	}
	return process.Host(), nil
}

func (self *ProcessMap) GetPortTcp(id *util.GUID) (int, error) {
	self.mutex.Lock()
	defer self.mutex.Unlock()
	process, ok := self.nodes[*id]
	if !ok {
		return 0, errors.New("Process does not exist")
	}
	return process.PortTcp(), nil
}

func (self *ProcessMap) GetPortHttp(id *util.GUID) (int, error) {
	self.mutex.Lock()
	defer self.mutex.Unlock()
	process, ok := self.nodes[*id]
	if !ok {
		return 0, errors.New("Process does not exist")
	}
	return process.PortHttp(), nil
}

func (self *ProcessMap) GetMetadata() map[string]map[string]interface{} {
	self.mutex.Lock()
	defer self.mutex.Unlock()
	out := make(map[string]map[string]interface{})
	for id, process := range self.nodes {
		pdata := make(map[string]interface{})
		pdata["host"] = process.Host()
		pdata["port_tcp"] = process.PortTcp()
		pdata["port_http"] = process.PortHttp()
		out[id.String()] = pdata
	}
	return out
}

type ProcessMapper struct {
	service   *Service
	receiver  chan etcd.Response
	commands  chan ProcessMapperCommand
	namespace string
}

func NewProcessMapper(service *Service, namespace string) *ProcessMapper {
	return &ProcessMapper{
		service:   service,
		receiver:  make(chan etcd.Response),
		commands:  make(chan ProcessMapperCommand),
		namespace: namespace,
	}
}

type ProcessMapperCommand struct {
	key string
}

func getKey(input string) string {
	bits := strings.Split(input, "/")
	return bits[len(bits)-1]
}

func (self *ProcessMapper) getnode(u *util.GUID) *Node {
	return new(Node)
}

func crash_on_error(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func (self *ProcessMapper) handlenode(node *etcd.Node) error {
	var err error
	var process *db.Process
	key := node.Key[len(self.namespace)+1:]
	bits := strings.Split(key, "/")
	if len(bits) <= 1 || bits[0] != "process" {
		return nil
	}
	if len(bits) >= 2 {
		id_string := bits[1]
		id, err := util.ParseGUID(id_string)
		if err != nil {
			return errors.New("Invalid GUID: " + id_string)
		}
		process = self.service.ProcessMap.GetOrAddProcess(&id)
	}
	if len(bits) >= 3 {
		switch bits[2] {
		case "port_tcp":
			port_tcp, _ := strconv.Atoi(node.Value)
			process.SetPortTcp(port_tcp)
		case "port_http":
			port_http, _ := strconv.Atoi(node.Value)
			process.SetPortHttp(port_http)
		case "host":
			host := node.Value
			process.SetHost(host)
		}
	}

	return err
}

func (self *ProcessMapper) Run() {
	id_string := self.service.Id.String()
	path := self.namespace + "/process"
	self_path := path + "/" + id_string

	log.Println("Writing configuration to etcd...")
	log.Println(self_path)

	var err error
	_, err = self.service.Etcd.Set(self_path+"/port_tcp", strconv.Itoa(config.GetInt("port_tcp")), 0)
	crash_on_error(err)
	_, err = self.service.Etcd.Set(self_path+"/port_http", strconv.Itoa(config.GetInt("port_http")), 0)
	crash_on_error(err)
	_, err = self.service.Etcd.Set(self_path+"/host", config.GetString("host"), 0)
	crash_on_error(err)

	response, err := self.service.Etcd.Get(path, false, true)
	for _, node := range flatten(response.Node) {
		err := self.handlenode(node)
		if err != nil {
			spew.Dump(node)
			log.Println(err)
		}
	}

	receiver := make(chan *etcd.Response)
	stop := make(chan bool)
	go func() {
		// TODO: error check and restart watcher
		// TODO: use modindex to make sure watch catches everything
		_, _ = self.service.Etcd.Watch(path, 0, true, receiver, stop)
	}()

	go func() {
		for response = range receiver {
			switch response.Action {
			case "set":
				self.handlenode(response.Node)
			}
			// TODO: handle deletes
		}
	}()
}
<|endoftext|>


<commit_before>package core

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"pilosa/config"
	"pilosa/db"
	"runtime"
	"strconv"

	"github.com/davecgh/go-spew/spew"
	"tux21b.org/v1/gocql/uuid"
)

type WebService struct {
	service *Service
}

func NewWebService(service *Service) *WebService {
	return &WebService{service}
}

func (self *WebService) Run() {
	port_string := strconv.Itoa(config.GetInt("port_http"))
	log.Printf("Serving HTTP on port %s...\n", port_string)
	mux := http.NewServeMux()
	mux.HandleFunc("/message", self.HandleMessage)
	mux.HandleFunc("/query", self.HandleQuery)
	mux.HandleFunc("/stats", self.HandleStats)
	mux.HandleFunc("/info", self.HandleInfo)
	mux.HandleFunc("/processes", self.HandleProcesses)
	mux.HandleFunc("/listen", self.HandleListen)
	mux.HandleFunc("/test", self.HandleTest)
	mux.HandleFunc("/version", self.HandleVersion)
	mux.HandleFunc("/ping", self.HandlePing)
	mux.HandleFunc("/batch", self.HandleBatch)
	s := &http.Server{
		Addr:    ":" + port_string,
		Handler: mux,
	}
	s.ListenAndServe()
}

func (self *WebService) HandleMessage(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
		return
	}
	var message db.Message
	decoder := json.NewDecoder(r.Body)
	if decoder.Decode(&message) != nil {
		http.Error(w, "Invalid JSON", http.StatusBadRequest)
		return
	}
	//service.Inbox <- &message
}

func (self *WebService) HandleBatch(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
		return
	}
	/*   	values.Set("db", database)
	values.Set("id", id)
	values.Set("frame", fragment_type)
	values.Set("slice", fmt.Sprintf("%d", slice))
	values.Set("bitmap", compressed_string)
	*/

	err := r.ParseForm()
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	database_name := r.Form.Get("db")
	if database_name == "" {
		http.Error(w, "Provide a database (db)", http.StatusNotFound)
		return
	}

	bitmap_id_string := r.Form.Get("id")
	if bitmap_id_string == "" {
		http.Error(w, "Provide a bitmap id (id)", http.StatusNotFound)
		return
	}
	bitmap_id, _ := strconv.ParseUint(bitmap_id_string, 10, 64)

	slice_string := r.Form.Get("slice")
	if bitmap_id_string == "" {
		http.Error(w, "Provide a slice (slice)", http.StatusNotFound)
		return
	}
	slice, _ := strconv.ParseInt(slice_string, 10, 32)

	frame := r.Form.Get("frame")
	if bitmap_id_string == "" {
		http.Error(w, "Provide a frame (frame)", http.StatusNotFound)
		return
	}

	compressed_bitmap := r.Form.Get("bitmap")
	if bitmap_id_string == "" {
		http.Error(w, "Provide a compressed base64 bitmap (bitmap)", http.StatusNotFound)
		return
	}

	results := self.service.Batch(database_name, frame, compressed_bitmap, bitmap_id, int(slice))

	encoder := json.NewEncoder(w)
	err = encoder.Encode(results)
	if err != nil {
		log.Println("Error Batch results")
	    log.Println(spew.Sdump(r.Form)
		err = encoder.Encode("Bad Batch Request")
	}

}

func (self *WebService) HandleQuery(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
		return
	}
	err := r.ParseForm()
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	database_name := r.Form.Get("db")
	if database_name == "" {
		database_name = config.GetString("default_db")
	}
	if database_name == "" {
		http.Error(w, "Provide a database (db)", http.StatusNotFound)
		return
	}
	pql := r.Form.Get("pql")
	if pql == "" {
		http.Error(w, "Provide a valid query string (pql)", http.StatusNotFound)
		return
	}

	results := self.service.Executor.RunPQL(database_name, pql)

	encoder := json.NewEncoder(w)
	err = encoder.Encode(results)
	if err != nil {
		log.Fatal("Error encoding stats")
	}

}

func (self *WebService) HandleStats(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
		return
	}
	encoder := json.NewEncoder(w)
	//stats := service.GetStats()
	//	stats := ""
	m := &runtime.MemStats{}
	runtime.ReadMemStats(m)
	stats := map[string]interface{}{"num_goroutines": runtime.NumGoroutine(), "Memory Aquired": m.Sys, "Memory Used": m.Alloc}

	err := encoder.Encode(stats)
	if err != nil {
		log.Fatal("Error encoding stats")
	}
}

func (self *WebService) HandleInfo(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
		return
	}
	spew.Fdump(w, self.service.Cluster)
}

func (self *WebService) HandleVersion(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
		return
	}

	fmt.Fprintf(w, "Pilosa v."+self.service.version+"\n")
}

func (self *WebService) HandleTest(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
		return
	}
	spew.Dump("TEST!")

	msg := new(db.Message)
	msg.Data = "mystring"
	self.service.Transport.Push(msg)

	msg2 := new(db.Message)
	msg2.Data = 789
	self.service.Transport.Push(msg2)
}

func (self *WebService) HandleProcesses(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
		return
	}
	encoder := json.NewEncoder(w)
	processes := self.service.ProcessMap.GetMetadata()
	err := encoder.Encode(processes)
	if err != nil {
		log.Fatal("Error encoding stats")
	}
}

func (self *WebService) HandlePing(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
		return
	}
	err := r.ParseForm()
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	process_string := r.Form.Get("process")
	process_id, err := uuid.ParseUUID(process_string)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	_, err = self.service.ProcessMap.GetProcess(&process_id)
	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	duration, err := self.service.Ping(&process_id)
	if err != nil {
		spew.Fdump(w, err)
		return
	}
	encoder := json.NewEncoder(w)
	encoder.Encode(map[string]float64{"duration": duration.Seconds()})
}

func (self *WebService) HandleListen(w http.ResponseWriter, r *http.Request) {
	//listener := service.NewListener()
	//encoder := json.NewEncoder(w)
	//for {
	//	select {
	//	case message := <-listener:
	//		err := encoder.Encode(message)
	//		if err != nil {
	//			log.Println("Error sending message")
	//			return
	//		}
	//	}
	//}
}
<commit_msg>fixed typo<commit_after>package core

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"pilosa/config"
	"pilosa/db"
	"runtime"
	"strconv"

	"github.com/davecgh/go-spew/spew"
	"tux21b.org/v1/gocql/uuid"
)

type WebService struct {
	service *Service
}

func NewWebService(service *Service) *WebService {
	return &WebService{service}
}

func (self *WebService) Run() {
	port_string := strconv.Itoa(config.GetInt("port_http"))
	log.Printf("Serving HTTP on port %s...\n", port_string)
	mux := http.NewServeMux()
	mux.HandleFunc("/message", self.HandleMessage)
	mux.HandleFunc("/query", self.HandleQuery)
	mux.HandleFunc("/stats", self.HandleStats)
	mux.HandleFunc("/info", self.HandleInfo)
	mux.HandleFunc("/processes", self.HandleProcesses)
	mux.HandleFunc("/listen", self.HandleListen)
	mux.HandleFunc("/test", self.HandleTest)
	mux.HandleFunc("/version", self.HandleVersion)
	mux.HandleFunc("/ping", self.HandlePing)
	mux.HandleFunc("/batch", self.HandleBatch)
	s := &http.Server{
		Addr:    ":" + port_string,
		Handler: mux,
	}
	s.ListenAndServe()
}

func (self *WebService) HandleMessage(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
		return
	}
	var message db.Message
	decoder := json.NewDecoder(r.Body)
	if decoder.Decode(&message) != nil {
		http.Error(w, "Invalid JSON", http.StatusBadRequest)
		return
	}
	//service.Inbox <- &message
}

func (self *WebService) HandleBatch(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
		return
	}
	/*   	values.Set("db", database)
	values.Set("id", id)
	values.Set("frame", fragment_type)
	values.Set("slice", fmt.Sprintf("%d", slice))
	values.Set("bitmap", compressed_string)
	*/

	err := r.ParseForm()
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	database_name := r.Form.Get("db")
	if database_name == "" {
		http.Error(w, "Provide a database (db)", http.StatusNotFound)
		return
	}

	bitmap_id_string := r.Form.Get("id")
	if bitmap_id_string == "" {
		http.Error(w, "Provide a bitmap id (id)", http.StatusNotFound)
		return
	}
	bitmap_id, _ := strconv.ParseUint(bitmap_id_string, 10, 64)

	slice_string := r.Form.Get("slice")
	if bitmap_id_string == "" {
		http.Error(w, "Provide a slice (slice)", http.StatusNotFound)
		return
	}
	slice, _ := strconv.ParseInt(slice_string, 10, 32)

	frame := r.Form.Get("frame")
	if bitmap_id_string == "" {
		http.Error(w, "Provide a frame (frame)", http.StatusNotFound)
		return
	}

	compressed_bitmap := r.Form.Get("bitmap")
	if bitmap_id_string == "" {
		http.Error(w, "Provide a compressed base64 bitmap (bitmap)", http.StatusNotFound)
		return
	}

	results := self.service.Batch(database_name, frame, compressed_bitmap, bitmap_id, int(slice))

	encoder := json.NewEncoder(w)
	err = encoder.Encode(results)
	if err != nil {
		log.Println("Error Batch results")
		log.Println(spew.Sdump(r.Form))
		err = encoder.Encode("Bad Batch Request")
	}

}

func (self *WebService) HandleQuery(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
		return
	}
	err := r.ParseForm()
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	database_name := r.Form.Get("db")
	if database_name == "" {
		database_name = config.GetString("default_db")
	}
	if database_name == "" {
		http.Error(w, "Provide a database (db)", http.StatusNotFound)
		return
	}
	pql := r.Form.Get("pql")
	if pql == "" {
		http.Error(w, "Provide a valid query string (pql)", http.StatusNotFound)
		return
	}

	results := self.service.Executor.RunPQL(database_name, pql)

	encoder := json.NewEncoder(w)
	err = encoder.Encode(results)
	if err != nil {
		log.Fatal("Error encoding stats")
	}

}

func (self *WebService) HandleStats(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
		return
	}
	encoder := json.NewEncoder(w)
	//stats := service.GetStats()
	//	stats := ""
	m := &runtime.MemStats{}
	runtime.ReadMemStats(m)
	stats := map[string]interface{}{"num_goroutines": runtime.NumGoroutine(), "Memory Aquired": m.Sys, "Memory Used": m.Alloc}

	err := encoder.Encode(stats)
	if err != nil {
		log.Fatal("Error encoding stats")
	}
}

func (self *WebService) HandleInfo(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
		return
	}
	spew.Fdump(w, self.service.Cluster)
}

func (self *WebService) HandleVersion(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
		return
	}

	fmt.Fprintf(w, "Pilosa v."+self.service.version+"\n")
}

func (self *WebService) HandleTest(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
		return
	}
	spew.Dump("TEST!")

	msg := new(db.Message)
	msg.Data = "mystring"
	self.service.Transport.Push(msg)

	msg2 := new(db.Message)
	msg2.Data = 789
	self.service.Transport.Push(msg2)
}

func (self *WebService) HandleProcesses(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
		return
	}
	encoder := json.NewEncoder(w)
	processes := self.service.ProcessMap.GetMetadata()
	err := encoder.Encode(processes)
	if err != nil {
		log.Fatal("Error encoding stats")
	}
}

func (self *WebService) HandlePing(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
		return
	}
	err := r.ParseForm()
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	process_string := r.Form.Get("process")
	process_id, err := uuid.ParseUUID(process_string)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	_, err = self.service.ProcessMap.GetProcess(&process_id)
	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	duration, err := self.service.Ping(&process_id)
	if err != nil {
		spew.Fdump(w, err)
		return
	}
	encoder := json.NewEncoder(w)
	encoder.Encode(map[string]float64{"duration": duration.Seconds()})
}

func (self *WebService) HandleListen(w http.ResponseWriter, r *http.Request) {
	//listener := service.NewListener()
	//encoder := json.NewEncoder(w)
	//for {
	//	select {
	//	case message := <-listener:
	//		err := encoder.Encode(message)
	//		if err != nil {
	//			log.Println("Error sending message")
	//			return
	//		}
	//	}
	//}
}
<|endoftext|>


<commit_before>package systests

import (
	"testing"

	"golang.org/x/net/context"

	client "github.com/keybase/client/go/client"
	keybase1 "github.com/keybase/client/go/protocol/keybase1"

	"github.com/stretchr/testify/require"
)

func TestSubteamChats(t *testing.T) {
	ctx := newSMUContext(t)
	defer ctx.cleanup()

	ann := ctx.installKeybaseForUser("ann", 3)
	ann.signup()
	t.Logf("Signed up ann (%s, %s)", ann.username, ann.uid())
	bob := ctx.installKeybaseForUser("bob", 5)
	bob.signup()
	t.Logf("Signed up bob (%s, %s)", bob.username, bob.uid())

	team := ann.createTeam([]*smuUser{})
	t.Logf("Team created %q", team.name)

	subteamName := team.name + ".subt"
	cli := ann.getTeamsClient()
	_, err := cli.TeamCreate(context.Background(), keybase1.TeamCreateArg{Name: subteamName})
	require.NoError(t, err)

	t.Logf("Subteam created %q", subteamName)

	subteam := smuTeam{name: subteamName}
	ann.addTeamMember(subteam, bob, keybase1.TeamRole_READER)

	// Can we chat after being added to subteam as sole reader? If
	// chats are not initialized in the subteam, sendChat will fail
	// with: "error creating conversation: error from chat server:
	// team readers are unable to create conversations". It's rather
	// imperfect test, ideally we would be checking if there is any
	// existing chat channel in the subteam instead of trying to send
	// chat messages.
	bob.sendChat(subteam, "0")
	bob.readChats(subteam, 1)
}

func TestSubteamCreateAndStay(t *testing.T) {
	// Test CreateTeam API w.r.t JoinSubteam. We should stay find
	// ourselves in new team as admin.
	tt := newTeamTester(t)
	defer tt.cleanup()

	ann := tt.addUser("ann")
	t.Logf("Signed up ann %s", ann.username)

	parentName := ann.createTeam()
	subteamName := parentName + ".test"

	var err error
	create := client.NewCmdTeamCreateRunner(ann.tc.G)
	create.TeamName, err = keybase1.TeamNameFromString(subteamName)
	require.NoError(t, err)
	create.JoinSubteam = true
	err = create.Run()
	require.NoError(t, err)

	t.Logf("Created subteam %s", subteamName)

	teamObj := ann.loadTeam(subteamName, false /* admin */)
	role, err := teamObj.MemberRole(context.Background(), ann.userVersion())
	require.NoError(t, err)
	require.Equal(t, keybase1.TeamRole_ADMIN, role, "got back wrong role")
	require.EqualValues(t, 1, teamObj.CurrentSeqno(), "expecting just one link in team")
}
<commit_msg>skip TestSubteamChats<commit_after>package systests

import (
	"testing"

	"golang.org/x/net/context"

	client "github.com/keybase/client/go/client"
	keybase1 "github.com/keybase/client/go/protocol/keybase1"

	"github.com/stretchr/testify/require"
)

func TestSubteamChats(t *testing.T) {
	t.Skip()
	ctx := newSMUContext(t)
	defer ctx.cleanup()

	ann := ctx.installKeybaseForUser("ann", 3)
	ann.signup()
	t.Logf("Signed up ann (%s, %s)", ann.username, ann.uid())
	bob := ctx.installKeybaseForUser("bob", 5)
	bob.signup()
	t.Logf("Signed up bob (%s, %s)", bob.username, bob.uid())

	team := ann.createTeam([]*smuUser{})
	t.Logf("Team created %q", team.name)

	subteamName := team.name + ".subt"
	cli := ann.getTeamsClient()
	_, err := cli.TeamCreate(context.Background(), keybase1.TeamCreateArg{Name: subteamName})
	require.NoError(t, err)

	t.Logf("Subteam created %q", subteamName)

	subteam := smuTeam{name: subteamName}
	ann.addTeamMember(subteam, bob, keybase1.TeamRole_READER)

	// Can we chat after being added to subteam as sole reader? If
	// chats are not initialized in the subteam, sendChat will fail
	// with: "error creating conversation: error from chat server:
	// team readers are unable to create conversations". It's rather
	// imperfect test, ideally we would be checking if there is any
	// existing chat channel in the subteam instead of trying to send
	// chat messages.
	bob.sendChat(subteam, "0")
	bob.readChats(subteam, 1)
}

func TestSubteamCreateAndStay(t *testing.T) {
	// Test CreateTeam API w.r.t JoinSubteam. We should stay find
	// ourselves in new team as admin.
	tt := newTeamTester(t)
	defer tt.cleanup()

	ann := tt.addUser("ann")
	t.Logf("Signed up ann %s", ann.username)

	parentName := ann.createTeam()
	subteamName := parentName + ".test"

	var err error
	create := client.NewCmdTeamCreateRunner(ann.tc.G)
	create.TeamName, err = keybase1.TeamNameFromString(subteamName)
	require.NoError(t, err)
	create.JoinSubteam = true
	err = create.Run()
	require.NoError(t, err)

	t.Logf("Created subteam %s", subteamName)

	teamObj := ann.loadTeam(subteamName, false /* admin */)
	role, err := teamObj.MemberRole(context.Background(), ann.userVersion())
	require.NoError(t, err)
	require.Equal(t, keybase1.TeamRole_ADMIN, role, "got back wrong role")
	require.EqualValues(t, 1, teamObj.CurrentSeqno(), "expecting just one link in team")
}
<|endoftext|>