<commit_before>package proto

import (
	"sync"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

// A safeContract protects a RenterContract with a mutex.
type safeContract struct {
	modules.RenterContract
	mu sync.Mutex
}

// A ContractSet provides safe concurrent access to a set of contracts. Its
// purpose is to serialize modifications to individual contracts, as well as
// to provide operations on the set as a whole.
type ContractSet struct {
	contracts map[types.FileContractID]*safeContract
	mu        sync.Mutex
}

// Len returns the number of contracts in the set.
func (cs *ContractSet) Len() int {
	cs.mu.Lock()
	defer cs.mu.Unlock()
	return len(cs.contracts)
}

// IDs returns the FileContractID of each contract in the set. The contracts
// are not locked.
func (cs *ContractSet) IDs() []types.FileContractID {
	cs.mu.Lock()
	defer cs.mu.Unlock()
	ids := make([]types.FileContractID, 0, len(cs.contracts))
	for id := range cs.contracts {
		ids = append(ids, id)
	}
	return ids
}

// ViewAll returns a copy of each contract in the set. The contracts are not
// locked. Certain fields, including the MerkleRoots, are set to nil for
// safety reasons.
func (cs *ContractSet) ViewAll() []modules.RenterContract {
	cs.mu.Lock()
	defer cs.mu.Unlock()
	contracts := make([]modules.RenterContract, 0, len(cs.contracts))
	for _, sc := range cs.contracts {
		// construct shallow copy, sans MerkleRoots
		c := sc.RenterContract
		c.MerkleRoots = nil
		contracts = append(contracts, c)
	}
	return contracts
}

// View returns a copy of the contract with the specified ID. The contracts is
// not locked. Certain fields, including the MerkleRoots, are set to nil for
// safety reasons. If the contract is not present in the set, View
// returns false and a zero-valued RenterContract.
func (cs *ContractSet) View(id types.FileContractID) (modules.RenterContract, bool) {
	cs.mu.Lock()
	defer cs.mu.Unlock()
	sc, ok := cs.contracts[id]
	if !ok {
		return modules.RenterContract{}, false
	}
	c := sc.RenterContract
	c.MerkleRoots = nil
	return c, true
}

// Insert adds a new contract to the set. It panics if the contract is already
// in the set.
func (cs *ContractSet) Insert(contract modules.RenterContract) {
	cs.mu.Lock()
	defer cs.mu.Unlock()
	if _, ok := cs.contracts[contract.ID]; ok {
		build.Critical("contract already in set")
	}
	cs.contracts[contract.ID] = &safeContract{RenterContract: contract}
}

// Acquire looks up the contract with the specified FileContractID and locks
// it before returning it. If the contract is not present in the set, Acquire
// returns false and a zero-valued RenterContract.
func (cs *ContractSet) Acquire(id types.FileContractID) (modules.RenterContract, bool) {
	cs.mu.Lock()
	sc, ok := cs.contracts[id]
	cs.mu.Unlock()
	if ok {
		sc.mu.Lock()
	}
	return sc.RenterContract, ok
}

// Return returns a locked contract to the set and unlocks it. The contract
// must have been previously acquired by Acquire. If the contract is not
// present in the set, Return panics.
func (cs *ContractSet) Return(contract modules.RenterContract) {
	cs.mu.Lock()
	defer cs.mu.Unlock()
	sc, ok := cs.contracts[contract.ID]
	if !ok {
		build.Critical("no contract with that id")
	}
	sc.RenterContract = contract
	cs.contracts[contract.ID] = sc
	sc.mu.Unlock()
}

// Delete removes a contract from the set. The contract must have been
// previously acquired by Acquire. If the contract is not present in the set,
// Delete is a no-op.
func (cs *ContractSet) Delete(contract modules.RenterContract) {
	cs.mu.Lock()
	defer cs.mu.Unlock()
	sc, ok := cs.contracts[contract.ID]
	if !ok {
		return
	}
	delete(cs.contracts, contract.ID)
	sc.mu.Unlock()
}

// NewContractSet returns a ContractSet populated with the provided slice of
// RenterContracts, which may be nil.
func NewContractSet(contracts []modules.RenterContract) ContractSet {
	set := make(map[types.FileContractID]*safeContract)
	for _, c := range contracts {
		set[c.ID] = &safeContract{RenterContract: c}
	}
	return ContractSet{
		contracts: set,
	}
}
<commit_msg>fix nil pointer bug in Acquire<commit_after>package proto

import (
	"sync"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

// A safeContract protects a RenterContract with a mutex.
type safeContract struct {
	modules.RenterContract
	mu sync.Mutex
}

// A ContractSet provides safe concurrent access to a set of contracts. Its
// purpose is to serialize modifications to individual contracts, as well as
// to provide operations on the set as a whole.
type ContractSet struct {
	contracts map[types.FileContractID]*safeContract
	mu        sync.Mutex
}

// Len returns the number of contracts in the set.
func (cs *ContractSet) Len() int {
	cs.mu.Lock()
	defer cs.mu.Unlock()
	return len(cs.contracts)
}

// IDs returns the FileContractID of each contract in the set. The contracts
// are not locked.
func (cs *ContractSet) IDs() []types.FileContractID {
	cs.mu.Lock()
	defer cs.mu.Unlock()
	ids := make([]types.FileContractID, 0, len(cs.contracts))
	for id := range cs.contracts {
		ids = append(ids, id)
	}
	return ids
}

// ViewAll returns a copy of each contract in the set. The contracts are not
// locked. Certain fields, including the MerkleRoots, are set to nil for
// safety reasons.
func (cs *ContractSet) ViewAll() []modules.RenterContract {
	cs.mu.Lock()
	defer cs.mu.Unlock()
	contracts := make([]modules.RenterContract, 0, len(cs.contracts))
	for _, sc := range cs.contracts {
		// construct shallow copy, sans MerkleRoots
		c := sc.RenterContract
		c.MerkleRoots = nil
		contracts = append(contracts, c)
	}
	return contracts
}

// View returns a copy of the contract with the specified ID. The contracts is
// not locked. Certain fields, including the MerkleRoots, are set to nil for
// safety reasons. If the contract is not present in the set, View
// returns false and a zero-valued RenterContract.
func (cs *ContractSet) View(id types.FileContractID) (modules.RenterContract, bool) {
	cs.mu.Lock()
	defer cs.mu.Unlock()
	sc, ok := cs.contracts[id]
	if !ok {
		return modules.RenterContract{}, false
	}
	c := sc.RenterContract
	c.MerkleRoots = nil
	return c, true
}

// Insert adds a new contract to the set. It panics if the contract is already
// in the set.
func (cs *ContractSet) Insert(contract modules.RenterContract) {
	cs.mu.Lock()
	defer cs.mu.Unlock()
	if _, ok := cs.contracts[contract.ID]; ok {
		build.Critical("contract already in set")
	}
	cs.contracts[contract.ID] = &safeContract{RenterContract: contract}
}

// Acquire looks up the contract with the specified FileContractID and locks
// it before returning it. If the contract is not present in the set, Acquire
// returns false and a zero-valued RenterContract.
func (cs *ContractSet) Acquire(id types.FileContractID) (modules.RenterContract, bool) {
	cs.mu.Lock()
	sc, ok := cs.contracts[id]
	cs.mu.Unlock()
	if !ok {
		return modules.RenterContract{}, false
	}
	sc.mu.Lock()
	return sc.RenterContract, ok
}

// Return returns a locked contract to the set and unlocks it. The contract
// must have been previously acquired by Acquire. If the contract is not
// present in the set, Return panics.
func (cs *ContractSet) Return(contract modules.RenterContract) {
	cs.mu.Lock()
	defer cs.mu.Unlock()
	sc, ok := cs.contracts[contract.ID]
	if !ok {
		build.Critical("no contract with that id")
	}
	sc.RenterContract = contract
	cs.contracts[contract.ID] = sc
	sc.mu.Unlock()
}

// Delete removes a contract from the set. The contract must have been
// previously acquired by Acquire. If the contract is not present in the set,
// Delete is a no-op.
func (cs *ContractSet) Delete(contract modules.RenterContract) {
	cs.mu.Lock()
	defer cs.mu.Unlock()
	sc, ok := cs.contracts[contract.ID]
	if !ok {
		return
	}
	delete(cs.contracts, contract.ID)
	sc.mu.Unlock()
}

// NewContractSet returns a ContractSet populated with the provided slice of
// RenterContracts, which may be nil.
func NewContractSet(contracts []modules.RenterContract) ContractSet {
	set := make(map[types.FileContractID]*safeContract)
	for _, c := range contracts {
		set[c.ID] = &safeContract{RenterContract: c}
	}
	return ContractSet{
		contracts: set,
	}
}
<|endoftext|>


<commit_before>package client

import (
	"errors"
	"fmt"
	"os"
	"time"

	"golang.org/x/net/context"

	"google.golang.org/grpc"
	"google.golang.org/grpc/metadata"

	log "github.com/Sirupsen/logrus"
	types "github.com/gogo/protobuf/types"

	"github.com/pachyderm/pachyderm/src/client/health"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/config"
	"github.com/pachyderm/pachyderm/src/client/pps"
)

// PfsAPIClient is an alias for pfs.APIClient.
type PfsAPIClient pfs.APIClient

// PpsAPIClient is an alias for pps.APIClient.
type PpsAPIClient pps.APIClient

// BlockAPIClient is an alias for pfs.BlockAPIClient.
type BlockAPIClient pfs.BlockAPIClient

// An APIClient is a wrapper around pfs, pps and block APIClients.
type APIClient struct {
	PfsAPIClient
	PpsAPIClient
	BlockAPIClient
	addr              string
	clientConn        *grpc.ClientConn
	healthClient      health.HealthClient
	_ctx              context.Context
	config            *config.Config
	cancel            func()
	reportUserMetrics bool
	metricsPrefix     string
	streamSemaphore   chan struct{}
}

// DefaultMaxConcurrentStreams defines the max number of Putfiles or Getfiles happening simultaneously
const DefaultMaxConcurrentStreams uint = 100

// NewMetricsClientFromAddress Creates a client that will report a user's Metrics
func NewMetricsClientFromAddress(addr string, metrics bool, prefix string) (*APIClient, error) {
	return NewMetricsClientFromAddress(addr, metrics, prefix,
		DefaultMaxConcurrentStreams)
}

// NewMetricsClientFromAddressWithConcurrency Creates a client that will report
// a user's Metrics, and sets the max concurrency of streaming requests (GetFile
// / PutFile)
func NewMetricsClientFromAddressWithConcurrency(addr string, metrics bool, prefix string, maxConcurrentStreams uint) (*APIClient, error) {
	c, err := NewFromAddress(addr)
	if err != nil {
		return nil, err
	}
	cfg, err := config.Read()
	if err != nil {
		// metrics errors are non fatal
		log.Errorf("error loading user config from ~/.pachderm/config: %v\n", err)
	} else {
		c.config = cfg
	}
	c.reportUserMetrics = metrics
	c.metricsPrefix = prefix
	return c, err
}

// NewFromAddressWithConcurrency constructs a new APIClient and sets the max
// concurrency of streaming requests (GetFile / PutFile)
func NewFromAddressWithConcurrency(addr string, maxConcurrentStreams uint) (*APIClient, error) {
	c := &APIClient{
		addr:            addr,
		streamSemaphore: make(chan struct{}, maxConcurrentStreams),
	}
	if err := c.connect(); err != nil {
		return nil, err
	}
	return c, nil
}

// NewFromAddress constructs a new APIClient for the server at addr.
func NewFromAddress(addr string) (*APIClient, error) {
	return NewFromAddressWithConcurrency(addr, DefaultMaxConcurrentStreams)
}

// NewInCluster constructs a new APIClient using env vars that Kubernetes creates.
// This should be used to access Pachyderm from within a Kubernetes cluster
// with Pachyderm running on it.
func NewInCluster() (*APIClient, error) {
	addr := os.Getenv("PACHD_PORT_650_TCP_ADDR")

	if addr == "" {
		return nil, fmt.Errorf("PACHD_PORT_650_TCP_ADDR not set")
	}

	return NewFromAddress(fmt.Sprintf("%v:650", addr))
}

// Close the connection to gRPC
func (c *APIClient) Close() error {
	return c.clientConn.Close()
}

// KeepConnected periodically health checks the connection and attempts to
// reconnect if it becomes unhealthy.
func (c *APIClient) KeepConnected(cancel chan bool) {
	for {
		select {
		case <-cancel:
			return
		case <-time.After(time.Second * 5):
			ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
			if _, err := c.healthClient.Health(ctx, &types.Empty{}); err != nil {
				c.cancel()
				c.connect()
			}
			cancel()
		}
	}
}

// DeleteAll deletes everything in the cluster.
// Use with caution, there is no undo.
func (c APIClient) DeleteAll() error {
	if _, err := c.PpsAPIClient.DeleteAll(
		c.ctx(),
		&types.Empty{},
	); err != nil {
		return sanitizeErr(err)
	}
	if _, err := c.PfsAPIClient.DeleteAll(
		c.ctx(),
		&types.Empty{},
	); err != nil {
		return sanitizeErr(err)
	}
	return nil
}

func (c *APIClient) connect() error {
	clientConn, err := grpc.Dial(c.addr, grpc.WithInsecure())
	if err != nil {
		return err
	}

	ctx, cancel := context.WithCancel(context.Background())
	c.PfsAPIClient = pfs.NewAPIClient(clientConn)
	c.PpsAPIClient = pps.NewAPIClient(clientConn)
	c.BlockAPIClient = pfs.NewBlockAPIClient(clientConn)
	c.clientConn = clientConn
	c.healthClient = health.NewHealthClient(clientConn)
	c._ctx = ctx
	c.cancel = cancel
	return nil
}

func (c *APIClient) addMetadata(ctx context.Context) context.Context {
	if !c.reportUserMetrics {
		return ctx
	}
	if c.config == nil {
		cfg, err := config.Read()
		if err != nil {
			// Don't report error if config fails to read
			// metrics errors are non fatal
			log.Errorf("Error loading config: %v\n", err)
			return ctx
		}
		c.config = cfg
	}
	// metadata API downcases all the key names
	return metadata.NewContext(
		ctx,
		metadata.Pairs(
			"userid", c.config.UserID,
			"prefix", c.metricsPrefix,
		),
	)
}

// TODO this method only exists because we initialize some APIClient in such a
// way that ctx will be nil
func (c *APIClient) ctx() context.Context {
	if c._ctx == nil {
		return c.addMetadata(context.Background())
	}
	return c.addMetadata(c._ctx)
}

func sanitizeErr(err error) error {
	if err == nil {
		return nil
	}

	return errors.New(grpc.ErrorDesc(err))
}
<commit_msg>Fix typo<commit_after>package client

import (
	"errors"
	"fmt"
	"os"
	"time"

	"golang.org/x/net/context"

	"google.golang.org/grpc"
	"google.golang.org/grpc/metadata"

	log "github.com/Sirupsen/logrus"
	types "github.com/gogo/protobuf/types"

	"github.com/pachyderm/pachyderm/src/client/health"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/config"
	"github.com/pachyderm/pachyderm/src/client/pps"
)

// PfsAPIClient is an alias for pfs.APIClient.
type PfsAPIClient pfs.APIClient

// PpsAPIClient is an alias for pps.APIClient.
type PpsAPIClient pps.APIClient

// BlockAPIClient is an alias for pfs.BlockAPIClient.
type BlockAPIClient pfs.BlockAPIClient

// An APIClient is a wrapper around pfs, pps and block APIClients.
type APIClient struct {
	PfsAPIClient
	PpsAPIClient
	BlockAPIClient
	addr              string
	clientConn        *grpc.ClientConn
	healthClient      health.HealthClient
	_ctx              context.Context
	config            *config.Config
	cancel            func()
	reportUserMetrics bool
	metricsPrefix     string
	streamSemaphore   chan struct{}
}

// DefaultMaxConcurrentStreams defines the max number of Putfiles or Getfiles happening simultaneously
const DefaultMaxConcurrentStreams uint = 100

// NewMetricsClientFromAddress Creates a client that will report a user's Metrics
func NewMetricsClientFromAddress(addr string, metrics bool, prefix string) (*APIClient, error) {
	return NewMetricsClientFromAddressWithConcurrency(addr, metrics, prefix,
		DefaultMaxConcurrentStreams)
}

// NewMetricsClientFromAddressWithConcurrency Creates a client that will report
// a user's Metrics, and sets the max concurrency of streaming requests (GetFile
// / PutFile)
func NewMetricsClientFromAddressWithConcurrency(addr string, metrics bool, prefix string, maxConcurrentStreams uint) (*APIClient, error) {
	c, err := NewFromAddress(addr)
	if err != nil {
		return nil, err
	}
	cfg, err := config.Read()
	if err != nil {
		// metrics errors are non fatal
		log.Errorf("error loading user config from ~/.pachderm/config: %v\n", err)
	} else {
		c.config = cfg
	}
	c.reportUserMetrics = metrics
	c.metricsPrefix = prefix
	return c, err
}

// NewFromAddressWithConcurrency constructs a new APIClient and sets the max
// concurrency of streaming requests (GetFile / PutFile)
func NewFromAddressWithConcurrency(addr string, maxConcurrentStreams uint) (*APIClient, error) {
	c := &APIClient{
		addr:            addr,
		streamSemaphore: make(chan struct{}, maxConcurrentStreams),
	}
	if err := c.connect(); err != nil {
		return nil, err
	}
	return c, nil
}

// NewFromAddress constructs a new APIClient for the server at addr.
func NewFromAddress(addr string) (*APIClient, error) {
	return NewFromAddressWithConcurrency(addr, DefaultMaxConcurrentStreams)
}

// NewInCluster constructs a new APIClient using env vars that Kubernetes creates.
// This should be used to access Pachyderm from within a Kubernetes cluster
// with Pachyderm running on it.
func NewInCluster() (*APIClient, error) {
	addr := os.Getenv("PACHD_PORT_650_TCP_ADDR")

	if addr == "" {
		return nil, fmt.Errorf("PACHD_PORT_650_TCP_ADDR not set")
	}

	return NewFromAddress(fmt.Sprintf("%v:650", addr))
}

// Close the connection to gRPC
func (c *APIClient) Close() error {
	return c.clientConn.Close()
}

// KeepConnected periodically health checks the connection and attempts to
// reconnect if it becomes unhealthy.
func (c *APIClient) KeepConnected(cancel chan bool) {
	for {
		select {
		case <-cancel:
			return
		case <-time.After(time.Second * 5):
			ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
			if _, err := c.healthClient.Health(ctx, &types.Empty{}); err != nil {
				c.cancel()
				c.connect()
			}
			cancel()
		}
	}
}

// DeleteAll deletes everything in the cluster.
// Use with caution, there is no undo.
func (c APIClient) DeleteAll() error {
	if _, err := c.PpsAPIClient.DeleteAll(
		c.ctx(),
		&types.Empty{},
	); err != nil {
		return sanitizeErr(err)
	}
	if _, err := c.PfsAPIClient.DeleteAll(
		c.ctx(),
		&types.Empty{},
	); err != nil {
		return sanitizeErr(err)
	}
	return nil
}

func (c *APIClient) connect() error {
	clientConn, err := grpc.Dial(c.addr, grpc.WithInsecure())
	if err != nil {
		return err
	}

	ctx, cancel := context.WithCancel(context.Background())
	c.PfsAPIClient = pfs.NewAPIClient(clientConn)
	c.PpsAPIClient = pps.NewAPIClient(clientConn)
	c.BlockAPIClient = pfs.NewBlockAPIClient(clientConn)
	c.clientConn = clientConn
	c.healthClient = health.NewHealthClient(clientConn)
	c._ctx = ctx
	c.cancel = cancel
	return nil
}

func (c *APIClient) addMetadata(ctx context.Context) context.Context {
	if !c.reportUserMetrics {
		return ctx
	}
	if c.config == nil {
		cfg, err := config.Read()
		if err != nil {
			// Don't report error if config fails to read
			// metrics errors are non fatal
			log.Errorf("Error loading config: %v\n", err)
			return ctx
		}
		c.config = cfg
	}
	// metadata API downcases all the key names
	return metadata.NewContext(
		ctx,
		metadata.Pairs(
			"userid", c.config.UserID,
			"prefix", c.metricsPrefix,
		),
	)
}

// TODO this method only exists because we initialize some APIClient in such a
// way that ctx will be nil
func (c *APIClient) ctx() context.Context {
	if c._ctx == nil {
		return c.addMetadata(context.Background())
	}
	return c.addMetadata(c._ctx)
}

func sanitizeErr(err error) error {
	if err == nil {
		return nil
	}

	return errors.New(grpc.ErrorDesc(err))
}
<|endoftext|>


<commit_before>/**
 * Copyright 2015 Comcast Cable Communications Management, LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// A simple proxy service to forward JSON events and transform or filter them along the way.
package main

import (
	"flag"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"time"

	. "github.com/Comcast/eel/eel/jtl"
	. "github.com/Comcast/eel/eel/util"
)

// build hint: go build -ldflags "-X main.Version 2.0"

var (
	Version = "1.0"
)

var (
	// proxy params
	env         = flag.String("env", "default", "environment name such as qa, prod for logging")
	basePath    = flag.String("path", "", "base path for config.json and handlers (optional)")
	configPath  = flag.String("config", "", "path to config.json (optional)")
	handlerPath = flag.String("handlers", "", "path to handlers (optional)")
	logLevel    = flag.String("loglevel", L_InfoLevel, "log level (optional)")
	// cmd params
	in    = flag.String("in", "", "incoming event string or @file")
	tf    = flag.String("tf", "", "transformation string or @file")
	istbe = flag.Bool("istbe", true, "is template by example flag")
)

// useCores if GOMAXPROCS not set use all cores you got.
func useCores(ctx Context) {
	cores := os.Getenv("GOMAXPROCS")
	if cores == "" {
		n := runtime.NumCPU()
		ctx.Log().Info("action", "use_cores", "cores", n)
		runtime.GOMAXPROCS(n)
		cores = strconv.Itoa(n)
	} else {
		ctx.Log().Info("action", "use_cores_from_env", "cores", cores)
	}
}

// initLogging sets up context and stats loop.
func initLogging() {
	if *basePath != "" {
		BasePath = *basePath
	}
	if *configPath != "" {
		ConfigPath = filepath.Join(BasePath, *configPath)
	} else {
		ConfigPath = filepath.Join(BasePath, EelConfigFile)
	}
	Gctx = NewDefaultContext(*logLevel)
	config := GetConfigFromFile(Gctx)
	if *handlerPath != "" {
		HandlerPath = *handlerPath
	} else if config.HandlerConfigPath != "" {
		HandlerPath = config.HandlerConfigPath
	}
	AppId = config.AppName
	Gctx.AddLogValue("app.id", AppId)
	InstanceName, _ = os.Hostname()
	Gctx.AddLogValue("instance.id", InstanceName)
	if *env != "" {
		EnvName = *env
		Gctx.AddLogValue("env.name", EnvName)
	}
	Gctx.AddValue(EelStartTime, time.Now().Local().Format("2006-01-02 15:04:05 +0800"))
	stats := new(ServiceStats)
	Gctx.AddValue(EelTotalStats, stats)
	Gctx.AddValue(Eel1MinStats, new(ServiceStats))
	Gctx.AddValue(Eel5MinStats, new(ServiceStats))
	Gctx.AddValue(Eel1hrStats, new(ServiceStats))
	Gctx.AddValue(Eel24hrStats, new(ServiceStats))

	Gctx.AddConfigValue(EelTraceLogger, NewTraceLogger(Gctx, config))

	getWorkQueueFillLevel := func() int {
		wd := GetWorkDispatcher(Gctx)
		if wd != nil {
			return len(wd.WorkQueue)
		}
		return -1
	}

	getNumWorkersIdle := func() int {
		wd := GetWorkDispatcher(Gctx)
		if wd != nil {
			return len(wd.WorkerQueue)
		}
		return -1
	}

	if config.LogStats {
		go Gctx.Log().RuntimeLogLoop(time.Duration(60)*time.Second, -1)
		go stats.StatsLoop(Gctx, 300*time.Second, -1, Eel5MinStats, getWorkQueueFillLevel, getNumWorkersIdle)
		go stats.StatsLoop(Gctx, 60*time.Second, -1, Eel1MinStats, getWorkQueueFillLevel, getNumWorkersIdle)
		go stats.StatsLoop(Gctx, 60*time.Minute, -1, Eel1hrStats, getWorkQueueFillLevel, getNumWorkersIdle)
		go stats.StatsLoop(Gctx, 24*time.Hour, -1, Eel24hrStats, getWorkQueueFillLevel, getNumWorkersIdle)
	}
}

func registerAdminServices() {
	http.HandleFunc("/health/shallow", NilHandler)
	http.HandleFunc("/health/deep", StatusHandler)
	http.HandleFunc("/health", StatusHandler)
	http.HandleFunc("/status", StatusHandler)
	http.HandleFunc("/pluginconfigs", PluginConfigHandler)
	http.HandleFunc("/plugins", ManagePluginsUIHandler)
	http.HandleFunc("/plugins/", ManagePluginsHandler)
	http.HandleFunc("/reload", ReloadConfigHandler)
	http.HandleFunc("/toggletracelogger", TraceLogConfigHandler)
	http.HandleFunc("/vet", VetHandler)
	http.HandleFunc("/test", TopicTestHandler)
	http.HandleFunc("/test/handlers", HandlersTestHandler)
	http.HandleFunc("/test/process/", ProcessExpressionHandler)
	http.HandleFunc("/test/ast", ParserDebugHandler)
	http.HandleFunc("/test/astjson/", GetASTJsonHandler)
	http.HandleFunc("/test/asttree/", ParserDebugVizHandler)
	http.HandleFunc("/event/dummy", DummyEventHandler)
	http.Handle("/img/", http.StripPrefix("/img/", http.FileServer(http.Dir(filepath.Join(BasePath, "mascot")))))
}

func main() {
	flag.Parse()
	if *tf != "" {
		eelCmd(*in, *tf, *istbe)
	} else {
		initLogging()
		ReloadConfig()
		GetConfig(Gctx).Version = Version
		InitHttpTransport(Gctx)
		ctx := Gctx.SubContext()
		ctx.Log().Info("action", "starting", "version", Version)
		useCores(ctx)
		dc := NewLocalInMemoryDupChecker(GetConfig(ctx).DuplicateTimeout, 10000)
		Gctx.AddValue(EelDuplicateChecker, dc)
		dp := NewWorkDispatcher(GetConfig(ctx).WorkerPoolSize, GetConfig(ctx).MessageQueueDepth)
		dp.Start(ctx)
		Gctx.AddValue(EelDispatcher, dp)
		registerAdminServices()
		// register inbound plugins
		RegisterInboundPluginType(NewStdinPlugin, "STDIN")
		RegisterInboundPluginType(NewWebhookPlugin, "WEBHOOK")
		LoadInboundPlugins(Gctx)
		// hang on channel forever
		c := make(chan int)
		<-c
	}
}
<commit_msg>added profile service to main.go<commit_after>/**
 * Copyright 2015 Comcast Cable Communications Management, LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// A simple proxy service to forward JSON events and transform or filter them along the way.
package main

import (
	"flag"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"time"

	_ "net/http/pprof"

	. "github.com/Comcast/eel/eel/jtl"
	. "github.com/Comcast/eel/eel/util"
)

// build hint: go build -ldflags "-X main.Version 2.0"

var (
	Version = "1.0"
)

var (
	// proxy params
	env         = flag.String("env", "default", "environment name such as qa, prod for logging")
	basePath    = flag.String("path", "", "base path for config.json and handlers (optional)")
	configPath  = flag.String("config", "", "path to config.json (optional)")
	handlerPath = flag.String("handlers", "", "path to handlers (optional)")
	logLevel    = flag.String("loglevel", L_InfoLevel, "log level (optional)")
	// cmd params
	in    = flag.String("in", "", "incoming event string or @file")
	tf    = flag.String("tf", "", "transformation string or @file")
	istbe = flag.Bool("istbe", true, "is template by example flag")
)

// useCores if GOMAXPROCS not set use all cores you got.
func useCores(ctx Context) {
	cores := os.Getenv("GOMAXPROCS")
	if cores == "" {
		n := runtime.NumCPU()
		ctx.Log().Info("action", "use_cores", "cores", n)
		runtime.GOMAXPROCS(n)
		cores = strconv.Itoa(n)
	} else {
		ctx.Log().Info("action", "use_cores_from_env", "cores", cores)
	}
}

// initLogging sets up context and stats loop.
func initLogging() {
	if *basePath != "" {
		BasePath = *basePath
	}
	if *configPath != "" {
		ConfigPath = filepath.Join(BasePath, *configPath)
	} else {
		ConfigPath = filepath.Join(BasePath, EelConfigFile)
	}
	Gctx = NewDefaultContext(*logLevel)
	config := GetConfigFromFile(Gctx)
	if *handlerPath != "" {
		HandlerPath = *handlerPath
	} else if config.HandlerConfigPath != "" {
		HandlerPath = config.HandlerConfigPath
	}
	AppId = config.AppName
	Gctx.AddLogValue("app.id", AppId)
	InstanceName, _ = os.Hostname()
	Gctx.AddLogValue("instance.id", InstanceName)
	if *env != "" {
		EnvName = *env
		Gctx.AddLogValue("env.name", EnvName)
	}
	Gctx.AddValue(EelStartTime, time.Now().Local().Format("2006-01-02 15:04:05 +0800"))
	stats := new(ServiceStats)
	Gctx.AddValue(EelTotalStats, stats)
	Gctx.AddValue(Eel1MinStats, new(ServiceStats))
	Gctx.AddValue(Eel5MinStats, new(ServiceStats))
	Gctx.AddValue(Eel1hrStats, new(ServiceStats))
	Gctx.AddValue(Eel24hrStats, new(ServiceStats))

	Gctx.AddConfigValue(EelTraceLogger, NewTraceLogger(Gctx, config))

	getWorkQueueFillLevel := func() int {
		wd := GetWorkDispatcher(Gctx)
		if wd != nil {
			return len(wd.WorkQueue)
		}
		return -1
	}

	getNumWorkersIdle := func() int {
		wd := GetWorkDispatcher(Gctx)
		if wd != nil {
			return len(wd.WorkerQueue)
		}
		return -1
	}

	if config.LogStats {
		go Gctx.Log().RuntimeLogLoop(time.Duration(60)*time.Second, -1)
		go stats.StatsLoop(Gctx, 300*time.Second, -1, Eel5MinStats, getWorkQueueFillLevel, getNumWorkersIdle)
		go stats.StatsLoop(Gctx, 60*time.Second, -1, Eel1MinStats, getWorkQueueFillLevel, getNumWorkersIdle)
		go stats.StatsLoop(Gctx, 60*time.Minute, -1, Eel1hrStats, getWorkQueueFillLevel, getNumWorkersIdle)
		go stats.StatsLoop(Gctx, 24*time.Hour, -1, Eel24hrStats, getWorkQueueFillLevel, getNumWorkersIdle)
	}
}

func registerAdminServices() {
	http.HandleFunc("/health/shallow", NilHandler)
	http.HandleFunc("/health/deep", StatusHandler)
	http.HandleFunc("/health", StatusHandler)
	http.HandleFunc("/status", StatusHandler)
	http.HandleFunc("/pluginconfigs", PluginConfigHandler)
	http.HandleFunc("/plugins", ManagePluginsUIHandler)
	http.HandleFunc("/plugins/", ManagePluginsHandler)
	http.HandleFunc("/reload", ReloadConfigHandler)
	http.HandleFunc("/toggletracelogger", TraceLogConfigHandler)
	http.HandleFunc("/vet", VetHandler)
	http.HandleFunc("/test", TopicTestHandler)
	http.HandleFunc("/test/handlers", HandlersTestHandler)
	http.HandleFunc("/test/process/", ProcessExpressionHandler)
	http.HandleFunc("/test/ast", ParserDebugHandler)
	http.HandleFunc("/test/astjson/", GetASTJsonHandler)
	http.HandleFunc("/test/asttree/", ParserDebugVizHandler)
	http.HandleFunc("/event/dummy", DummyEventHandler)
	http.Handle("/img/", http.StripPrefix("/img/", http.FileServer(http.Dir(filepath.Join(BasePath, "mascot")))))
}

func main() {
	flag.Parse()
	if *tf != "" {
		eelCmd(*in, *tf, *istbe)
	} else {
		initLogging()
		ReloadConfig()
		GetConfig(Gctx).Version = Version
		InitHttpTransport(Gctx)
		ctx := Gctx.SubContext()
		ctx.Log().Info("action", "starting", "version", Version)
		useCores(ctx)
		dc := NewLocalInMemoryDupChecker(GetConfig(ctx).DuplicateTimeout, 10000)
		Gctx.AddValue(EelDuplicateChecker, dc)
		dp := NewWorkDispatcher(GetConfig(ctx).WorkerPoolSize, GetConfig(ctx).MessageQueueDepth)
		dp.Start(ctx)
		Gctx.AddValue(EelDispatcher, dp)
		registerAdminServices()

		// resgister profile service
		go func() {
			ctx.Log().Error(http.ListenAndServe("localhost:6060", nil))
		}()

		// register inbound plugins
		RegisterInboundPluginType(NewStdinPlugin, "STDIN")
		RegisterInboundPluginType(NewWebhookPlugin, "WEBHOOK")
		LoadInboundPlugins(Gctx)

		// hang on channel forever
		c := make(chan int)
		<-c
	}
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/acm"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsAcmCertificate() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsAcmCertificateCreate,
		Read:   resourceAwsAcmCertificateRead,
		Update: resourceAwsAcmCertificateUpdate,
		Delete: resourceAwsAcmCertificateDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"domain_name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"subject_alternative_names": {
				Type:     schema.TypeList,
				Optional: true,
				ForceNew: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
			},
			"validation_method": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"arn": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"domain_validation_options": {
				Type:     schema.TypeList,
				Computed: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"domain_name": {
							Type:     schema.TypeString,
							Computed: true,
						},
						"resource_record_name": {
							Type:     schema.TypeString,
							Computed: true,
						},
						"resource_record_type": {
							Type:     schema.TypeString,
							Computed: true,
						},
						"resource_record_value": {
							Type:     schema.TypeString,
							Computed: true,
						},
					},
				},
			},
			"validation_emails": {
				Type:     schema.TypeList,
				Computed: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
			},
			"tags": tagsSchema(),
		},
	}
}

func resourceAwsAcmCertificateCreate(d *schema.ResourceData, meta interface{}) error {
	acmconn := meta.(*AWSClient).acmconn
	params := &acm.RequestCertificateInput{
		DomainName:       aws.String(d.Get("domain_name").(string)),
		ValidationMethod: aws.String(d.Get("validation_method").(string)),
	}

	sans, ok := d.GetOk("subject_alternative_names")
	if ok {
		sanStrings := sans.([]interface{})
		params.SubjectAlternativeNames = expandStringList(sanStrings)
	}

	log.Printf("[DEBUG] ACM Certificate Request: %#v", params)
	resp, err := acmconn.RequestCertificate(params)

	if err != nil {
		return fmt.Errorf("Error requesting certificate: %s", err)
	}

	d.SetId(*resp.CertificateArn)
	if v, ok := d.GetOk("tags"); ok {
		params := &acm.AddTagsToCertificateInput{
			CertificateArn: resp.CertificateArn,
			Tags:           tagsFromMapACM(v.(map[string]interface{})),
		}
		_, err := acmconn.AddTagsToCertificate(params)

		if err != nil {
			return fmt.Errorf("Error requesting certificate: %s", err)
		}
	}

	return resourceAwsAcmCertificateRead(d, meta)
}

func resourceAwsAcmCertificateRead(d *schema.ResourceData, meta interface{}) error {
	acmconn := meta.(*AWSClient).acmconn

	params := &acm.DescribeCertificateInput{
		CertificateArn: aws.String(d.Id()),
	}

	return resource.Retry(time.Duration(1)*time.Minute, func() *resource.RetryError {
		resp, err := acmconn.DescribeCertificate(params)

		if err != nil {
			if isAWSErr(err, acm.ErrCodeResourceNotFoundException, "") {
				d.SetId("")
				return nil
			}
			return resource.NonRetryableError(fmt.Errorf("Error describing certificate: %s", err))
		}

		d.Set("domain_name", resp.Certificate.DomainName)
		d.Set("arn", resp.Certificate.CertificateArn)

		if err := d.Set("subject_alternative_names", cleanUpSubjectAlternativeNames(resp.Certificate)); err != nil {
			return resource.NonRetryableError(err)
		}

		domainValidationOptions, emailValidationOptions, err := convertValidationOptions(resp.Certificate)

		if err != nil {
			return resource.RetryableError(err)
		}

		if err := d.Set("domain_validation_options", domainValidationOptions); err != nil {
			return resource.NonRetryableError(err)
		}
		if err := d.Set("validation_emails", emailValidationOptions); err != nil {
			return resource.NonRetryableError(err)
		}
		d.Set("validation_method", resourceAwsAcmCertificateGuessValidationMethod(domainValidationOptions, emailValidationOptions))

		params := &acm.ListTagsForCertificateInput{
			CertificateArn: aws.String(d.Id()),
		}

		tagResp, err := acmconn.ListTagsForCertificate(params)
		if err := d.Set("tags", tagsToMapACM(tagResp.Tags)); err != nil {
			return resource.NonRetryableError(err)
		}

		return nil
	})
}
func resourceAwsAcmCertificateGuessValidationMethod(domainValidationOptions []map[string]interface{}, emailValidationOptions []string) string {
	// The DescribeCertificate Response doesn't have information on what validation method was used
	// so we need to guess from the validation options we see...
	if len(domainValidationOptions) > 0 {
		return acm.ValidationMethodDns
	} else if len(emailValidationOptions) > 0 {
		return acm.ValidationMethodEmail
	} else {
		return "NONE"
	}
}

func resourceAwsAcmCertificateUpdate(d *schema.ResourceData, meta interface{}) error {
	if d.HasChange("tags") {
		acmconn := meta.(*AWSClient).acmconn
		err := setTagsACM(acmconn, d)
		if err != nil {
			return err
		}
	}
	return nil
}

func cleanUpSubjectAlternativeNames(cert *acm.CertificateDetail) []string {
	sans := cert.SubjectAlternativeNames
	vs := make([]string, 0, len(sans)-1)
	for _, v := range sans {
		if *v != *cert.DomainName {
			vs = append(vs, *v)
		}
	}
	return vs

}

func convertValidationOptions(certificate *acm.CertificateDetail) ([]map[string]interface{}, []string, error) {
	var domainValidationResult []map[string]interface{}
	var emailValidationResult []string

	if *certificate.Type == acm.CertificateTypeAmazonIssued {
		for _, o := range certificate.DomainValidationOptions {
			if o.ResourceRecord != nil {
				validationOption := map[string]interface{}{
					"domain_name":           *o.DomainName,
					"resource_record_name":  *o.ResourceRecord.Name,
					"resource_record_type":  *o.ResourceRecord.Type,
					"resource_record_value": *o.ResourceRecord.Value,
				}
				domainValidationResult = append(domainValidationResult, validationOption)
			} else if o.ValidationEmails != nil && len(o.ValidationEmails) > 0 {
				for _, validationEmail := range o.ValidationEmails {
					emailValidationResult = append(emailValidationResult, *validationEmail)
				}
			} else {
				log.Printf("[DEBUG] No validation options need to retry: %#v", o)
				return nil, nil, fmt.Errorf("No validation options need to retry: %#v", o)
			}
		}
	}

	return domainValidationResult, emailValidationResult, nil
}

func resourceAwsAcmCertificateDelete(d *schema.ResourceData, meta interface{}) error {
	acmconn := meta.(*AWSClient).acmconn

	log.Printf("[INFO] Deleting ACM Certificate: %s", d.Id())

	params := &acm.DeleteCertificateInput{
		CertificateArn: aws.String(d.Id()),
	}

	err := resource.Retry(10*time.Minute, func() *resource.RetryError {
		_, err := acmconn.DeleteCertificate(params)
		if err != nil {
			if isAWSErr(err, acm.ErrCodeResourceInUseException, "") {
				log.Printf("[WARN] Conflict deleting certificate in use: %s, retrying", err.Error())
				return resource.RetryableError(err)
			}
			return resource.NonRetryableError(err)
		}
		return nil
	})

	if err != nil && !isAWSErr(err, acm.ErrCodeResourceNotFoundException, "") {
		return fmt.Errorf("Error deleting certificate: %s", err)
	}

	return nil
}
<commit_msg>Fix read for non-pending aws_acm_certificate<commit_after>package aws

import (
	"fmt"
	"log"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/acm"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsAcmCertificate() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsAcmCertificateCreate,
		Read:   resourceAwsAcmCertificateRead,
		Update: resourceAwsAcmCertificateUpdate,
		Delete: resourceAwsAcmCertificateDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"domain_name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"subject_alternative_names": {
				Type:     schema.TypeList,
				Optional: true,
				ForceNew: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
			},
			"validation_method": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"arn": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"domain_validation_options": {
				Type:     schema.TypeList,
				Computed: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"domain_name": {
							Type:     schema.TypeString,
							Computed: true,
						},
						"resource_record_name": {
							Type:     schema.TypeString,
							Computed: true,
						},
						"resource_record_type": {
							Type:     schema.TypeString,
							Computed: true,
						},
						"resource_record_value": {
							Type:     schema.TypeString,
							Computed: true,
						},
					},
				},
			},
			"validation_emails": {
				Type:     schema.TypeList,
				Computed: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
			},
			"tags": tagsSchema(),
		},
	}
}

func resourceAwsAcmCertificateCreate(d *schema.ResourceData, meta interface{}) error {
	acmconn := meta.(*AWSClient).acmconn
	params := &acm.RequestCertificateInput{
		DomainName:       aws.String(d.Get("domain_name").(string)),
		ValidationMethod: aws.String(d.Get("validation_method").(string)),
	}

	sans, ok := d.GetOk("subject_alternative_names")
	if ok {
		sanStrings := sans.([]interface{})
		params.SubjectAlternativeNames = expandStringList(sanStrings)
	}

	log.Printf("[DEBUG] ACM Certificate Request: %#v", params)
	resp, err := acmconn.RequestCertificate(params)

	if err != nil {
		return fmt.Errorf("Error requesting certificate: %s", err)
	}

	d.SetId(*resp.CertificateArn)
	if v, ok := d.GetOk("tags"); ok {
		params := &acm.AddTagsToCertificateInput{
			CertificateArn: resp.CertificateArn,
			Tags:           tagsFromMapACM(v.(map[string]interface{})),
		}
		_, err := acmconn.AddTagsToCertificate(params)

		if err != nil {
			return fmt.Errorf("Error requesting certificate: %s", err)
		}
	}

	return resourceAwsAcmCertificateRead(d, meta)
}

func resourceAwsAcmCertificateRead(d *schema.ResourceData, meta interface{}) error {
	acmconn := meta.(*AWSClient).acmconn

	params := &acm.DescribeCertificateInput{
		CertificateArn: aws.String(d.Id()),
	}

	return resource.Retry(time.Duration(1)*time.Minute, func() *resource.RetryError {
		resp, err := acmconn.DescribeCertificate(params)

		if err != nil {
			if isAWSErr(err, acm.ErrCodeResourceNotFoundException, "") {
				d.SetId("")
				return nil
			}
			return resource.NonRetryableError(fmt.Errorf("Error describing certificate: %s", err))
		}

		d.Set("domain_name", resp.Certificate.DomainName)
		d.Set("arn", resp.Certificate.CertificateArn)

		if err := d.Set("subject_alternative_names", cleanUpSubjectAlternativeNames(resp.Certificate)); err != nil {
			return resource.NonRetryableError(err)
		}

		domainValidationOptions, emailValidationOptions, err := convertValidationOptions(resp.Certificate)

		if err != nil {
			return resource.RetryableError(err)
		}

		if err := d.Set("domain_validation_options", domainValidationOptions); err != nil {
			return resource.NonRetryableError(err)
		}
		if err := d.Set("validation_emails", emailValidationOptions); err != nil {
			return resource.NonRetryableError(err)
		}
		d.Set("validation_method", resourceAwsAcmCertificateGuessValidationMethod(domainValidationOptions, emailValidationOptions))

		params := &acm.ListTagsForCertificateInput{
			CertificateArn: aws.String(d.Id()),
		}

		tagResp, err := acmconn.ListTagsForCertificate(params)
		if err := d.Set("tags", tagsToMapACM(tagResp.Tags)); err != nil {
			return resource.NonRetryableError(err)
		}

		return nil
	})
}
func resourceAwsAcmCertificateGuessValidationMethod(domainValidationOptions []map[string]interface{}, emailValidationOptions []string) string {
	// The DescribeCertificate Response doesn't have information on what validation method was used
	// so we need to guess from the validation options we see...
	if len(domainValidationOptions) > 0 {
		return acm.ValidationMethodDns
	} else if len(emailValidationOptions) > 0 {
		return acm.ValidationMethodEmail
	} else {
		return "NONE"
	}
}

func resourceAwsAcmCertificateUpdate(d *schema.ResourceData, meta interface{}) error {
	if d.HasChange("tags") {
		acmconn := meta.(*AWSClient).acmconn
		err := setTagsACM(acmconn, d)
		if err != nil {
			return err
		}
	}
	return nil
}

func cleanUpSubjectAlternativeNames(cert *acm.CertificateDetail) []string {
	sans := cert.SubjectAlternativeNames
	vs := make([]string, 0, len(sans)-1)
	for _, v := range sans {
		if *v != *cert.DomainName {
			vs = append(vs, *v)
		}
	}
	return vs

}

func convertValidationOptions(certificate *acm.CertificateDetail) ([]map[string]interface{}, []string, error) {
	var domainValidationResult []map[string]interface{}
	var emailValidationResult []string

	if *certificate.Type == acm.CertificateTypeAmazonIssued {
		for _, o := range certificate.DomainValidationOptions {
			if o.ResourceRecord != nil {
				validationOption := map[string]interface{}{
					"domain_name":           *o.DomainName,
					"resource_record_name":  *o.ResourceRecord.Name,
					"resource_record_type":  *o.ResourceRecord.Type,
					"resource_record_value": *o.ResourceRecord.Value,
				}
				domainValidationResult = append(domainValidationResult, validationOption)
			} else if o.ValidationEmails != nil && len(o.ValidationEmails) > 0 {
				for _, validationEmail := range o.ValidationEmails {
					emailValidationResult = append(emailValidationResult, *validationEmail)
				}
			} else if *o.ValidationStatus == acm.DomainStatusPendingValidation {
				log.Printf("[DEBUG] No validation options need to retry: %#v", o)
				return nil, nil, fmt.Errorf("No validation options need to retry: %#v", o)
			}
		}
	}

	return domainValidationResult, emailValidationResult, nil
}

func resourceAwsAcmCertificateDelete(d *schema.ResourceData, meta interface{}) error {
	acmconn := meta.(*AWSClient).acmconn

	log.Printf("[INFO] Deleting ACM Certificate: %s", d.Id())

	params := &acm.DeleteCertificateInput{
		CertificateArn: aws.String(d.Id()),
	}

	err := resource.Retry(10*time.Minute, func() *resource.RetryError {
		_, err := acmconn.DeleteCertificate(params)
		if err != nil {
			if isAWSErr(err, acm.ErrCodeResourceInUseException, "") {
				log.Printf("[WARN] Conflict deleting certificate in use: %s, retrying", err.Error())
				return resource.RetryableError(err)
			}
			return resource.NonRetryableError(err)
		}
		return nil
	})

	if err != nil && !isAWSErr(err, acm.ErrCodeResourceNotFoundException, "") {
		return fmt.Errorf("Error deleting certificate: %s", err)
	}

	return nil
}
<|endoftext|>


<commit_before>package notifiers

import (
	"time"

	"github.com/grafana/grafana/pkg/bus"
	"github.com/grafana/grafana/pkg/components/simplejson"
	"github.com/grafana/grafana/pkg/log"
	m "github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/services/alerting"
)

func init() {
	alerting.RegisterNotifier(&alerting.NotifierPlugin{
		Type:        "alertmanager",
		Name:        "alertmanager",
		Description: "Sends alert to Alertmanager",
		Factory:     NewAlertmanagerNotifier,
		OptionsTemplate: `

Alertmanager settings

        Url
        <input type="text" required class="gf-form-input max-width-26" ng-model="ctrl.model.settings.url" placeholder="http://localhost:9093"></input>

    `,
	})
}

func NewAlertmanagerNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
	url := model.Settings.Get("url").MustString()
	if url == "" {
		return nil, alerting.ValidationError{Reason: "Could not find url property in settings"}
	}

	return &AlertmanagerNotifier{
		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings),
		Url:          url,
		log:          log.New("alerting.notifier.alertmanager"),
	}, nil
}

type AlertmanagerNotifier struct {
	NotifierBase
	Url string
	log log.Logger
}

func (this *AlertmanagerNotifier) ShouldNotify(evalContext *alerting.EvalContext) bool {
	if evalContext.Rule.State == m.AlertStateAlerting {
		return true
	}
	return false
}

func (this *AlertmanagerNotifier) Notify(evalContext *alerting.EvalContext) error {
	this.log.Info("Sending alertmanager")

	alerts := make([]interface{}, 0)
	for _, match := range evalContext.EvalMatches {
		alertJSON := simplejson.New()
		alertJSON.Set("startsAt", evalContext.StartTime.UTC().Format(time.RFC3339))
		// Rule state should always be alerting if notifying.
		alertJSON.Set("endsAt", "0001-01-01T00:00:00Z")

		ruleUrl, err := evalContext.GetRuleUrl()
		if err == nil {
			alertJSON.Set("generatorURL", ruleUrl)
		}

		if evalContext.Rule.Message != "" {
			alertJSON.SetPath([]string{"annotations", "description"}, evalContext.Rule.Message)
		}

		tags := make(map[string]string)
		for k, v := range match.Tags {
			tags[k] = v
		}
		tags["alertname"] = evalContext.Rule.Name
		alertJSON.Set("labels", tags)

		alerts = append(alerts, alertJSON)
	}

	bodyJSON := simplejson.NewFromAny(alerts)
	body, _ := bodyJSON.MarshalJSON()

	cmd := &m.SendWebhookSync{
		Url:        this.Url + "/api/v1/alerts",
		HttpMethod: "POST",
		Body:       string(body),
	}

	if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil {
		this.log.Error("Failed to send alertmanager", "error", err, "alertmanager", this.Name)
		return err
	}

	return nil
}
<commit_msg>Alertmanager notifier: add "metric" labels if no tags<commit_after>package notifiers

import (
	"time"

	"github.com/grafana/grafana/pkg/bus"
	"github.com/grafana/grafana/pkg/components/simplejson"
	"github.com/grafana/grafana/pkg/log"
	m "github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/services/alerting"
)

func init() {
	alerting.RegisterNotifier(&alerting.NotifierPlugin{
		Type:        "alertmanager",
		Name:        "alertmanager",
		Description: "Sends alert to Alertmanager",
		Factory:     NewAlertmanagerNotifier,
		OptionsTemplate: `

Alertmanager settings

        Url
        <input type="text" required class="gf-form-input max-width-26" ng-model="ctrl.model.settings.url" placeholder="http://localhost:9093"></input>

    `,
	})
}

func NewAlertmanagerNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
	url := model.Settings.Get("url").MustString()
	if url == "" {
		return nil, alerting.ValidationError{Reason: "Could not find url property in settings"}
	}

	return &AlertmanagerNotifier{
		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings),
		Url:          url,
		log:          log.New("alerting.notifier.alertmanager"),
	}, nil
}

type AlertmanagerNotifier struct {
	NotifierBase
	Url string
	log log.Logger
}

func (this *AlertmanagerNotifier) ShouldNotify(evalContext *alerting.EvalContext) bool {
	if evalContext.Rule.State == m.AlertStateAlerting {
		return true
	}
	return false
}

func (this *AlertmanagerNotifier) Notify(evalContext *alerting.EvalContext) error {
	this.log.Info("Sending alertmanager")

	alerts := make([]interface{}, 0)
	for _, match := range evalContext.EvalMatches {
		alertJSON := simplejson.New()
		alertJSON.Set("startsAt", evalContext.StartTime.UTC().Format(time.RFC3339))
		// Rule state should always be alerting if notifying.
		alertJSON.Set("endsAt", "0001-01-01T00:00:00Z")

		ruleUrl, err := evalContext.GetRuleUrl()
		if err == nil {
			alertJSON.Set("generatorURL", ruleUrl)
		}

		if evalContext.Rule.Message != "" {
			alertJSON.SetPath([]string{"annotations", "description"}, evalContext.Rule.Message)
		}

		tags := make(map[string]string)
		if len(match.Tags) == 0 {
			tags["metric"] = match.Metric
		} else {
			for k, v := range match.Tags {
				tags[k] = v
			}
		}
		tags["alertname"] = evalContext.Rule.Name
		alertJSON.Set("labels", tags)

		alerts = append(alerts, alertJSON)
	}

	bodyJSON := simplejson.NewFromAny(alerts)
	body, _ := bodyJSON.MarshalJSON()

	cmd := &m.SendWebhookSync{
		Url:        this.Url + "/api/v1/alerts",
		HttpMethod: "POST",
		Body:       string(body),
	}

	if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil {
		this.log.Error("Failed to send alertmanager", "error", err, "alertmanager", this.Name)
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package shell

import (
	"context"
	"sync/atomic"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/wdclient"
)

const (
	RenewInteval     = 4 * time.Second
	SafeRenewInteval = 3 * time.Second
	InitLockInteval  = 1 * time.Second
)

type ExclusiveLocker struct {
	masterClient *wdclient.MasterClient
	token        int64
	lockTsNs     int64
	isLocking    bool
}

func NewExclusiveLocker(masterClient *wdclient.MasterClient) *ExclusiveLocker {
	return &ExclusiveLocker{
		masterClient: masterClient,
	}
}

func (l *ExclusiveLocker) GetToken() (token int64, lockTsNs int64) {
	for time.Unix(0, atomic.LoadInt64(&l.lockTsNs)).Add(SafeRenewInteval).Before(time.Now()) {
		// wait until now is within the safe lock period, no immediate renewal to change the token
		time.Sleep(100 * time.Millisecond)
	}
	return atomic.LoadInt64(&l.token), atomic.LoadInt64(&l.lockTsNs)
}

func (l *ExclusiveLocker) RequestLock() {
	// retry to get the lease
	for {
		if err := l.masterClient.WithClient(func(client master_pb.SeaweedClient) error {
			resp, err := client.LeaseAdminToken(context.Background(), &master_pb.LeaseAdminTokenRequest{
				PreviousToken:    atomic.LoadInt64(&l.token),
				PreviousLockTime: atomic.LoadInt64(&l.lockTsNs),
			})
			if err == nil {
				atomic.StoreInt64(&l.token, resp.Token)
				atomic.StoreInt64(&l.lockTsNs, resp.LockTsNs)
			}
			return err
		}); err != nil {
			// println("leasing problem", err.Error())
			time.Sleep(InitLockInteval)
		} else {
			break
		}
	}

	l.isLocking = true

	// start a goroutine to renew the lease
	go func() {
		for l.isLocking {
			if err := l.masterClient.WithClient(func(client master_pb.SeaweedClient) error {
				resp, err := client.LeaseAdminToken(context.Background(), &master_pb.LeaseAdminTokenRequest{
					PreviousToken:    atomic.LoadInt64(&l.token),
					PreviousLockTime: atomic.LoadInt64(&l.lockTsNs),
				})
				if err == nil {
					atomic.StoreInt64(&l.token, resp.Token)
					atomic.StoreInt64(&l.lockTsNs, resp.LockTsNs)
					// println("ts", l.lockTsNs, "token", l.token)
				}
				return err
			}); err != nil {
				glog.Error("failed to renew lock: %v", err)
				return
			} else {
				time.Sleep(RenewInteval)
			}

		}
	}()

}

func (l *ExclusiveLocker) ReleaseLock() {
	l.isLocking = false
	l.masterClient.WithClient(func(client master_pb.SeaweedClient) error {
		client.ReleaseAdminToken(context.Background(), &master_pb.ReleaseAdminTokenRequest{
			PreviousToken:    atomic.LoadInt64(&l.token),
			PreviousLockTime: atomic.LoadInt64(&l.lockTsNs),
		})
		return nil
	})
	atomic.StoreInt64(&l.token, 0)
	atomic.StoreInt64(&l.lockTsNs, 0)
}
<commit_msg>fix builds<commit_after>package shell

import (
	"context"
	"sync/atomic"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/wdclient"
)

const (
	RenewInteval     = 4 * time.Second
	SafeRenewInteval = 3 * time.Second
	InitLockInteval  = 1 * time.Second
)

type ExclusiveLocker struct {
	masterClient *wdclient.MasterClient
	token        int64
	lockTsNs     int64
	isLocking    bool
}

func NewExclusiveLocker(masterClient *wdclient.MasterClient) *ExclusiveLocker {
	return &ExclusiveLocker{
		masterClient: masterClient,
	}
}

func (l *ExclusiveLocker) GetToken() (token int64, lockTsNs int64) {
	for time.Unix(0, atomic.LoadInt64(&l.lockTsNs)).Add(SafeRenewInteval).Before(time.Now()) {
		// wait until now is within the safe lock period, no immediate renewal to change the token
		time.Sleep(100 * time.Millisecond)
	}
	return atomic.LoadInt64(&l.token), atomic.LoadInt64(&l.lockTsNs)
}

func (l *ExclusiveLocker) RequestLock() {
	// retry to get the lease
	for {
		if err := l.masterClient.WithClient(func(client master_pb.SeaweedClient) error {
			resp, err := client.LeaseAdminToken(context.Background(), &master_pb.LeaseAdminTokenRequest{
				PreviousToken:    atomic.LoadInt64(&l.token),
				PreviousLockTime: atomic.LoadInt64(&l.lockTsNs),
			})
			if err == nil {
				atomic.StoreInt64(&l.token, resp.Token)
				atomic.StoreInt64(&l.lockTsNs, resp.LockTsNs)
			}
			return err
		}); err != nil {
			// println("leasing problem", err.Error())
			time.Sleep(InitLockInteval)
		} else {
			break
		}
	}

	l.isLocking = true

	// start a goroutine to renew the lease
	go func() {
		for l.isLocking {
			if err := l.masterClient.WithClient(func(client master_pb.SeaweedClient) error {
				resp, err := client.LeaseAdminToken(context.Background(), &master_pb.LeaseAdminTokenRequest{
					PreviousToken:    atomic.LoadInt64(&l.token),
					PreviousLockTime: atomic.LoadInt64(&l.lockTsNs),
				})
				if err == nil {
					atomic.StoreInt64(&l.token, resp.Token)
					atomic.StoreInt64(&l.lockTsNs, resp.LockTsNs)
					// println("ts", l.lockTsNs, "token", l.token)
				}
				return err
			}); err != nil {
				glog.Errorf("failed to renew lock: %v", err)
				return
			} else {
				time.Sleep(RenewInteval)
			}

		}
	}()

}

func (l *ExclusiveLocker) ReleaseLock() {
	l.isLocking = false
	l.masterClient.WithClient(func(client master_pb.SeaweedClient) error {
		client.ReleaseAdminToken(context.Background(), &master_pb.ReleaseAdminTokenRequest{
			PreviousToken:    atomic.LoadInt64(&l.token),
			PreviousLockTime: atomic.LoadInt64(&l.lockTsNs),
		})
		return nil
	})
	atomic.StoreInt64(&l.token, 0)
	atomic.StoreInt64(&l.lockTsNs, 0)
}
<|endoftext|>


<commit_before>// Copyright 2015-2020 Bret Jordan, All rights reserved.
//
// Use of this source code is governed by an Apache 2.0 license that can be
// found in the LICENSE file in the root of the source tree.

package collections

import (
	"github.com/freetaxii/libstix2/objects/envelope"
	"github.com/freetaxii/libstix2/objects/manifest"
	"github.com/freetaxii/libstix2/objects/properties"
	"github.com/freetaxii/libstix2/objects/versions"
)

// ----------------------------------------------------------------------
// Define Message Type
// ----------------------------------------------------------------------

/*
Collections - This type implements the TAXII 2 Collections Resource and defines
all of the properties and methods needed to create and work with the TAXII Collections
Resource. All of the methods not defined local to this type are inherited from
the individual properties.

The following information comes directly from the TAXII 2 specification documents.

This Endpoint provides information about the Collections hosted under this API
Root. This is similar to the response to get a Collection (see section 5.2), but
rather than providing information about one Collection it provides information
about all of the Collections. Most importantly, it provides the Collection's id,
which is used to request objects or manifest entries from the Collection.

The collections resource is a simple wrapper around a list of collection
resources.
*/
type Collections struct {
	Collections []Collection `json:"collections,omitempty"`
}

/*
Collection - This type implements the TAXII 2 Collection Resource and defines
all of the properties and methods needed to create and work with the TAXII
Collection Resource. All of the methods not defined local to this type are
inherited from the individual properties.

DatastoreID = A unique integer that represents this collection
DateAdded   = The date that this collection was added to the system
Enabled     = Is this collection currently enabled
Hidden      = Is this collection currently hidden for the directory listing
Size        = The current size of the collection
ID 		    = The collection ID, a UUIDv4 value
Title 	    = The title of this collection
Description = A long description about this collection
CanRead     = A boolean flag that indicates if one can read from this collection
CanWrite    = A boolean flag that indicates if one can write to this collection
MediaTypes  = A slice of strings of the media types that are found in this collection

The following information comes directly from the TAXII 2 specification documents.

This Endpoint provides general information about a Collection, which can be used
to help users and clients decide whether and how they want to interact with it.
For example, it will tell clients what it's called and what permissions they
have to it.

The collection resource contains general information about a Collection, such as
its id, a human-readable title and description, an optional list of supported
media_types (representing the media type of objects can be requested from or
added to it), and whether the TAXII Client, as authenticated, can get objects
from the Collection and/or add objects to it.
*/
type Collection struct {
	DatastoreID int    `json:"-"`
	DateAdded   string `json:"-"`
	Enabled     bool   `json:"-"`
	Hidden      bool   `json:"-"`
	Size        int    `json:"-"`
	properties.IDProperty
	properties.TitleProperty
	properties.DescriptionProperty
	CanRead    bool     `json:"can_read"`
	CanWrite   bool     `json:"can_write"`
	MediaTypes []string `json:"media_types,omitempty"`
}

/*
CollectionQuery - This struct will hold all of the variables that a user can
use to query a collection.
*/
type CollectionQuery struct {
	CollectionUUID        string
	CollectionDatastoreID int
	STIXID                []string // Passed in from the URL
	STIXType              []string // Passed in from the URL
	STIXVersion           []string // Passed in from the URL
	AddedAfter            []string // Passed in from the URL
	AddedBefore           []string // Passed in from the URL
	Limit                 []string // Passed in from the URL
	SpecVersion           []string // Passed in from the URL
	ServerRecordLimit     int      // Server defined value in the configuration file
}

/*
CollectionQueryResult - This struct contains the various bits of meta data
that are returned from a query against a collection on a TAXII server. This is
done so that the method signatures do not need to change as time goes on and we
add more meta data that needs to be returned. It is important to note that a
collection may have more entries than the server or client wants to transmit. So
it is important to keep track of which records are actually being delivered to
the client.

Size           = The total size of the dataset returned from the database query.
DateAddedFirst = The added date of the first record being sent to the client.
DateAddedLast  = The added date of the last record being sent to the client.
BundleData     = The STIX bundle that contains the requested data from the collection.
ManifestData   = The TAXII manifest resource that contains the requested data from the collection.
RangeBegin     = The range value of the first record being sent to the client.
RangeEnd       = The range value of the last record being sent to the client.
*/
type CollectionQueryResult struct {
	Size           int
	DateAddedFirst string
	DateAddedLast  string
	ObjectData     envelope.Envelope
	VersionsData   versions.Versions
	ManifestData   manifest.Manifest
	// RangeBegin     int
	// RangeEnd       int
}

// ----------------------------------------------------------------------
// Initialization Functions
// ----------------------------------------------------------------------

/*
New - This function will create a new TAXII Collections object and return
it as a pointer.
*/
func New() *Collections {
	var obj Collections
	return &obj
}

/*
NewCollection - This function will create a new TAXII Collection object and return
it as a pointer.
*/
func NewCollection() *Collection {
	var obj Collection
	return &obj
}

/*
NewCollectionQuery - This function will take in a collection ID as a string
and the Server Record Limit and return a CollectionQueryType object.
*/
func NewCollectionQuery(id string, limit int) *CollectionQuery {
	var obj CollectionQuery
	obj.CollectionUUID = id
	obj.ServerRecordLimit = limit
	return &obj
}

// ----------------------------------------------------------------------
// Public Methods - Collections
// ----------------------------------------------------------------------

/*
AddCollection - This method takes in an object that represents a collection
and adds it to the list in the collections property and returns an integer of
the location in the slice where the collection object was added. This method
would be used if the collection was created separately and it just needs to be
added in whole to the collections list.
*/
func (o *Collections) AddCollection(c *Collection) (int, error) {
	//o.initCollectionsProperty()
	positionThatAppendWillUse := len(o.Collections)
	o.Collections = append(o.Collections, *c)
	return positionThatAppendWillUse, nil
}

/*
NewCollection - This method is used to create a collection and automatically
add it to the collections array. It returns a resources.Collection which
is a pointer to the actual Collection that was created in the collections
slice.
*/
func (o *Collections) NewCollection() (*Collection, error) {
	//o.initCollectionsProperty()
	c := NewCollection()
	positionThatAppendWillUse := len(o.Collections)
	o.Collections = append(o.Collections, *c)
	return &o.Collections[positionThatAppendWillUse], nil
}

// ----------------------------------------------------------------------
// Private Methods - Collections
// ----------------------------------------------------------------------

/*
initCollectionsProperty - This method will initialize the Collections
slice if it has not already been initialized.
*/
// func (o *Collections) initCollectionsProperty() error {
// 	if o.Collections == nil {
// 		a := make([]Collection, 0)
// 		o.Collections = a
// 	}
// 	return nil
// }

// ----------------------------------------------------------------------
// Public Methods - Collection
// ----------------------------------------------------------------------

/*
SetEnabled - This method will set the collection to be enabled.
*/
func (o *Collection) SetEnabled() error {
	o.Enabled = true
	return nil
}

/*
SetDisabled - This method will set the collection to be disabled.
*/
func (o *Collection) SetDisabled() error {
	o.Enabled = false
	return nil
}

/*
SetHidden - This method will set the collection to be hidden.
*/
func (o *Collection) SetHidden() error {
	o.Hidden = true
	return nil
}

/*
SetVisible - This method will set the collection to be visible.
*/
func (o *Collection) SetVisible() error {
	o.Hidden = false
	return nil
}

/*
SetCanRead - This method will set the can_read boolean to true.
*/
func (o *Collection) SetCanRead() error {
	o.CanRead = true
	return nil
}

/*
GetCanRead - This method will return the value of Can Read.
*/
func (o *Collection) GetCanRead() bool {
	return o.CanRead
}

/*
SetCanWrite - This method will set the can_write boolean to true.
*/
func (o *Collection) SetCanWrite() error {
	o.CanWrite = true
	return nil
}

/*
GetCanWrite - This method will return the value of Can Write.
*/
func (o *Collection) GetCanWrite() bool {
	return o.CanWrite
}

/*
AddMediaType - This method takes in a string value that represents a version
of the TAXII api that is supported and adds it to the list in media types
property.
*/
func (o *Collection) AddMediaType(s string) error {
	if o.MediaTypes == nil {
		a := make([]string, 0)
		o.MediaTypes = a
	}
	o.MediaTypes = append(o.MediaTypes, s)
	return nil
}
<commit_msg>fix import paths<commit_after>// Copyright 2015-2022 Bret Jordan, All rights reserved.
//
// Use of this source code is governed by an Apache 2.0 license that can be
// found in the LICENSE file in the root of the source tree.

package collections

import (
	"github.com/freetaxii/libstix2/objects/properties"
	"github.com/freetaxii/libstix2/objects/taxii/envelope"
	"github.com/freetaxii/libstix2/objects/taxii/manifest"
	"github.com/freetaxii/libstix2/objects/taxii/versions"
)

// ----------------------------------------------------------------------
// Define Message Type
// ----------------------------------------------------------------------

/*
Collections - This type implements the TAXII 2 Collections Resource and defines
all of the properties and methods needed to create and work with the TAXII Collections
Resource. All of the methods not defined local to this type are inherited from
the individual properties.

The following information comes directly from the TAXII 2 specification documents.

This Endpoint provides information about the Collections hosted under this API
Root. This is similar to the response to get a Collection (see section 5.2), but
rather than providing information about one Collection it provides information
about all of the Collections. Most importantly, it provides the Collection's id,
which is used to request objects or manifest entries from the Collection.

The collections resource is a simple wrapper around a list of collection
resources.
*/
type Collections struct {
	Collections []Collection `json:"collections,omitempty"`
}

/*
Collection - This type implements the TAXII 2 Collection Resource and defines
all of the properties and methods needed to create and work with the TAXII
Collection Resource. All of the methods not defined local to this type are
inherited from the individual properties.

DatastoreID = A unique integer that represents this collection
DateAdded   = The date that this collection was added to the system
Enabled     = Is this collection currently enabled
Hidden      = Is this collection currently hidden for the directory listing
Size        = The current size of the collection
ID 		    = The collection ID, a UUIDv4 value
Title 	    = The title of this collection
Description = A long description about this collection
CanRead     = A boolean flag that indicates if one can read from this collection
CanWrite    = A boolean flag that indicates if one can write to this collection
MediaTypes  = A slice of strings of the media types that are found in this collection

The following information comes directly from the TAXII 2 specification documents.

This Endpoint provides general information about a Collection, which can be used
to help users and clients decide whether and how they want to interact with it.
For example, it will tell clients what it's called and what permissions they
have to it.

The collection resource contains general information about a Collection, such as
its id, a human-readable title and description, an optional list of supported
media_types (representing the media type of objects can be requested from or
added to it), and whether the TAXII Client, as authenticated, can get objects
from the Collection and/or add objects to it.
*/
type Collection struct {
	DatastoreID int    `json:"-"`
	DateAdded   string `json:"-"`
	Enabled     bool   `json:"-"`
	Hidden      bool   `json:"-"`
	Size        int    `json:"-"`
	properties.IDProperty
	properties.TitleProperty
	properties.DescriptionProperty
	CanRead    bool     `json:"can_read"`
	CanWrite   bool     `json:"can_write"`
	MediaTypes []string `json:"media_types,omitempty"`
}

/*
CollectionQuery - This struct will hold all of the variables that a user can
use to query a collection.
*/
type CollectionQuery struct {
	CollectionUUID        string
	CollectionDatastoreID int
	STIXID                []string // Passed in from the URL
	STIXType              []string // Passed in from the URL
	STIXVersion           []string // Passed in from the URL
	AddedAfter            []string // Passed in from the URL
	AddedBefore           []string // Passed in from the URL
	Limit                 []string // Passed in from the URL
	SpecVersion           []string // Passed in from the URL
	ServerRecordLimit     int      // Server defined value in the configuration file
}

/*
CollectionQueryResult - This struct contains the various bits of meta data
that are returned from a query against a collection on a TAXII server. This is
done so that the method signatures do not need to change as time goes on and we
add more meta data that needs to be returned. It is important to note that a
collection may have more entries than the server or client wants to transmit. So
it is important to keep track of which records are actually being delivered to
the client.

Size           = The total size of the dataset returned from the database query.
DateAddedFirst = The added date of the first record being sent to the client.
DateAddedLast  = The added date of the last record being sent to the client.
BundleData     = The STIX bundle that contains the requested data from the collection.
ManifestData   = The TAXII manifest resource that contains the requested data from the collection.
RangeBegin     = The range value of the first record being sent to the client.
RangeEnd       = The range value of the last record being sent to the client.
*/
type CollectionQueryResult struct {
	Size           int
	DateAddedFirst string
	DateAddedLast  string
	ObjectData     envelope.Envelope
	VersionsData   versions.Versions
	ManifestData   manifest.Manifest
	// RangeBegin     int
	// RangeEnd       int
}

// ----------------------------------------------------------------------
// Initialization Functions
// ----------------------------------------------------------------------

/*
New - This function will create a new TAXII Collections object and return
it as a pointer.
*/
func New() *Collections {
	var obj Collections
	return &obj
}

/*
NewCollection - This function will create a new TAXII Collection object and return
it as a pointer.
*/
func NewCollection() *Collection {
	var obj Collection
	return &obj
}

/*
NewCollectionQuery - This function will take in a collection ID as a string
and the Server Record Limit and return a CollectionQueryType object.
*/
func NewCollectionQuery(id string, limit int) *CollectionQuery {
	var obj CollectionQuery
	obj.CollectionUUID = id
	obj.ServerRecordLimit = limit
	return &obj
}

// ----------------------------------------------------------------------
// Public Methods - Collections
// ----------------------------------------------------------------------

/*
AddCollection - This method takes in an object that represents a collection
and adds it to the list in the collections property and returns an integer of
the location in the slice where the collection object was added. This method
would be used if the collection was created separately and it just needs to be
added in whole to the collections list.
*/
func (o *Collections) AddCollection(c *Collection) (int, error) {
	//o.initCollectionsProperty()
	positionThatAppendWillUse := len(o.Collections)
	o.Collections = append(o.Collections, *c)
	return positionThatAppendWillUse, nil
}

/*
NewCollection - This method is used to create a collection and automatically
add it to the collections array. It returns a resources.Collection which
is a pointer to the actual Collection that was created in the collections
slice.
*/
func (o *Collections) NewCollection() (*Collection, error) {
	//o.initCollectionsProperty()
	c := NewCollection()
	positionThatAppendWillUse := len(o.Collections)
	o.Collections = append(o.Collections, *c)
	return &o.Collections[positionThatAppendWillUse], nil
}

// ----------------------------------------------------------------------
// Private Methods - Collections
// ----------------------------------------------------------------------

/*
initCollectionsProperty - This method will initialize the Collections
slice if it has not already been initialized.
*/
// func (o *Collections) initCollectionsProperty() error {
// 	if o.Collections == nil {
// 		a := make([]Collection, 0)
// 		o.Collections = a
// 	}
// 	return nil
// }

// ----------------------------------------------------------------------
// Public Methods - Collection
// ----------------------------------------------------------------------

/*
SetEnabled - This method will set the collection to be enabled.
*/
func (o *Collection) SetEnabled() error {
	o.Enabled = true
	return nil
}

/*
SetDisabled - This method will set the collection to be disabled.
*/
func (o *Collection) SetDisabled() error {
	o.Enabled = false
	return nil
}

/*
SetHidden - This method will set the collection to be hidden.
*/
func (o *Collection) SetHidden() error {
	o.Hidden = true
	return nil
}

/*
SetVisible - This method will set the collection to be visible.
*/
func (o *Collection) SetVisible() error {
	o.Hidden = false
	return nil
}

/*
SetCanRead - This method will set the can_read boolean to true.
*/
func (o *Collection) SetCanRead() error {
	o.CanRead = true
	return nil
}

/*
GetCanRead - This method will return the value of Can Read.
*/
func (o *Collection) GetCanRead() bool {
	return o.CanRead
}

/*
SetCanWrite - This method will set the can_write boolean to true.
*/
func (o *Collection) SetCanWrite() error {
	o.CanWrite = true
	return nil
}

/*
GetCanWrite - This method will return the value of Can Write.
*/
func (o *Collection) GetCanWrite() bool {
	return o.CanWrite
}

/*
AddMediaType - This method takes in a string value that represents a version
of the TAXII api that is supported and adds it to the list in media types
property.
*/
func (o *Collection) AddMediaType(s string) error {
	if o.MediaTypes == nil {
		a := make([]string, 0)
		o.MediaTypes = a
	}
	o.MediaTypes = append(o.MediaTypes, s)
	return nil
}
<|endoftext|>


<commit_before>package scheduler

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"reflect"
	"strconv"
	"sync"

	"github.com/chrislusf/glow/driver/plan"
	"github.com/chrislusf/glow/driver/scheduler/market"
	"github.com/chrislusf/glow/flow"
	"github.com/chrislusf/glow/netchan"
	"github.com/chrislusf/glow/resource"
)

type SubmitTaskGroup struct {
	FlowContext *flow.FlowContext
	TaskGroup   *plan.TaskGroup
	Bid         float64
	WaitGroup   *sync.WaitGroup
}

type ReleaseTaskGroupInputs struct {
	FlowContext *flow.FlowContext
	TaskGroups  []*plan.TaskGroup
	WaitGroup   *sync.WaitGroup
}

/*
resources are leased to driver, expires every X miniute unless renewed.
1. request resource
2. release resource
*/
func (s *Scheduler) EventLoop() {
	for {
		event := <-s.EventChan
		switch event := event.(type) {
		default:
		case SubmitTaskGroup:
			// fmt.Printf("processing %+v\n", event)
			taskGroup := event.TaskGroup
			pickedServerChan := make(chan market.Supply, 1)
			go func() {
				defer event.WaitGroup.Done()
				tasks := event.TaskGroup.Tasks

				// wait until inputs are registed
				s.shardLocator.waitForInputDatasetShardLocations(tasks[0])
				// fmt.Printf("inputs of %s is %s\n", tasks[0].Name(), s.allInputLocations(tasks[0]))

				s.Market.AddDemand(market.Requirement(taskGroup), event.Bid, pickedServerChan)

				// get assigned executor location
				supply := <-pickedServerChan
				allocation := supply.Object.(resource.Allocation)
				defer s.Market.ReturnSupply(supply)

				s.setupInputChannels(event.FlowContext, tasks[0], allocation.Location, event.WaitGroup)

				for _, shard := range tasks[len(tasks)-1].Outputs {
					s.shardLocator.SetShardLocation(shard.Name(), allocation.Location)
				}
				s.setupOutputChannels(tasks[len(tasks)-1].Outputs, event.WaitGroup)

				// fmt.Printf("allocated %s on %v\n", tasks[0].Name(), allocation.Location)
				// create reqeust
				args := []string{
					"-glow.flow.id",
					strconv.Itoa(event.FlowContext.Id),
					"-glow.taskGroup.id",
					strconv.Itoa(taskGroup.Id),
					"-glow.task.name",
					tasks[0].Name(),
					"-glow.agent.port",
					strconv.Itoa(allocation.Location.Port),
					"-glow.taskGroup.inputs",
					s.shardLocator.allInputLocations(tasks[0]),
				}
				for _, arg := range os.Args[1:] {
					args = append(args, arg)
				}
				request := NewStartRequest(
					filepath.Join(".", filepath.Base(os.Args[0])),
					s.option.Module,
					args,
					allocation.Allocated,
					os.Environ(),
					int32(s.option.DriverPort),
				)

				// fmt.Printf("starting on %s: %v\n", allocation.Allocated, request)
				if err := RemoteDirectExecute(allocation.Location.URL(), request); err != nil {
					log.Printf("exeuction error %v: %v", err, request)
				}
			}()
		case ReleaseTaskGroupInputs:
			go func() {
				defer event.WaitGroup.Done()

				for _, taskGroup := range event.TaskGroups {
					tasks := taskGroup.Tasks
					for _, ds := range tasks[len(tasks)-1].Outputs {
						location, _ := s.shardLocator.GetShardLocation(ds.Name())
						request := NewDeleteDatasetShardRequest(ds.Name())
						// println("deleting", ds.Name(), "on", location.URL())
						if err := RemoteDirectExecute(location.URL(), request); err != nil {
							println("exeuction error:", err.Error())
						}
					}
				}

			}()
		}
	}
}

func (s *Scheduler) setupInputChannels(fc *flow.FlowContext, task *flow.Task, location resource.Location, waitGroup *sync.WaitGroup) {
	if len(task.Inputs) > 0 {
		return
	}
	ds := task.Outputs[0].Parent
	if len(ds.ExternalInputChans) == 0 {
		return
	}
	// connect local typed chan to remote raw chan
	// write to the dataset location in the cluster so that the task can be retried if needed.
	for i, inChan := range ds.ExternalInputChans {
		inputChanName := fmt.Sprintf("ct-%d-input-%d-p-%d", fc.Id, ds.Id, i)
		// println("setup input channel for", task.Name(), "on", location.URL())
		s.shardLocator.SetShardLocation(inputChanName, location)
		rawChan, err := netchan.GetDirectSendChannel(inputChanName, location.URL(), waitGroup)
		if err != nil {
			log.Panic(err)
		}
		// println("writing", inputChanName, "to", location.URL())
		netchan.ConnectTypedWriteChannelToRaw(inChan, rawChan, waitGroup)
	}
}

func (s *Scheduler) setupOutputChannels(shards []*flow.DatasetShard, waitGroup *sync.WaitGroup) {
	for _, shard := range shards {
		ds := shard.Parent
		if len(ds.ExternalOutputChans) == 0 {
			continue
		}
		// connect remote raw chan to local typed chan
		readChanName := shard.Name()
		location, _ := s.shardLocator.GetShardLocation(readChanName)
		rawChan, err := netchan.GetDirectReadChannel(readChanName, location.URL(), 1024)
		if err != nil {
			log.Panic(err)
		}
		for _, out := range ds.ExternalOutputChans {
			ch := make(chan reflect.Value)
			netchan.ConnectRawReadChannelToTyped(rawChan, ch, ds.Type, waitGroup)
			waitGroup.Add(1)
			go func() {
				defer waitGroup.Done()
				for v := range ch {
					v = netchan.CleanObject(v, ds.Type, out.Type().Elem())
					out.Send(v)
				}
			}()
		}
	}
}
<commit_msg>revert executable file path<commit_after>package scheduler

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"reflect"
	"strconv"
	"sync"

	"github.com/chrislusf/glow/driver/plan"
	"github.com/chrislusf/glow/driver/scheduler/market"
	"github.com/chrislusf/glow/flow"
	"github.com/chrislusf/glow/netchan"
	"github.com/chrislusf/glow/resource"
)

type SubmitTaskGroup struct {
	FlowContext *flow.FlowContext
	TaskGroup   *plan.TaskGroup
	Bid         float64
	WaitGroup   *sync.WaitGroup
}

type ReleaseTaskGroupInputs struct {
	FlowContext *flow.FlowContext
	TaskGroups  []*plan.TaskGroup
	WaitGroup   *sync.WaitGroup
}

/*
resources are leased to driver, expires every X miniute unless renewed.
1. request resource
2. release resource
*/
func (s *Scheduler) EventLoop() {
	for {
		event := <-s.EventChan
		switch event := event.(type) {
		default:
		case SubmitTaskGroup:
			// fmt.Printf("processing %+v\n", event)
			taskGroup := event.TaskGroup
			pickedServerChan := make(chan market.Supply, 1)
			go func() {
				defer event.WaitGroup.Done()
				tasks := event.TaskGroup.Tasks

				// wait until inputs are registed
				s.shardLocator.waitForInputDatasetShardLocations(tasks[0])
				// fmt.Printf("inputs of %s is %s\n", tasks[0].Name(), s.allInputLocations(tasks[0]))

				s.Market.AddDemand(market.Requirement(taskGroup), event.Bid, pickedServerChan)

				// get assigned executor location
				supply := <-pickedServerChan
				allocation := supply.Object.(resource.Allocation)
				defer s.Market.ReturnSupply(supply)

				s.setupInputChannels(event.FlowContext, tasks[0], allocation.Location, event.WaitGroup)

				for _, shard := range tasks[len(tasks)-1].Outputs {
					s.shardLocator.SetShardLocation(shard.Name(), allocation.Location)
				}
				s.setupOutputChannels(tasks[len(tasks)-1].Outputs, event.WaitGroup)

				// fmt.Printf("allocated %s on %v\n", tasks[0].Name(), allocation.Location)
				// create reqeust
				args := []string{
					"-glow.flow.id",
					strconv.Itoa(event.FlowContext.Id),
					"-glow.taskGroup.id",
					strconv.Itoa(taskGroup.Id),
					"-glow.task.name",
					tasks[0].Name(),
					"-glow.agent.port",
					strconv.Itoa(allocation.Location.Port),
					"-glow.taskGroup.inputs",
					s.shardLocator.allInputLocations(tasks[0]),
				}
				for _, arg := range os.Args[1:] {
					args = append(args, arg)
				}
				request := NewStartRequest(
					"./"+filepath.Base(os.Args[0]),
					// filepath.Join(".", filepath.Base(os.Args[0])),
					s.option.Module,
					args,
					allocation.Allocated,
					os.Environ(),
					int32(s.option.DriverPort),
				)

				// fmt.Printf("starting on %s: %v\n", allocation.Allocated, request)
				if err := RemoteDirectExecute(allocation.Location.URL(), request); err != nil {
					log.Printf("exeuction error %v: %v", err, request)
				}
			}()
		case ReleaseTaskGroupInputs:
			go func() {
				defer event.WaitGroup.Done()

				for _, taskGroup := range event.TaskGroups {
					tasks := taskGroup.Tasks
					for _, ds := range tasks[len(tasks)-1].Outputs {
						location, _ := s.shardLocator.GetShardLocation(ds.Name())
						request := NewDeleteDatasetShardRequest(ds.Name())
						// println("deleting", ds.Name(), "on", location.URL())
						if err := RemoteDirectExecute(location.URL(), request); err != nil {
							println("exeuction error:", err.Error())
						}
					}
				}

			}()
		}
	}
}

func (s *Scheduler) setupInputChannels(fc *flow.FlowContext, task *flow.Task, location resource.Location, waitGroup *sync.WaitGroup) {
	if len(task.Inputs) > 0 {
		return
	}
	ds := task.Outputs[0].Parent
	if len(ds.ExternalInputChans) == 0 {
		return
	}
	// connect local typed chan to remote raw chan
	// write to the dataset location in the cluster so that the task can be retried if needed.
	for i, inChan := range ds.ExternalInputChans {
		inputChanName := fmt.Sprintf("ct-%d-input-%d-p-%d", fc.Id, ds.Id, i)
		// println("setup input channel for", task.Name(), "on", location.URL())
		s.shardLocator.SetShardLocation(inputChanName, location)
		rawChan, err := netchan.GetDirectSendChannel(inputChanName, location.URL(), waitGroup)
		if err != nil {
			log.Panic(err)
		}
		// println("writing", inputChanName, "to", location.URL())
		netchan.ConnectTypedWriteChannelToRaw(inChan, rawChan, waitGroup)
	}
}

func (s *Scheduler) setupOutputChannels(shards []*flow.DatasetShard, waitGroup *sync.WaitGroup) {
	for _, shard := range shards {
		ds := shard.Parent
		if len(ds.ExternalOutputChans) == 0 {
			continue
		}
		// connect remote raw chan to local typed chan
		readChanName := shard.Name()
		location, _ := s.shardLocator.GetShardLocation(readChanName)
		rawChan, err := netchan.GetDirectReadChannel(readChanName, location.URL(), 1024)
		if err != nil {
			log.Panic(err)
		}
		for _, out := range ds.ExternalOutputChans {
			ch := make(chan reflect.Value)
			netchan.ConnectRawReadChannelToTyped(rawChan, ch, ds.Type, waitGroup)
			waitGroup.Add(1)
			go func() {
				defer waitGroup.Done()
				for v := range ch {
					v = netchan.CleanObject(v, ds.Type, out.Type().Elem())
					out.Send(v)
				}
			}()
		}
	}
}
<|endoftext|>


<commit_before>package action

import (
	"fmt"

	"github.com/fatih/color"
	"github.com/justwatchcom/gopass/gpg"
	"github.com/urfave/cli"
)

// Initialized returns an error if the store is not properly
// prepared.
func (s *Action) Initialized(*cli.Context) error {
	if !s.Store.Initialized() {
		return fmt.Errorf("password-store is not initialized. Try '%s init'", s.Name)
	}
	return nil
}

// Init a new password store with a first gpg id
func (s *Action) Init(c *cli.Context) error {
	path := c.String("store")
	alias := c.String("alias")
	nogit := c.Bool("nogit")

	return s.init(alias, path, nogit, c.Args()...)
}

func (s *Action) init(alias, path string, nogit bool, keys ...string) error {
	if !hasConfig() {
		// when creating a new config we set some sensible defaults
		s.Store.AutoPush = true
		s.Store.AutoPull = true
		s.Store.AutoImport = false
		s.Store.NoConfirm = false
		s.Store.PersistKeys = true
		s.Store.LoadKeys = false
		s.Store.ClipTimeout = 45
		s.Store.SafeContent = false
	}
	if path == "" {
		path = s.Store.Path
	}

	if len(keys) < 1 {
		nk, err := askForPrivateKey(color.CyanString("Please select a private key for encryption:"))
		if err != nil {
			return err
		}
		keys = []string{nk}
	}

	if err := s.Store.Init(alias, path, keys...); err != nil {
		return err
	}

	if !nogit {
		if err := s.gitInit(path, ""); err != nil {
			color.Yellow("Failed to init git: %s", err)
		}
	}

	if alias != "" && path != "" {
		if err := s.Store.AddMount(alias, path); err != nil {
			return err
		}
	}

	fmt.Fprint(color.Output, color.GreenString("Password store %s initialized for:\n", path))
	for _, recipient := range s.Store.ListRecipients(alias) {
		r := "0x" + recipient
		if kl, err := gpg.ListPublicKeys(recipient); err == nil && len(kl) > 0 {
			r = kl[0].OneLine()
		}
		color.Yellow("  " + r)
	}
	fmt.Println("")

	// write config
	if err := writeConfig(s.Store); err != nil {
		color.Red(fmt.Sprintf("Failed to write config: %s", err))
	}

	return nil
}
<commit_msg>Fix substore git init (#130)<commit_after>package action

import (
	"fmt"

	"github.com/fatih/color"
	"github.com/justwatchcom/gopass/gpg"
	"github.com/urfave/cli"
)

// Initialized returns an error if the store is not properly
// prepared.
func (s *Action) Initialized(*cli.Context) error {
	if !s.Store.Initialized() {
		return fmt.Errorf("password-store is not initialized. Try '%s init'", s.Name)
	}
	return nil
}

// Init a new password store with a first gpg id
func (s *Action) Init(c *cli.Context) error {
	path := c.String("store")
	alias := c.String("alias")
	nogit := c.Bool("nogit")

	return s.init(alias, path, nogit, c.Args()...)
}

func (s *Action) init(alias, path string, nogit bool, keys ...string) error {
	if path != "" && alias == "" {
		return fmt.Errorf("need mount alias when using path")
	}
	if !hasConfig() {
		// when creating a new config we set some sensible defaults
		s.Store.AutoPush = true
		s.Store.AutoPull = true
		s.Store.AutoImport = false
		s.Store.NoConfirm = false
		s.Store.PersistKeys = true
		s.Store.LoadKeys = false
		s.Store.ClipTimeout = 45
		s.Store.SafeContent = false
	}
	if path == "" {
		path = s.Store.Path
	}

	if len(keys) < 1 {
		nk, err := askForPrivateKey(color.CyanString("Please select a private key for encryption:"))
		if err != nil {
			return err
		}
		keys = []string{nk}
	}

	if err := s.Store.Init(alias, path, keys...); err != nil {
		return err
	}

	if alias != "" && path != "" {
		if err := s.Store.AddMount(alias, path); err != nil {
			return err
		}
	}

	if !nogit {
		if err := s.gitInit(alias, ""); err != nil {
			color.Yellow("Failed to init git: %s", err)
		}
	}

	fmt.Fprint(color.Output, color.GreenString("Password store %s initialized for:\n", path))
	for _, recipient := range s.Store.ListRecipients(alias) {
		r := "0x" + recipient
		if kl, err := gpg.ListPublicKeys(recipient); err == nil && len(kl) > 0 {
			r = kl[0].OneLine()
		}
		color.Yellow("  " + r)
	}
	fmt.Println("")

	// write config
	if err := writeConfig(s.Store); err != nil {
		color.Red(fmt.Sprintf("Failed to write config: %s", err))
	}

	return nil
}
<|endoftext|>