<commit_before>package providers

import (
	"github.com/zclconf/go-cty/cty"

	"github.com/hashicorp/terraform/configs/configschema"
	"github.com/hashicorp/terraform/states"
	"github.com/hashicorp/terraform/tfdiags"
)

// Interface represents the set of methods required for a complete resource
// provider plugin.
type Interface interface {
	// GetSchema returns the complete schema for the provider.
	GetSchema() GetSchemaResponse

	// ValidateProviderConfig allows the provider to validate the configuration
	// values.
	ValidateProviderConfig(ValidateProviderConfigRequest) ValidateProviderConfigResponse

	// ValidateResourceTypeConfig allows the provider to validate the resource
	// configuration values.
	ValidateResourceTypeConfig(ValidateResourceTypeConfigRequest) ValidateResourceTypeConfigResponse

	// ValidateDataSource allows the provider to validate the data source
	// configuration values.
	ValidateDataSourceConfig(ValidateDataSourceConfigRequest) ValidateDataSourceConfigResponse

	// UpgradeResourceState is called when the state loader encounters an
	// instance state whose schema version is less than the one reported by the
	// currently-used version of the corresponding provider, and the upgraded
	// result is used for any further processing.
	UpgradeResourceState(UpgradeResourceStateRequest) UpgradeResourceStateResponse

	// Configure configures and initialized the provider.
	Configure(ConfigureRequest) ConfigureResponse

	// Stop is called when the provider should halt any in-flight actions.
	//
	// Stop should not block waiting for in-flight actions to complete. It
	// should take any action it wants and return immediately acknowledging it
	// has received the stop request. Terraform will not make any further API
	// calls to the provider after Stop is called.
	//
	// The error returned, if non-nil, is assumed to mean that signaling the
	// stop somehow failed and that the user should expect potentially waiting
	// a longer period of time.
	Stop() error

	// ReadResource refreshes a resource and returns its current state.
	ReadResource(ReadResourceRequest) ReadResourceResponse

	// PlanResourceChange takes the current state and proposed state of a
	// resource, and returns the planned final state.
	PlanResourceChange(PlanResourceChangeRequest) PlanResourceChangeResponse

	// ApplyResourceChange takes the planned state for a resource, which may
	// yet contain unknown computed values, and applies the changes returning
	// the final state.
	ApplyResourceChange(ApplyResourceChangeRequest) ApplyResourceChangeResponse

	// ImportResourceState requests that the given resource be imported.
	ImportResourceState(ImportResourceStateRequest) ImportResourceStateResponse

	// ReadDataSource returns the data source's current state.
	ReadDataSource(ReadDataSourceRequest) ReadDataSourceResponse

	// Close shuts down the plugin process if applicable.
	Close() error
}

type GetSchemaResponse struct {
	// Provider is the schema for the provider itself.
	Provider Schema

	// ResourceTypes map the resource type name to that type's schema.
	ResourceTypes map[string]Schema

	// DataSources maps the data source name to that data source's schema.
	DataSources map[string]Schema

	// Diagnostics contains any warnings or errors from the method call.
	Diagnostics tfdiags.Diagnostics
}

// Schema pairs a provider or resource schema with that schema's version.
// This is used to be able to upgrade the schema in UpgradeResourceState.
type Schema struct {
	Version uint64
	Block   *configschema.Block
}

type ValidateProviderConfigRequest struct {
	// Config is the complete configuration value for the provider.
	Config cty.Value
}

type ValidateProviderConfigResponse struct {
	// Diagnostics contains any warnings or errors from the method call.
	Diagnostics tfdiags.Diagnostics
}

type ValidateResourceTypeConfigRequest struct {
	// TypeName is the name of the resource type to validate.
	TypeName string

	// Config is the configuration value to validate, which may contain unknown
	// values.
	Config cty.Value
}

type ValidateResourceTypeConfigResponse struct {
	// Diagnostics contains any warnings or errors from the method call.
	Diagnostics tfdiags.Diagnostics
}

type ValidateDataSourceConfigRequest struct {
	// TypeName is the name of the data source type to validate.
	TypeName string

	// Config is the configuration value to validate, which may contain unknown
	// values.
	Config cty.Value
}

type ValidateDataSourceConfigResponse struct {
	// Diagnostics contains any warnings or errors from the method call.
	Diagnostics tfdiags.Diagnostics
}

type UpgradeResourceStateRequest struct {
	// TypeName is the name of the resource type being upgraded
	TypeName string

	// Version is version of the schema that created the current state.
	Version int

	// RawStateJSON and RawStateFlatmap contiain the state that needs to be
	// upgraded to match the current schema version. Because the schema is
	// unknown, this contains only the raw data as stored in the state.
	// RawStateJSON is the current json state encoding.
	// RawStateFlatmap is the legacy flatmap encoding.
	// Only on of these fields may be set for the upgrade request.
	RawStateJSON    []byte
	RawStateFlatmap map[string]string
}

type UpgradeResourceStateResponse struct {
	// UpgradedState is the newly upgraded resource state.
	UpgradedState cty.Value

	// Diagnostics contains any warnings or errors from the method call.
	Diagnostics tfdiags.Diagnostics
}

type ConfigureRequest struct {
	// Terraform version is the version string from the running instance of
	// terraform. Providers can use TerraformVersion to verify compatibility,
	// and to store for informational purposes.
	TerraformVersion string

	// Config is the complete configuration value for the provider.
	Config cty.Value
}

type ConfigureResponse struct {
	// Diagnostics contains any warnings or errors from the method call.
	Diagnostics tfdiags.Diagnostics
}

type ReadResourceRequest struct {
	// TypeName is the name of the resource type being read.
	TypeName string

	// PriorState contains the previously saved state value for this resource.
	PriorState cty.Value
}

type ReadResourceResponse struct {
	// NewState contains the current state of the resource.
	NewState cty.Value

	// Diagnostics contains any warnings or errors from the method call.
	Diagnostics tfdiags.Diagnostics
}

type PlanResourceChangeRequest struct {
	// TypeName is the name of the resource type to plan.
	TypeName string

	// PriorState is the previously saved state value for this resource.
	PriorState cty.Value

	// ProposedNewState is the expected state after the new configuration is
	// applied. This is created by directly applying the configuration to the
	// PriorState. The provider is then responsible for applying any further
	// changes required to create the proposed final state.
	ProposedNewState cty.Value

	// Config is the resource configuration, before being merged with the
	// PriorState. Any value not explicitly set in the configuration will be
	// null. Config is supplied for reference, but Provider implementations
	// should prefer the ProposedNewState in most circumstances.
	Config cty.Value

	// PriorPrivate is the previously saved private data returned from the
	// provider during the last apply.
	PriorPrivate []byte
}

type PlanResourceChangeResponse struct {
	// PlannedState is the expected state of the resource once the current
	// configuration is applied.
	PlannedState cty.Value

	// RequiresReplace is the list of thee attributes that are requiring
	// resource replacement.
	RequiresReplace []cty.Path

	// PlannedPrivate is an opaque blob that is not interpreted by terraform
	// core. This will be saved and relayed back to the provider during
	// ApplyResourceChange.
	PlannedPrivate []byte

	// Diagnostics contains any warnings or errors from the method call.
	Diagnostics tfdiags.Diagnostics
}

type ApplyResourceChangeRequest struct {
	// TypeName is the name of the resource type being applied.
	TypeName string

	// PriorState is the current state of resource.
	PriorState cty.Value

	// Planned state is the state returned from PlanResourceChange, and should
	// represent the new state, minus any remaining computed attributes.
	PlannedState cty.Value

	// Config is the resource configuration, before being merged with the
	// PriorState. Any value not explicitly set in the configuration will be
	// null. Config is supplied for reference, but Provider implementations
	// should prefer the PlannedState in most circumstances.
	Config cty.Value

	// PlannedPrivate is the same value as returned by PlanResourceChange.
	PlannedPrivate []byte
}

type ApplyResourceChangeResponse struct {
	// NewState is the new complete state after applying the planned change.
	// In the event of an error, NewState should represent the most recent
	// known state of the resource, if it exists.
	NewState cty.Value

	// Private is an opaque blob that will be stored in state along with the
	// resource. It is intended only for interpretation by the provider itself.
	Private []byte

	// Diagnostics contains any warnings or errors from the method call.
	Diagnostics tfdiags.Diagnostics
}

type ImportResourceStateRequest struct {
	// TypeName is the name of the resource type to be imported.
	TypeName string

	// ID is a string with which the provider can identify the resource to be
	// imported.
	ID string
}

type ImportResourceStateResponse struct {
	// ImportedResources contains one or more state values related to the
	// imported resource. It is not required that these be complete, only that
	// there is enough identifying information for the provider to successfully
	// update the states in ReadResource.
	ImportedResources []ImportedResource

	// Diagnostics contains any warnings or errors from the method call.
	Diagnostics tfdiags.Diagnostics
}

// ImportedResource represents an object being imported into Terraform with the
// help of a provider. An ImportedObject is a RemoteObject that has been read
// by the provider's import handler but hasn't yet been committed to state.
type ImportedResource struct {
	// TypeName is the name of the resource type associated with the
	// returned state. It's possible for providers to import multiple related
	// types with a single import request.
	TypeName string

	// State is the state of the remote object being imported. This may not be
	// complete, but must contain enough information to uniquely identify the
	// resource.
	State cty.Value

	// Private is an opaque blob that will be stored in state along with the
	// resource. It is intended only for interpretation by the provider itself.
	Private []byte
}

// AsInstanceObject converts the receiving ImportedObject into a
// ResourceInstanceObject that has status ObjectReady.
//
// The returned object does not know its own resource type, so the caller must
// retain the ResourceType value from the source object if this information is
// needed.
//
// The returned object also has no dependency addresses, but the caller may
// freely modify the direct fields of the returned object without affecting
// the receiver.
func (ir ImportedResource) AsInstanceObject() *states.ResourceInstanceObject {
	return &states.ResourceInstanceObject{
		Status:  states.ObjectReady,
		Value:   ir.State,
		Private: ir.Private,
	}
}

type ReadDataSourceRequest struct {
	// TypeName is the name of the data source type to Read.
	TypeName string

	// Config is the complete configuration for the requested data source.
	Config cty.Value
}

type ReadDataSourceResponse struct {
	// State is the current state of the requested data source.
	State cty.Value

	// Diagnostics contains any warnings or errors from the method call.
	Diagnostics tfdiags.Diagnostics
}
<commit_msg>providers PrepareProviderConfig<commit_after>package providers

import (
	"github.com/zclconf/go-cty/cty"

	"github.com/hashicorp/terraform/configs/configschema"
	"github.com/hashicorp/terraform/states"
	"github.com/hashicorp/terraform/tfdiags"
)

// Interface represents the set of methods required for a complete resource
// provider plugin.
type Interface interface {
	// GetSchema returns the complete schema for the provider.
	GetSchema() GetSchemaResponse

	// PrepareProviderConfig allows the provider to validate the configuration
	// values, and set or override any values with defaults.
	PrepareProviderConfig(PrepareProviderConfigRequest) PrepareProviderConfigResponse

	// ValidateResourceTypeConfig allows the provider to validate the resource
	// configuration values.
	ValidateResourceTypeConfig(ValidateResourceTypeConfigRequest) ValidateResourceTypeConfigResponse

	// ValidateDataSource allows the provider to validate the data source
	// configuration values.
	ValidateDataSourceConfig(ValidateDataSourceConfigRequest) ValidateDataSourceConfigResponse

	// UpgradeResourceState is called when the state loader encounters an
	// instance state whose schema version is less than the one reported by the
	// currently-used version of the corresponding provider, and the upgraded
	// result is used for any further processing.
	UpgradeResourceState(UpgradeResourceStateRequest) UpgradeResourceStateResponse

	// Configure configures and initialized the provider.
	Configure(ConfigureRequest) ConfigureResponse

	// Stop is called when the provider should halt any in-flight actions.
	//
	// Stop should not block waiting for in-flight actions to complete. It
	// should take any action it wants and return immediately acknowledging it
	// has received the stop request. Terraform will not make any further API
	// calls to the provider after Stop is called.
	//
	// The error returned, if non-nil, is assumed to mean that signaling the
	// stop somehow failed and that the user should expect potentially waiting
	// a longer period of time.
	Stop() error

	// ReadResource refreshes a resource and returns its current state.
	ReadResource(ReadResourceRequest) ReadResourceResponse

	// PlanResourceChange takes the current state and proposed state of a
	// resource, and returns the planned final state.
	PlanResourceChange(PlanResourceChangeRequest) PlanResourceChangeResponse

	// ApplyResourceChange takes the planned state for a resource, which may
	// yet contain unknown computed values, and applies the changes returning
	// the final state.
	ApplyResourceChange(ApplyResourceChangeRequest) ApplyResourceChangeResponse

	// ImportResourceState requests that the given resource be imported.
	ImportResourceState(ImportResourceStateRequest) ImportResourceStateResponse

	// ReadDataSource returns the data source's current state.
	ReadDataSource(ReadDataSourceRequest) ReadDataSourceResponse

	// Close shuts down the plugin process if applicable.
	Close() error
}

type GetSchemaResponse struct {
	// Provider is the schema for the provider itself.
	Provider Schema

	// ResourceTypes map the resource type name to that type's schema.
	ResourceTypes map[string]Schema

	// DataSources maps the data source name to that data source's schema.
	DataSources map[string]Schema

	// Diagnostics contains any warnings or errors from the method call.
	Diagnostics tfdiags.Diagnostics
}

// Schema pairs a provider or resource schema with that schema's version.
// This is used to be able to upgrade the schema in UpgradeResourceState.
type Schema struct {
	Version uint64
	Block   *configschema.Block
}

type PrepareProviderConfigRequest struct {
	// Config is the raw configuration value for the provider.
	Config cty.Value
}

type PrepareProviderConfigResponse struct {
	// PreparedConfig is the configuration as prepared by the provider.
	PreparedConfig cty.Value
	// Diagnostics contains any warnings or errors from the method call.
	Diagnostics tfdiags.Diagnostics
}

type ValidateResourceTypeConfigRequest struct {
	// TypeName is the name of the resource type to validate.
	TypeName string

	// Config is the configuration value to validate, which may contain unknown
	// values.
	Config cty.Value
}

type ValidateResourceTypeConfigResponse struct {
	// Diagnostics contains any warnings or errors from the method call.
	Diagnostics tfdiags.Diagnostics
}

type ValidateDataSourceConfigRequest struct {
	// TypeName is the name of the data source type to validate.
	TypeName string

	// Config is the configuration value to validate, which may contain unknown
	// values.
	Config cty.Value
}

type ValidateDataSourceConfigResponse struct {
	// Diagnostics contains any warnings or errors from the method call.
	Diagnostics tfdiags.Diagnostics
}

type UpgradeResourceStateRequest struct {
	// TypeName is the name of the resource type being upgraded
	TypeName string

	// Version is version of the schema that created the current state.
	Version int

	// RawStateJSON and RawStateFlatmap contiain the state that needs to be
	// upgraded to match the current schema version. Because the schema is
	// unknown, this contains only the raw data as stored in the state.
	// RawStateJSON is the current json state encoding.
	// RawStateFlatmap is the legacy flatmap encoding.
	// Only on of these fields may be set for the upgrade request.
	RawStateJSON    []byte
	RawStateFlatmap map[string]string
}

type UpgradeResourceStateResponse struct {
	// UpgradedState is the newly upgraded resource state.
	UpgradedState cty.Value

	// Diagnostics contains any warnings or errors from the method call.
	Diagnostics tfdiags.Diagnostics
}

type ConfigureRequest struct {
	// Terraform version is the version string from the running instance of
	// terraform. Providers can use TerraformVersion to verify compatibility,
	// and to store for informational purposes.
	TerraformVersion string

	// Config is the complete configuration value for the provider.
	Config cty.Value
}

type ConfigureResponse struct {
	// Diagnostics contains any warnings or errors from the method call.
	Diagnostics tfdiags.Diagnostics
}

type ReadResourceRequest struct {
	// TypeName is the name of the resource type being read.
	TypeName string

	// PriorState contains the previously saved state value for this resource.
	PriorState cty.Value
}

type ReadResourceResponse struct {
	// NewState contains the current state of the resource.
	NewState cty.Value

	// Diagnostics contains any warnings or errors from the method call.
	Diagnostics tfdiags.Diagnostics
}

type PlanResourceChangeRequest struct {
	// TypeName is the name of the resource type to plan.
	TypeName string

	// PriorState is the previously saved state value for this resource.
	PriorState cty.Value

	// ProposedNewState is the expected state after the new configuration is
	// applied. This is created by directly applying the configuration to the
	// PriorState. The provider is then responsible for applying any further
	// changes required to create the proposed final state.
	ProposedNewState cty.Value

	// Config is the resource configuration, before being merged with the
	// PriorState. Any value not explicitly set in the configuration will be
	// null. Config is supplied for reference, but Provider implementations
	// should prefer the ProposedNewState in most circumstances.
	Config cty.Value

	// PriorPrivate is the previously saved private data returned from the
	// provider during the last apply.
	PriorPrivate []byte
}

type PlanResourceChangeResponse struct {
	// PlannedState is the expected state of the resource once the current
	// configuration is applied.
	PlannedState cty.Value

	// RequiresReplace is the list of thee attributes that are requiring
	// resource replacement.
	RequiresReplace []cty.Path

	// PlannedPrivate is an opaque blob that is not interpreted by terraform
	// core. This will be saved and relayed back to the provider during
	// ApplyResourceChange.
	PlannedPrivate []byte

	// Diagnostics contains any warnings or errors from the method call.
	Diagnostics tfdiags.Diagnostics
}

type ApplyResourceChangeRequest struct {
	// TypeName is the name of the resource type being applied.
	TypeName string

	// PriorState is the current state of resource.
	PriorState cty.Value

	// Planned state is the state returned from PlanResourceChange, and should
	// represent the new state, minus any remaining computed attributes.
	PlannedState cty.Value

	// Config is the resource configuration, before being merged with the
	// PriorState. Any value not explicitly set in the configuration will be
	// null. Config is supplied for reference, but Provider implementations
	// should prefer the PlannedState in most circumstances.
	Config cty.Value

	// PlannedPrivate is the same value as returned by PlanResourceChange.
	PlannedPrivate []byte
}

type ApplyResourceChangeResponse struct {
	// NewState is the new complete state after applying the planned change.
	// In the event of an error, NewState should represent the most recent
	// known state of the resource, if it exists.
	NewState cty.Value

	// Private is an opaque blob that will be stored in state along with the
	// resource. It is intended only for interpretation by the provider itself.
	Private []byte

	// Diagnostics contains any warnings or errors from the method call.
	Diagnostics tfdiags.Diagnostics
}

type ImportResourceStateRequest struct {
	// TypeName is the name of the resource type to be imported.
	TypeName string

	// ID is a string with which the provider can identify the resource to be
	// imported.
	ID string
}

type ImportResourceStateResponse struct {
	// ImportedResources contains one or more state values related to the
	// imported resource. It is not required that these be complete, only that
	// there is enough identifying information for the provider to successfully
	// update the states in ReadResource.
	ImportedResources []ImportedResource

	// Diagnostics contains any warnings or errors from the method call.
	Diagnostics tfdiags.Diagnostics
}

// ImportedResource represents an object being imported into Terraform with the
// help of a provider. An ImportedObject is a RemoteObject that has been read
// by the provider's import handler but hasn't yet been committed to state.
type ImportedResource struct {
	// TypeName is the name of the resource type associated with the
	// returned state. It's possible for providers to import multiple related
	// types with a single import request.
	TypeName string

	// State is the state of the remote object being imported. This may not be
	// complete, but must contain enough information to uniquely identify the
	// resource.
	State cty.Value

	// Private is an opaque blob that will be stored in state along with the
	// resource. It is intended only for interpretation by the provider itself.
	Private []byte
}

// AsInstanceObject converts the receiving ImportedObject into a
// ResourceInstanceObject that has status ObjectReady.
//
// The returned object does not know its own resource type, so the caller must
// retain the ResourceType value from the source object if this information is
// needed.
//
// The returned object also has no dependency addresses, but the caller may
// freely modify the direct fields of the returned object without affecting
// the receiver.
func (ir ImportedResource) AsInstanceObject() *states.ResourceInstanceObject {
	return &states.ResourceInstanceObject{
		Status:  states.ObjectReady,
		Value:   ir.State,
		Private: ir.Private,
	}
}

type ReadDataSourceRequest struct {
	// TypeName is the name of the data source type to Read.
	TypeName string

	// Config is the complete configuration for the requested data source.
	Config cty.Value
}

type ReadDataSourceResponse struct {
	// State is the current state of the requested data source.
	State cty.Value

	// Diagnostics contains any warnings or errors from the method call.
	Diagnostics tfdiags.Diagnostics
}
<|endoftext|>


<commit_before>// Copyright 2011, Bryan Matsuo. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
/*
 *  Filename:    godirs.go
 *  Author:      Bryan Matsuo <bmatsuo@soe.ucsc.edu>
 *  Created:     Tue Jul  5 22:13:49 PDT 2011
 *  Description:
 *  Usage:       godirs [options] ARGUMENT ...
 */

//  Package dispatch provides goroutine dispatch and concurrency limiting.
//  It provides an object Dispatch which is a queueing system for concurrent
//  functions. It implements a dynamic limit on the number of routines that
//  it runs simultaneously. It also uses a Queue interface, allowing for
//  alternate queue implementations.
//
//  See github.com/bmatsuo/dispatch/queues for more about the Queue interface.
//
//  See github.com/bmatsuo/dispatch/examples for usage examples.
package dispatch
import (
    "sync"
    "github.com/bmatsuo/dispatch/queues"
)

//  A Dispatch is an automated function dispatch queue with a limited
//  number of concurrent gorountines.
type Dispatch struct {
    // The maximum number of goroutines can be changed while the queue is
    // processing.
    MaxGo      int

    // Handle waiting when the limit of concurrent goroutines has been reached.
    waitingToRun bool
    //nextWake     chan bool
    nextWait     *sync.WaitGroup

    // Handle waiting when function queue is empty.
    waitingOnQ   bool
    restart      *sync.WaitGroup

    // Manage the Start()'ing of a Dispatch, avoiding race conditions.
    startLock    *sync.Mutex
    started      bool

    // Handle goroutine-safe queue operations.
    qLock        *sync.Mutex
    queue        queues.Queue

    // Handle goroutine-safe limiting and identifier operations.
    pLock        *sync.Mutex
    processing   int         // Number of QueueTasks running
    idcount      int64       // pid counter

    // Handle stopping of the Start() method.
    kill         chan bool
}

//  Create a new queue object with a specified limit on concurrency.
func New(maxroutines int) *Dispatch {
    return NewCustom(maxroutines, queues.NewFIFO())
}
func NewCustom(maxroutines int, queue queues.Queue) *Dispatch {
    var rl = new(Dispatch)
    rl.startLock = new(sync.Mutex)
    rl.qLock     = new(sync.Mutex)
    rl.pLock     = new(sync.Mutex)
    rl.restart   = new(sync.WaitGroup)
    rl.kill      = make(chan bool)
    //rl.nextWake  = make(chan bool)
    rl.nextWait  = new(sync.WaitGroup)
    rl.queue     = queue
    rl.MaxGo     = maxroutines
    rl.idcount   = 0
    return rl
}

//  Goroutines called from a Dispatch are given an int identifier unique
//  to that routine.
type StdTask struct {
    F func(id int64)
}
func (dt StdTask) Type() string {
    return "StdTask"
}
func (dt StdTask) SetFunc(f func(id int64)) {
    dt.F = f
}
func (dt StdTask) Func() func(id int64) {
    return dt.F
}
type dispatchTaskWrapper struct {
    id int64
    t  queues.Task
}
func (dtw dispatchTaskWrapper) Func() func(id int64) {
    return dtw.t.Func()
}
func (dtw dispatchTaskWrapper) Id() int64 {
    return dtw.id
}
func (dtw dispatchTaskWrapper) Task() queues.Task {
    return dtw.t
}

//  Enqueue a task for execution as a goroutine.
func (gq *Dispatch) Enqueue(t queues.Task) int64 {
    // Wrap the function so it works with the goroutine limiting code.
    var f = t.Func()
    var dtFunc = func (id int64) {
        // Run the given function.
        f(id)

        // Decrement the process counter.
        gq.pLock.Lock()
        gq.processing--
        var procWaiting = gq.waitingToRun
        if procWaiting {
            gq.waitingToRun = false
        }
        gq.pLock.Unlock()

        // Start any waiting process.
        if procWaiting {
            //gq.nextWake<-true
            gq.nextWait.Done()
        }
    }
    t.SetFunc(dtFunc)

    // Lock the queue and enqueue a new task.
    gq.qLock.Lock()
    gq.idcount++
    var id = gq.idcount
    gq.queue.Enqueue(dispatchTaskWrapper{id, t})
    var loopWaiting = gq.waitingOnQ
    if loopWaiting {
        gq.waitingOnQ = false
    }
    gq.qLock.Unlock()

    // Restart the Start() loop if it was deemed necessary.
    if loopWaiting {
        gq.restart.Done()
    }

    return id
}

//  Stop the queue after gq.Start() has been called. Any goroutines which
//  have not already been dequeued will not be executed until gq.Start()
//  is called again.
func (gq *Dispatch) Stop() {
    // Lock out Start() and queue ops for the entire call.
    gq.startLock.Lock()
    defer gq.startLock.Unlock()
    gq.qLock.Lock()
    defer gq.qLock.Unlock()

    if !gq.started {
        return
    }

    // Clear channel flags and close channels, stoping further processing.
    close(gq.kill)
    gq.started = false
    if gq.waitingOnQ {
        gq.waitingOnQ = false
        gq.restart.Done()
    }
    if gq.waitingToRun {
        gq.waitingToRun = false
        gq.nextWait.Done()
    }
    //close(gq.nextWake)
}

//  Start the next task in the queue. It's assumed that the queue is non-
//  empty. Furthermore, there should only be one goroutine in this method
//  (for this object) at a time. Both conditions are enforced in
//  gq.Start(), which calls gq.next() exclusively.
func (gq *Dispatch) next() {
    for true {
        // Attempt to start processing the file.
        gq.pLock.Lock()
        if gq.processing >= gq.MaxGo {
            gq.waitingToRun = true
            gq.nextWait.Add(1)
            gq.pLock.Unlock()
            gq.nextWait.Wait()
            /*
            var cont, ok =<-gq.nextWake
            if !ok {
                gq.nextWake = make(chan bool)
                return
            }
            if !cont {
                return
            }
            */
            continue
        }
        // Keep the books and reset wait time before unlocking.
        gq.waitingToRun = false
        gq.processing++
        gq.pLock.Unlock()

        // Get an element from the queue.
        gq.qLock.Lock()
        var wrapper = gq.queue.Dequeue().(queues.RegisteredTask)
        gq.qLock.Unlock()

        // Begin processing and asyncronously return.
        //var task = taskelm.Value.(dispatchTaskWrapper)
        var task = wrapper.Func()
        go task(wrapper.Id())
        return
    }
}

//  Start executing goroutines. Don't stop until gq.Stop() is called.
func (gq *Dispatch) Start() {
    // Avoid multiple gq.Start() methods and avoid race conditions.
    gq.startLock.Lock()
    if gq.started {
        panic("already started")
    }
    gq.started = true
    gq.startLock.Unlock()

    // Recreate any channels that were closed by a previous Stop().
    var inited = false
    for !inited {
        select {
        case _, okKill :=<-gq.kill:
            if !okKill {
                gq.kill = make(chan bool)
            }
        /*
        case _, okWake :=<-gq.nextWake:
            if !okWake {
                gq.nextWake = make(chan bool)
            }
        */
        default:
            inited = true
        }
    }

    // Process the queue
    for true {
        select {
        case die, ok :=<-gq.kill:
            // If something came out of this channel, we must stop.
            if !ok {
                // Recreate the channel on a closure.
                gq.kill = make(chan bool)
                return
            }
            if die {
                return
            }
        default:
            // Check the queue size and determine if we need to wait.
            gq.qLock.Lock()
            if gq.waitingOnQ = gq.queue.Len() == 0 ; gq.waitingOnQ {
                gq.restart.Add(1)
            }
            gq.qLock.Unlock()

            if gq.waitingOnQ {
                // Wait for a restart signal from gq.Enqueue
                gq.restart.Wait()
            } else {
                // Process the head of the queue and start the loop again.
                gq.next()
            }
        }
    }
}
<commit_msg>Still trying to solve deadlock.<commit_after>// Copyright 2011, Bryan Matsuo. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
/*
 *  Filename:    godirs.go
 *  Author:      Bryan Matsuo <bmatsuo@soe.ucsc.edu>
 *  Created:     Tue Jul  5 22:13:49 PDT 2011
 *  Description:
 *  Usage:       godirs [options] ARGUMENT ...
 */

//  Package dispatch provides goroutine dispatch and concurrency limiting.
//  It provides an object Dispatch which is a queueing system for concurrent
//  functions. It implements a dynamic limit on the number of routines that
//  it runs simultaneously. It also uses a Queue interface, allowing for
//  alternate queue implementations.
//
//  See github.com/bmatsuo/dispatch/queues for more about the Queue interface.
//
//  See github.com/bmatsuo/dispatch/examples for usage examples.
package dispatch
import (
    "sync"
    "github.com/bmatsuo/dispatch/queues"
)

//  A Dispatch is an automated function dispatch queue with a limited
//  number of concurrent gorountines.
type Dispatch struct {
    // The maximum number of goroutines can be changed while the queue is
    // processing.
    MaxGo      int

    // Handle waiting when the limit of concurrent goroutines has been reached.
    waitingToRun bool
    //nextWake     chan bool
    nextWait     *sync.WaitGroup

    // Handle waiting when function queue is empty.
    waitingOnQ   bool
    restart      *sync.WaitGroup

    // Manage the Start()'ing of a Dispatch, avoiding race conditions.
    startLock    *sync.Mutex
    started      bool

    // Handle goroutine-safe queue operations.
    qLock        *sync.Mutex
    queue        queues.Queue

    // Handle goroutine-safe limiting and identifier operations.
    pLock        *sync.Mutex
    processing   int         // Number of QueueTasks running
    idcount      int64       // pid counter

    // Handle stopping of the Start() method.
    kill         chan bool
}

//  Create a new queue object with a specified limit on concurrency.
func New(maxroutines int) *Dispatch {
    return NewCustom(maxroutines, queues.NewFIFO())
}
func NewCustom(maxroutines int, queue queues.Queue) *Dispatch {
    var rl = new(Dispatch)
    rl.startLock = new(sync.Mutex)
    rl.qLock     = new(sync.Mutex)
    rl.pLock     = new(sync.Mutex)
    rl.restart   = new(sync.WaitGroup)
    rl.kill      = make(chan bool)
    //rl.nextWake  = make(chan bool)
    rl.nextWait  = new(sync.WaitGroup)
    rl.queue     = queue
    rl.MaxGo     = maxroutines
    rl.idcount   = 0
    return rl
}

//  Goroutines called from a Dispatch are given an int identifier unique
//  to that routine.
type StdTask struct {
    F func(id int64)
}
func (dt StdTask) Type() string {
    return "StdTask"
}
func (dt StdTask) SetFunc(f func(id int64)) {
    dt.F = f
}
func (dt StdTask) Func() func(id int64) {
    return dt.F
}
type dispatchTaskWrapper struct {
    id int64
    t  queues.Task
}
func (dtw dispatchTaskWrapper) Func() func(id int64) {
    return dtw.t.Func()
}
func (dtw dispatchTaskWrapper) Id() int64 {
    return dtw.id
}
func (dtw dispatchTaskWrapper) Task() queues.Task {
    return dtw.t
}

//  Enqueue a task for execution as a goroutine.
func (gq *Dispatch) Enqueue(t queues.Task) int64 {
    // Wrap the function so it works with the goroutine limiting code.
    var f = t.Func()
    var dtFunc = func (id int64) {
        // Run the given function.
        f(id)

        // Decrement the process counter.
        gq.pLock.Lock()
        gq.processing--
        var procWaiting = gq.waitingToRun
        if procWaiting {
            gq.waitingToRun = false
        }
        gq.pLock.Unlock()

        // Start any waiting process.
        if procWaiting {
            //gq.nextWake<-true
            gq.nextWait.Done()
        }
    }
    t.SetFunc(dtFunc)

    // Lock the queue and enqueue a new task.
    gq.qLock.Lock()
    gq.idcount++
    var id = gq.idcount
    gq.queue.Enqueue(dispatchTaskWrapper{id, t})
    var loopWaiting = gq.waitingOnQ
    if loopWaiting {
        gq.waitingOnQ = false
    }
    gq.qLock.Unlock()

    // Restart the Start() loop if it was deemed necessary.
    if loopWaiting {
        gq.restart.Done()
    }

    return id
}

//  Stop the queue after gq.Start() has been called. Any goroutines which
//  have not already been dequeued will not be executed until gq.Start()
//  is called again.
func (gq *Dispatch) Stop() {
    // Lock out Start() and queue ops for the entire call.
    gq.startLock.Lock()
    defer gq.startLock.Unlock()
    gq.qLock.Lock()
    defer gq.qLock.Unlock()

    if !gq.started {
        return
    }

    // Clear channel flags and close channels, stoping further processing.
    close(gq.kill)
    gq.started = false
    if gq.waitingOnQ {
        gq.waitingOnQ = false
        gq.restart.Done()
    }
    if gq.waitingToRun {
        gq.waitingToRun = false
        gq.nextWait.Done()
    }
    //close(gq.nextWake)
}

//  Start the next task in the queue. It's assumed that the queue is non-
//  empty. Furthermore, there should only be one goroutine in this method
//  (for this object) at a time. Both conditions are enforced in
//  gq.Start(), which calls gq.next() exclusively.
func (gq *Dispatch) next() {
    for true {
        // Attempt to start processing the file.
        gq.pLock.Lock()
        if gq.processing >= gq.MaxGo {
            gq.waitingToRun = true
            gq.nextWait.Add(1)
            gq.pLock.Unlock()
            gq.nextWait.Wait()
            /*
            var cont, ok =<-gq.nextWake
            if !ok {
                gq.nextWake = make(chan bool)
                return
            }
            if !cont {
                return
            }
            */
            continue
        }
        // Keep the books and reset wait time before unlocking.
        gq.processing++
        gq.pLock.Unlock()

        // Get an element from the queue.
        gq.qLock.Lock()
        var wrapper = gq.queue.Dequeue().(queues.RegisteredTask)
        gq.qLock.Unlock()

        // Begin processing and asyncronously return.
        //var task = taskelm.Value.(dispatchTaskWrapper)
        var task = wrapper.Func()
        go task(wrapper.Id())
        return
    }
}

//  Start executing goroutines. Don't stop until gq.Stop() is called.
func (gq *Dispatch) Start() {
    // Avoid multiple gq.Start() methods and avoid race conditions.
    gq.startLock.Lock()
    if gq.started {
        panic("already started")
    }
    gq.started = true
    gq.startLock.Unlock()

    // Recreate any channels that were closed by a previous Stop().
    var inited = false
    for !inited {
        select {
        case _, okKill :=<-gq.kill:
            if !okKill {
                gq.kill = make(chan bool)
            }
        /*
        case _, okWake :=<-gq.nextWake:
            if !okWake {
                gq.nextWake = make(chan bool)
            }
        */
        default:
            inited = true
        }
    }

    // Process the queue
    for true {
        select {
        case die, ok :=<-gq.kill:
            // If something came out of this channel, we must stop.
            if !ok {
                // Recreate the channel on a closure.
                gq.kill = make(chan bool)
                return
            }
            if die {
                return
            }
        default:
            // Check the queue size and determine if we need to wait.
            gq.qLock.Lock()
            if gq.waitingOnQ = gq.queue.Len() == 0 ; gq.waitingOnQ {
                gq.restart.Add(1)
            }
            gq.qLock.Unlock()

            if gq.waitingOnQ {
                // Wait for a restart signal from gq.Enqueue
                gq.restart.Wait()
            } else {
                // Process the head of the queue and start the loop again.
                gq.next()
            }
        }
    }
}
<|endoftext|>


<commit_before>package mrepo

import (
	"os/exec"
	"strings"
)

//Make invoke make on the prj with the target as argument.
func Make(prj, target string) (result string, err error) {
	cmd := exec.Command("make", target)
	cmd.Dir = prj
	out, err := cmd.CombinedOutput()
	if err != nil {
		return
	}
	result = strings.Trim(string(out), "\n \t")
	return result, nil
}
<commit_msg>Make cmd now suport buffer passing<commit_after>package mrepo

import (
	"io"
	"os/exec"
)

//Make invoke make on the prj with the target as argument.
func Make(prj, target string, buf io.Writer) (err error) {
	cmd := exec.Command("make", target)
	cmd.Dir = prj
	cmd.Stdout = buf
	cmd.Stderr = buf
	return cmd.Run()
}
<|endoftext|>


<commit_before>// Go language map drawing library
package carto

import (
	"bufio"
	"code.google.com/p/draw2d/draw2d"
	"fmt"
	"github.com/pmylund/go-cache"
	"github.com/twpayne/gogeom/geom"
	"image"
	"image/color"
	"image/draw"
	"image/png"
	"io"
	"math"
	"reflect"
	"time"
)

type Mapper interface {
	DrawVector(geom.T, color.NRGBA, color.NRGBA, float64, float64)
	Save()
}

type RasterMap struct {
	bounds        *geom.Bounds // geographic boundaries of map
	width, height int          // pixel dimensions of map
	dx, dy        float64
	f             io.Writer
	I             draw.Image
	GC            draw2d.GraphicContext
	MarkerFunc    func(*RasterMap, float64, float64, float64) // function that specifies the shape of the marker for points
}

type MarkerFunction func(*RasterMap, float64, float64, float64) // Function for specifying the shape of the marker for points

var (
	Circle MarkerFunction = func(m *RasterMap, x, y, markersize float64) {
		m.GC.ArcTo(x, y, markersize, markersize, 0, 2*math.Pi)
	}
	Square MarkerFunction = func(m *RasterMap, x, y, markersize float64) {
		adjMS := markersize / 1.2 // ratio to adjust the markersize
		// to make the area be the same as the circle
		m.GC.MoveTo(x-adjMS, y-adjMS)
		m.GC.LineTo(x+adjMS, y-adjMS)
		m.GC.LineTo(x+adjMS, y+adjMS)
		m.GC.LineTo(x-adjMS, y+adjMS)
		m.GC.LineTo(x-adjMS, y-adjMS)
	}
	Triangle MarkerFunction = func(m *RasterMap, x, y, markersize float64) {
		adjMS := markersize / 0.75 // ratio to adjust the markersize
		// to make the area be the same as the circle
		cosval := math.Cos(0.125 * math.Pi)
		sinval := math.Sin(0.125 * math.Pi)
		m.GC.MoveTo(x-adjMS*cosval, y+adjMS*sinval)
		m.GC.LineTo(x+adjMS*cosval, y+adjMS*sinval)
		m.GC.LineTo(x, y-adjMS)
		m.GC.LineTo(x-adjMS*cosval, y+adjMS*sinval)
	}
	Star MarkerFunction = func(m *RasterMap, x, y, markersize float64) {
		adjMS := markersize / 0.75 // ratio to adjust the markersize
		// to make the area be the same as the circle
		var alpha = (2 * math.Pi) / 10
		// works out the angle between each vertex (5 external + 5 internal = 10)
		var r_concave = adjMS / 2.25 // r_point is the radius to the external point
		for i := 11; i != 0; i-- {
			var ra float64
			if i%2 == 1 {
				ra = adjMS
			} else {
				ra = r_concave
			}
			omega := alpha * float64(i) //omega is the angle of the current point
			//cx and cy are the center point of the star.
			if i == 11 {
				m.GC.MoveTo(x+(ra*math.Sin(omega)), y+(ra*math.Cos(omega)))
			} else {
				m.GC.LineTo(x+(ra*math.Sin(omega)), y+(ra*math.Cos(omega)))
			}
		}
	}
)

func NewRasterMap(N, S, E, W float64, width int, f io.Writer) *RasterMap {
	r := new(RasterMap)
	r.f = f
	r.bounds = geom.NewBoundsPoint(geom.Point{W, S})
	r.bounds.ExtendPoint(geom.Point{E, N})
	r.width, r.height = width, int(float64(width)*(N-S)/(E-W))
	r.dx = (E - W) / float64(r.width)
	r.dy = (N - S) / float64(r.height)
	r.I = image.NewRGBA(image.Rect(0, 0, r.width, r.height))
	r.GC = draw2d.NewGraphicContext(r.I)
	r.GC.SetFillRule(draw2d.FillRuleWinding)
	r.MarkerFunc = Circle
	return r
}

// Make a new raster map from raster data.
// It is assumed that the outer axis in the data is the Y-axis
// (north-south) and the inner axis is the X-axis (west-east)
// (i.e., len(data)==nx*ny &&  val[j,i] = data[j*nx+i]).
func NewRasterMapFromRaster(S, W, dy, dx float64, ny, nx int,
	data []float64, cmap *ColorMap, f io.Writer, flipVertical, flipHorizontal bool) *RasterMap {
	N := S + float64(ny)*dy
	E := W + float64(nx)*dx
	r := NewRasterMap(N, S, E, W, nx, f)
	r.height = ny
	if !flipVertical && !flipHorizontal {
		for i := 0; i < nx; i++ {
			for j := 0; j < ny; j++ {
				val := data[j*nx+i]
				r.I.Set(i, j, cmap.GetColor(val))
			}
		}
	} else if flipVertical && !flipHorizontal {
		for i := 0; i < nx; i++ {
			for j := 0; j < ny; j++ {
				val := data[j*nx+i]
				r.I.Set(i, ny-1-j, cmap.GetColor(val))
			}
		}
	} else if !flipVertical && flipHorizontal {
		for i := 0; i < nx; i++ {
			for j := 0; j < ny; j++ {
				val := data[j*nx+i]
				r.I.Set(nx-1-i, j, cmap.GetColor(val))
			}
		}
	} else if flipVertical && flipHorizontal {
		for i := 0; i < nx; i++ {
			for j := 0; j < ny; j++ {
				val := data[j*nx+i]
				r.I.Set(nx-1-i, ny-1-j, cmap.GetColor(val))
			}
		}
	}
	return r
}

// Draw a vector on a raster map when given the geometry,
// stroke and fill colors, the width of the bounding line,
// and the size of the marker (only used for point shapes).
func (r *RasterMap) DrawVector(g geom.T, strokeColor,
	fillColor color.NRGBA, linewidth, markersize float64) {
	// check bounding box
	if g == nil {
		return
	}
	gbounds := g.Bounds(nil)
	if !gbounds.Overlaps(r.bounds) {
		return
	}
	r.GC.SetStrokeColor(strokeColor)
	r.GC.SetFillColor(fillColor)
	r.GC.SetLineWidth(linewidth)
	switch g.(type) {
	case geom.Point:
		p := g.(geom.Point)
		x, y := r.coordinates(p.X, p.Y)
		r.MarkerFunc(r, x, y, markersize)
	//case geom.PointZ:
	//case geom.PointM:
	//case geom.PointZM:
	case geom.MultiPoint:
		for _, p := range g.(geom.MultiPoint).Points {
			x, y := r.coordinates(p.X, p.Y)
			r.GC.MoveTo(x, y)
			r.MarkerFunc(r, x, y, markersize)
		}
	//case geom.MultiPointZ:
	//case geom.MultiPointM:
	//case geom.MultiPointZM:
	case geom.LineString:
		l := g.(geom.LineString)
		for i, p := range l.Points {
			x, y := r.coordinates(p.X, p.Y)
			if i == 0 {
				r.GC.MoveTo(x, y)
			} else {
				r.GC.LineTo(x, y)
			}
		}
	//case geom.LineStringZ:
	//case geom.LineStringM:
	//case geom.LineStringZM:
	case geom.MultiLineString:
		l := g.(geom.MultiLineString)
		for _, ls := range l.LineStrings {
			r.DrawVector(ls, strokeColor,
				fillColor, linewidth, markersize)
		}
	//case geom.MultiLineStringZ:
	//case geom.MultiLineStringM:
	//case geom.MultiLineStringZM:
	case geom.Polygon:
		pg := g.(geom.Polygon)
		for _, ring := range pg.Rings {
			for i, p := range ring {
				x, y := r.coordinates(p.X, p.Y)
				if i == 0 {
					r.GC.MoveTo(x, y)
				} else {
					r.GC.LineTo(x, y)
				}
			}
		}
	//case geom.PolygonZ:
	//case geom.PolygonM:
	//case geom.PolygonZM:
	case geom.MultiPolygon:
		mpg := g.(geom.MultiPolygon)
		for _, pg := range mpg.Polygons {
			r.DrawVector(pg, strokeColor,
				fillColor, linewidth, markersize)
		}
	//case geom.MultiPolygonZ:
	//case geom.MultiPolygonM:
	//case geom.MultiPolygonZM:
	default:
		panic(&UnsupportedGeometryError{reflect.TypeOf(g)})
	}
	r.GC.FillStroke()
}

func (r *RasterMap) Save() {
	b := bufio.NewWriter(r.f)
	err := png.Encode(b, r.I)
	if err != nil {
		panic(err)
	}
	err = b.Flush()
	if err != nil {
		panic(err)
	}
}

// transform geographic coordinates to raster map coordinates
func (r *RasterMap) coordinates(X, Y float64) (
	x, y float64) {
	x = (X - r.bounds.Min.X) / r.dx
	y = float64(r.height) - 1. - (Y-r.bounds.Min.Y)/r.dy
	return
}

type MapData struct {
	Cmap      *ColorMap
	Shapes    []geom.T
	Data      []float64
	tileCache *cache.Cache
	DrawEdges bool
	EdgeWidth float64
}

func NewMapData(numShapes int, Type ColorMapType) *MapData {
	m := new(MapData)
	m.Cmap = NewColorMap(Type)
	m.Shapes = make([]geom.T, numShapes)
	m.Data = make([]float64, numShapes)
	m.tileCache = cache.New(1*time.Hour, 10*time.Minute)
	m.EdgeWidth = 0.5
	return m
}

func (m *MapData) WriteGoogleMapTile(w io.Writer, zoom, x, y int) error {
	// Check if image is already in the cache.
	cacheKey := fmt.Sprintf("%v_%v_%v", zoom, x, y)
	if img, found := m.tileCache.Get(cacheKey); found {
		err := png.Encode(w, img.(image.Image))
		if err != nil {
			return err
		}
		return nil
	}
	//strokeColor := color.NRGBA{0, 0, 0, 255}
	N, S, E, W := getGoogleTileBounds(zoom, x, y)
	maptile := NewRasterMap(N, S, E, W, 256, w)

	var strokeColor color.NRGBA
	for i, shp := range m.Shapes {
		fillColor := m.Cmap.GetColor(m.Data[i])
		if m.DrawEdges {
			strokeColor = color.NRGBA{0, 0, 0, 255}
		} else {
			strokeColor = fillColor
		}
		// use the fill color for both the fill and the stroke
		// to avoid unsightly gaps between shapes.
		maptile.DrawVector(shp, strokeColor, fillColor, m.EdgeWidth, 0)
	}
	err := png.Encode(w, maptile.I)
	if err != nil {
		return err
	}
	m.tileCache.Set(cacheKey, maptile.I, 0)
	return nil
}

func getGoogleTileBounds(zoom, x, y int) (N, S, E, W float64) {
	const originShift = math.Pi * 6378137. // for mercator projection
	// get boundaries in lat/lon
	n := math.Pow(2, float64(zoom))
	W_lon := float64(x)/n*360.0 - 180.0
	E_lon := float64(x+1)/n*360.0 - 180.0
	N_rad := math.Atan(math.Sinh(math.Pi * (1 - 2*float64(y)/n)))
	N_lat := N_rad * 180.0 / math.Pi
	S_rad := math.Atan(math.Sinh(math.Pi * (1 - 2*float64(y+1)/n)))
	S_lat := S_rad * 180.0 / math.Pi
	// convert to Mercator meters
	W = W_lon * originShift / 180.0
	E = E_lon * originShift / 180.0
	N = math.Log(math.Tan((90+N_lat)*math.Pi/360.0)) /
		(math.Pi / 180.0) * originShift / 180.0
	S = math.Log(math.Tan((90+S_lat)*math.Pi/360.0)) /
		(math.Pi / 180.0) * originShift / 180.0
	return
}

// convert from long/lat to google mercator (or EPSG:4326 to EPSG:900913)
func Degrees2meters(lon, lat float64) (x, y float64) {
	x = lon * 20037508.34 / 180.
	y = math.Log(math.Tan((90.+lat)*math.Pi/360.)) / (math.Pi / 180.)
	y *= 20037508.34 / 180.
	return x, y
}

type UnsupportedGeometryError struct {
	Type reflect.Type
}

func (e UnsupportedGeometryError) Error() string {
	return "Unsupported geometry type: " + e.Type.String()
}

// Convenience function for making a simple map.
func DrawShapes(f io.Writer, strokeColor, fillColor []color.NRGBA,
	linewidth, markersize float64, shapes ...geom.T) {
	bounds := geom.NewBounds()
	for _, s := range shapes {
		if s != nil {
			bounds = s.Bounds(bounds)
		}
	}
	m := NewRasterMap(bounds.Max.Y, bounds.Min.Y,
		bounds.Max.X, bounds.Min.X, 500, f)
	for i, s := range shapes {
		m.DrawVector(s, strokeColor[i], fillColor[i], linewidth, markersize)
	}
	m.Save()
}
<commit_msg>removed tile cache<commit_after>// Go language map drawing library
package carto

import (
	"bufio"
	"code.google.com/p/draw2d/draw2d"
	"github.com/twpayne/gogeom/geom"
	"image"
	"image/color"
	"image/draw"
	"image/png"
	"io"
	"math"
	"reflect"
)

type Mapper interface {
	DrawVector(geom.T, color.NRGBA, color.NRGBA, float64, float64)
	Save()
}

type RasterMap struct {
	bounds        *geom.Bounds // geographic boundaries of map
	width, height int          // pixel dimensions of map
	dx, dy        float64
	f             io.Writer
	I             draw.Image
	GC            draw2d.GraphicContext
	MarkerFunc    func(*RasterMap, float64, float64, float64) // function that specifies the shape of the marker for points
}

type MarkerFunction func(*RasterMap, float64, float64, float64) // Function for specifying the shape of the marker for points

var (
	Circle MarkerFunction = func(m *RasterMap, x, y, markersize float64) {
		m.GC.ArcTo(x, y, markersize, markersize, 0, 2*math.Pi)
	}
	Square MarkerFunction = func(m *RasterMap, x, y, markersize float64) {
		adjMS := markersize / 1.2 // ratio to adjust the markersize
		// to make the area be the same as the circle
		m.GC.MoveTo(x-adjMS, y-adjMS)
		m.GC.LineTo(x+adjMS, y-adjMS)
		m.GC.LineTo(x+adjMS, y+adjMS)
		m.GC.LineTo(x-adjMS, y+adjMS)
		m.GC.LineTo(x-adjMS, y-adjMS)
	}
	Triangle MarkerFunction = func(m *RasterMap, x, y, markersize float64) {
		adjMS := markersize / 0.75 // ratio to adjust the markersize
		// to make the area be the same as the circle
		cosval := math.Cos(0.125 * math.Pi)
		sinval := math.Sin(0.125 * math.Pi)
		m.GC.MoveTo(x-adjMS*cosval, y+adjMS*sinval)
		m.GC.LineTo(x+adjMS*cosval, y+adjMS*sinval)
		m.GC.LineTo(x, y-adjMS)
		m.GC.LineTo(x-adjMS*cosval, y+adjMS*sinval)
	}
	Star MarkerFunction = func(m *RasterMap, x, y, markersize float64) {
		adjMS := markersize / 0.75 // ratio to adjust the markersize
		// to make the area be the same as the circle
		var alpha = (2 * math.Pi) / 10
		// works out the angle between each vertex (5 external + 5 internal = 10)
		var r_concave = adjMS / 2.25 // r_point is the radius to the external point
		for i := 11; i != 0; i-- {
			var ra float64
			if i%2 == 1 {
				ra = adjMS
			} else {
				ra = r_concave
			}
			omega := alpha * float64(i) //omega is the angle of the current point
			//cx and cy are the center point of the star.
			if i == 11 {
				m.GC.MoveTo(x+(ra*math.Sin(omega)), y+(ra*math.Cos(omega)))
			} else {
				m.GC.LineTo(x+(ra*math.Sin(omega)), y+(ra*math.Cos(omega)))
			}
		}
	}
)

func NewRasterMap(N, S, E, W float64, width int, f io.Writer) *RasterMap {
	r := new(RasterMap)
	r.f = f
	r.bounds = geom.NewBoundsPoint(geom.Point{W, S})
	r.bounds.ExtendPoint(geom.Point{E, N})
	r.width, r.height = width, int(float64(width)*(N-S)/(E-W))
	r.dx = (E - W) / float64(r.width)
	r.dy = (N - S) / float64(r.height)
	r.I = image.NewRGBA(image.Rect(0, 0, r.width, r.height))
	r.GC = draw2d.NewGraphicContext(r.I)
	r.GC.SetFillRule(draw2d.FillRuleWinding)
	r.MarkerFunc = Circle
	return r
}

// Make a new raster map from raster data.
// It is assumed that the outer axis in the data is the Y-axis
// (north-south) and the inner axis is the X-axis (west-east)
// (i.e., len(data)==nx*ny &&  val[j,i] = data[j*nx+i]).
func NewRasterMapFromRaster(S, W, dy, dx float64, ny, nx int,
	data []float64, cmap *ColorMap, f io.Writer, flipVertical, flipHorizontal bool) *RasterMap {
	N := S + float64(ny)*dy
	E := W + float64(nx)*dx
	r := NewRasterMap(N, S, E, W, nx, f)
	r.height = ny
	if !flipVertical && !flipHorizontal {
		for i := 0; i < nx; i++ {
			for j := 0; j < ny; j++ {
				val := data[j*nx+i]
				r.I.Set(i, j, cmap.GetColor(val))
			}
		}
	} else if flipVertical && !flipHorizontal {
		for i := 0; i < nx; i++ {
			for j := 0; j < ny; j++ {
				val := data[j*nx+i]
				r.I.Set(i, ny-1-j, cmap.GetColor(val))
			}
		}
	} else if !flipVertical && flipHorizontal {
		for i := 0; i < nx; i++ {
			for j := 0; j < ny; j++ {
				val := data[j*nx+i]
				r.I.Set(nx-1-i, j, cmap.GetColor(val))
			}
		}
	} else if flipVertical && flipHorizontal {
		for i := 0; i < nx; i++ {
			for j := 0; j < ny; j++ {
				val := data[j*nx+i]
				r.I.Set(nx-1-i, ny-1-j, cmap.GetColor(val))
			}
		}
	}
	return r
}

// Draw a vector on a raster map when given the geometry,
// stroke and fill colors, the width of the bounding line,
// and the size of the marker (only used for point shapes).
func (r *RasterMap) DrawVector(g geom.T, strokeColor,
	fillColor color.NRGBA, linewidth, markersize float64) {
	// check bounding box
	if g == nil {
		return
	}
	gbounds := g.Bounds(nil)
	if !gbounds.Overlaps(r.bounds) {
		return
	}
	r.GC.SetStrokeColor(strokeColor)
	r.GC.SetFillColor(fillColor)
	r.GC.SetLineWidth(linewidth)
	switch g.(type) {
	case geom.Point:
		p := g.(geom.Point)
		x, y := r.coordinates(p.X, p.Y)
		r.MarkerFunc(r, x, y, markersize)
	//case geom.PointZ:
	//case geom.PointM:
	//case geom.PointZM:
	case geom.MultiPoint:
		for _, p := range g.(geom.MultiPoint).Points {
			x, y := r.coordinates(p.X, p.Y)
			r.GC.MoveTo(x, y)
			r.MarkerFunc(r, x, y, markersize)
		}
	//case geom.MultiPointZ:
	//case geom.MultiPointM:
	//case geom.MultiPointZM:
	case geom.LineString:
		l := g.(geom.LineString)
		for i, p := range l.Points {
			x, y := r.coordinates(p.X, p.Y)
			if i == 0 {
				r.GC.MoveTo(x, y)
			} else {
				r.GC.LineTo(x, y)
			}
		}
	//case geom.LineStringZ:
	//case geom.LineStringM:
	//case geom.LineStringZM:
	case geom.MultiLineString:
		l := g.(geom.MultiLineString)
		for _, ls := range l.LineStrings {
			r.DrawVector(ls, strokeColor,
				fillColor, linewidth, markersize)
		}
	//case geom.MultiLineStringZ:
	//case geom.MultiLineStringM:
	//case geom.MultiLineStringZM:
	case geom.Polygon:
		pg := g.(geom.Polygon)
		for _, ring := range pg.Rings {
			for i, p := range ring {
				x, y := r.coordinates(p.X, p.Y)
				if i == 0 {
					r.GC.MoveTo(x, y)
				} else {
					r.GC.LineTo(x, y)
				}
			}
		}
	//case geom.PolygonZ:
	//case geom.PolygonM:
	//case geom.PolygonZM:
	case geom.MultiPolygon:
		mpg := g.(geom.MultiPolygon)
		for _, pg := range mpg.Polygons {
			r.DrawVector(pg, strokeColor,
				fillColor, linewidth, markersize)
		}
	//case geom.MultiPolygonZ:
	//case geom.MultiPolygonM:
	//case geom.MultiPolygonZM:
	default:
		panic(&UnsupportedGeometryError{reflect.TypeOf(g)})
	}
	r.GC.FillStroke()
}

func (r *RasterMap) Save() {
	b := bufio.NewWriter(r.f)
	err := png.Encode(b, r.I)
	if err != nil {
		panic(err)
	}
	err = b.Flush()
	if err != nil {
		panic(err)
	}
}

// transform geographic coordinates to raster map coordinates
func (r *RasterMap) coordinates(X, Y float64) (
	x, y float64) {
	x = (X - r.bounds.Min.X) / r.dx
	y = float64(r.height) - 1. - (Y-r.bounds.Min.Y)/r.dy
	return
}

type MapData struct {
	Cmap      *ColorMap
	Shapes    []geom.T
	Data      []float64
	DrawEdges bool
	EdgeWidth float64
}

func NewMapData(numShapes int, Type ColorMapType) *MapData {
	m := new(MapData)
	m.Cmap = NewColorMap(Type)
	m.Shapes = make([]geom.T, numShapes)
	m.Data = make([]float64, numShapes)
	m.EdgeWidth = 0.5
	return m
}

func (m *MapData) WriteGoogleMapTile(w io.Writer, zoom, x, y int) error {
	//strokeColor := color.NRGBA{0, 0, 0, 255}
	N, S, E, W := getGoogleTileBounds(zoom, x, y)
	maptile := NewRasterMap(N, S, E, W, 256, w)

	var strokeColor color.NRGBA
	for i, shp := range m.Shapes {
		fillColor := m.Cmap.GetColor(m.Data[i])
		if m.DrawEdges {
			strokeColor = color.NRGBA{0, 0, 0, 255}
		} else {
			strokeColor = fillColor
		}
		// use the fill color for both the fill and the stroke
		// to avoid unsightly gaps between shapes.
		maptile.DrawVector(shp, strokeColor, fillColor, m.EdgeWidth, 0)
	}
	err := png.Encode(w, maptile.I)
	if err != nil {
		return err
	}
	return nil
}

func getGoogleTileBounds(zoom, x, y int) (N, S, E, W float64) {
	const originShift = math.Pi * 6378137. // for mercator projection
	// get boundaries in lat/lon
	n := math.Pow(2, float64(zoom))
	W_lon := float64(x)/n*360.0 - 180.0
	E_lon := float64(x+1)/n*360.0 - 180.0
	N_rad := math.Atan(math.Sinh(math.Pi * (1 - 2*float64(y)/n)))
	N_lat := N_rad * 180.0 / math.Pi
	S_rad := math.Atan(math.Sinh(math.Pi * (1 - 2*float64(y+1)/n)))
	S_lat := S_rad * 180.0 / math.Pi
	// convert to Mercator meters
	W = W_lon * originShift / 180.0
	E = E_lon * originShift / 180.0
	N = math.Log(math.Tan((90+N_lat)*math.Pi/360.0)) /
		(math.Pi / 180.0) * originShift / 180.0
	S = math.Log(math.Tan((90+S_lat)*math.Pi/360.0)) /
		(math.Pi / 180.0) * originShift / 180.0
	return
}

// convert from long/lat to google mercator (or EPSG:4326 to EPSG:900913)
func Degrees2meters(lon, lat float64) (x, y float64) {
	x = lon * 20037508.34 / 180.
	y = math.Log(math.Tan((90.+lat)*math.Pi/360.)) / (math.Pi / 180.)
	y *= 20037508.34 / 180.
	return x, y
}

type UnsupportedGeometryError struct {
	Type reflect.Type
}

func (e UnsupportedGeometryError) Error() string {
	return "Unsupported geometry type: " + e.Type.String()
}

// Convenience function for making a simple map.
func DrawShapes(f io.Writer, strokeColor, fillColor []color.NRGBA,
	linewidth, markersize float64, shapes ...geom.T) {
	bounds := geom.NewBounds()
	for _, s := range shapes {
		if s != nil {
			bounds = s.Bounds(bounds)
		}
	}
	m := NewRasterMap(bounds.Max.Y, bounds.Min.Y,
		bounds.Max.X, bounds.Min.X, 500, f)
	for i, s := range shapes {
		m.DrawVector(s, strokeColor[i], fillColor[i], linewidth, markersize)
	}
	m.Save()
}
<|endoftext|>


<commit_before>/*
   Copyright (c) 2014-2015, Percona LLC and/or its affiliates. All rights reserved.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>
*/

package slowlog

import (
	"fmt"
	"os"
	"time"

	"github.com/percona/cloud-protocol/proto"
	"github.com/percona/go-mysql/event"
	"github.com/percona/go-mysql/log"
	parser "github.com/percona/go-mysql/log/slow"
	"github.com/percona/go-mysql/query"
	"github.com/percona/percona-agent/mysql"
	"github.com/percona/percona-agent/pct"
	"github.com/percona/percona-agent/qan"
)

type WorkerFactory interface {
	Make(name string, config qan.Config, mysqlConn mysql.Connector) *Worker
}

type RealWorkerFactory struct {
	logChan chan *proto.LogEntry
}

func NewRealWorkerFactory(logChan chan *proto.LogEntry) *RealWorkerFactory {
	f := &RealWorkerFactory{
		logChan: logChan,
	}
	return f
}

func (f *RealWorkerFactory) Make(name string, config qan.Config, mysqlConn mysql.Connector) *Worker {
	return NewWorker(pct.NewLogger(f.logChan, name), config, mysqlConn)
}

// --------------------------------------------------------------------------

type Job struct {
	Id             string
	SlowLogFile    string
	RunTime        time.Duration
	StartOffset    int64
	EndOffset      int64
	ExampleQueries bool
}

func (j *Job) String() string {
	return fmt.Sprintf("%s %d-%d", j.SlowLogFile, j.StartOffset, j.EndOffset)
}

type Worker struct {
	logger    *pct.Logger
	config    qan.Config
	mysqlConn mysql.Connector
	// --
	ZeroRunTime bool // testing
	// --
	name            string
	status          *pct.Status
	queryChan       chan string
	fingerprintChan chan string
	errChan         chan interface{}
	doneChan        chan bool
	oldSlowLogs     map[int]string
	job             *Job
	sync            *pct.SyncChan
	running         bool
	logParser       log.LogParser
}

func NewWorker(logger *pct.Logger, config qan.Config, mysqlConn mysql.Connector) *Worker {
	name := logger.Service()
	w := &Worker{
		logger:    logger,
		config:    config,
		mysqlConn: mysqlConn,
		// --
		name:            name,
		status:          pct.NewStatus([]string{name}),
		queryChan:       make(chan string, 1),
		fingerprintChan: make(chan string, 1),
		errChan:         make(chan interface{}, 1),
		doneChan:        make(chan bool, 1),
		oldSlowLogs:     make(map[int]string),
		sync:            pct.NewSyncChan(),
	}
	return w
}

func (w *Worker) Setup(interval *qan.Interval) error {
	w.logger.Debug("Setup:call")
	defer w.logger.Debug("Setup:return")
	w.logger.Debug("Setup:", interval)
	if interval.EndOffset >= w.config.MaxSlowLogSize {
		w.logger.Info(fmt.Sprintf("Rotating slow log: %s >= %s",
			pct.Bytes(uint64(interval.EndOffset)),
			pct.Bytes(uint64(w.config.MaxSlowLogSize))))
		if err := w.rotateSlowLog(interval); err != nil {
			w.logger.Error(err)
		}
	}
	w.job = &Job{
		Id:             fmt.Sprintf("%d", interval.Number),
		SlowLogFile:    interval.Filename,
		StartOffset:    interval.StartOffset,
		EndOffset:      interval.EndOffset,
		RunTime:        time.Duration(w.config.WorkerRunTime) * time.Second,
		ExampleQueries: w.config.ExampleQueries,
	}
	w.logger.Debug("Setup:", w.job)
	return nil
}

func (w *Worker) Run() (*qan.Result, error) {
	w.logger.Debug("Run:call")
	defer w.logger.Debug("Run:return")

	w.status.Update(w.name, "Starting job "+w.job.Id)
	defer w.status.Update(w.name, "Idle")

	stopped := false
	w.running = true
	defer func() {
		if stopped {
			w.sync.Done()
		}
		w.running = false
	}()

	// Open the slow log file. Be sure to close it else we'll leak fd.
	file, err := os.Open(w.job.SlowLogFile)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	// Create a slow log parser and run it.  It sends log.Event via its channel.
	// Be sure to stop it when done, else we'll leak goroutines.
	result := &qan.Result{}
	opts := log.Options{
		StartOffset: uint64(w.job.StartOffset),
		FilterAdminCommand: map[string]bool{
			"Binlog Dump":      true,
			"Binlog Dump GTID": true,
		},
	}
	p := w.MakeLogParser(file, opts)
	go func() {
		defer func() {
			if err := recover(); err != nil {
				errMsg := fmt.Sprintf("Slow log parser for %s crashed: %s", w.job, err)
				w.logger.Error(errMsg)
				result.Error = errMsg
			}
		}()
		if err := p.Start(); err != nil {
			w.logger.Warn(err)
			result.Error = err.Error()
		}
	}()
	defer p.Stop()

	// Make an event aggregate to do all the heavy lifting: fingerprint
	// queries, group, and aggregate.
	a := event.NewEventAggregator(w.job.ExampleQueries)

	// Misc runtime meta data.
	jobSize := w.job.EndOffset - w.job.StartOffset
	runtime := time.Duration(0)
	progress := "Not started"
	rateType := ""
	rateLimit := uint(0)

	// Do fingerprinting in a separate Go routine so we can recover in case
	// query.Fingerprint() crashes. We don't want one bad fingerprint to stop
	// parsing the entire interval. Also, we want to log crashes and hopefully
	// fix the fingerprinter.
	go w.fingerprinter()
	defer func() { w.doneChan <- true }()

	t0 := time.Now()
EVENT_LOOP:
	for event := range p.EventChan() {
		runtime = time.Now().Sub(t0)
		progress = fmt.Sprintf("%.1f%% %d/%d %d %.1fs",
			float64(event.Offset)/float64(w.job.EndOffset)*100, event.Offset, w.job.EndOffset, jobSize, runtime.Seconds())
		w.status.Update(w.name, fmt.Sprintf("Parsing %s: %s", w.job.SlowLogFile, progress))

		// Stop if Stop() called.
		select {
		case <-w.sync.StopChan:
			w.logger.Debug("Run:stop")
			stopped = true
			break EVENT_LOOP
		default:
		}

		// Stop if runtime exceeded.
		if runtime >= w.job.RunTime {
			errMsg := fmt.Sprintf("Timeout parsing %s: %s", w.job, progress)
			w.logger.Warn(errMsg)
			result.Error = errMsg
			break EVENT_LOOP
		}

		// Stop if past file end offset. This happens often because we parse
		// only a slice of the slow log, and it's growing (if MySQL is busy),
		// so typical case is, for example, parsing from offset 100 to 5000
		// but slow log is already 7000 bytes large and growing. So the first
		// event with offset > 5000 marks the end (StopOffset) of this slice.
		if int64(event.Offset) >= w.job.EndOffset {
			result.StopOffset = int64(event.Offset)
			break EVENT_LOOP
		}

		// Stop if rate limits are mixed. This shouldn't happen. If it does,
		// another program or person might have reconfigured the rate limit.
		// We don't handle by design this because it's too much of an edge case.
		if event.RateType != "" {
			if rateType != "" {
				if rateType != event.RateType || rateLimit != event.RateLimit {
					errMsg := fmt.Sprintf("Slow log has mixed rate limits: %s/%d and %s/%d",
						rateType, rateLimit, event.RateType, event.RateLimit)
					w.logger.Warn(errMsg)
					result.Error = errMsg
					break EVENT_LOOP
				}
			} else {
				rateType = event.RateType
				rateLimit = event.RateLimit
			}
		}

		// Fingerprint the query and add it to the event aggregator. If the
		// fingerprinter crashes, start it again and skip this event.
		var fingerprint string
		w.queryChan <- event.Query
		select {
		case fingerprint = <-w.fingerprintChan:
			id := query.Id(fingerprint)
			a.AddEvent(event, id, fingerprint)
		case _ = <-w.errChan:
			w.logger.Warn(fmt.Sprintf("Cannot fingerprint '%s'", event.Query))
			go w.fingerprinter()
		}
	}

	// If StopOffset isn't set above it means we reached the end of the slow log
	// file. This happens if MySQL isn't busy so the slow log didn't grow any,
	// or we rotated the slow log in Setup() so we're finishing the rotated slow
	// log file. So the StopOffset is the end of the file which we're already
	// at, so use SEEK_CUR.
	if result.StopOffset == 0 {
		result.StopOffset, _ = file.Seek(0, os.SEEK_CUR)
	}

	// Finalize the global and class metrics, i.e. calculate metric stats.
	w.status.Update(w.name, "Finalizing job "+w.job.Id)
	r := a.Finalize()

	// The aggregator result is a map, but we need an array of classes for
	// the query report, so convert it.
	n := len(r.Class)
	classes := make([]*event.QueryClass, n)
	for _, class := range r.Class {
		n-- // can't classes[--n] in Go
		classes[n] = class
	}
	result.Global = r.Global
	result.Class = classes

	// Zero the runtime for testing.
	if !w.ZeroRunTime {
		result.RunTime = time.Now().Sub(t0).Seconds()
	}

	w.logger.Info(fmt.Sprintf("Parsed %s: %s", w.job, progress))
	return result, nil
}

func (w *Worker) Stop() error {
	w.logger.Debug("Stop:call")
	defer w.logger.Debug("Stop:return")
	if w.running {
		w.sync.Stop()
		w.sync.Wait()
	}
	return nil
}

func (w *Worker) Cleanup() error {
	w.logger.Debug("Cleanup:call")
	defer w.logger.Debug("Cleanup:return")
	for i, file := range w.oldSlowLogs {
		w.status.Update(w.name, "Removing slow log "+file)
		if err := os.Remove(file); err != nil {
			w.logger.Warn(err)
			continue
		}
		delete(w.oldSlowLogs, i)
		w.logger.Info("Removed " + file)
	}
	return nil
}

func (w *Worker) Status() map[string]string {
	return w.status.All()
}

func (w *Worker) SetLogParser(p log.LogParser) {
	// This is just for testing, so tests can inject a parser that does
	// abnormal things like be slow, crash, etc.
	w.logParser = p
}

func (w *Worker) MakeLogParser(file *os.File, opts log.Options) log.LogParser {
	if w.logParser != nil {
		p := w.logParser
		w.logParser = nil
		return p
	}
	return parser.NewSlowLogParser(file, opts)
}

// --------------------------------------------------------------------------

func (w *Worker) fingerprinter() {
	w.logger.Debug("fingerprinter:call")
	defer w.logger.Debug("fingerprinter:return")
	defer func() {
		if err := recover(); err != nil {
			w.errChan <- err
		}
	}()
	for {
		select {
		case q := <-w.queryChan:
			f := query.Fingerprint(q)
			w.fingerprintChan <- f
		case <-w.doneChan:
			return
		}
	}
}

func (w *Worker) rotateSlowLog(interval *qan.Interval) error {
	w.logger.Debug("rotateSlowLog:call")
	defer w.logger.Debug("rotateSlowLog:return")

	w.status.Update(w.name, "Rotating slow log")
	defer w.status.Update(w.name, "Idle")

	if err := w.mysqlConn.Connect(2); err != nil {
		return err
	}
	defer w.mysqlConn.Close()

	// Stop slow log so we don't move it while MySQL is using it.
	if err := w.mysqlConn.Set(w.config.Stop); err != nil {
		return err
	}

	// Move current slow log by renaming it.
	newSlowLogFile := fmt.Sprintf("%s-%d", interval.Filename, time.Now().UTC().Unix())
	if err := os.Rename(interval.Filename, newSlowLogFile); err != nil {
		return err
	}

	// Re-enable slow log.
	if err := w.mysqlConn.Set(w.config.Start); err != nil {
		return err
	}

	// Modify interval so worker parses the rest of the old slow log.
	interval.Filename = newSlowLogFile
	interval.EndOffset, _ = pct.FileSize(newSlowLogFile) // todo: handle err

	// Save old slow log and remove later if configured to do so.
	if w.config.RemoveOldSlowLogs {
		w.oldSlowLogs[interval.Number] = newSlowLogFile
	}

	return nil
}
<commit_msg>Restore 'query.ReplaceNumbersInWords = true'.<commit_after>/*
   Copyright (c) 2014-2015, Percona LLC and/or its affiliates. All rights reserved.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>
*/

package slowlog

import (
	"fmt"
	"os"
	"time"

	"github.com/percona/cloud-protocol/proto"
	"github.com/percona/go-mysql/event"
	"github.com/percona/go-mysql/log"
	parser "github.com/percona/go-mysql/log/slow"
	"github.com/percona/go-mysql/query"
	"github.com/percona/percona-agent/mysql"
	"github.com/percona/percona-agent/pct"
	"github.com/percona/percona-agent/qan"
)

type WorkerFactory interface {
	Make(name string, config qan.Config, mysqlConn mysql.Connector) *Worker
}

type RealWorkerFactory struct {
	logChan chan *proto.LogEntry
}

func NewRealWorkerFactory(logChan chan *proto.LogEntry) *RealWorkerFactory {
	f := &RealWorkerFactory{
		logChan: logChan,
	}
	return f
}

func (f *RealWorkerFactory) Make(name string, config qan.Config, mysqlConn mysql.Connector) *Worker {
	return NewWorker(pct.NewLogger(f.logChan, name), config, mysqlConn)
}

// --------------------------------------------------------------------------

type Job struct {
	Id             string
	SlowLogFile    string
	RunTime        time.Duration
	StartOffset    int64
	EndOffset      int64
	ExampleQueries bool
}

func (j *Job) String() string {
	return fmt.Sprintf("%s %d-%d", j.SlowLogFile, j.StartOffset, j.EndOffset)
}

type Worker struct {
	logger    *pct.Logger
	config    qan.Config
	mysqlConn mysql.Connector
	// --
	ZeroRunTime bool // testing
	// --
	name            string
	status          *pct.Status
	queryChan       chan string
	fingerprintChan chan string
	errChan         chan interface{}
	doneChan        chan bool
	oldSlowLogs     map[int]string
	job             *Job
	sync            *pct.SyncChan
	running         bool
	logParser       log.LogParser
}

func NewWorker(logger *pct.Logger, config qan.Config, mysqlConn mysql.Connector) *Worker {
	// By default replace numbers in words with ?
	query.ReplaceNumbersInWords = true

	name := logger.Service()
	w := &Worker{
		logger:    logger,
		config:    config,
		mysqlConn: mysqlConn,
		// --
		name:            name,
		status:          pct.NewStatus([]string{name}),
		queryChan:       make(chan string, 1),
		fingerprintChan: make(chan string, 1),
		errChan:         make(chan interface{}, 1),
		doneChan:        make(chan bool, 1),
		oldSlowLogs:     make(map[int]string),
		sync:            pct.NewSyncChan(),
	}
	return w
}

func (w *Worker) Setup(interval *qan.Interval) error {
	w.logger.Debug("Setup:call")
	defer w.logger.Debug("Setup:return")
	w.logger.Debug("Setup:", interval)
	if interval.EndOffset >= w.config.MaxSlowLogSize {
		w.logger.Info(fmt.Sprintf("Rotating slow log: %s >= %s",
			pct.Bytes(uint64(interval.EndOffset)),
			pct.Bytes(uint64(w.config.MaxSlowLogSize))))
		if err := w.rotateSlowLog(interval); err != nil {
			w.logger.Error(err)
		}
	}
	w.job = &Job{
		Id:             fmt.Sprintf("%d", interval.Number),
		SlowLogFile:    interval.Filename,
		StartOffset:    interval.StartOffset,
		EndOffset:      interval.EndOffset,
		RunTime:        time.Duration(w.config.WorkerRunTime) * time.Second,
		ExampleQueries: w.config.ExampleQueries,
	}
	w.logger.Debug("Setup:", w.job)
	return nil
}

func (w *Worker) Run() (*qan.Result, error) {
	w.logger.Debug("Run:call")
	defer w.logger.Debug("Run:return")

	w.status.Update(w.name, "Starting job "+w.job.Id)
	defer w.status.Update(w.name, "Idle")

	stopped := false
	w.running = true
	defer func() {
		if stopped {
			w.sync.Done()
		}
		w.running = false
	}()

	// Open the slow log file. Be sure to close it else we'll leak fd.
	file, err := os.Open(w.job.SlowLogFile)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	// Create a slow log parser and run it.  It sends log.Event via its channel.
	// Be sure to stop it when done, else we'll leak goroutines.
	result := &qan.Result{}
	opts := log.Options{
		StartOffset: uint64(w.job.StartOffset),
		FilterAdminCommand: map[string]bool{
			"Binlog Dump":      true,
			"Binlog Dump GTID": true,
		},
	}
	p := w.MakeLogParser(file, opts)
	go func() {
		defer func() {
			if err := recover(); err != nil {
				errMsg := fmt.Sprintf("Slow log parser for %s crashed: %s", w.job, err)
				w.logger.Error(errMsg)
				result.Error = errMsg
			}
		}()
		if err := p.Start(); err != nil {
			w.logger.Warn(err)
			result.Error = err.Error()
		}
	}()
	defer p.Stop()

	// Make an event aggregate to do all the heavy lifting: fingerprint
	// queries, group, and aggregate.
	a := event.NewEventAggregator(w.job.ExampleQueries)

	// Misc runtime meta data.
	jobSize := w.job.EndOffset - w.job.StartOffset
	runtime := time.Duration(0)
	progress := "Not started"
	rateType := ""
	rateLimit := uint(0)

	// Do fingerprinting in a separate Go routine so we can recover in case
	// query.Fingerprint() crashes. We don't want one bad fingerprint to stop
	// parsing the entire interval. Also, we want to log crashes and hopefully
	// fix the fingerprinter.
	go w.fingerprinter()
	defer func() { w.doneChan <- true }()

	t0 := time.Now()
EVENT_LOOP:
	for event := range p.EventChan() {
		runtime = time.Now().Sub(t0)
		progress = fmt.Sprintf("%.1f%% %d/%d %d %.1fs",
			float64(event.Offset)/float64(w.job.EndOffset)*100, event.Offset, w.job.EndOffset, jobSize, runtime.Seconds())
		w.status.Update(w.name, fmt.Sprintf("Parsing %s: %s", w.job.SlowLogFile, progress))

		// Stop if Stop() called.
		select {
		case <-w.sync.StopChan:
			w.logger.Debug("Run:stop")
			stopped = true
			break EVENT_LOOP
		default:
		}

		// Stop if runtime exceeded.
		if runtime >= w.job.RunTime {
			errMsg := fmt.Sprintf("Timeout parsing %s: %s", w.job, progress)
			w.logger.Warn(errMsg)
			result.Error = errMsg
			break EVENT_LOOP
		}

		// Stop if past file end offset. This happens often because we parse
		// only a slice of the slow log, and it's growing (if MySQL is busy),
		// so typical case is, for example, parsing from offset 100 to 5000
		// but slow log is already 7000 bytes large and growing. So the first
		// event with offset > 5000 marks the end (StopOffset) of this slice.
		if int64(event.Offset) >= w.job.EndOffset {
			result.StopOffset = int64(event.Offset)
			break EVENT_LOOP
		}

		// Stop if rate limits are mixed. This shouldn't happen. If it does,
		// another program or person might have reconfigured the rate limit.
		// We don't handle by design this because it's too much of an edge case.
		if event.RateType != "" {
			if rateType != "" {
				if rateType != event.RateType || rateLimit != event.RateLimit {
					errMsg := fmt.Sprintf("Slow log has mixed rate limits: %s/%d and %s/%d",
						rateType, rateLimit, event.RateType, event.RateLimit)
					w.logger.Warn(errMsg)
					result.Error = errMsg
					break EVENT_LOOP
				}
			} else {
				rateType = event.RateType
				rateLimit = event.RateLimit
			}
		}

		// Fingerprint the query and add it to the event aggregator. If the
		// fingerprinter crashes, start it again and skip this event.
		var fingerprint string
		w.queryChan <- event.Query
		select {
		case fingerprint = <-w.fingerprintChan:
			id := query.Id(fingerprint)
			a.AddEvent(event, id, fingerprint)
		case _ = <-w.errChan:
			w.logger.Warn(fmt.Sprintf("Cannot fingerprint '%s'", event.Query))
			go w.fingerprinter()
		}
	}

	// If StopOffset isn't set above it means we reached the end of the slow log
	// file. This happens if MySQL isn't busy so the slow log didn't grow any,
	// or we rotated the slow log in Setup() so we're finishing the rotated slow
	// log file. So the StopOffset is the end of the file which we're already
	// at, so use SEEK_CUR.
	if result.StopOffset == 0 {
		result.StopOffset, _ = file.Seek(0, os.SEEK_CUR)
	}

	// Finalize the global and class metrics, i.e. calculate metric stats.
	w.status.Update(w.name, "Finalizing job "+w.job.Id)
	r := a.Finalize()

	// The aggregator result is a map, but we need an array of classes for
	// the query report, so convert it.
	n := len(r.Class)
	classes := make([]*event.QueryClass, n)
	for _, class := range r.Class {
		n-- // can't classes[--n] in Go
		classes[n] = class
	}
	result.Global = r.Global
	result.Class = classes

	// Zero the runtime for testing.
	if !w.ZeroRunTime {
		result.RunTime = time.Now().Sub(t0).Seconds()
	}

	w.logger.Info(fmt.Sprintf("Parsed %s: %s", w.job, progress))
	return result, nil
}

func (w *Worker) Stop() error {
	w.logger.Debug("Stop:call")
	defer w.logger.Debug("Stop:return")
	if w.running {
		w.sync.Stop()
		w.sync.Wait()
	}
	return nil
}

func (w *Worker) Cleanup() error {
	w.logger.Debug("Cleanup:call")
	defer w.logger.Debug("Cleanup:return")
	for i, file := range w.oldSlowLogs {
		w.status.Update(w.name, "Removing slow log "+file)
		if err := os.Remove(file); err != nil {
			w.logger.Warn(err)
			continue
		}
		delete(w.oldSlowLogs, i)
		w.logger.Info("Removed " + file)
	}
	return nil
}

func (w *Worker) Status() map[string]string {
	return w.status.All()
}

func (w *Worker) SetLogParser(p log.LogParser) {
	// This is just for testing, so tests can inject a parser that does
	// abnormal things like be slow, crash, etc.
	w.logParser = p
}

func (w *Worker) MakeLogParser(file *os.File, opts log.Options) log.LogParser {
	if w.logParser != nil {
		p := w.logParser
		w.logParser = nil
		return p
	}
	return parser.NewSlowLogParser(file, opts)
}

// --------------------------------------------------------------------------

func (w *Worker) fingerprinter() {
	w.logger.Debug("fingerprinter:call")
	defer w.logger.Debug("fingerprinter:return")
	defer func() {
		if err := recover(); err != nil {
			w.errChan <- err
		}
	}()
	for {
		select {
		case q := <-w.queryChan:
			f := query.Fingerprint(q)
			w.fingerprintChan <- f
		case <-w.doneChan:
			return
		}
	}
}

func (w *Worker) rotateSlowLog(interval *qan.Interval) error {
	w.logger.Debug("rotateSlowLog:call")
	defer w.logger.Debug("rotateSlowLog:return")

	w.status.Update(w.name, "Rotating slow log")
	defer w.status.Update(w.name, "Idle")

	if err := w.mysqlConn.Connect(2); err != nil {
		return err
	}
	defer w.mysqlConn.Close()

	// Stop slow log so we don't move it while MySQL is using it.
	if err := w.mysqlConn.Set(w.config.Stop); err != nil {
		return err
	}

	// Move current slow log by renaming it.
	newSlowLogFile := fmt.Sprintf("%s-%d", interval.Filename, time.Now().UTC().Unix())
	if err := os.Rename(interval.Filename, newSlowLogFile); err != nil {
		return err
	}

	// Re-enable slow log.
	if err := w.mysqlConn.Set(w.config.Start); err != nil {
		return err
	}

	// Modify interval so worker parses the rest of the old slow log.
	interval.Filename = newSlowLogFile
	interval.EndOffset, _ = pct.FileSize(newSlowLogFile) // todo: handle err

	// Save old slow log and remove later if configured to do so.
	if w.config.RemoveOldSlowLogs {
		w.oldSlowLogs[interval.Number] = newSlowLogFile
	}

	return nil
}
<|endoftext|>


<commit_before>package rethinkgo

import (
	"fmt"
	test "launchpad.net/gocheck"
)

func (s *RethinkSuite) TestControlExecList(c *test.C) {
	tree := Expr(List{
		1, 2, 3, 4, 5, 6, List{
			7.1, 7.2, 7.3,
		},
	})
	fmt.Println(tree.String())
}
<commit_msg>Added some more tests for Expr/Do<commit_after>package rethinkgo

import (
	"fmt"
	test "launchpad.net/gocheck"
)

func (s *RethinkSuite) TestControlExecList(c *test.C) {
	query := Expr(List{
		1, 2, 3, 4, 5, 6, List{
			7.1, 7.2, 7.3,
		},
	})
	fmt.Println(query.String())
}

// func (s *RethinkSuite) TestControlExecObj(c *test.C) {
// 	query := Expr(Obj{
// 		"A": 1,
// 		"B": 2,
// 		"C": Obj{
// 			"1": 3,
// 			"2": 4,
// 		},
// 	})
// 	fmt.Println(query.String())
// }

func (s *RethinkSuite) TestControlDo(c *test.C) {
	query := Do(Obj{"a": 1}, Obj{"a": 2}, Obj{"a": 3}, func(row RqlTerm) RqlTerm {
		// return row.Field("a")
		return row
	})
	fmt.Println(query.String())
}

func (s *RethinkSuite) TestControlDoWithExpr(c *test.C) {
	query := Expr(List{
		Obj{"a": 1},
		Obj{"a": 2},
		Obj{"a": 3},
	}).Do(func(row RqlTerm) RqlTerm {
		// return row.Field("a")
		return row
	})
	fmt.Println(query.String())
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"os"
	_ "strings"
	"github.com/jessevdk/go-flags"
	"time"
	"path/filepath"
	"io/ioutil"
)

type Cookbook struct {
	Location string
}

type Artifact struct {
	Id string
	Version string
	Location string
}

type Release struct {
	Time      int
	Cookbooks []Cookbook
	Artifacts []Artifact
}

var opts struct {
	Verbose []bool `short:"v" long:"verbose" description:"Show verbose debug information"`
	ArtifactIds []string `short:"i" long:"artifact-id" description:"An artifact id"`
	ArtifactVersions []string `short:"v" long:"artifact-version" description:"An artifact version"`
	ArtifactLocations []string `short:"l" long:"artifact-location" description:"An artifact location"`
	Cookbooks []string `short:"c" long:"cookbook" description:"A cookbook location"`
}

func help() {
	fmt.Printf(`Usage: menu <command> <arguments and options>

Commands:
	help
	create
	show
	artifacts
	cookbooks
	list
`)
}

func helpHelp() {
	fmt.Printf(`Usage: menu help <command>

Commands:
	help
	create
	show
	artifacts
	cookbooks
	list
`)
}

func helpCreate() {
	fmt.Printf(`Usage: menu create

Options:
	-ai,--artifact-id
	-av,--artifact-version
	-al,--artifact-location
	-c,--cookbook
`)
}

func helpShow() {
	fmt.Printf(`Usage: menu show <path>
`)
}

func helpArtifacts() {
	fmt.Printf(`Usage: menu artifacts <path>
`)
}

func helpCookbooks() {
	fmt.Printf(`Usage: menu cookbooks <path>
`)
}

func helpList() {
	fmt.Printf(`Usage: menu list <options>

Options:
	-ai,--artifact-id
	-av,--artifact-version
	-al,--artifact-location
	-c,--cookbook
`)
}

func main() {
	args, err := flags.Parse(&opts)

	if err != nil {
		panic(err)
		help()
		os.Exit(1)
	}
	if len(args) == 0 {
		help()
		os.Exit(1)
	}

	if args[0] == "help" {
		handleHelp(args)
		os.Exit(0)
	}
	if args[0] == "create" {
		handleCreate(args)
		os.Exit(0)
	}

	/*
	fmt.Printf("Verbosity: %v\n", opts.Verbose)
	fmt.Printf("Cookbooks: %v\n", opts.Cookbooks)
	fmt.Printf("Command: %s\n", args[0])
	fmt.Printf("Remaining args: %s\n", strings.Join(args[1:], " "))
	*/
}

func handleHelp(args []string) {
	if len(args) > 1 {
		if args[1] == "create" {
			helpCreate()
			return
		}
		if args[1] == "show" {
			helpShow()
			return
		}
		if args[1] == "cookbooks" {
			helpCookbooks()
			return
		}
		if args[1] == "artifacts" {
			helpArtifacts()
			return
		}
		if args[1] == "list" {
			helpList()
			return
		}
	}
	helpHelp()
}

func handleCreate(args []string) {
	// [todo] Validate the options.
	if len(opts.ArtifactIds) != len(opts.ArtifactVersions) {
		fmt.Println("An equal number of artifact ids, versions and locations are required.")
		helpCreate()
		os.Exit(1)
	}
	if len(opts.ArtifactVersions) != len(opts.ArtifactLocations) {
		fmt.Println("An equal number of artifact ids, versions and locations are required.")
		helpCreate()
		os.Exit(1)
	}

	artifacts := make([]Artifact, len(opts.ArtifactVersions))
	cookbooks := make([]Cookbook, len(opts.Cookbooks))

	for i := 0; i < len(opts.ArtifactIds); i++ {
		artifact := Artifact{
			Id: opts.ArtifactIds[i],
			Version: opts.ArtifactVersions[i],
			Location: opts.ArtifactLocations[i],
		}
		artifacts[i] = artifact
	}

	for i := 0; i < len(opts.Cookbooks); i++ {
		cookbook := Cookbook{
			Location: opts.Cookbooks[i],
		}
		cookbooks[i] = cookbook
	}

	createdAt := time.Now().Unix()

	release := Release{
		Time: int(createdAt),
		Artifacts: artifacts,
		Cookbooks: cookbooks,
	}

	b, err := json.Marshal(release)
	if err != nil {
		fmt.Println("error:", err)
		os.Exit(1)
	}
	fmt.Println(string(b))

	path := getCreatePath(release, args)
	dir, fileName := filepath.Split(path)
	fmt.Println("Writing file ", fileName, " to directory ", dir)

	dirExists := false
	dirExists, err = exists(dir); if err != nil {
		panic(err)
	}
	if dirExists == false {
		err = os.MkdirAll(dir, os.ModeDir); if err != nil {
			panic(err)
		}
	}
	err = ioutil.WriteFile(path, b, 0644); if err != nil {
		panic(err)
	}
}

func exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

func getCreatePath(release Release, args []string) string {
	if len(args) > 1 {
		return args[1]
	}
	cwd, err := os.Getwd(); if err != nil {
		panic(err)
	}
	fileName := fmt.Sprintf("%v.menu", release.Time)
	return filepath.Join(cwd, fileName)
}

func handleShow(args []string) {
	helpShow()
}

func handleArtifacts(args []string) {
	helpArtifacts()
}

func handleCookbooks(args []string) {
	helpCookbooks()
}

func handleList(args []string) {
	helpList()
}
<commit_msg>minor code cleanup<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"os"
	_ "strings"
	"github.com/jessevdk/go-flags"
	"time"
	"path/filepath"
	"io/ioutil"
)

type Cookbook struct {
	Location string
}

type Artifact struct {
	Id string
	Version string
	Location string
}

type Release struct {
	Time      int
	Cookbooks []Cookbook
	Artifacts []Artifact
}

var opts struct {
	Verbose []bool `short:"v" long:"verbose" description:"Show verbose debug information"`
	ArtifactIds []string `short:"i" long:"artifact-id" description:"An artifact id"`
	ArtifactVersions []string `short:"v" long:"artifact-version" description:"An artifact version"`
	ArtifactLocations []string `short:"l" long:"artifact-location" description:"An artifact location"`
	Cookbooks []string `short:"c" long:"cookbook" description:"A cookbook location"`
}

func help() {
	fmt.Printf(`Usage: menu <command> <arguments and options>

Commands:
	help
	create
	show
	artifacts
	cookbooks
	list
`)
}

func helpHelp() {
	fmt.Printf(`Usage: menu help <command>

Commands:
	help
	create
	show
	artifacts
	cookbooks
	list
`)
}

func helpCreate() {
	fmt.Printf(`Usage: menu create

Options:
	-ai,--artifact-id
	-av,--artifact-version
	-al,--artifact-location
	-c,--cookbook
`)
}

func helpShow() {
	fmt.Printf(`Usage: menu show <path>
`)
}

func helpArtifacts() {
	fmt.Printf(`Usage: menu artifacts <path>
`)
}

func helpCookbooks() {
	fmt.Printf(`Usage: menu cookbooks <path>
`)
}

func helpList() {
	fmt.Printf(`Usage: menu list <options>

Options:
	-ai,--artifact-id
	-av,--artifact-version
	-al,--artifact-location
	-c,--cookbook
`)
}

func main() {
	args, err := flags.Parse(&opts)

	if err != nil {
		panic(err)
		help()
		os.Exit(1)
	}
	if len(args) == 0 {
		help()
		os.Exit(1)
	}

	if args[0] == "help" {
		handleHelp(args)
		os.Exit(0)
	}
	if args[0] == "create" {
		handleCreate(args)
		os.Exit(0)
	}

	/*
	fmt.Printf("Verbosity: %v\n", opts.Verbose)
	fmt.Printf("Cookbooks: %v\n", opts.Cookbooks)
	fmt.Printf("Command: %s\n", args[0])
	fmt.Printf("Remaining args: %s\n", strings.Join(args[1:], " "))
	*/
}

func handleHelp(args []string) {
	if len(args) > 1 {
		if args[1] == "create" {
			helpCreate()
			return
		}
		if args[1] == "show" {
			helpShow()
			return
		}
		if args[1] == "cookbooks" {
			helpCookbooks()
			return
		}
		if args[1] == "artifacts" {
			helpArtifacts()
			return
		}
		if args[1] == "list" {
			helpList()
			return
		}
	}
	helpHelp()
}

func handleCreate(args []string) {
	// [todo] Validate the options.
	if len(opts.ArtifactIds) != len(opts.ArtifactVersions) {
		fmt.Println("An equal number of artifact ids, versions and locations are required.")
		helpCreate()
		os.Exit(1)
	}
	if len(opts.ArtifactVersions) != len(opts.ArtifactLocations) {
		fmt.Println("An equal number of artifact ids, versions and locations are required.")
		helpCreate()
		os.Exit(1)
	}

	artifacts := make([]Artifact, len(opts.ArtifactVersions))
	cookbooks := make([]Cookbook, len(opts.Cookbooks))

	for i := 0; i < len(opts.ArtifactIds); i++ {
		artifact := Artifact{
			Id: opts.ArtifactIds[i],
			Version: opts.ArtifactVersions[i],
			Location: opts.ArtifactLocations[i],
		}
		artifacts[i] = artifact
	}

	for i := 0; i < len(opts.Cookbooks); i++ {
		cookbook := Cookbook{
			Location: opts.Cookbooks[i],
		}
		cookbooks[i] = cookbook
	}

	createdAt := time.Now().Unix()

	release := Release{
		Time: int(createdAt),
		Artifacts: artifacts,
		Cookbooks: cookbooks,
	}

	b, err := json.Marshal(release)
	if err != nil {
		fmt.Println("error:", err)
		os.Exit(1)
	}
	if len(opts.Verbose) > 0 {
		fmt.Println(string(b))
	}

	path := getCreatePath(release, args)
	dir, fileName := filepath.Split(path)

	if len(opts.Verbose) > 0 {
		fmt.Println("Writing file ", fileName, " to directory ", dir)
	}

	dirExists := false
	dirExists, err = exists(dir); if err != nil {
		panic(err)
	}
	if dirExists == false {
		err = os.MkdirAll(dir, os.ModeDir); if err != nil {
			panic(err)
		}
	}
	err = ioutil.WriteFile(path, b, 0644); if err != nil {
		panic(err)
	}
}

func exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

func getCreatePath(release Release, args []string) string {
	if len(args) > 1 {
		return args[1]
	}
	cwd, err := os.Getwd(); if err != nil {
		panic(err)
	}
	fileName := fmt.Sprintf("%v.menu", release.Time)
	return filepath.Join(cwd, fileName)
}

func handleShow(args []string) {
	helpShow()
}

func handleArtifacts(args []string) {
	helpArtifacts()
}

func handleCookbooks(args []string) {
	helpCookbooks()
}

func handleList(args []string) {
	helpList()
}
<|endoftext|>