<commit_before>package clarifai

import (
	"net/http"
)

const (
	// Public pre-defined models.
	// Source: https://developer-preview.clarifai.com/guide/publicmodels#public-models

	// PublicModelGeneral is a public model "general".
	PublicModelGeneral = "aaa03c23b3724a16a56b629203edc62c"

	// PublicModelFood is a public model "food".
	PublicModelFood = "bd367be194cf45149e75f01d59f77ba7"

	// PublicModelTravel is a public model "travel".
	PublicModelTravel = "eee28c313d69466f836ab83287a54ed9"

	// PublicModelNSFW is a public model "NSFW" (Not Safe For Work).
	PublicModelNSFW = "e9576d86d2004ed1a38ba0cf39ecb4b1"

	// PublicModelWeddings is a public model "weddings".
	PublicModelWeddings = "c386b7a870114f4a87477c0824499348"

	// PublicModelColor is a public model "color".
	PublicModelColor = "eeed0b6733a644cea07cf4c60f87ebb7"
)

type ModelRequest struct {
	Model Model `json:"model"`
}

type Model struct {
	Name         *string       `json:"name,omitempty"`
	ID           *string       `json:"id,omitempty"`
	CreatedAt    *string       `json:"created_at,omitempty"`
	AppID        *string       `json:"app_id,omitempty"`
	OutputInfo   *OutputInfo   `json:"output_info,omitempty"`
	ModelVersion *ModelVersion `json:"model_version,omitempty"`
}

type OutputInfo struct {
	Message      string        `json:"message,omitempty"`
	Type         string        `json:"type,omitempty"`
	OutputConfig *OutputConfig `json:"output_config,omitempty"`
	OutputData   *OutputData   `json:"data,omitempty"`
}

type ModelVersion struct {
	ID        string         `json:"id"`
	CreatedAt string         `json:"created_at"`
	Status    *ServiceStatus `json:"status"`
}

type OutputConfig struct {
	ConceptsMutuallyExclusive bool `json:"concepts_mutually_exclusive"`
	ClosedEnvironment         bool `json:"closed_environment"`
}

// modelOptions is a model configuration object used to set optional settings for a new model.
type modelOptions struct {
	ID                        string   // Model ID. If not set, wil be generated automatically.
	Concepts                  []string // Optional concepts to associated with this model
	ConceptsMutuallyExclusive bool     // True or False, whether concepts are mutually exclusive
	ClosedEnvironment         bool     // True or False, whether use negatives for prediction
}

// ModelQuery is a model search payload.
type ModelQuery struct {
	Name string `json:"name,omitempty"`
	Type string `json:"type,omitempty"`
}

// NewModelOptions returns default model options.
func NewModelOptions() *modelOptions {
	return &modelOptions{}
}

// Predict fetches prediction info for a provided asset from a given model.
func (s *Session) Predict(i *Inputs) *Request {

	r := NewRequest(s, http.MethodPost, "models/"+i.modelID+"/outputs")
	r.SetPayload(i)

	return r
}

// CreateModel creates a new model. If ID is empty, it will be created automatically by Clarifai API.
func (s *Session) CreateModel(name string, opt *modelOptions) *Request {

	p := ModelRequest{
		Model: Model{
			Name: &name,
		},
	}

	if p.Model.OutputInfo == nil {
		p.Model.OutputInfo = &OutputInfo{
			OutputData:   &OutputData{},
			OutputConfig: &OutputConfig{},
		}
	}

	if opt.ID != "" {
		p.Model.ID = &opt.ID
	}

	if len(opt.Concepts) > 0 {
		var concepts []*OutputConcept

		for _, id := range opt.Concepts {
			concepts = append(concepts, &OutputConcept{
				ID: id,
			})
		}

		p.Model.OutputInfo.OutputData.Concepts = concepts
	}

	if opt.Concepts != nil {
		p.Model.OutputInfo.OutputConfig.ConceptsMutuallyExclusive = opt.ConceptsMutuallyExclusive
	}

	if opt.Concepts != nil {
		p.Model.OutputInfo.OutputConfig.ClosedEnvironment = opt.ClosedEnvironment
	}

	r := NewRequest(s, http.MethodPost, "models")
	r.SetPayload(p)

	return r
}

// AddModelConcepts adds new concepts to an existing model.
func (s *Session) AddModelConcepts(ID string, c []string) *Request {

	r := NewRequest(s, http.MethodPatch, "models/"+ID+"/output_info/data/concepts")

	p := struct {
		Concepts []*OutputConcept `json:"concepts"`
		Action   string           `json:"action"`
	}{
		Concepts: sliceToConcepts(c),
		Action:   "merge_concepts",
	}

	r.SetPayload(p)

	return r
}

// GetModels fetches a list of all models, including custom and public.
func (s *Session) GetModels() *Request {

	return NewRequest(s, http.MethodGet, "models")
}

// GetModel fetches a single model by its ID.
func (s *Session) GetModel(ID string) *Request {

	return NewRequest(s, http.MethodGet, "models/"+ID)
}

// GetModelOutput fetches a single model by its ID with output_info data.
func (s *Session) GetModelOutput(ID string) *Request {

	return NewRequest(s, http.MethodGet, "models/"+ID+"/output_info")
}

// GetModelVersion fetches version data of a single model .
func (s *Session) GetModelVersion(m, v string) *Request {

	return NewRequest(s, http.MethodGet, "models/"+m+"/versions/"+v)
}

// GetModelVersionInputs fetches inputs used to train a specific model version.
func (s *Session) GetModelVersionInputs(m, v string) *Request {

	return NewRequest(s, http.MethodGet, "models/"+m+"/versions/"+v+"/inputs")
}

// GetModelVersions fetches a single model by its ID with versions data.
func (s *Session) GetModelVersions(ID string) *Request {

	return NewRequest(s, http.MethodGet, "models/"+ID+"/versions")
}

// GetModelInputs fetches inputs of a single model by its ID.
func (s *Session) GetModelInputs(ID string) *Request {

	return NewRequest(s, http.MethodGet, "models/"+ID+"/inputs")
}

// DeleteModelVersion deletes a specific version of a model.
func (s *Session) DeleteModelVersion(m, v string) *Request {

	return NewRequest(s, http.MethodDelete, "models/"+m+"/versions/"+v)
}

// DeleteModel deletes a single model by ID.
func (s *Session) DeleteModel(ID string) *Request {

	return NewRequest(s, http.MethodDelete, "models/"+ID)
}

// DeleteAllModels deletes all models associated with your application.
func (s *Session) DeleteAllModels() *Request {

	return NewRequest(s, http.MethodDelete, "models")
}

// TrainModel starts a model training operation.
// When you train a model, you are telling the system to look at all the images with concepts you've provided and learn from them.
// This train operation is asynchronous. It may take a few seconds for your model to be fully trained and ready.
func (s *Session) TrainModel(ID string) *Request {

	r := NewRequest(s, http.MethodPost, "models/"+ID+"/versions")

	return r
}

// DeleteModelConcepts removes concepts from a model.
func (s *Session) DeleteModelConcepts(ID string, c []string) *Request {

	r := NewRequest(s, http.MethodPatch, "models/"+ID+"/output_info/data/concepts")

	p := struct {
		Concepts []*OutputConcept `json:"concepts"`
		Action   string           `json:"action"`
	}{
		Concepts: sliceToConcepts(c),
		Action:   "delete_concepts",
	}

	r.SetPayload(p)

	return r
}

// SearchModel searches models by name and/or type.
func (s *Session) SearchModel(n, t string) *Request {

	r := NewRequest(s, http.MethodPost, "models/searches")

	mq := ModelQuery{}
	if n != "" {
		mq.Name = n
	}
	if t != "" {
		mq.Type = t
	}

	p := struct {
		ModelQuery ModelQuery `json:"model_query"`
	}{
		ModelQuery: mq,
	}

	r.SetPayload(p)

	return r
}

// Convert a slice of concepts into a request format.
func sliceToConcepts(c []string) []*OutputConcept {
	var concepts []*OutputConcept

	for _, id := range c {
		concepts = append(concepts, &OutputConcept{
			ID: id,
		})
	}

	return concepts
}
<commit_msg>updated add models<commit_after>package clarifai

import (
	"net/http"
)

const (
	// Public pre-defined models.
	// Source: https://developer-preview.clarifai.com/guide/publicmodels#public-models

	// PublicModelGeneral is a public model "general".
	PublicModelGeneral = "aaa03c23b3724a16a56b629203edc62c"

	// PublicModelFood is a public model "food".
	PublicModelFood = "bd367be194cf45149e75f01d59f77ba7"

	// PublicModelTravel is a public model "travel".
	PublicModelTravel = "eee28c313d69466f836ab83287a54ed9"

	// PublicModelNSFW is a public model "NSFW" (Not Safe For Work).
	PublicModelNSFW = "e9576d86d2004ed1a38ba0cf39ecb4b1"

	// PublicModelWeddings is a public model "weddings".
	PublicModelWeddings = "c386b7a870114f4a87477c0824499348"

	// PublicModelColor is a public model "color".
	PublicModelColor = "eeed0b6733a644cea07cf4c60f87ebb7"
)

type ModelRequest struct {
	Model Model `json:"model"`
}

type Model struct {
	Name         *string       `json:"name,omitempty"`
	ID           *string       `json:"id,omitempty"`
	CreatedAt    *string       `json:"created_at,omitempty"`
	AppID        *string       `json:"app_id,omitempty"`
	OutputInfo   *OutputInfo   `json:"output_info,omitempty"`
	ModelVersion *ModelVersion `json:"model_version,omitempty"`
}

type OutputInfo struct {
	Message      string        `json:"message,omitempty"`
	Type         string        `json:"type,omitempty"`
	OutputConfig *OutputConfig `json:"output_config,omitempty"`
	OutputData   *OutputData   `json:"data,omitempty"`
}

type ModelVersion struct {
	ID        string         `json:"id"`
	CreatedAt string         `json:"created_at"`
	Status    *ServiceStatus `json:"status"`
}

type OutputConfig struct {
	ConceptsMutuallyExclusive bool `json:"concepts_mutually_exclusive"`
	ClosedEnvironment         bool `json:"closed_environment"`
}

// modelOptions is a model configuration object used to set optional settings for a new model.
type modelOptions struct {
	ID                        string   // Model ID. If not set, wil be generated automatically.
	Concepts                  []string // Optional concepts to associated with this model
	ConceptsMutuallyExclusive bool     // True or False, whether concepts are mutually exclusive
	ClosedEnvironment         bool     // True or False, whether use negatives for prediction
}

// ModelQuery is a model search payload.
type ModelQuery struct {
	Name string `json:"name,omitempty"`
	Type string `json:"type,omitempty"`
}

// NewModelOptions returns default model options.
func NewModelOptions() *modelOptions {
	return &modelOptions{}
}

// Predict fetches prediction info for a provided asset from a given model.
func (s *Session) Predict(i *Inputs) *Request {

	r := NewRequest(s, http.MethodPost, "models/"+i.modelID+"/outputs")
	r.SetPayload(i)

	return r
}

// CreateModel creates a new model. If ID is empty, it will be created automatically by Clarifai API.
func (s *Session) CreateModel(name string, opt *modelOptions) *Request {

	p := ModelRequest{
		Model: Model{
			Name: &name,
		},
	}

	if p.Model.OutputInfo == nil {
		p.Model.OutputInfo = &OutputInfo{
			OutputData:   &OutputData{},
			OutputConfig: &OutputConfig{},
		}
	}

	if opt.ID != "" {
		p.Model.ID = &opt.ID
	}

	if len(opt.Concepts) > 0 {
		var concepts []*OutputConcept

		for _, id := range opt.Concepts {
			concepts = append(concepts, &OutputConcept{
				ID: id,
			})
		}

		p.Model.OutputInfo.OutputData.Concepts = concepts
	}

	if opt.Concepts != nil {
		p.Model.OutputInfo.OutputConfig.ConceptsMutuallyExclusive = opt.ConceptsMutuallyExclusive
	}

	if opt.Concepts != nil {
		p.Model.OutputInfo.OutputConfig.ClosedEnvironment = opt.ClosedEnvironment
	}

	r := NewRequest(s, http.MethodPost, "models")
	r.SetPayload(p)

	return r
}

// AddModelConcepts adds new concepts to an existing model.
func (s *Session) AddModelConcepts(ID string, c []string) *Request {

	r := NewRequest(s, http.MethodPatch, "models/")

	p := struct {
		Models []Model `json:"models"`
		Action string  `json:"action"`
	}{
		Models: []Model{
			{
				ID: &ID,
				OutputInfo: &OutputInfo{
					OutputData: &OutputData{
						Concepts: sliceToConcepts(c),
					},
				},
			},
		},
		Action: "merge",
	}

	r.SetPayload(p)

	return r
}

// GetModels fetches a list of all models, including custom and public.
func (s *Session) GetModels() *Request {

	return NewRequest(s, http.MethodGet, "models")
}

// GetModel fetches a single model by its ID.
func (s *Session) GetModel(ID string) *Request {

	return NewRequest(s, http.MethodGet, "models/"+ID)
}

// GetModelOutput fetches a single model by its ID with output_info data.
func (s *Session) GetModelOutput(ID string) *Request {

	return NewRequest(s, http.MethodGet, "models/"+ID+"/output_info")
}

// GetModelVersion fetches version data of a single model .
func (s *Session) GetModelVersion(m, v string) *Request {

	return NewRequest(s, http.MethodGet, "models/"+m+"/versions/"+v)
}

// GetModelVersionInputs fetches inputs used to train a specific model version.
func (s *Session) GetModelVersionInputs(m, v string) *Request {

	return NewRequest(s, http.MethodGet, "models/"+m+"/versions/"+v+"/inputs")
}

// GetModelVersions fetches a single model by its ID with versions data.
func (s *Session) GetModelVersions(ID string) *Request {

	return NewRequest(s, http.MethodGet, "models/"+ID+"/versions")
}

// GetModelInputs fetches inputs of a single model by its ID.
func (s *Session) GetModelInputs(ID string) *Request {

	return NewRequest(s, http.MethodGet, "models/"+ID+"/inputs")
}

// DeleteModelVersion deletes a specific version of a model.
func (s *Session) DeleteModelVersion(m, v string) *Request {

	return NewRequest(s, http.MethodDelete, "models/"+m+"/versions/"+v)
}

// DeleteModel deletes a single model by ID.
func (s *Session) DeleteModel(ID string) *Request {

	return NewRequest(s, http.MethodDelete, "models/"+ID)
}

// DeleteAllModels deletes all models associated with your application.
func (s *Session) DeleteAllModels() *Request {

	return NewRequest(s, http.MethodDelete, "models")
}

// TrainModel starts a model training operation.
// When you train a model, you are telling the system to look at all the images with concepts you've provided and learn from them.
// This train operation is asynchronous. It may take a few seconds for your model to be fully trained and ready.
func (s *Session) TrainModel(ID string) *Request {

	r := NewRequest(s, http.MethodPost, "models/"+ID+"/versions")

	return r
}

// DeleteModelConcepts removes concepts from a model.
func (s *Session) DeleteModelConcepts(ID string, c []string) *Request {

	r := NewRequest(s, http.MethodPatch, "models/"+ID+"/output_info/data/concepts")

	p := struct {
		Concepts []*OutputConcept `json:"concepts"`
		Action   string           `json:"action"`
	}{
		Concepts: sliceToConcepts(c),
		Action:   "delete_concepts",
	}

	r.SetPayload(p)

	return r
}

// SearchModel searches models by name and/or type.
func (s *Session) SearchModel(n, t string) *Request {

	r := NewRequest(s, http.MethodPost, "models/searches")

	mq := ModelQuery{}
	if n != "" {
		mq.Name = n
	}
	if t != "" {
		mq.Type = t
	}

	p := struct {
		ModelQuery ModelQuery `json:"model_query"`
	}{
		ModelQuery: mq,
	}

	r.SetPayload(p)

	return r
}

// Convert a slice of concepts into a request format.
func sliceToConcepts(c []string) []*OutputConcept {
	var concepts []*OutputConcept

	for _, id := range c {
		concepts = append(concepts, &OutputConcept{
			ID: id,
		})
	}

	return concepts
}
<|endoftext|>


<commit_before>// Package usl provides functionality to build Universal Scalability Law models
// from sets of observed measurements.
package usl

import (
	"errors"
	"fmt"
	"math"

	"github.com/maorshutman/lm"
)

var (
	// ErrInsufficientMeasurements is returned when fewer than 6 measurements were provided.
	ErrInsufficientMeasurements = errors.New("usl: need at least 6 measurements")
)

// Model is a Universal Scalability Law model.
type Model struct {
	Sigma  float64 // The model's coefficient of contention, σ.
	Kappa  float64 // The model's coefficient of crosstalk/coherency, κ.
	Lambda float64 // The model's coefficient of performance, λ.
}

func (m *Model) String() string {
	return fmt.Sprintf("Model{σ=%v,κ=%v,λ=%v}", m.Sigma, m.Kappa, m.Lambda)
}

// ThroughputAtConcurrency returns the expected throughput given a number of concurrent events,
// X(N).
//
// See "Practical Scalability Analysis with the Universal Scalability Law, Equation 3".
func (m *Model) ThroughputAtConcurrency(n float64) float64 {
	return (m.Lambda * n) / (1 + (m.Sigma * (n - 1)) + (m.Kappa * n * (n - 1)))
}

// LatencyAtConcurrency returns the expected mean latency given a number of concurrent events,
// R(N).
//
// See "Practical Scalability Analysis with the Universal Scalability Law, Equation 6".
func (m *Model) LatencyAtConcurrency(n float64) float64 {
	return (1 + (m.Sigma * (n - 1)) + (m.Kappa * n * (n - 1))) / m.Lambda
}

// MaxConcurrency returns the maximum expected number of concurrent events the system can handle,
// Nmax.
//
// See "Practical Scalability Analysis with the Universal Scalability Law, Equation 4".
func (m *Model) MaxConcurrency() float64 {
	return math.Floor(math.Sqrt((1 - m.Sigma) / m.Kappa))
}

// MaxThroughput returns the maximum expected throughput the system can handle, Xmax.
func (m Model) MaxThroughput() float64 {
	return m.ThroughputAtConcurrency(m.MaxConcurrency())
}

// LatencyAtThroughput returns the expected mean latency given a throughput, R(X).
//
// See "Practical Scalability Analysis with the Universal Scalability Law, Equation 8".
func (m *Model) LatencyAtThroughput(x float64) float64 {
	return (m.Sigma - 1) / (m.Sigma*x - m.Lambda)
}

// ThroughputAtLatency returns the expected throughput given a mean latency, X(R).
//
// See "Practical Scalability Analysis with the Universal Scalability Law, Equation 9".
func (m *Model) ThroughputAtLatency(r float64) float64 {
	return (math.Sqrt(math.Pow(m.Sigma, 2)+math.Pow(m.Kappa, 2)+
		2*m.Kappa*(2*m.Lambda*r+m.Sigma-2)) - m.Kappa + m.Sigma) / (2.0 * m.Kappa * r)
}

// ConcurrencyAtLatency returns the expected number of concurrent events at a particular mean
// latency, N(R).
//
// See "Practical Scalability Analysis with the Universal Scalability Law, Equation 10".
func (m *Model) ConcurrencyAtLatency(r float64) float64 {
	return (m.Kappa - m.Sigma +
		math.Sqrt(math.Pow(m.Sigma, 2)+
			math.Pow(m.Kappa, 2)+
			2*m.Kappa*((2*m.Lambda*r)+m.Sigma-2))) / (2 * m.Kappa)
}

// ConcurrencyAtThroughput returns the expected number of concurrent events at a particular
// throughput, N(X).
func (m *Model) ConcurrencyAtThroughput(x float64) float64 {
	return m.LatencyAtThroughput(x) * x
}

// ContentionConstrained returns true if the system is constrained by contention.
func (m *Model) ContentionConstrained() bool {
	return m.Sigma > m.Kappa
}

// CoherencyConstrained returns true if the system is constrained by coherency costs.
func (m *Model) CoherencyConstrained() bool {
	return m.Sigma < m.Kappa
}

// Limitless returns true if the system is linearly scalable.
func (m *Model) Limitless() bool {
	return m.Kappa == 0
}

const minMeasurements = 6

// Build returns a model whose parameters are generated from the given measurements.
//
// Finds a set of coefficients for the equation y = λx/(1+σ(x-1)+κx(x-1)) which best fit the
// observed values using unconstrained least-squares regression. The resulting values for λ, κ, and
// σ are the parameters of the returned model.
func Build(measurements []Measurement) (m *Model, err error) {
	if len(measurements) < minMeasurements {
		return nil, ErrInsufficientMeasurements
	}

	// Calculate an initial guess at the model parameters.
	init := []float64{0.1, 0.01, 0}

	// Use max(x/n) as initial lambda.
	for _, m := range measurements {
		v := m.Throughput / m.Concurrency
		if v > init[2] {
			init[2] = v
		}
	}

	// Calculate the residuals of a possible model.
	f := func(dst, x []float64) {
		model := Model{Sigma: x[0], Kappa: x[1], Lambda: x[2]}

		for i, v := range measurements {
			dst[i] = v.Throughput - model.ThroughputAtConcurrency(v.Concurrency)
		}
	}
	j := lm.NumJac{Func: f}

	// Formulate an LM problem.
	p := lm.LMProblem{
		Dim:        3,                 // Three parameters in the model.
		Size:       len(measurements), // Use all measurements to calculate residuals.
		Func:       f,                 // Reduce the residuals of model predictions to observations.
		Jac:        j.Jac,             // Approximate the Jacobian by finite differences.
		InitParams: init,              // Use our initial guesses at parameters.
		Tau:        1e-6,              // Need a non-zero initial damping factor.
		Eps1:       1e-8,              // Small but non-zero values here prevent singular matrices.
		Eps2:       1e-8,
	}

	// Calculate the model parameters.
	results, err := lm.LM(p, nil)
	if err != nil {
		return nil, fmt.Errorf("unable to build model: %w", err)
	}

	// Return the model.
	return &Model{
		Sigma:  results.X[0],
		Kappa:  results.X[1],
		Lambda: results.X[2],
	}, nil
}
<commit_msg>fmtfix<commit_after>// Package usl provides functionality to build Universal Scalability Law models
// from sets of observed measurements.
package usl

import (
	"fmt"
	"math"

	"github.com/maorshutman/lm"
)

// Model is a Universal Scalability Law model.
type Model struct {
	Sigma  float64 // The model's coefficient of contention, σ.
	Kappa  float64 // The model's coefficient of crosstalk/coherency, κ.
	Lambda float64 // The model's coefficient of performance, λ.
}

func (m *Model) String() string {
	return fmt.Sprintf("Model{σ=%v,κ=%v,λ=%v}", m.Sigma, m.Kappa, m.Lambda)
}

// ThroughputAtConcurrency returns the expected throughput given a number of concurrent events,
// X(N).
//
// See "Practical Scalability Analysis with the Universal Scalability Law, Equation 3".
func (m *Model) ThroughputAtConcurrency(n float64) float64 {
	return (m.Lambda * n) / (1 + (m.Sigma * (n - 1)) + (m.Kappa * n * (n - 1)))
}

// LatencyAtConcurrency returns the expected mean latency given a number of concurrent events,
// R(N).
//
// See "Practical Scalability Analysis with the Universal Scalability Law, Equation 6".
func (m *Model) LatencyAtConcurrency(n float64) float64 {
	return (1 + (m.Sigma * (n - 1)) + (m.Kappa * n * (n - 1))) / m.Lambda
}

// MaxConcurrency returns the maximum expected number of concurrent events the system can handle,
// Nmax.
//
// See "Practical Scalability Analysis with the Universal Scalability Law, Equation 4".
func (m *Model) MaxConcurrency() float64 {
	return math.Floor(math.Sqrt((1 - m.Sigma) / m.Kappa))
}

// MaxThroughput returns the maximum expected throughput the system can handle, Xmax.
func (m Model) MaxThroughput() float64 {
	return m.ThroughputAtConcurrency(m.MaxConcurrency())
}

// LatencyAtThroughput returns the expected mean latency given a throughput, R(X).
//
// See "Practical Scalability Analysis with the Universal Scalability Law, Equation 8".
func (m *Model) LatencyAtThroughput(x float64) float64 {
	return (m.Sigma - 1) / (m.Sigma*x - m.Lambda)
}

// ThroughputAtLatency returns the expected throughput given a mean latency, X(R).
//
// See "Practical Scalability Analysis with the Universal Scalability Law, Equation 9".
func (m *Model) ThroughputAtLatency(r float64) float64 {
	return (math.Sqrt(math.Pow(m.Sigma, 2)+math.Pow(m.Kappa, 2)+
		2*m.Kappa*(2*m.Lambda*r+m.Sigma-2)) - m.Kappa + m.Sigma) / (2.0 * m.Kappa * r)
}

// ConcurrencyAtLatency returns the expected number of concurrent events at a particular mean
// latency, N(R).
//
// See "Practical Scalability Analysis with the Universal Scalability Law, Equation 10".
func (m *Model) ConcurrencyAtLatency(r float64) float64 {
	return (m.Kappa - m.Sigma +
		math.Sqrt(math.Pow(m.Sigma, 2)+
			math.Pow(m.Kappa, 2)+
			2*m.Kappa*((2*m.Lambda*r)+m.Sigma-2))) / (2 * m.Kappa)
}

// ConcurrencyAtThroughput returns the expected number of concurrent events at a particular
// throughput, N(X).
func (m *Model) ConcurrencyAtThroughput(x float64) float64 {
	return m.LatencyAtThroughput(x) * x
}

// ContentionConstrained returns true if the system is constrained by contention.
func (m *Model) ContentionConstrained() bool {
	return m.Sigma > m.Kappa
}

// CoherencyConstrained returns true if the system is constrained by coherency costs.
func (m *Model) CoherencyConstrained() bool {
	return m.Sigma < m.Kappa
}

// Limitless returns true if the system is linearly scalable.
func (m *Model) Limitless() bool {
	return m.Kappa == 0
}

// Build returns a model whose parameters are generated from the given measurements.
//
// Finds a set of coefficients for the equation y = λx/(1+σ(x-1)+κx(x-1)) which best fit the
// observed values using unconstrained least-squares regression. The resulting values for λ, κ, and
// σ are the parameters of the returned model.
func Build(measurements []Measurement) (m *Model, err error) {
	if len(measurements) < minMeasurements {
		return nil, ErrInsufficientMeasurements
	}

	// Calculate an initial guess at the model parameters.
	init := []float64{0.1, 0.01, 0}

	// Use max(x/n) as initial lambda.
	for _, m := range measurements {
		v := m.Throughput / m.Concurrency
		if v > init[2] {
			init[2] = v
		}
	}

	// Calculate the residuals of a possible model.
	f := func(dst, x []float64) {
		model := Model{Sigma: x[0], Kappa: x[1], Lambda: x[2]}

		for i, v := range measurements {
			dst[i] = v.Throughput - model.ThroughputAtConcurrency(v.Concurrency)
		}
	}
	j := lm.NumJac{Func: f}

	// Formulate an LM problem.
	p := lm.LMProblem{
		Dim:        3,                 // Three parameters in the model.
		Size:       len(measurements), // Use all measurements to calculate residuals.
		Func:       f,                 // Reduce the residuals of model predictions to observations.
		Jac:        j.Jac,             // Approximate the Jacobian by finite differences.
		InitParams: init,              // Use our initial guesses at parameters.
		Tau:        1e-6,              // Need a non-zero initial damping factor.
		Eps1:       1e-8,              // Small but non-zero values here prevent singular matrices.
		Eps2:       1e-8,
	}

	// Calculate the model parameters.
	results, err := lm.LM(p, nil)
	if err != nil {
		return nil, fmt.Errorf("unable to build model: %w", err)
	}

	// Return the model.
	return &Model{
		Sigma:  results.X[0],
		Kappa:  results.X[1],
		Lambda: results.X[2],
	}, nil
}

const (
	// minMeasurement is the smallest number of measurements from which a useful model can be
	// created.
	minMeasurements = 6
)

var (
	// ErrInsufficientMeasurements is returned when fewer than 6 measurements were provided.
	ErrInsufficientMeasurements = fmt.Errorf("usl: need at least %d measurements", minMeasurements)
)
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/llgcode/draw2d/draw2dimg"
	"github.com/llgcode/draw2d"
	"github.com/llgcode/draw2d/draw2dpdf"
	"image"
	"image/color"
	"image/png"
	"io"
	"math"
	"net/http"
	_ "net/http/pprof"
	"strconv"
)

func init() {
	http.HandleFunc("/png", handler)
	http.Handle("/", http.FileServer(http.Dir("./static")))
}

type pt struct {
	w    float64
	d    float64
	d_in float64
}

func computeD(r float64, R float64, D float64, phi_deg int, theta float64) float64 {
	phi := float64(phi_deg) / 360.0 * 2*math.Pi
	x_disp := (r*math.Sin(theta))
	d := D
	if math.Abs(x_disp) < R {
		d -= (math.Sqrt(R*R - x_disp*x_disp))
	}

	if phi_deg != 90 {
		d += (r - r*math.Cos(theta)) / math.Tan(phi)
	}
	return d
}

func computeIntersection(r float64, t float64, R float64, D float64, phi_deg int) (pts []pt) {
	for w := float64(0.0); w < r*2*math.Pi; w += 0.025 {
		theta := float64(w/r);
		d_out := computeD(r, R, D, phi_deg, theta);
		d_in := computeD(r - t, R, D, phi_deg, theta);
		pts = append(pts, pt{w, d_out, d_in})
	}
	return pts
}

func dumpText(writer http.ResponseWriter, pts []pt) {
	for _, p := range pts {
		io.WriteString(writer, fmt.Sprintf("%f %f\n", p.w, p.d));
	}
}

func DrawPattern(gc draw2d.GraphicContext,
	r float64,
	R float64,
	D float64,
	phi_deg int,
	pts []pt,
	scale float64) {
	gc.SetStrokeColor(color.Gray{0x00})
	gc.SetLineWidth(0.02 * scale)

	// outside profile:
	gc.MoveTo(0, 0)
	for _,p := range pts {
		gc.LineTo(p.d * scale, p.w * scale)
	}
	gc.Stroke()

	// inside profile:
	gc.SetStrokeColor(color.Gray{0xaa})
	gc.MoveTo(0, 0)
	for _,p := range pts {
		gc.LineTo(p.d_in * scale, p.w * scale)
	}
	gc.Stroke()

	// quarter rotation lines:
	gc.SetStrokeColor(color.Gray{0x00})
	for i := float64(1.0); i <= 4; i++ {
		theta := float64(math.Pi*i/2.0)
		quarter_d := computeD(r, R, D, int(phi_deg), theta)
		quarter_w := theta*r
		gc.MoveTo(0, quarter_w * scale)
		gc.LineTo(quarter_d * scale, quarter_w * scale)

	}
	gc.Stroke()
}

func dumpPng(
	writer http.ResponseWriter,
	r float64,
	R float64,
	D float64,
	phi_deg int,
	pts []pt) {

	dest := image.NewRGBA(image.Rect(0, 0, 1000, 1000))
	gc := draw2dimg.NewGraphicContext(dest)

	DrawPattern(gc,r,R,D,phi_deg,pts, 100.0)

	// text output does not appear to be portable across PNG/PDF
	draw2d.SetFontFolder(".")
	gc.SetFontData(draw2d.FontData{"go", draw2d.FontFamilyMono, draw2d.FontStyleBold})
	gc.SetFontSize(12)
	gc.SetFillColor(color.Gray{0x00})
	gc.FillStringAt(fmt.Sprintf("Edge of pattern is %0.2fin from center of tube", D),
		10, 30)
	gc.Fill()

	png.Encode(writer, dest)
}

func dumpPdf(
	writer http.ResponseWriter,
	r float64,
	R float64,
	D float64,
	phi_deg int,
	pts []pt) {

	dest := draw2dpdf.NewPdf("L", "in", "Letter")
	gc := draw2dpdf.NewGraphicContext(dest)

	dest.SetFont("Courier", "B", 12)
	dest.Cell(0.5, 0.5, fmt.Sprintf("Edge of pattern is %0.2fin from center of tube", D))

	DrawPattern(gc,r,R,D,phi_deg,pts,1.0)
	e := dest.Output(writer)
	if e != nil {
		fmt.Fprint(writer, "Error: ", e);
	}
}

func handler(writer http.ResponseWriter, req *http.Request) {
	if (req.ParseForm() != nil) {
		fmt.Fprint(writer, "Error parsing request")
		return
	}

	R,_ := strconv.ParseFloat(req.FormValue("R"), 64)
	r,_ := strconv.ParseFloat(req.FormValue("r"), 64)
	phi_deg,_ := strconv.ParseFloat(req.FormValue("phi"), 64)
	thick,_ := strconv.ParseFloat(req.FormValue("t"), 64);
	format := req.FormValue("f");

	R = R/2;
	r = r/2;

	D := math.Trunc(R + 4)
	pts := computeIntersection(r, thick, R, D, int(phi_deg))

	if format == "text" {
		dumpText(writer, pts)
	} else if format == "png" {
		dumpPng(writer, r, R, D, int(phi_deg), pts)
	} else if format == "pdf" {
		dumpPdf(writer, r, R, D, int(phi_deg), pts)
	}

}
<commit_msg>Mime types<commit_after>package main

import (
	"fmt"
	"github.com/llgcode/draw2d/draw2dimg"
	"github.com/llgcode/draw2d"
	"github.com/llgcode/draw2d/draw2dpdf"
	"image"
	"image/color"
	"image/png"
	"io"
	"math"
	"net/http"
	_ "net/http/pprof"
	"strconv"
)

func init() {
	http.HandleFunc("/png", handler)
	http.Handle("/", http.FileServer(http.Dir("./static")))
}

type pt struct {
	w    float64
	d    float64
	d_in float64
}

func computeD(r float64, R float64, D float64, phi_deg int, theta float64) float64 {
	phi := float64(phi_deg) / 360.0 * 2*math.Pi
	x_disp := (r*math.Sin(theta))
	d := D
	if math.Abs(x_disp) < R {
		d -= (math.Sqrt(R*R - x_disp*x_disp))
	}

	if phi_deg != 90 {
		d += (r - r*math.Cos(theta)) / math.Tan(phi)
	}
	return d
}

func computeIntersection(r float64, t float64, R float64, D float64, phi_deg int) (pts []pt) {
	for w := float64(0.0); w < r*2*math.Pi; w += 0.025 {
		theta := float64(w/r);
		d_out := computeD(r, R, D, phi_deg, theta);
		d_in := computeD(r - t, R, D, phi_deg, theta);
		pts = append(pts, pt{w, d_out, d_in})
	}
	return pts
}

func dumpText(writer http.ResponseWriter, pts []pt) {
	for _, p := range pts {
		io.WriteString(writer, fmt.Sprintf("%f %f\n", p.w, p.d));
	}
}

func DrawPattern(gc draw2d.GraphicContext,
	r float64,
	R float64,
	D float64,
	phi_deg int,
	pts []pt,
	scale float64) {
	gc.SetStrokeColor(color.Gray{0x00})
	gc.SetLineWidth(0.02 * scale)

	// outside profile:
	gc.MoveTo(0, 0)
	for _,p := range pts {
		gc.LineTo(p.d * scale, p.w * scale)
	}
	gc.Stroke()

	// inside profile:
	gc.SetStrokeColor(color.Gray{0xaa})
	gc.MoveTo(0, 0)
	for _,p := range pts {
		gc.LineTo(p.d_in * scale, p.w * scale)
	}
	gc.Stroke()

	// quarter rotation lines:
	gc.SetStrokeColor(color.Gray{0x00})
	for i := float64(1.0); i <= 4; i++ {
		theta := float64(math.Pi*i/2.0)
		quarter_d := computeD(r, R, D, int(phi_deg), theta)
		quarter_w := theta*r
		gc.MoveTo(0, quarter_w * scale)
		gc.LineTo(quarter_d * scale, quarter_w * scale)

	}
	gc.Stroke()
}

func dumpPng(
	writer http.ResponseWriter,
	r float64,
	R float64,
	D float64,
	phi_deg int,
	pts []pt) {

	dest := image.NewRGBA(image.Rect(0, 0, 1000, 1000))
	gc := draw2dimg.NewGraphicContext(dest)

	DrawPattern(gc,r,R,D,phi_deg,pts, 100.0)

	// text output does not appear to be portable across PNG/PDF
	draw2d.SetFontFolder(".")
	gc.SetFontData(draw2d.FontData{"go", draw2d.FontFamilyMono, draw2d.FontStyleBold})
	gc.SetFontSize(12)
	gc.SetFillColor(color.Gray{0x00})
	gc.FillStringAt(fmt.Sprintf("Edge of pattern is %0.2fin from center of tube", D),
		10, 30)
	gc.Fill()

	png.Encode(writer, dest)
}

func dumpPdf(
	writer http.ResponseWriter,
	r float64,
	R float64,
	D float64,
	phi_deg int,
	pts []pt) {

	dest := draw2dpdf.NewPdf("L", "in", "Letter")
	gc := draw2dpdf.NewGraphicContext(dest)

	dest.SetFont("Courier", "B", 12)
	dest.Cell(0.5, 0.5, fmt.Sprintf("Edge of pattern is %0.2fin from center of tube", D))

	DrawPattern(gc,r,R,D,phi_deg,pts,1.0)
	e := dest.Output(writer)
	if e != nil {
		fmt.Fprint(writer, "Error: ", e);
	}
}

func handler(writer http.ResponseWriter, req *http.Request) {
	if (req.ParseForm() != nil) {
		fmt.Fprint(writer, "Error parsing request")
		return
	}

	R,_ := strconv.ParseFloat(req.FormValue("R"), 64)
	r,_ := strconv.ParseFloat(req.FormValue("r"), 64)
	phi_deg,_ := strconv.ParseFloat(req.FormValue("phi"), 64)
	thick,_ := strconv.ParseFloat(req.FormValue("t"), 64);
	format := req.FormValue("f");

	R = R/2;
	r = r/2;

	D := math.Trunc(R + 4)
	pts := computeIntersection(r, thick, R, D, int(phi_deg))

	if format == "text" {
		writer.Header().Set("Content-type", "text/plain")
		dumpText(writer, pts)
	} else if format == "png" {
		writer.Header().Set("Content-type", "image/png")
		dumpPng(writer, r, R, D, int(phi_deg), pts)
	} else if format == "pdf" {
		writer.Header().Set("Content-type", "application/pdf")
		dumpPdf(writer, r, R, D, int(phi_deg), pts)
	}

}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backup

import (
	"errors"
	"fmt"
	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/fs"
	"github.com/jacobsa/comeback/repr"
	"log"
	"path"
	"regexp"
)

// An object that knows how to save directories to some underlying storage.
type DirectorySaver interface {
	// Recursively save the contents of the supplied directory (defined by a base
	// path of a backup and a relative path within the backup) to the underlying
	// storage, returning the score of a blob representing the directory's
	// listing in a format that can be recovered with repr.Unmarshal.
	//
	// Recursively exclude from backup relative paths that match any of the
	// supplied exclusion regexps. This is *not* tested against the initial
	// relative path.
	Save(basePath, relPath string, exclusions []*regexp.Regexp) (blob.Score, error)
}

// A directory saver that creates a new directory saver for each call to Save.
// This breaks a self-dependency that would be needed to make use of
// NewNonRecursiveDirectorySaver.
type onDemandDirSaver struct {
	createSaver func(wrapped DirectorySaver) DirectorySaver
}

func (s *onDemandDirSaver) Save(
	basePath,
	relPath string,
	exclusions []*regexp.Regexp) (
	score blob.Score,
	err error) {
	return s.createSaver(s).Save(basePath, relPath, exclusions)
}

// Return a directory saver that makes use of the supplied dependencies.
func NewDirectorySaver(
	blobStore blob.Store,
	fileSystem fs.FileSystem,
	fileSaver FileSaver) (DirectorySaver, error) {
	linkResolver := NewLinkResolver()
	createSaver := func(wrapped DirectorySaver) DirectorySaver {
		saver, err := NewNonRecursiveDirectorySaver(
			blobStore,
			fileSystem,
			fileSaver,
			wrapped,
			linkResolver)

		if err != nil {
			panic(err)
		}

		return saver
	}

	return &onDemandDirSaver{createSaver}, nil
}

// Equivalent to NewDirectorySaver, but with an injectable wrapped directory
// saver and link resolver to aid with testability. You should not use this
// function.
func NewNonRecursiveDirectorySaver(
	store blob.Store,
	fileSystem fs.FileSystem,
	fileSaver FileSaver,
	wrapped DirectorySaver,
	linkResolver LinkResolver) (DirectorySaver, error) {
	return &dirSaver{
		blobStore:    store,
		fileSystem:   fileSystem,
		fileSaver:    fileSaver,
		wrapped:      wrapped,
		linkResolver: linkResolver,
	}, nil
}

type dirSaver struct {
	blobStore    blob.Store
	fileSystem   fs.FileSystem
	fileSaver    FileSaver
	wrapped      DirectorySaver
	linkResolver LinkResolver
}

func (s *dirSaver) saveDir(
	basePath string,
	relPath string,
	exclusions []*regexp.Regexp,
	entry *fs.DirectoryEntry) (
	[]blob.Score,
	error) {
	// Recurse.
	score, err := s.wrapped.Save(
		basePath,
		path.Join(relPath, entry.Name),
		exclusions)

	if err != nil {
		return nil, err
	}

	return []blob.Score{score}, nil
}

func (s *dirSaver) saveFile(parent string, entry *fs.DirectoryEntry) ([]blob.Score, error) {
	// Defer to the file saver.
	return s.fileSaver.Save(path.Join(parent, entry.Name))
}

func shouldExclude(exclusions []*regexp.Regexp, relPath string) bool {
	for _, re := range exclusions {
		if re.MatchString(relPath) {
			return true
		}
	}

	return false
}

func (s *dirSaver) Save(
	basePath,
	relPath string,
	exclusions []*regexp.Regexp) (
	score blob.Score,
	err error) {
	dirpath := path.Join(basePath, relPath)

	// Grab a listing for the directory.
	entries, err := s.fileSystem.ReadDir(dirpath)
	if err != nil {
		err = fmt.Errorf("Listing directory: %v", err)
		return
	}

	// Filter the entries according to the list of exclusions.
	var tmp []*fs.DirectoryEntry
	for _, entry := range entries {
		if !shouldExclude(exclusions, path.Join(relPath, entry.Name)) {
			tmp = append(tmp, entry)
		}
	}

	entries = tmp

	// Save the data for each entry.
	for _, entry := range entries {
		log.Println("Processing:", path.Join(relPath, entry.Name))

		// Call the appropriate method based on this entry's type.
		switch entry.Type {
		case fs.TypeFile:
			// Make sure this isn't a hard link to something we've already saved.
			entry.HardLinkTarget = s.linkResolver.Register(
				entry.ContainingDevice,
				entry.Inode,
				path.Join(relPath, entry.Name))

			if entry.HardLinkTarget == nil {
				entry.Scores, err = s.saveFile(dirpath, entry)
			}

		case fs.TypeDirectory:
			entry.Scores, err = s.saveDir(basePath, relPath, exclusions, entry)
		case fs.TypeSymlink:
		case fs.TypeBlockDevice:
		case fs.TypeCharDevice:
		case fs.TypeNamedPipe:
		default:
			err = fmt.Errorf("Unhandled type: %v", entry.Type)
		}

		if err != nil {
			return
		}
	}

	// Create a serialized version of this information.
	data, err := repr.Marshal(entries)
	if err != nil {
		err = fmt.Errorf("Marshaling: %v", err)
		return
	}

	// Store that serialized version.
	score, err = s.blobStore.Store(data)
	if err != nil {
		err = errors.New("Storing dir blob: " + err.Error())
		return
	}

	return
}
<commit_msg>Fixed a long line.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backup

import (
	"errors"
	"fmt"
	"log"
	"path"
	"regexp"

	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/fs"
	"github.com/jacobsa/comeback/repr"
)

// An object that knows how to save directories to some underlying storage.
type DirectorySaver interface {
	// Recursively save the contents of the supplied directory (defined by a base
	// path of a backup and a relative path within the backup) to the underlying
	// storage, returning the score of a blob representing the directory's
	// listing in a format that can be recovered with repr.Unmarshal.
	//
	// Recursively exclude from backup relative paths that match any of the
	// supplied exclusion regexps. This is *not* tested against the initial
	// relative path.
	Save(basePath, relPath string, exclusions []*regexp.Regexp) (blob.Score, error)
}

// A directory saver that creates a new directory saver for each call to Save.
// This breaks a self-dependency that would be needed to make use of
// NewNonRecursiveDirectorySaver.
type onDemandDirSaver struct {
	createSaver func(wrapped DirectorySaver) DirectorySaver
}

func (s *onDemandDirSaver) Save(
	basePath,
	relPath string,
	exclusions []*regexp.Regexp) (
	score blob.Score,
	err error) {
	return s.createSaver(s).Save(basePath, relPath, exclusions)
}

// Return a directory saver that makes use of the supplied dependencies.
func NewDirectorySaver(
	blobStore blob.Store,
	fileSystem fs.FileSystem,
	fileSaver FileSaver) (DirectorySaver, error) {
	linkResolver := NewLinkResolver()
	createSaver := func(wrapped DirectorySaver) DirectorySaver {
		saver, err := NewNonRecursiveDirectorySaver(
			blobStore,
			fileSystem,
			fileSaver,
			wrapped,
			linkResolver)

		if err != nil {
			panic(err)
		}

		return saver
	}

	return &onDemandDirSaver{createSaver}, nil
}

// Equivalent to NewDirectorySaver, but with an injectable wrapped directory
// saver and link resolver to aid with testability. You should not use this
// function.
func NewNonRecursiveDirectorySaver(
	store blob.Store,
	fileSystem fs.FileSystem,
	fileSaver FileSaver,
	wrapped DirectorySaver,
	linkResolver LinkResolver) (DirectorySaver, error) {
	return &dirSaver{
		blobStore:    store,
		fileSystem:   fileSystem,
		fileSaver:    fileSaver,
		wrapped:      wrapped,
		linkResolver: linkResolver,
	}, nil
}

type dirSaver struct {
	blobStore    blob.Store
	fileSystem   fs.FileSystem
	fileSaver    FileSaver
	wrapped      DirectorySaver
	linkResolver LinkResolver
}

func (s *dirSaver) saveDir(
	basePath string,
	relPath string,
	exclusions []*regexp.Regexp,
	entry *fs.DirectoryEntry) (
	[]blob.Score,
	error) {
	// Recurse.
	score, err := s.wrapped.Save(
		basePath,
		path.Join(relPath, entry.Name),
		exclusions)

	if err != nil {
		return nil, err
	}

	return []blob.Score{score}, nil
}

func (s *dirSaver) saveFile(
	parent string,
	entry *fs.DirectoryEntry) ([]blob.Score, error) {
	// Defer to the file saver.
	return s.fileSaver.Save(path.Join(parent, entry.Name))
}

func shouldExclude(exclusions []*regexp.Regexp, relPath string) bool {
	for _, re := range exclusions {
		if re.MatchString(relPath) {
			return true
		}
	}

	return false
}

func (s *dirSaver) Save(
	basePath,
	relPath string,
	exclusions []*regexp.Regexp) (
	score blob.Score,
	err error) {
	dirpath := path.Join(basePath, relPath)

	// Grab a listing for the directory.
	entries, err := s.fileSystem.ReadDir(dirpath)
	if err != nil {
		err = fmt.Errorf("Listing directory: %v", err)
		return
	}

	// Filter the entries according to the list of exclusions.
	var tmp []*fs.DirectoryEntry
	for _, entry := range entries {
		if !shouldExclude(exclusions, path.Join(relPath, entry.Name)) {
			tmp = append(tmp, entry)
		}
	}

	entries = tmp

	// Save the data for each entry.
	for _, entry := range entries {
		log.Println("Processing:", path.Join(relPath, entry.Name))

		// Call the appropriate method based on this entry's type.
		switch entry.Type {
		case fs.TypeFile:
			// Make sure this isn't a hard link to something we've already saved.
			entry.HardLinkTarget = s.linkResolver.Register(
				entry.ContainingDevice,
				entry.Inode,
				path.Join(relPath, entry.Name))

			if entry.HardLinkTarget == nil {
				entry.Scores, err = s.saveFile(dirpath, entry)
			}

		case fs.TypeDirectory:
			entry.Scores, err = s.saveDir(basePath, relPath, exclusions, entry)
		case fs.TypeSymlink:
		case fs.TypeBlockDevice:
		case fs.TypeCharDevice:
		case fs.TypeNamedPipe:
		default:
			err = fmt.Errorf("Unhandled type: %v", entry.Type)
		}

		if err != nil {
			return
		}
	}

	// Create a serialized version of this information.
	data, err := repr.Marshal(entries)
	if err != nil {
		err = fmt.Errorf("Marshaling: %v", err)
		return
	}

	// Store that serialized version.
	score, err = s.blobStore.Store(data)
	if err != nil {
		err = errors.New("Storing dir blob: " + err.Error())
		return
	}

	return
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"expvar"
	"flag"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"runtime"
	"runtime/pprof"
	"strings"
	"time"

	"github.com/gorilla/mux"

	"github.com/blevesearch/bleve"
	bleveHttp "github.com/blevesearch/bleve/http"
	bleveRegistry "github.com/blevesearch/bleve/registry"

	log "github.com/couchbase/clog"
	"github.com/couchbase/go-couchbase"
	"github.com/couchbaselabs/cbft"
)

var VERSION = "0.0.0"

var bindAddr = flag.String("addr", "localhost:8095",
	"\n\thttp listen address:port")
var dataDir = flag.String("dataDir", "data",
	"\n\tdirectory path where index data and\n"+
		"\tlocal configuration files will be stored")
var logFlags = flag.String("logFlags", "",
	"\n\tcomma-separated logging control flags")
var staticDir = flag.String("staticDir", "static",
	"\n\tdirectory for static web UI content")
var staticETag = flag.String("staticETag", "",
	"\n\tstatic etag value")
var server = flag.String("server", "",
	"\n\turl to datasource server;\n"+
		"\texample for couchbase: http://localhost:8091")
var tags = flag.String("tags", "",
	"\n\tcomma-separated list of tags (or roles) for this node")
var container = flag.String("container", "",
	"\n\tslash separated path of parent containers for this node,\n"+
		"\tfor shelf/rack/row/zone awareness")
var weight = flag.Int("weight", 1,
	"\n\tweight of this node (a more capable node has higher weight)")
var register = flag.String("register", "wanted",
	"\n\tregister this node as wanted, wantedForce,\n\t"+
		" known, knownForce,"+
		" unwanted, unknown or unchanged")
var cfgConnect = flag.String("cfgConnect", "simple",
	"\n\tconnection string/info to configuration provider")

var expvars = expvar.NewMap("stats")

func init() {
	expvars.Set("indexes", bleveHttp.IndexStats())

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr,
			"%s: couchbase full-text server\n\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "more information is available at:\n"+
			"  http://github.com/couchbaselabs/cbft\n\n")
		fmt.Fprintf(os.Stderr, "usage:\n  %s [flags]\n\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "flags:\n")
		flag.PrintDefaults()
	}
}

func main() {
	flag.Parse()

	log.Printf("main: %s started (%s/%s)", os.Args[0], VERSION, cbft.VERSION)

	rand.Seed(time.Now().UTC().UnixNano())

	go dumpOnSignalForPlatform()

	mr, err := cbft.NewMsgRing(os.Stderr, 1000)
	if err != nil {
		log.Fatalf("main: could not create MsgRing, err: %v", err)
	}
	log.SetOutput(mr)

	MainWelcome()

	// TODO: If cfg goes down, should we stop?  How do we reconnect?
	//
	cfg, err := MainCfg(*cfgConnect, *dataDir)
	if err != nil {
		log.Fatalf("main: could not start cfg, cfgConnect: %s, err: %v",
			*cfgConnect, err)
		return
	}

	uuid, err := MainUUID(*dataDir)
	if err != nil {
		log.Fatalf(fmt.Sprintf("%v", err))
		return
	}

	var tagsArr []string
	if *tags != "" {
		tagsArr = strings.Split(*tags, ",")
	}

	router, err := MainStart(cfg, uuid, tagsArr, *container, *weight,
		*bindAddr, *dataDir, *staticDir, *staticETag, *server, *register, mr)
	if err != nil {
		log.Fatal(err)
	}

	http.Handle("/", router)
	log.Printf("main: listening on: %v", *bindAddr)
	log.Fatal(http.ListenAndServe(*bindAddr, nil))
}

func MainWelcome() {
	if *logFlags != "" {
		log.ParseLogFlag(*logFlags)
	}
	flag.VisitAll(func(f *flag.Flag) {
		log.Printf("  -%s=%s\n", f.Name, f.Value)
	})
	log.Printf("  GOMAXPROCS=%d", runtime.GOMAXPROCS(-1))

	log.Printf("main: registered bleve stores")
	types, instances := bleveRegistry.KVStoreTypesAndInstances()
	for _, s := range types {
		log.Printf("  %s", s)
	}
	for _, s := range instances {
		log.Printf("  %s", s)
	}
}

func MainUUID(dataDir string) (string, error) {
	uuid := cbft.NewUUID()
	uuidPath := dataDir + string(os.PathSeparator) + "cbft.uuid"
	uuidBuf, err := ioutil.ReadFile(uuidPath)
	if err == nil {
		uuid = strings.TrimSpace(string(uuidBuf))
		if uuid == "" {
			return "", fmt.Errorf("error: could not parse uuidPath: %s",
				uuidPath)
		}
		log.Printf("main: manager uuid: %s", uuid)
		log.Printf("main: manager uuid was reloaded")
	} else {
		log.Printf("main: manager uuid: %s", uuid)
		log.Printf("main: manager uuid was generated")
	}
	err = ioutil.WriteFile(uuidPath, []byte(uuid), 0600)
	if err != nil {
		return "", fmt.Errorf("error: could not write uuidPath: %s", uuidPath)
	}
	return uuid, nil
}

func MainStart(cfg cbft.Cfg, uuid string, tags []string, container string,
	weight int, bindAddr, dataDir, staticDir, staticETag, server string,
	register string, mr *cbft.MsgRing) (
	*mux.Router, error) {
	if server == "" {
		return nil, fmt.Errorf("error: server URL required (-server)")
	}

	_, err := couchbase.Connect(server)
	if err != nil {
		return nil, fmt.Errorf("error: could not connect to server,"+
			" URL: %s, err: %v",
			server, err)
	}

	mgr := cbft.NewManager(cbft.VERSION, cfg, uuid, tags, container, weight,
		bindAddr, dataDir, server, &MainHandlers{})
	err = mgr.Start(register)
	if err != nil {
		return nil, err
	}

	router, _, err :=
		cbft.NewManagerRESTRouter(VERSION, mgr, staticDir, staticETag, mr)

	return router, err
}

type MainHandlers struct{}

func (meh *MainHandlers) OnRegisterPIndex(pindex *cbft.PIndex) {
	bindex, ok := pindex.Impl.(bleve.Index)
	if ok {
		bleveHttp.RegisterIndexName(pindex.Name, bindex)
	}
}

func (meh *MainHandlers) OnUnregisterPIndex(pindex *cbft.PIndex) {
	bleveHttp.UnregisterIndexByName(pindex.Name)
}

func dumpOnSignal(signals ...os.Signal) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, signals...)
	for _ = range c {
		log.Printf("dump: goroutine...")
		pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
		log.Printf("dump: heap...")
		pprof.Lookup("heap").WriteTo(os.Stderr, 1)
	}
}
<commit_msg>use spaces instead of tabs in usage/help<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"expvar"
	"flag"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"runtime"
	"runtime/pprof"
	"strings"
	"time"

	"github.com/gorilla/mux"

	"github.com/blevesearch/bleve"
	bleveHttp "github.com/blevesearch/bleve/http"
	bleveRegistry "github.com/blevesearch/bleve/registry"

	log "github.com/couchbase/clog"
	"github.com/couchbase/go-couchbase"
	"github.com/couchbaselabs/cbft"
)

var VERSION = "0.0.0"

var bindAddr = flag.String("addr", "localhost:8095",
	adesc("http listen address:port"))
var dataDir = flag.String("dataDir", "data",
	adesc("directory path where index data and"+
		"\nlocal configuration files will be stored"))
var logFlags = flag.String("logFlags", "",
	adesc("comma-separated logging control flags"))
var staticDir = flag.String("staticDir", "static",
	adesc("directory for static web UI content"))
var staticETag = flag.String("staticETag", "",
	adesc("static etag value"))
var server = flag.String("server", "",
	adesc("url to datasource server;"+
		"\nexample for couchbase: http://localhost:8091"))
var tags = flag.String("tags", "",
	adesc("comma-separated list of tags (or roles) for this node"))
var container = flag.String("container", "",
	adesc("slash separated path of parent containers for this node,"+
		"\nfor shelf/rack/row/zone awareness"))
var weight = flag.Int("weight", 1,
	adesc("weight of this node (a more capable node has higher weight)"))
var register = flag.String("register", "wanted",
	adesc("register this node as wanted, wantedForce,"+
		"\nknown, knownForce, unwanted, unknown or unchanged"))
var cfgConnect = flag.String("cfgConnect", "simple",
	adesc("connection string/info to configuration provider"))

var expvars = expvar.NewMap("stats")

func init() {
	expvars.Set("indexes", bleveHttp.IndexStats())

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr,
			"%s: couchbase full-text server\n\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "more information is available at:\n"+
			"  http://github.com/couchbaselabs/cbft\n\n")
		fmt.Fprintf(os.Stderr, "usage:\n  %s [flags]\n\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "flags:\n")
		flag.PrintDefaults()
	}
}

func main() {
	flag.Parse()

	log.Printf("main: %s started (%s/%s)", os.Args[0], VERSION, cbft.VERSION)

	rand.Seed(time.Now().UTC().UnixNano())

	go dumpOnSignalForPlatform()

	mr, err := cbft.NewMsgRing(os.Stderr, 1000)
	if err != nil {
		log.Fatalf("main: could not create MsgRing, err: %v", err)
	}
	log.SetOutput(mr)

	MainWelcome()

	// TODO: If cfg goes down, should we stop?  How do we reconnect?
	//
	cfg, err := MainCfg(*cfgConnect, *dataDir)
	if err != nil {
		log.Fatalf("main: could not start cfg, cfgConnect: %s, err: %v",
			*cfgConnect, err)
		return
	}

	uuid, err := MainUUID(*dataDir)
	if err != nil {
		log.Fatalf(fmt.Sprintf("%v", err))
		return
	}

	var tagsArr []string
	if *tags != "" {
		tagsArr = strings.Split(*tags, ",")
	}

	router, err := MainStart(cfg, uuid, tagsArr, *container, *weight,
		*bindAddr, *dataDir, *staticDir, *staticETag, *server, *register, mr)
	if err != nil {
		log.Fatal(err)
	}

	http.Handle("/", router)
	log.Printf("main: listening on: %v", *bindAddr)
	log.Fatal(http.ListenAndServe(*bindAddr, nil))
}

func MainWelcome() {
	if *logFlags != "" {
		log.ParseLogFlag(*logFlags)
	}
	flag.VisitAll(func(f *flag.Flag) {
		log.Printf("  -%s=%s\n", f.Name, f.Value)
	})
	log.Printf("  GOMAXPROCS=%d", runtime.GOMAXPROCS(-1))

	log.Printf("main: registered bleve stores")
	types, instances := bleveRegistry.KVStoreTypesAndInstances()
	for _, s := range types {
		log.Printf("  %s", s)
	}
	for _, s := range instances {
		log.Printf("  %s", s)
	}
}

func MainUUID(dataDir string) (string, error) {
	uuid := cbft.NewUUID()
	uuidPath := dataDir + string(os.PathSeparator) + "cbft.uuid"
	uuidBuf, err := ioutil.ReadFile(uuidPath)
	if err == nil {
		uuid = strings.TrimSpace(string(uuidBuf))
		if uuid == "" {
			return "", fmt.Errorf("error: could not parse uuidPath: %s",
				uuidPath)
		}
		log.Printf("main: manager uuid: %s", uuid)
		log.Printf("main: manager uuid was reloaded")
	} else {
		log.Printf("main: manager uuid: %s", uuid)
		log.Printf("main: manager uuid was generated")
	}
	err = ioutil.WriteFile(uuidPath, []byte(uuid), 0600)
	if err != nil {
		return "", fmt.Errorf("error: could not write uuidPath: %s", uuidPath)
	}
	return uuid, nil
}

func MainStart(cfg cbft.Cfg, uuid string, tags []string, container string,
	weight int, bindAddr, dataDir, staticDir, staticETag, server string,
	register string, mr *cbft.MsgRing) (
	*mux.Router, error) {
	if server == "" {
		return nil, fmt.Errorf("error: server URL required (-server)")
	}

	_, err := couchbase.Connect(server)
	if err != nil {
		return nil, fmt.Errorf("error: could not connect to server,"+
			" URL: %s, err: %v",
			server, err)
	}

	mgr := cbft.NewManager(cbft.VERSION, cfg, uuid, tags, container, weight,
		bindAddr, dataDir, server, &MainHandlers{})
	err = mgr.Start(register)
	if err != nil {
		return nil, err
	}

	router, _, err :=
		cbft.NewManagerRESTRouter(VERSION, mgr, staticDir, staticETag, mr)

	return router, err
}

type MainHandlers struct{}

func (meh *MainHandlers) OnRegisterPIndex(pindex *cbft.PIndex) {
	bindex, ok := pindex.Impl.(bleve.Index)
	if ok {
		bleveHttp.RegisterIndexName(pindex.Name, bindex)
	}
}

func (meh *MainHandlers) OnUnregisterPIndex(pindex *cbft.PIndex) {
	bleveHttp.UnregisterIndexByName(pindex.Name)
}

func dumpOnSignal(signals ...os.Signal) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, signals...)
	for _ = range c {
		log.Printf("dump: goroutine...")
		pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
		log.Printf("dump: heap...")
		pprof.Lookup("heap").WriteTo(os.Stderr, 1)
	}
}

func adesc(s string) string {
	return "\n      " + strings.Replace(s, "\n", "\n      ", -1)
}
<|endoftext|>


<commit_before>// Copyright 2016 The Vulcan Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"

	"github.com/digitalocean/vulcan/forwarder"
	"github.com/digitalocean/vulcan/kafka"

	log "github.com/Sirupsen/logrus"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

// Forwarder handles parsing the command line options, initializes, and starts the
// forwarder service accordingling.  It is the entry point for the forwarder
// service.
func Forwarder() *cobra.Command {
	f := &cobra.Command{
		Use:   "forwarder",
		Short: "forwards metric received from prometheus to message bus",
		RunE: func(cmd *cobra.Command, args []string) error {
			var (
				reg     = prometheus.NewRegistry()
				signals = make(chan os.Signal, 1)
			)

			signal.Notify(signals, os.Interrupt, syscall.SIGTERM)

			// create upstream kafka writer to receive data
			w, err := kafka.NewWriter(&kafka.WriterConfig{
				ClientID:    viper.GetString(flagKafkaClientID),
				Topic:       viper.GetString(flagKafkaTopic),
				Addrs:       strings.Split(viper.GetString(flagKafkaAddrs), ","),
				TrackWrites: viper.GetBool(flagKafkaTrackWrites),
				BatchSize:   viper.GetInt(flagKafkaBatchSize),
			})
			if err != nil {
				return err
			}

			err = reg.Register(w)
			if err != nil {
				return err
			}

			log.WithFields(log.Fields{
				"kafka_client_id": viper.GetString(flagKafkaClientID),
				"kafka_topic":     viper.GetString(flagKafkaTopic),
				"kafka_addresses": viper.GetString(flagKafkaAddrs),
			}).Info("registered as kafka producer")

			fwd := forwarder.NewForwarder(&forwarder.Config{
				Writer: w,
			})

			// listen for signals so can gracefully stop kakfa producer
			go func() {
				<-signals
				// tell forwarder to stop handling in incoming requests
				fwd.Stop()
				// tell kafka writer to stop sending out producer messages
				w.Stop()
				os.Exit(0)
			}()

			http.Handle("/", forwarder.WriteHandler(fwd, "snappy"))
			http.Handle(viper.GetString(flagTelemetryPath), promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))

			log.WithFields(log.Fields{
				"listening_address": viper.GetString(flagAddress),
			}).Info("starting vulcan forwarder http service")

			return http.ListenAndServe(viper.GetString(flagAddress), nil)
		},
	}

	f.Flags().String(flagAddress, ":8888", "server listening address")
	f.Flags().String(flagKafkaTopic, "vulcan", "kafka topic to write to")
	f.Flags().String(flagKafkaAddrs, "", "one.example.com:9092,two.example.com:9092")
	f.Flags().String(flagKafkaClientID, "vulcan-forwarder", "set the kafka client id")
	f.Flags().String(flagTelemetryPath, "/metrics", "path under which to expose metrics")
	f.Flags().Bool(flagKafkaTrackWrites, false, "track kafka writes for metric scraping and logging")
	f.Flags().Int(flagKafkaBatchSize, 65536, "batch size of each send of kafka producer messages")

	return f
}
<commit_msg>forwarder: register forwarder to prometheus (#76)<commit_after>// Copyright 2016 The Vulcan Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"

	"github.com/digitalocean/vulcan/forwarder"
	"github.com/digitalocean/vulcan/kafka"

	log "github.com/Sirupsen/logrus"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

// Forwarder handles parsing the command line options, initializes, and starts the
// forwarder service accordingling.  It is the entry point for the forwarder
// service.
func Forwarder() *cobra.Command {
	f := &cobra.Command{
		Use:   "forwarder",
		Short: "forwards metric received from prometheus to message bus",
		RunE: func(cmd *cobra.Command, args []string) error {
			var (
				reg     = prometheus.NewRegistry()
				signals = make(chan os.Signal, 1)
			)

			signal.Notify(signals, os.Interrupt, syscall.SIGTERM)

			// create upstream kafka writer to receive data
			w, err := kafka.NewWriter(&kafka.WriterConfig{
				ClientID:    viper.GetString(flagKafkaClientID),
				Topic:       viper.GetString(flagKafkaTopic),
				Addrs:       strings.Split(viper.GetString(flagKafkaAddrs), ","),
				TrackWrites: viper.GetBool(flagKafkaTrackWrites),
				BatchSize:   viper.GetInt(flagKafkaBatchSize),
			})
			if err != nil {
				return err
			}
			reg.MustRegister(w)

			log.WithFields(log.Fields{
				"kafka_client_id": viper.GetString(flagKafkaClientID),
				"kafka_topic":     viper.GetString(flagKafkaTopic),
				"kafka_addresses": viper.GetString(flagKafkaAddrs),
			}).Info("registered as kafka producer")

			fwd := forwarder.NewForwarder(&forwarder.Config{
				Writer: w,
			})
			reg.MustRegister(fwd)

			// listen for signals so can gracefully stop kakfa producer
			go func() {
				<-signals
				// tell forwarder to stop handling in incoming requests
				fwd.Stop()
				// tell kafka writer to stop sending out producer messages
				w.Stop()
				os.Exit(0)
			}()

			http.Handle("/", forwarder.WriteHandler(fwd, "snappy"))
			http.Handle(viper.GetString(flagTelemetryPath), promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))

			log.WithFields(log.Fields{
				"listening_address": viper.GetString(flagAddress),
			}).Info("starting vulcan forwarder http service")

			return http.ListenAndServe(viper.GetString(flagAddress), nil)
		},
	}

	f.Flags().String(flagAddress, ":8888", "server listening address")
	f.Flags().String(flagKafkaTopic, "vulcan", "kafka topic to write to")
	f.Flags().String(flagKafkaAddrs, "", "one.example.com:9092,two.example.com:9092")
	f.Flags().String(flagKafkaClientID, "vulcan-forwarder", "set the kafka client id")
	f.Flags().String(flagTelemetryPath, "/metrics", "path under which to expose metrics")
	f.Flags().Bool(flagKafkaTrackWrites, false, "track kafka writes for metric scraping and logging")
	f.Flags().Int(flagKafkaBatchSize, 65536, "batch size of each send of kafka producer messages")

	return f
}
<|endoftext|>


<commit_before>// lissajous2 solution for task 1.5
package main

import (
	"image"
	"image/color"
	"image/gif"
	"io"
	"log"
	"math"
	"math/rand"
	"os"
	"time"
)

var palette = []color.Color{color.Black, color.RGBA{0, 128, 0, 255}}

const (
	blackIndex = 0
	greenIndex = 1
)

func main() {
	rand.Seed(time.Now().UTC().UnixNano())
	lissajous(os.Stdout)
}

func lissajous(out io.Writer) {
	const (
		cycles  = 5
		res     = 0.001
		size    = 200
		nframes = 64
		delay   = 8
	)
	freq := rand.Float64() * 3.0
	anim := gif.GIF{LoopCount: nframes}
	phase := 0.0
	for i := 0; i < nframes; i++ {
		rect := image.Rect(0, 0, 2*size+1, 2*size+1)
		img := image.NewPaletted(rect, palette)
		for t := 0.0; t < cycles*2*math.Pi; t += res {
			x := math.Sin(t)
			y := math.Sin(t*freq + phase)
			img.SetColorIndex(size+int(x*size+0.5), size+int(y*size+0.5),
				greenIndex)
		}
		phase += 0.1
		anim.Delay = append(anim.Delay, delay)
		anim.Image = append(anim.Image, img)
	}
	err := gif.EncodeAll(out, &anim)
	if err != nil {
		log.Fatalf("error encoding gif image: %v", err)
	}
}
<commit_msg>Updated soluton for gif for task 1.6.<commit_after>// lissajous2 solution for task 1.6
package main

import (
	"image"
	"image/color"
	"image/gif"
	"io"
	"log"
	"math"
	"math/rand"
	"os"
	"time"
)

func main() {
	rand.Seed(time.Now().UTC().UnixNano())
	lissajous(os.Stdout)
}

func lissajous(out io.Writer) {
	const (
		colours = 128
		cycles  = 5
		res     = 0.001
		size    = 200
		nframes = 64
		delay   = 8
	)

	freq := rand.Float64() * 3.0
	anim := gif.GIF{LoopCount: nframes}
	phase := 0.0
	for i := 0; i < nframes; i++ {
		palette := []color.Color{color.Black}
		rect := image.Rect(0, 0, 2*size+1, 2*size+1)
		img := image.NewPaletted(rect, append(palette, rainbow()))

		for t := 0.0; t < cycles*2*math.Pi; t += res {
			x := math.Sin(t)
			y := math.Sin(t*freq + phase)
			img.SetColorIndex(size+int(x*size+0.5), size+int(y*size+0.5), 1)
		}
		phase += 0.1
		anim.Delay = append(anim.Delay, delay)
		anim.Image = append(anim.Image, img)
	}
	err := gif.EncodeAll(out, &anim)
	if err != nil {
		log.Fatalf("error encoding gif image: %v", err)
	}
}

func rainbow() color.Color {

	var vibgyor = []color.Color{
		color.RGBA{0x94, 0x00, 0xD3, 0xFF}, // violet
		color.RGBA{0x4B, 0x00, 0x82, 0xFF}, // indigo
		color.RGBA{0x00, 0x00, 0xFF, 0xFF}, // blue
		color.RGBA{0x00, 0xFF, 0x00, 0xFF}, // green
		color.RGBA{0xFF, 0xFF, 0x00, 0xFF}, // yellow
		color.RGBA{0xFF, 0x7F, 0x00, 0xFF}, // orange
		color.RGBA{0xFF, 0x00, 0x00, 0xFF}, // red
	}
	return vibgyor[rand.Intn(len(vibgyor))]
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Reposeed Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"

	"gopkg.in/yaml.v2"
)

type config struct {
	Project struct {
		Name        string `yaml:"name"`
		Description string `yaml:"description"`
		State       string `yaml:"state,omitempty"`
		OneLiner    string `yaml:"oneLiner,omitempty"`
		Image       string `yaml:"image,omitempty"`
		Website     string `yaml:"website,omitempty"`
	} `yaml:"project"`
	Vision struct {
		Type     string   `yaml:"type"`
		Items    []string `yaml:"items"`
		Concept  string   `yaml:"concept"`
		Overview string   `yaml:"overview"`
		Aim      string   `yaml:"aim"`
	} `yaml:"vision"`
	Repo struct {
		Type string `yaml:"type"`
		Link string `yaml:"link"`
	} `yaml:"repo"`
	Copyright struct {
		Owner string `yaml:"owner"`
		Year  string `yaml:"year"`
	} `yaml:"copyright"`
	Cla struct {
		CopyrightHolder string `yaml:"copyrightHolder"`
	} `yaml:"cla"`
	Maintainers []struct {
		Name string `yaml:"name"`
		Nick string `yaml:"nick,omitempty"`
	} `yaml:"maintainers,omitempty"`
	Emails struct {
		CommercialSupport string `yaml:"commercialSupport,omitempty"`
		Security          string `yaml:"security"`
		Coc               string `yaml:"coc"`
	} `yaml:"emails"`
	Badges []struct {
		Image string `yaml:"image,omitempty"`
		Link  string `yaml:"link,omitempty"`
		Alt   string `yaml:"alt,omitempty"`
	} `yaml:"badges"`
	SupportLinks struct {
		Documentation   string `yaml:"documentation,omitempty"`
		Examples        string `yaml:"examples,omitempty"`
		Troubleshooting string `yaml:"troubleshooting,omitempty"`
	} `yaml:"supportLinks,omitempty"`
	Readme struct {
		UsageExample string `yaml:"usageExample,omitempty"`
	} `yaml:"readme,omitempty"`
	SupportPlatforms []struct {
		Service string `yaml:"service"`
		Link    string `yaml:"link"`
	} `yaml:"supportPlatforms,omitempty"`
	IssueTemplate struct {
		Questions []string `yaml:"questions"`
	} `yaml:"issueTemplate"`
	ContributionLinks struct {
		IssueTemplate string `yaml:"issueTemplate,omitempty"`
		StarterIssues string `yaml:"starterIssues,omitempty"`
	} `yaml:"contributionLinks"`
}

func parseConfig(path string) config {
	var conf config

	filename, _ := filepath.Abs(path)
	file, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(err)
	}
	err = yaml.Unmarshal(file, &conf)
	if err != nil {
		panic(err)
	}
	return conf
}

func generateFile(config config, path string, newPath string, overwrite bool) error {
	temp, err := template.ParseFiles(path)
	if err != nil {
		return fmt.Errorf("unable to parse file: %s", err)
	}

	if _, e := os.Stat(newPath); os.IsNotExist(e) {
		os.MkdirAll(filepath.Dir(newPath), os.ModePerm)
	}

	if !overwrite {
		if _, e := os.Stat(newPath); !os.IsNotExist(e) {
			return fmt.Errorf("file %s not overwritten", newPath)
		}
	}

	file, err := os.Create(newPath)
	defer file.Close()
	if err != nil {
		return fmt.Errorf("unable to create file: %s", err)
	}

	err = temp.Execute(file, config)
	if err != nil {
		return fmt.Errorf("unable to parse template: %s", err)
	}

	return nil
}

func main() {
	var tempDir, conf string
	var overwrite bool

	flag.StringVar(&tempDir, "input", "templates", "Template directory")
	flag.StringVar(&conf, "conf", ".seed-config.yaml", "Config file")
	flag.BoolVar(&overwrite, "overwrite", false, "Force overwrite files")

	flag.Parse()

	config := parseConfig(conf)
	tempDir, _ = filepath.Abs(tempDir)

	bl := make(map[string]bool)
	bl["seed-config.example.yaml"] = true

	filepath.Walk(tempDir, func(path string, info os.FileInfo, err error) error {
		if bl[info.Name()] {
			return filepath.SkipDir
		}

		if !info.IsDir() {
			newPath, _ := filepath.Rel(tempDir, path)
			err := generateFile(config, path, newPath, overwrite)
			if err != nil {
				log.Println(err)
			}
		}

		return nil
	})

}
<commit_msg>Add output directory option<commit_after>/*
Copyright 2017 The Reposeed Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"

	"gopkg.in/yaml.v2"
)

type config struct {
	Project struct {
		Name        string `yaml:"name"`
		Description string `yaml:"description"`
		State       string `yaml:"state,omitempty"`
		OneLiner    string `yaml:"oneLiner,omitempty"`
		Image       string `yaml:"image,omitempty"`
		Website     string `yaml:"website,omitempty"`
	} `yaml:"project"`
	Vision struct {
		Type     string   `yaml:"type"`
		Items    []string `yaml:"items"`
		Concept  string   `yaml:"concept"`
		Overview string   `yaml:"overview"`
		Aim      string   `yaml:"aim"`
	} `yaml:"vision"`
	Repo struct {
		Type string `yaml:"type"`
		Link string `yaml:"link"`
	} `yaml:"repo"`
	Copyright struct {
		Owner string `yaml:"owner"`
		Year  string `yaml:"year"`
	} `yaml:"copyright"`
	Cla struct {
		CopyrightHolder string `yaml:"copyrightHolder"`
	} `yaml:"cla"`
	Maintainers []struct {
		Name string `yaml:"name"`
		Nick string `yaml:"nick,omitempty"`
	} `yaml:"maintainers,omitempty"`
	Emails struct {
		CommercialSupport string `yaml:"commercialSupport,omitempty"`
		Security          string `yaml:"security"`
		Coc               string `yaml:"coc"`
	} `yaml:"emails"`
	Badges []struct {
		Image string `yaml:"image,omitempty"`
		Link  string `yaml:"link,omitempty"`
		Alt   string `yaml:"alt,omitempty"`
	} `yaml:"badges"`
	SupportLinks struct {
		Documentation   string `yaml:"documentation,omitempty"`
		Examples        string `yaml:"examples,omitempty"`
		Troubleshooting string `yaml:"troubleshooting,omitempty"`
	} `yaml:"supportLinks,omitempty"`
	Readme struct {
		UsageExample string `yaml:"usageExample,omitempty"`
	} `yaml:"readme,omitempty"`
	SupportPlatforms []struct {
		Service string `yaml:"service"`
		Link    string `yaml:"link"`
	} `yaml:"supportPlatforms,omitempty"`
	IssueTemplate struct {
		Questions []string `yaml:"questions"`
	} `yaml:"issueTemplate"`
	ContributionLinks struct {
		IssueTemplate string `yaml:"issueTemplate,omitempty"`
		StarterIssues string `yaml:"starterIssues,omitempty"`
	} `yaml:"contributionLinks"`
}

func parseConfig(path string) config {
	var conf config

	filename, _ := filepath.Abs(path)
	file, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(err)
	}
	err = yaml.Unmarshal(file, &conf)
	if err != nil {
		panic(err)
	}
	return conf
}

func generateFile(config config, path string, newPath string, overwrite bool) error {
	temp, err := template.ParseFiles(path)
	if err != nil {
		return fmt.Errorf("unable to parse file: %s", err)
	}

	if _, e := os.Stat(newPath); os.IsNotExist(e) {
		os.MkdirAll(filepath.Dir(newPath), os.ModePerm)
	}

	if !overwrite {
		if _, e := os.Stat(newPath); !os.IsNotExist(e) {
			return fmt.Errorf("file %s not overwritten", newPath)
		}
	}

	file, err := os.Create(newPath)
	defer file.Close()
	if err != nil {
		return fmt.Errorf("unable to create file: %s", err)
	}

	err = temp.Execute(file, config)
	if err != nil {
		return fmt.Errorf("unable to parse template: %s", err)
	}

	return nil
}

func main() {
	var tempDir, outputDir, conf string
	var overwrite bool

	flag.StringVar(&tempDir, "input", "templates", "Template directory")
	flag.StringVar(&outputDir, "output", "", "Output directory")
	flag.StringVar(&conf, "conf", ".seed-config.yaml", "Config file")
	flag.BoolVar(&overwrite, "overwrite", false, "Force overwrite files")

	flag.Parse()

	config := parseConfig(conf)
	tempDir, _ = filepath.Abs(tempDir)

	bl := make(map[string]bool)
	bl["seed-config.example.yaml"] = true

	filepath.Walk(tempDir, func(path string, info os.FileInfo, err error) error {
		if bl[info.Name()] {
			return filepath.SkipDir
		}

		if !info.IsDir() {
			relPath, _ := filepath.Rel(tempDir, path)
			newPath := relPath
			if outputDir != "" {
				newPath = filepath.Join(outputDir, relPath)
			}
			err := generateFile(config, path, newPath, overwrite)
			if err != nil {
				log.Println(err)
			}
		}

		return nil
	})

}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"os"
	"strings"

	"github.com/spf13/cobra"
	"google.golang.org/grpc"
)

var helmHome string

// flagVerbose is a signal that the user wants additional output.
var flagVerbose bool

var globalUsage = `The Kubernetes package manager

To begin working with Helm, run the 'helm init' command:

$ helm init

This will install Tiller to your running Kubernetes cluster.
It will also set up any necessary local configuration.

Commond actions from this point on include:

- helm search: search for charts
- helm fetch: download a chart to your local directory to view
- helm install: upload the chart to Kubernetes
- helm list: list releases of charts

ENVIRONMENT:
$HELM_HOME: Set an alternative location for Helm files.
            By default, these are stored in ~/.helm
`

// RootCommand is the top-level command for Helm.
var RootCommand = &cobra.Command{
	Use:   "helm",
	Short: "The Helm package manager for Kubernetes.",
	Long:  globalUsage,
}

func init() {
	RootCommand.PersistentFlags().StringVar(&helmHome, "home", "$HOME/.helm", "location of you Helm files [$HELM_HOME]")
	RootCommand.PersistentFlags().BoolVarP(&flagVerbose, "verbose", "v", false, "enable verbose output")
}

func main() {
	if err := RootCommand.Execute(); err != nil {
		os.Exit(1)
	}
}

func checkArgsLength(expectedNum, actualNum int, requiredArgs ...string) error {
	if actualNum != expectedNum {
		arg := "arguments"
		if expectedNum == 1 {
			arg = "argument"
		}
		return fmt.Errorf("This command needs %v %s: %s", expectedNum, arg, strings.Join(requiredArgs, ", "))
	}
	return nil
}

// prettyError unwraps or rewrites certain errors to make them more user-friendly.
func prettyError(err error) error {
	// This is ridiculous. Why is 'grpc.rpcError' not exported? The least they
	// could do is throw an interface on the lib that would let us get back
	// the desc. Instead, we have to pass ALL errors through this.
	return errors.New(grpc.ErrorDesc(err))
}
<commit_msg>fix(helm): update main help text<commit_after>package main

import (
	"errors"
	"fmt"
	"os"
	"strings"

	"github.com/spf13/cobra"
	"google.golang.org/grpc"
)

var helmHome string

// flagVerbose is a signal that the user wants additional output.
var flagVerbose bool

var globalUsage = `The Kubernetes package manager

To begin working with Helm, run the 'helm init' command:

	$ helm init

This will install Tiller to your running Kubernetes cluster.
It will also set up any necessary local configuration.

Common actions from this point include:

- helm search: search for charts
- helm fetch: download a chart to your local directory to view
- helm install: upload the chart to Kubernetes
- helm list: list releases of charts

Environment:
  $HELM_HOME    Set an alternative location for Helm files. By default, these are stored in ~/.helm
`

// RootCommand is the top-level command for Helm.
var RootCommand = &cobra.Command{
	Use:   "helm",
	Short: "The Helm package manager for Kubernetes.",
	Long:  globalUsage,
}

func init() {
	RootCommand.PersistentFlags().StringVar(&helmHome, "home", "$HOME/.helm", "location of you Helm files [$HELM_HOME]")
	RootCommand.PersistentFlags().BoolVarP(&flagVerbose, "verbose", "v", false, "enable verbose output")
}

func main() {
	if err := RootCommand.Execute(); err != nil {
		os.Exit(1)
	}
}

func checkArgsLength(expectedNum, actualNum int, requiredArgs ...string) error {
	if actualNum != expectedNum {
		arg := "arguments"
		if expectedNum == 1 {
			arg = "argument"
		}
		return fmt.Errorf("This command needs %v %s: %s", expectedNum, arg, strings.Join(requiredArgs, ", "))
	}
	return nil
}

// prettyError unwraps or rewrites certain errors to make them more user-friendly.
func prettyError(err error) error {
	// This is ridiculous. Why is 'grpc.rpcError' not exported? The least they
	// could do is throw an interface on the lib that would let us get back
	// the desc. Instead, we have to pass ALL errors through this.
	return errors.New(grpc.ErrorDesc(err))
}
<|endoftext|>


<commit_before>/*
Command lark executes project tasks defined with lua scripts.  The Lua
interpreter included in lark in embedded and is not affected by an existing Lua
installation on the host system.  Furthermore, Lark isolates the lua modules
available to scripts to modules in the relative directory path ./lark_modules/
to ensure portability of project tasks across developer machines.

As of the current release the embedded Lua interpreter is compliant with
Lua5.1.

The lark command locates tasks defined in the ./lark.lua file or otherwise
directly under the directory ./lark_tasks/.  Tasks can have names (either
explicitly given or otherwise inferred) or patterns.  Pattern matching tasks
will match a set of names defined by a regular expression.

Names are matched against available tasks with a strict precedence.  Explicitly
named tasks will match the same name with the highest priority.  Any task with
an inferred name will match the same name with the second highest priority.
Pattern matching tasks have the lowest priority and will match names in the
order they were defined.

Tasks can be executed by calling the lua function lark.run() in a script, using
the lark subcommand "run".  When given no arguments, run will execute the first
named task that was defined, or a task specified by setting the "default"
variable in the "lark.task" lua module.

	local task = require('lark.task')
	task1 = task .. function() print('task1') end
	task2 = task .. function() print('task2') end
	lark.run()
	task.default = 'task2'
	lark.run()
	lark.run('task1')

The above script will print a line containing text "task1" followed by a line
containing "task2" and finally a line containing "task1" again.

Command Reference

Command reference documentation is available through the "help" subcommand.

	lark help

The documentation for a specific subcommand is available through the help
command or by passing the subcommand the -h (or --help) flag.

	lark run -h
	lark help run

Lua Reference

Lua API documentation is available through the help() function in the embedded REPL.

	lark repl
	> help()
	> help(lark)
*/
package main

import (
	"log"
	"os"

	"github.com/bmatsuo/lark/larkmeta"
	"github.com/codegangsta/cli"
	"github.com/mattn/go-isatty"
	"github.com/yuin/gopher-lua"
)

// IsTTY is true if standard error is connected to a terminal. This is taken to
// mean that lark was executed from the command line and is not being logged to
// a file.
//
// BUG:
// The assumptions made due to IsTTY cannot be overridden (e.g. by a command
// line flag).
var (
	IsTTYStderr = isatty.IsTerminal(os.Stderr.Fd())
	IsTTYStdout = isatty.IsTerminal(os.Stdout.Fd())
	IsTTYStdin  = isatty.IsTerminal(os.Stdin.Fd())
)

// MainHelp is the top-level hop documentation.
var MainHelp = `

    If no builtin command is provided the "run" command is executed with any
    task arguments provided.  For more information see the command
    documentation

        lark run -h
`

func main() {
	if IsTTYStderr {
		logflags := log.Flags()
		logflags &^= log.Ldate | log.Ltime
		log.SetFlags(logflags)
	}

	// Set search path for lua modules.  The search path must be completely
	// contained by the working directory to help ensure repeatable builds
	// across machines.
	lua.LuaPathDefault = "./lark_modules/?.lua;./lark_modules/?/init.lua"
	os.Setenv(lua.LuaPath, "")

	cli.VersionFlag.Name = "version"

	app := cli.NewApp()
	app.Name = "lark"
	app.Usage = "Run repeated project tasks"
	app.ArgsUsage = MainHelp
	app.Version = larkmeta.Version
	app.Authors = larkmeta.Authors
	app.Action = func(c *cli.Context) {
		args := []string{os.Args[0], "run"}
		args = append(args, c.Args()...)
		app.Run(args)
	}
	app.Commands = Commands

	app.Run(os.Args)
}
<commit_msg>docs: replace reference to `lark repl` with `lark lua`<commit_after>/*
Command lark executes project tasks defined with lua scripts.  The Lua
interpreter included in lark in embedded and is not affected by an existing Lua
installation on the host system.  Furthermore, Lark isolates the lua modules
available to scripts to modules in the relative directory path ./lark_modules/
to ensure portability of project tasks across developer machines.

As of the current release the embedded Lua interpreter is compliant with
Lua5.1.

The lark command locates tasks defined in the ./lark.lua file or otherwise
directly under the directory ./lark_tasks/.  Tasks can have names (either
explicitly given or otherwise inferred) or patterns.  Pattern matching tasks
will match a set of names defined by a regular expression.

Names are matched against available tasks with a strict precedence.  Explicitly
named tasks will match the same name with the highest priority.  Any task with
an inferred name will match the same name with the second highest priority.
Pattern matching tasks have the lowest priority and will match names in the
order they were defined.

Tasks can be executed by calling the lua function lark.run() in a script, using
the lark subcommand "run".  When given no arguments, run will execute the first
named task that was defined, or a task specified by setting the "default"
variable in the "lark.task" lua module.

	local task = require('lark.task')
	task1 = task .. function() print('task1') end
	task2 = task .. function() print('task2') end
	lark.run()
	task.default = 'task2'
	lark.run()
	lark.run('task1')

The above script will print a line containing text "task1" followed by a line
containing "task2" and finally a line containing "task1" again.

Command Reference

Command reference documentation is available through the "help" subcommand.

	lark help

The documentation for a specific subcommand is available through the help
command or by passing the subcommand the -h (or --help) flag.

	lark run -h
	lark help run

Lua Reference

Lua API documentation is available through the help() function in the embedded REPL.

	lark lua
	> help()
	> help(lark)
*/
package main

import (
	"log"
	"os"

	"github.com/bmatsuo/lark/larkmeta"
	"github.com/codegangsta/cli"
	"github.com/mattn/go-isatty"
	"github.com/yuin/gopher-lua"
)

// IsTTY is true if standard error is connected to a terminal. This is taken to
// mean that lark was executed from the command line and is not being logged to
// a file.
//
// BUG:
// The assumptions made due to IsTTY cannot be overridden (e.g. by a command
// line flag).
var (
	IsTTYStderr = isatty.IsTerminal(os.Stderr.Fd())
	IsTTYStdout = isatty.IsTerminal(os.Stdout.Fd())
	IsTTYStdin  = isatty.IsTerminal(os.Stdin.Fd())
)

// MainHelp is the top-level hop documentation.
var MainHelp = `

    If no builtin command is provided the "run" command is executed with any
    task arguments provided.  For more information see the command
    documentation

        lark run -h
`

func main() {
	if IsTTYStderr {
		logflags := log.Flags()
		logflags &^= log.Ldate | log.Ltime
		log.SetFlags(logflags)
	}

	// Set search path for lua modules.  The search path must be completely
	// contained by the working directory to help ensure repeatable builds
	// across machines.
	lua.LuaPathDefault = "./lark_modules/?.lua;./lark_modules/?/init.lua"
	os.Setenv(lua.LuaPath, "")

	cli.VersionFlag.Name = "version"

	app := cli.NewApp()
	app.Name = "lark"
	app.Usage = "Run repeated project tasks"
	app.ArgsUsage = MainHelp
	app.Version = larkmeta.Version
	app.Authors = larkmeta.Authors
	app.Action = func(c *cli.Context) {
		args := []string{os.Args[0], "run"}
		args = append(args, c.Args()...)
		app.Run(args)
	}
	app.Commands = Commands

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>// Copyright 2018 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

/*
Command mdot turns an mtail program AST into a graphviz graph on standard output.

To use, run it like (assuming your shell is in the same directory as this file)

  go run github.com/google/mtail/cmd/mdot --prog ../../examples/dhcpd.mtail | xdot -

or

  go run github.com/google/mtail/cmd/mdot --prog ../../examples/dhcpd.mtail --http_port 8080

to view the dot output visit http://localhost:8080

You'll need the graphviz `dot' command installed.
*/
package main

import (
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"strings"

	"github.com/golang/glog"
	"github.com/google/mtail/internal/mtail"
	"github.com/google/mtail/internal/vm/ast"
	"github.com/google/mtail/internal/vm/checker"
	"github.com/google/mtail/internal/vm/parser"
)

var (
	prog     = flag.String("prog", "", "Name of the program source to parse.")
	httpPort = flag.String("http_port", "", "Port number to run HTTP server on.")
)

type dotter struct {
	w        io.Writer
	id       int
	parentID []int // id of the parent node
}

func (d *dotter) nextID() int {
	d.id++
	return d.id
}

func (d *dotter) emitNode(id int, node ast.Node) {
	attrs := map[string]string{
		"label":   strings.Split(fmt.Sprintf("%T", node), ".")[1] + "\n",
		"shape":   "box",
		"style":   "filled",
		"tooltip": node.Type().String(),
	}
	switch n := node.(type) {
	case *ast.VarDecl, *ast.DecoDecl:
		attrs["fillcolor"] = "green"
		switch n := n.(type) {
		case *ast.VarDecl:
			attrs["label"] += fmt.Sprintf("%s %s", n.Kind, n.Name)
		case *ast.DecoDecl:
			attrs["label"] += n.Name
		}
	case *ast.IdTerm, *ast.CaprefTerm:
		attrs["fillcolor"] = "pink"
		attrs["shape"] = "ellipse"
		switch n := n.(type) {
		case *ast.IdTerm:
			attrs["label"] += n.Name
		case *ast.CaprefTerm:
			attrs["label"] += fmt.Sprintf("$%s", n.Name)
		}
	case *ast.IntLit, *ast.FloatLit, *ast.PatternLit, *ast.StringLit:
		attrs["fillcolor"] = "pink"
		attrs["shape"] = "ellipse"
		switch n := n.(type) {
		case *ast.IntLit:
			attrs["label"] += fmt.Sprintf("%d", n.I)
		case *ast.FloatLit:
			attrs["label"] += fmt.Sprintf("%g", n.F)
		case *ast.PatternLit:
			attrs["label"] += fmt.Sprintf("/%s/", n.Pattern)
		case *ast.StringLit:
			attrs["label"] += n.Text
		}
	case *ast.IndexedExpr, *ast.BinaryExpr, *ast.UnaryExpr, *ast.PatternExpr, *ast.BuiltinExpr:
		attrs["fillcolor"] = "lightblue"
		switch n := n.(type) {
		case *ast.BinaryExpr:
			attrs["label"] += parser.Kind(n.Op).String()
		case *ast.UnaryExpr:
			attrs["label"] += parser.Kind(n.Op).String()
		case *ast.BuiltinExpr:
			attrs["label"] += n.Name
		}
	}
	pos := node.Pos()
	if pos != nil {
		attrs["xlabel"] = pos.String()
	}
	fmt.Fprintf(d.w, "n%d [", id)
	for k, v := range attrs {
		fmt.Fprintf(d.w, "%s=\"%s\" ", k, v)
	}
	fmt.Fprintf(d.w, "]\n")
}

func (d *dotter) emitLine(src, dst int) {
	fmt.Fprintf(d.w, "n%d -> n%d\n", src, dst)
}

func (d *dotter) VisitBefore(node ast.Node) (ast.Visitor, ast.Node) {
	id := d.nextID()
	d.emitNode(id, node)
	if len(d.parentID) > 0 {
		parentID := d.parentID[len(d.parentID)-1]
		d.emitLine(parentID, id)
	}
	d.parentID = append(d.parentID, id)
	return d, node
}

func (d *dotter) VisitAfter(node ast.Node) ast.Node {
	d.parentID = d.parentID[:len(d.parentID)-1]
	return node
}

func makeDot(n ast.Node, w io.Writer) error {
	fmt.Fprintf(w, "digraph \"%s\" {\n", *prog)
	dot := &dotter{w: w}
	ast.Walk(dot, n)
	fmt.Fprintf(w, "}\n")
	return nil
}

func main() {
	flag.Parse()

	if *prog == "" {
		glog.Exitf("No -prog given")
	}

	f, err := os.Open(*prog)
	if err != nil {
		glog.Fatal(err)
	}
	n, err := parser.Parse(*prog, f)
	if err != nil {
		glog.Exit(err)
	}
	n, err = checker.Check(n)
	if err != nil {
		glog.Exit(err)
	}
	if *httpPort == "" {
		makeDot(n, os.Stdout)
		return
	}

	http.HandleFunc("/",
		func(w http.ResponseWriter, r *http.Request) {
			dot := exec.Command("dot", "-Tsvg")
			in, err := dot.StdinPipe()
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			out, err := dot.StdoutPipe()
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			err = dot.Start()
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			err = makeDot(n, in)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			err = in.Close()
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			w.Header().Add("Content-type", "image/svg+xml")
			w.WriteHeader(http.StatusOK)
			_, err = io.Copy(w, out)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
			}
			err = dot.Wait()
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
			}
		})
	http.HandleFunc("/favicon.ico", mtail.FaviconHandler)
	http.ListenAndServe(fmt.Sprintf(":%s", *httpPort), nil)
}
<commit_msg>Let a page reload also regenerate the graph so you can edit the source code but leave mdot running.<commit_after>// Copyright 2018 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

/*
Command mdot turns an mtail program AST into a graphviz graph on standard output.

To use, run it like (assuming your shell is in the same directory as this file)

  go run github.com/google/mtail/cmd/mdot --prog ../../examples/dhcpd.mtail | xdot -

or

  go run github.com/google/mtail/cmd/mdot --prog ../../examples/dhcpd.mtail --http_port 8080

to view the dot output visit http://localhost:8080

You'll need the graphviz `dot' command installed.
*/
package main

import (
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"strings"

	"github.com/golang/glog"
	"github.com/google/mtail/internal/mtail"
	"github.com/google/mtail/internal/vm/ast"
	"github.com/google/mtail/internal/vm/checker"
	"github.com/google/mtail/internal/vm/parser"
)

var (
	prog     = flag.String("prog", "", "Name of the program source to parse.")
	httpPort = flag.String("http_port", "", "Port number to run HTTP server on.")
)

type dotter struct {
	w        io.Writer
	id       int
	parentID []int // id of the parent node
}

func (d *dotter) nextID() int {
	d.id++
	return d.id
}

func (d *dotter) emitNode(id int, node ast.Node) {
	attrs := map[string]string{
		"label":   strings.Split(fmt.Sprintf("%T", node), ".")[1] + "\n",
		"shape":   "box",
		"style":   "filled",
		"tooltip": node.Type().String(),
	}
	switch n := node.(type) {
	case *ast.VarDecl, *ast.DecoDecl:
		attrs["fillcolor"] = "green"
		switch n := n.(type) {
		case *ast.VarDecl:
			attrs["label"] += fmt.Sprintf("%s %s", n.Kind, n.Name)
		case *ast.DecoDecl:
			attrs["label"] += n.Name
		}
	case *ast.IdTerm, *ast.CaprefTerm:
		attrs["fillcolor"] = "pink"
		attrs["shape"] = "ellipse"
		switch n := n.(type) {
		case *ast.IdTerm:
			attrs["label"] += n.Name
		case *ast.CaprefTerm:
			attrs["label"] += fmt.Sprintf("$%s", n.Name)
		}
	case *ast.IntLit, *ast.FloatLit, *ast.PatternLit, *ast.StringLit:
		attrs["fillcolor"] = "pink"
		attrs["shape"] = "ellipse"
		switch n := n.(type) {
		case *ast.IntLit:
			attrs["label"] += fmt.Sprintf("%d", n.I)
		case *ast.FloatLit:
			attrs["label"] += fmt.Sprintf("%g", n.F)
		case *ast.PatternLit:
			attrs["label"] += fmt.Sprintf("/%s/", n.Pattern)
		case *ast.StringLit:
			attrs["label"] += n.Text
		}
	case *ast.IndexedExpr, *ast.BinaryExpr, *ast.UnaryExpr, *ast.PatternExpr, *ast.BuiltinExpr:
		attrs["fillcolor"] = "lightblue"
		switch n := n.(type) {
		case *ast.BinaryExpr:
			attrs["label"] += parser.Kind(n.Op).String()
		case *ast.UnaryExpr:
			attrs["label"] += parser.Kind(n.Op).String()
		case *ast.BuiltinExpr:
			attrs["label"] += n.Name
		}
	}
	pos := node.Pos()
	if pos != nil {
		attrs["xlabel"] = pos.String()
	}
	fmt.Fprintf(d.w, "n%d [", id)
	for k, v := range attrs {
		fmt.Fprintf(d.w, "%s=\"%s\" ", k, v)
	}
	fmt.Fprintf(d.w, "]\n")
}

func (d *dotter) emitLine(src, dst int) {
	fmt.Fprintf(d.w, "n%d -> n%d\n", src, dst)
}

func (d *dotter) VisitBefore(node ast.Node) (ast.Visitor, ast.Node) {
	id := d.nextID()
	d.emitNode(id, node)
	if len(d.parentID) > 0 {
		parentID := d.parentID[len(d.parentID)-1]
		d.emitLine(parentID, id)
	}
	d.parentID = append(d.parentID, id)
	return d, node
}

func (d *dotter) VisitAfter(node ast.Node) ast.Node {
	d.parentID = d.parentID[:len(d.parentID)-1]
	return node
}

func makeDot(name string, w io.Writer) error {
	f, err := os.Open(name)
	if err != nil {
		return err
	}
	n, err := parser.Parse(name, f)
	if err != nil {
		return err
	}
	n, err = checker.Check(n)
	if err != nil {
		return err
	}
	fmt.Fprintf(w, "digraph \"%s\" {\n", *prog)
	dot := &dotter{w: w}
	ast.Walk(dot, n)
	fmt.Fprintf(w, "}\n")
	return nil
}

func main() {
	flag.Parse()

	if *prog == "" {
		glog.Exitf("No -prog given")
	}

	if *httpPort == "" {
		glog.Exit(makeDot(*prog, os.Stdout))
	}

	http.HandleFunc("/",
		func(w http.ResponseWriter, r *http.Request) {
			dot := exec.Command("dot", "-Tsvg")
			in, err := dot.StdinPipe()
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			out, err := dot.StdoutPipe()
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			err = dot.Start()
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			err = makeDot(*prog, in)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			err = in.Close()
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			w.Header().Add("Content-type", "image/svg+xml")
			w.WriteHeader(http.StatusOK)
			_, err = io.Copy(w, out)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
			}
			err = dot.Wait()
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
			}
		})
	http.HandleFunc("/favicon.ico", mtail.FaviconHandler)
	http.ListenAndServe(fmt.Sprintf(":%s", *httpPort), nil)
}
<|endoftext|>