<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ledcmd

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"

	"go.chromium.org/luci/led/job"
)

// EditRecipeBundleOpts are user-provided options for the recipe bundling
// process.
type EditRecipeBundleOpts struct {
	// Path on disk to the repo to extract the recipes from. May be a subdirectory
	// of the repo, as long as `git rev-parse --show-toplevel` can find the root
	// of the repository.
	//
	// If empty, uses the current working directory.
	RepoDir string

	// Overrides is a mapping of recipe project id (e.g. "recipe_engine") to
	// a local path to a checkout of that repo (e.g. "/path/to/recipes-py.git").
	//
	// When the bundle is created, this local repo will be used instead of the
	// pinned version of this recipe project id. This is helpful for preparing
	// bundles which have code changes in multiple recipe repos.
	Overrides map[string]string

	// DebugSleep is the amount of time to wait after the recipe completes
	// execution (either success or failure). This is injected into the generated
	// recipe bundle as a 'sleep X' command after the invocation of the recipe
	// itself.
	DebugSleep time.Duration
}

// RecipeDirectory is a very unfortunate constant which is here for
// a combination of reasons:
//   1) swarming doesn't allow you to 'checkout' an isolate relative to any path
//      in the task (other than the task root). This means that whatever value
//      we pick for EditRecipeBundle must be used EVERYWHERE the isolated hash
//      is used.
//   2) Currently the 'recipe_engine/led' module will blindly take the isolated
//      input and 'inject' it into further uses of led. This module currently
//      doesn't specify the checkout dir, relying on kitchen's default value of
//      (you guessed it) "kitchen-checkout".
//
// In order to fix this (and it will need to be fixed for bbagent support):
//   * The 'recipe_engine/led' module needs to accept 'checkout-dir' as
//     a parameter in its input properties.
//   * led needs to start passing the checkout dir to the led module's input
//     properties.
//   * `led edit` needs a way to manipulate the checkout directory in a job
//   * The 'recipe_engine/led' module needs to set this in the job
//     alongside the isolate hash when it's doing the injection.
//
// For now, we just hard-code it.
//
// TODO(crbug.com/1072117): Fix this, it's weird.
const RecipeDirectory = "kitchen-checkout"

// EditRecipeBundle overrides the recipe bundle in the given job with one
// located on disk.
//
// It isolates the recipes from the repository in the given working directory
// into the UserPayload under the directory "kitchen-checkout/". If there's an
// existing directory in the UserPayload at that location, it will be removed.
func EditRecipeBundle(ctx context.Context, authClient *http.Client,  authOpts auth.Options, jd *job.Definition, opts *EditRecipeBundleOpts) error {
	if jd.GetSwarming() != nil {
		return errors.New("ledcmd.EditRecipeBundle is only available for Buildbucket tasks")
	}

	if opts == nil {
		opts = &EditRecipeBundleOpts{}
	}

	recipesPy, err := findRecipesPy(ctx, opts.RepoDir)
	if err != nil {
		return err
	}
	logging.Debugf(ctx, "using recipes.py: %q", recipesPy)

	err = EditIsolated(ctx, authClient, authOpts, jd, func(ctx context.Context, dir string) error {
		logging.Infof(ctx, "bundling recipes")
		bundlePath := filepath.Join(dir, RecipeDirectory)
		// Remove existing bundled recipes, if any. Ignore the error.
		os.RemoveAll(bundlePath)
		if err := opts.prepBundle(ctx, opts.RepoDir, recipesPy, bundlePath); err != nil {
			return err
		}
		logging.Infof(ctx, "isolating recipes")
		return nil
	})
	if err != nil {
		return err
	}

	return jd.HighLevelEdit(func(je job.HighLevelEditor) {
		je.TaskPayloadSource("", "")
		je.TaskPayloadPath(RecipeDirectory)
	})
}

func logCmd(ctx context.Context, inDir string, arg0 string, args ...string) *exec.Cmd {
	ret := exec.CommandContext(ctx, arg0, args...)
	ret.Dir = inDir
	logging.Debugf(ctx, "Running (from %q) - %s %v", inDir, arg0, args)
	return ret
}

func cmdErr(err error, reason string) error {
	if err != nil {
		ee, _ := err.(*exec.ExitError)
		outErr := ""
		if ee != nil {
			outErr = strings.TrimSpace(string(ee.Stderr))
			if len(outErr) > 128 {
				outErr = outErr[:128] + "..."
			}
		}
		err = errors.Annotate(err, reason+": %s", outErr).Err()
	}
	return err
}

func appendText(path, fmtStr string, items ...interface{}) error {
	file, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0666)
	if err != nil {
		return err
	}
	defer file.Close()
	_, err = fmt.Fprintf(file, fmtStr, items...)
	return err
}

func (opts *EditRecipeBundleOpts) prepBundle(ctx context.Context, inDir, recipesPy, toDirectory string) (err error) {
	args := []string{
		recipesPy,
	}
	if logging.GetLevel(ctx) < logging.Info {
		args = append(args, "-v")
	}
	for projID, path := range opts.Overrides {
		args = append(args, "-O", fmt.Sprintf("%s=%s", projID, path))
	}
	args = append(args, "bundle", "--destination", filepath.Join(toDirectory))
	cmd := logCmd(ctx, inDir, "python", args...)
	if logging.GetLevel(ctx) < logging.Info {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	}
	if err = cmdErr(cmd.Run(), "creating bundle"); err != nil {
		return
	}
	if opts.DebugSleep != 0 {
		fname := filepath.Join(toDirectory, "recipes")
		seconds := opts.DebugSleep / time.Second
		msg := "echo ENTERING DEBUG SLEEP. SSH to the bot to debug."

		if err = appendText(fname, "\n%s\nsleep %d\n", msg, seconds); err != nil {
			return
		}
		// Wait for a bogus event that won't occur... Windows sucks, amirite?
		if err = appendText(fname+".bat", "\r\n%s\r\nwaitfor /t %d DebugSessionEnd\r\n", msg, seconds); err != nil {
			return
		}
	}

	return
}

// findRecipesPy locates the current repo's `recipes.py`. It does this by:
//   * invoking git to find the repo root
//   * loading the recipes.cfg at infra/config/recipes.cfg
//   * stat'ing the recipes.py implied by the recipes_path in that cfg file.
//
// Failure will return an error.
//
// On success, the absolute path to recipes.py is returned.
func findRecipesPy(ctx context.Context, inDir string) (string, error) {
	cmd := logCmd(ctx, inDir, "git", "rev-parse", "--show-toplevel")
	out, err := cmd.Output()
	if err = cmdErr(err, "finding git repo"); err != nil {
		return "", err
	}

	repoRoot := strings.TrimSpace(string(out))

	pth := filepath.Join(repoRoot, "infra", "config", "recipes.cfg")
	switch st, err := os.Stat(pth); {
	case err != nil:
		return "", errors.Annotate(err, "reading recipes.cfg").Err()

	case !st.Mode().IsRegular():
		return "", errors.Reason("%q is not a regular file", pth).Err()
	}

	type recipesJSON struct {
		RecipesPath string `json:"recipes_path"`
	}
	rj := &recipesJSON{}

	f, err := os.Open(pth)
	if err != nil {
		return "", errors.Reason("reading recipes.cfg: %q", pth).Err()
	}
	defer f.Close()

	if err := json.NewDecoder(f).Decode(rj); err != nil {
		return "", errors.Reason("parsing recipes.cfg: %q", pth).Err()
	}

	return filepath.Join(
		repoRoot, filepath.FromSlash(rj.RecipesPath), "recipes.py"), nil
}
<commit_msg>[led] Inject debug_sleep into luciexe wrappers, too.<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ledcmd

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"

	"go.chromium.org/luci/led/job"
)

// EditRecipeBundleOpts are user-provided options for the recipe bundling
// process.
type EditRecipeBundleOpts struct {
	// Path on disk to the repo to extract the recipes from. May be a subdirectory
	// of the repo, as long as `git rev-parse --show-toplevel` can find the root
	// of the repository.
	//
	// If empty, uses the current working directory.
	RepoDir string

	// Overrides is a mapping of recipe project id (e.g. "recipe_engine") to
	// a local path to a checkout of that repo (e.g. "/path/to/recipes-py.git").
	//
	// When the bundle is created, this local repo will be used instead of the
	// pinned version of this recipe project id. This is helpful for preparing
	// bundles which have code changes in multiple recipe repos.
	Overrides map[string]string

	// DebugSleep is the amount of time to wait after the recipe completes
	// execution (either success or failure). This is injected into the generated
	// recipe bundle as a 'sleep X' command after the invocation of the recipe
	// itself.
	DebugSleep time.Duration
}

// RecipeDirectory is a very unfortunate constant which is here for
// a combination of reasons:
//   1) swarming doesn't allow you to 'checkout' an isolate relative to any path
//      in the task (other than the task root). This means that whatever value
//      we pick for EditRecipeBundle must be used EVERYWHERE the isolated hash
//      is used.
//   2) Currently the 'recipe_engine/led' module will blindly take the isolated
//      input and 'inject' it into further uses of led. This module currently
//      doesn't specify the checkout dir, relying on kitchen's default value of
//      (you guessed it) "kitchen-checkout".
//
// In order to fix this (and it will need to be fixed for bbagent support):
//   * The 'recipe_engine/led' module needs to accept 'checkout-dir' as
//     a parameter in its input properties.
//   * led needs to start passing the checkout dir to the led module's input
//     properties.
//   * `led edit` needs a way to manipulate the checkout directory in a job
//   * The 'recipe_engine/led' module needs to set this in the job
//     alongside the isolate hash when it's doing the injection.
//
// For now, we just hard-code it.
//
// TODO(crbug.com/1072117): Fix this, it's weird.
const RecipeDirectory = "kitchen-checkout"

// EditRecipeBundle overrides the recipe bundle in the given job with one
// located on disk.
//
// It isolates the recipes from the repository in the given working directory
// into the UserPayload under the directory "kitchen-checkout/". If there's an
// existing directory in the UserPayload at that location, it will be removed.
func EditRecipeBundle(ctx context.Context, authClient *http.Client, authOpts auth.Options, jd *job.Definition, opts *EditRecipeBundleOpts) error {
	if jd.GetSwarming() != nil {
		return errors.New("ledcmd.EditRecipeBundle is only available for Buildbucket tasks")
	}

	if opts == nil {
		opts = &EditRecipeBundleOpts{}
	}

	recipesPy, err := findRecipesPy(ctx, opts.RepoDir)
	if err != nil {
		return err
	}
	logging.Debugf(ctx, "using recipes.py: %q", recipesPy)

	err = EditIsolated(ctx, authClient, authOpts, jd, func(ctx context.Context, dir string) error {
		logging.Infof(ctx, "bundling recipes")
		bundlePath := filepath.Join(dir, RecipeDirectory)
		// Remove existing bundled recipes, if any. Ignore the error.
		os.RemoveAll(bundlePath)
		if err := opts.prepBundle(ctx, opts.RepoDir, recipesPy, bundlePath); err != nil {
			return err
		}
		logging.Infof(ctx, "isolating recipes")
		return nil
	})
	if err != nil {
		return err
	}

	return jd.HighLevelEdit(func(je job.HighLevelEditor) {
		je.TaskPayloadSource("", "")
		je.TaskPayloadPath(RecipeDirectory)
	})
}

func logCmd(ctx context.Context, inDir string, arg0 string, args ...string) *exec.Cmd {
	ret := exec.CommandContext(ctx, arg0, args...)
	ret.Dir = inDir
	logging.Debugf(ctx, "Running (from %q) - %s %v", inDir, arg0, args)
	return ret
}

func cmdErr(err error, reason string) error {
	if err != nil {
		ee, _ := err.(*exec.ExitError)
		outErr := ""
		if ee != nil {
			outErr = strings.TrimSpace(string(ee.Stderr))
			if len(outErr) > 128 {
				outErr = outErr[:128] + "..."
			}
		}
		err = errors.Annotate(err, reason+": %s", outErr).Err()
	}
	return err
}

func appendText(path, fmtStr string, items ...interface{}) error {
	file, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0666)
	if err != nil {
		return err
	}
	defer file.Close()
	_, err = fmt.Fprintf(file, fmtStr, items...)
	return err
}

func (opts *EditRecipeBundleOpts) prepBundle(ctx context.Context, inDir, recipesPy, toDirectory string) (err error) {
	args := []string{
		recipesPy,
	}
	if logging.GetLevel(ctx) < logging.Info {
		args = append(args, "-v")
	}
	for projID, path := range opts.Overrides {
		args = append(args, "-O", fmt.Sprintf("%s=%s", projID, path))
	}
	args = append(args, "bundle", "--destination", filepath.Join(toDirectory))
	cmd := logCmd(ctx, inDir, "python", args...)
	if logging.GetLevel(ctx) < logging.Info {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	}
	if err = cmdErr(cmd.Run(), "creating bundle"); err != nil {
		return
	}
	if opts.DebugSleep != 0 {
		for _, basename := range []string{"recipes", "luciexe"} {
			fname := filepath.Join(toDirectory, basename)
			seconds := opts.DebugSleep / time.Second
			msg := "echo ENTERING DEBUG SLEEP. SSH to the bot to debug."

			if err = appendText(fname, "\n%s\nsleep %d\n", msg, seconds); err != nil {
				return
			}
			// Wait for a bogus event that won't occur... Windows sucks, amirite?
			if err = appendText(fname+".bat", "\r\n%s\r\nwaitfor /t %d DebugSessionEnd\r\n", msg, seconds); err != nil {
				return
			}
		}
	}

	return
}

// findRecipesPy locates the current repo's `recipes.py`. It does this by:
//   * invoking git to find the repo root
//   * loading the recipes.cfg at infra/config/recipes.cfg
//   * stat'ing the recipes.py implied by the recipes_path in that cfg file.
//
// Failure will return an error.
//
// On success, the absolute path to recipes.py is returned.
func findRecipesPy(ctx context.Context, inDir string) (string, error) {
	cmd := logCmd(ctx, inDir, "git", "rev-parse", "--show-toplevel")
	out, err := cmd.Output()
	if err = cmdErr(err, "finding git repo"); err != nil {
		return "", err
	}

	repoRoot := strings.TrimSpace(string(out))

	pth := filepath.Join(repoRoot, "infra", "config", "recipes.cfg")
	switch st, err := os.Stat(pth); {
	case err != nil:
		return "", errors.Annotate(err, "reading recipes.cfg").Err()

	case !st.Mode().IsRegular():
		return "", errors.Reason("%q is not a regular file", pth).Err()
	}

	type recipesJSON struct {
		RecipesPath string `json:"recipes_path"`
	}
	rj := &recipesJSON{}

	f, err := os.Open(pth)
	if err != nil {
		return "", errors.Reason("reading recipes.cfg: %q", pth).Err()
	}
	defer f.Close()

	if err := json.NewDecoder(f).Decode(rj); err != nil {
		return "", errors.Reason("parsing recipes.cfg: %q", pth).Err()
	}

	return filepath.Join(
		repoRoot, filepath.FromSlash(rj.RecipesPath), "recipes.py"), nil
}
<|endoftext|>


<commit_before>/*

Hidden Markov model.

 α β γ ζ Φ

*/
package hmm

import (
	"code.google.com/p/biogo.matrix"
	"fmt"
	"github.com/akualab/gjoa/model"
	"github.com/golang/glog"
	"math"
)

const (
	SMALL_NUMBER = 0.000001
)

type HMM struct {

	// q(t) is the state at time t
	// states are labeled {0,2,...,N-1}

	// Number fo hidden states.
	// N
	nstates int

	// State-transition probability distribution matrixi (log scale).
	// [nstates x nstates]
	// a(i,j) = log(P[q(t+1) = j | q(t) = i]); 0 <= i,j <= N-1
	// when transitions between states are not allowed: a(i,j) = 0
	// => saved in log domain.
	logTransProbs *matrix.Dense

	// Observation probability distribution functions. [nstates x 1]
	// b(j,k) = P[o(t) | q(t) = j]
	obsModels []model.Modeler

	// Initial state distribution. [nstates x 1]
	// π(i) = P[q(0) = i]; 0<=i<N
	// => saved in log domain.
	logInitProbs *matrix.Dense

	// Num elements in obs vector.
	numElements int

	// Complete HMM model.
	// Φ = (A, B, π)
}

// Define functions for elementwise transformations.
var log = func(r, c int, v float64) float64 { return math.Log(v) }

// Create a new HMM.
func NewHMM(transProbs, initialStateProbs *matrix.Dense, obsModels []model.Modeler) (hmm *HMM, e error) {

	r, c := transProbs.Dims()
	if r != c {
		e = fmt.Errorf("Matrix transProbs must be square. rows=[%d], cols=[%d]", r, c)
		return
	}

	glog.Infof("New HMM. Num states = [%d].", r)

	// init logTransProbs and logInitProbs
	logTransProbs := matrix.MustDense(matrix.ZeroDense(r, r))
	logInitProbs := matrix.MustDense(matrix.ZeroDense(r, 1))

	// apply log to transProbs and initialStateProbs
	logTransProbs = transProbs.ApplyDense(log, logTransProbs)
	logInitProbs = initialStateProbs.ApplyDense(log, logInitProbs)

	hmm = &HMM{
		nstates:       r,
		logTransProbs: logTransProbs,
		obsModels:     obsModels,
		logInitProbs:  logInitProbs,
		numElements:   obsModels[0].NumElements(),
	}

	return
}

// Compute alphas. Indices are: α(state, time)
//
// α = | α(0,0),   α(0,1)   ... α(0,T-1)   |
//     | α(1,0),   α(1,1)   ... α(1,T-1)   |
//     ...
//     | α(N-1,0), α(N-1,1) ... α(N-1,T-1) |
//
//
// 1. Initialization: α(i,0) =  π(i) b(i,o(0)); 0<=i<N
// 2. Induction:      α(j,t+1) =  sum_{i=0}^{N-1}[α(i,t)a(i,j)] b(j,o(t+1)); 0<=t<T-1; 0<=j<N
// 3. Termination:    P(O/Φ) = sum_{i=0}^{N-1} α(i,T-1)
// For scaling details see Rabiner/Juang
func (hmm *HMM) alpha(observations *matrix.Dense) (α *matrix.Dense, logProb float64, e error) {

	// Num states.
	N := hmm.nstates

	// expected num rows: numElements
	// expected num cols: T
	ne, T := observations.Dims()

	if ne != hmm.numElements {
		e = fmt.Errorf("Mismatch in num elements in observations [%d] expected [%d].", ne, hmm.numElements)
		return
	}

	// Allocate log-alpha matrix.
	// TODO: use a reusable data structure to minimize garbage.
	α = matrix.MustDense(matrix.ZeroDense(N, T))

	// 1. Initialization. Add in the log doman.
	for i := 0; i < N; i++ {
		α.Set(i, 0, hmm.logInitProbs.At(i, 0)+hmm.obsModels[i].LogProb(ColumnAt(observations, 0)))
	}

	// 2. Induction.
	var sumAlphas, sum float64
	for t := 0; t < T-1; t++ {
		sumAlphas = 0
		for j := 0; j < N; j++ {

			sum = 0
			for i := 0; i < N; i++ {
				sum += math.Exp(α.At(i, t) + hmm.logTransProbs.At(i, j))
			}
			v := math.Log(sum) + hmm.obsModels[j].LogProb(ColumnAt(observations, t+1))
			α.Set(j, t+1, v)

			sumAlphas += math.Exp(v)
			if glog.V(4) {
				glog.Infof("t: %4d | j: %2d | logAlpha: %5e | sumAlphas: %5e", t, j, v, sumAlphas)
			}
		}
		// Applied scale for t independent of j.
		logSumAlphas := math.Log(sumAlphas)
		for j := 0; j < N; j++ {
			v := α.At(j, t+1) - logSumAlphas
			α.Set(j, t+1, v)
		}
		logProb += logSumAlphas
	}

	return
}

// Compute betas. Indices are: β(state, time)
//
// β = | β(0,0),   β(0,1)   ... β(0,T-1)   |
//     | β(1,0),   β(1,1)   ... β(1,T-1)   |
//     ...
//     | β(N-1,0), β(N-1,1) ... β(N-1,T-1) |
//
//
// 1. Initialization: β(i,T-1) = 1;  0<=i<N
// 2. Induction:      β(i,t) =  sum_{j=0}^{N-1} a(i,j) b(j,o(t+1)) β(j,t+1); t=T-2,T-3,...,0; 0<=i<N
func (hmm *HMM) beta(observations *matrix.Dense) (β *matrix.Dense, e error) {

	// Num states.
	N := hmm.nstates

	// expected num rows: numElements
	// expected num cols: T
	ne, T := observations.Dims()

	if ne != hmm.numElements {
		e = fmt.Errorf("Mismatch in num elements in observations [%d] expected [%d].", ne, hmm.numElements)
		return
	}

	// Allocate log-beta matrix.
	// TODO: use a reusable data structure to minimize garbage.
	β = matrix.MustDense(matrix.ZeroDense(N, T))

	// 1. Initialization. Add in the log doman.
	for i := 0; i < N; i++ {
		β.Set(i, T-1, 1.0)
	}

	// 2. Induction.
	var sumBetas float64
	for t := T - 2; t >= 0; t-- {
		sumBetas = 0
		for i := 0; i < N; i++ {

			var sum float64
			for j := 0; j < N; j++ {

				sum += math.Exp(hmm.logTransProbs.At(i, j) + // a(i,j)
					hmm.obsModels[j].LogProb(ColumnAt(observations, t+1)) + // b(j,o(t+1))
					β.At(j, t+1)) // β(j,t+1)
			}
			β.Set(i, t, math.Log(sum))
			sumBetas += sum
		}
		// Applied scale for t independent of i.
		logSumBetas := math.Log(sumBetas)
		for i := 0; i < N; i++ {
			v := β.At(i, t) - logSumBetas
			β.Set(i, t, v)
		}
	}

	return
}

// Compute gammas. Indices are: γ(state, time)
//
// γ(i,t) =  α(i,t)β(i,t) / sum_{j=0}^{N-1} α(j,t)β(j,t);  0<=j<N
func (hmm *HMM) gamma(α, β *matrix.Dense) (γ *matrix.Dense, e error) {

	αr, αc := α.Dims()
	βr, βc := β.Dims()

	if αr != βr || αc != βc {
		e = fmt.Errorf("Shape mismatch: alpha[%d,%d] beta[%d,%d]", αr, αc, βr, βc)
		return
	}

	T := αc
	N := hmm.nstates
	if αr != N {
		e = fmt.Errorf("Num rows [%d] doesn't match num states [%d].", αr, N)
	}

	// Allocate log-gamma matrix.
	// TODO: use a reusable data structure to minimize garbage.
	γ = matrix.MustDense(matrix.ZeroDense(N, T))

	for t := 0; t < T; t++ {
		var sum float64 = 0.0
		for i := 0; i < N; i++ {
			x := α.At(i, t) + β.At(i, t)
			γ.Set(i, t, x)
			sum += math.Exp(x)
		}

		// Normalize.
		for i := 0; i < N; i++ {
			x := γ.At(i, t) - math.Log(sum)
			γ.Set(i, t, x)
		}
	}
	return
}

/*
   Compute xi. Indices are: ζ(from, to, time)

                         α(i,t) a(i,j) b(j,o(t+1)) β(j,t+1)
 ζ(i,j,t) = ------------------------------------------------------------------
            sum_{i=0}^{N-1} sum_{j=0}^{N-1} α(i,t) a(i,j) b(j,o(t+1)) β(j,t+1)

*/
func (hmm *HMM) xi(observations, α, β *matrix.Dense) (ζ [][][]float64, e error) {

	a := hmm.logTransProbs
	αr, αc := α.Dims()
	βr, βc := β.Dims()
	or, oc := observations.Dims()

	if αr != βr || αc != βc {
		e = fmt.Errorf("Shape mismatch: alpha[%d,%d] beta[%d,%d]", αr, αc, βr, βc)
		return
	}

	T := αc
	N := hmm.nstates
	if or != hmm.numElements {
		e = fmt.Errorf("Mismatch in num elements in observations [%d] expected [%d].",
			or, hmm.numElements)
		return
	}
	if oc != T {
		e = fmt.Errorf("Mismatch in T observations has [%d], expected [%d].", oc, T)
		return
	}
	if αr != N {
		e = fmt.Errorf("Num rows [%d] doesn't match num states [%d].", αr, N)
	}

	// Allocate log-xi matrix.
	// TODO: use a reusable data structure to minimize garbage.
	ζ = make([][][]float64, N)
	for i := 0; i < N; i++ {
		ζ[i] = make([][]float64, N)
		for j := 0; j < N; j++ {
			for t := 0; t < N; t++ {
				ζ[i][j] = make([]float64, T)
			}
		}
	}

	for t := 0; t < T-1; t++ {
		var sum float64 = 0.0
		for j := 0; j < N; j++ {
			b := hmm.obsModels[j].LogProb(ColumnAt(observations, t+1))
			for i := 0; i < N; i++ {
				x := α.At(i, t) + a.At(i, j) + b + β.At(j, t+1)
				ζ[i][j][t] = x
				sum += math.Exp(x)
			}
		}
		// Normalize.
		for i := 0; i < N; i++ {
			for j := 0; j < N; j++ {
				ζ[i][j][t] -= math.Log(sum)
			}
		}
	}
	return
}

// Update model statistics.
// sequence is a matrix
func (hmm *HMM) Update(observations *matrix.Dense) (e error) {

	var α, β, γ *matrix.Dense
	var ζ [][][]float64
	var logProb float64

	// Compute  α, β, γ, ζ
	// TODO: compute α, β, concurrently using go routines.
	α, logProb, e = hmm.alpha(observations)
	if e != nil {
		return
	}
	β, e = hmm.beta(observations)
	if e != nil {
		return
	}
	// TODO: compute γ, ζ concurrently using go routines.
	// Can we compute ζ more efficiently using γ?
	γ, e = hmm.gamma(α, β)
	if e != nil {
		return
	}
	ζ, e = hmm.xi(observations, α, β)
	if e != nil {
		return
	}

	fmt.Println(α, β, γ, ζ, logProb)
	return
}

// ColumnAt returns a *matrix.Dense column that is a copy of the values at column c of the matrix.
// Column will panic with ErrIndexOutOfRange is c is not a valid column index.
func ColumnAt(d *matrix.Dense, c int) *matrix.Dense {

	sl := d.Column(c)
	nrows := len(sl)

	col := matrix.MustDense(matrix.ZeroDense(nrows, 1))

	for i := 0; i < nrows; i++ {
		col.Set(i, 0, sl[i])
	}

	return col
}

func (hmm *HMM) NumElements() int {
	return hmm.numElements
}
<commit_msg>Add comment.<commit_after>/*

Hidden Markov model.

 α β γ ζ Φ

*/
package hmm

import (
	"code.google.com/p/biogo.matrix"
	"fmt"
	"github.com/akualab/gjoa/model"
	"github.com/golang/glog"
	"math"
)

const (
	SMALL_NUMBER = 0.000001
)

type HMM struct {

	// q(t) is the state at time t
	// states are labeled {0,2,...,N-1}

	// Number fo hidden states.
	// N
	nstates int

	// State-transition probability distribution matrixi (log scale).
	// [nstates x nstates]
	// a(i,j) = log(P[q(t+1) = j | q(t) = i]); 0 <= i,j <= N-1
	// when transitions between states are not allowed: a(i,j) = 0
	// => saved in log domain.
	logTransProbs *matrix.Dense

	// Observation probability distribution functions. [nstates x 1]
	// b(j,k) = P[o(t) | q(t) = j]
	obsModels []model.Modeler

	// Initial state distribution. [nstates x 1]
	// π(i) = P[q(0) = i]; 0<=i<N
	// => saved in log domain.
	logInitProbs *matrix.Dense

	// Num elements in obs vector.
	numElements int

	// Complete HMM model.
	// Φ = (A, B, π)
}

// Define functions for elementwise transformations.
var log = func(r, c int, v float64) float64 { return math.Log(v) }

// Create a new HMM.
func NewHMM(transProbs, initialStateProbs *matrix.Dense, obsModels []model.Modeler) (hmm *HMM, e error) {

	r, c := transProbs.Dims()
	if r != c {
		e = fmt.Errorf("Matrix transProbs must be square. rows=[%d], cols=[%d]", r, c)
		return
	}

	glog.Infof("New HMM. Num states = [%d].", r)

	// init logTransProbs and logInitProbs
	logTransProbs := matrix.MustDense(matrix.ZeroDense(r, r))
	logInitProbs := matrix.MustDense(matrix.ZeroDense(r, 1))

	// apply log to transProbs and initialStateProbs
	logTransProbs = transProbs.ApplyDense(log, logTransProbs)
	logInitProbs = initialStateProbs.ApplyDense(log, logInitProbs)

	hmm = &HMM{
		nstates:       r,
		logTransProbs: logTransProbs,
		obsModels:     obsModels,
		logInitProbs:  logInitProbs,
		numElements:   obsModels[0].NumElements(),
	}

	return
}

// Compute alphas. Indices are: α(state, time)
//
// α = | α(0,0),   α(0,1)   ... α(0,T-1)   |
//     | α(1,0),   α(1,1)   ... α(1,T-1)   |
//     ...
//     | α(N-1,0), α(N-1,1) ... α(N-1,T-1) |
//
//
// 1. Initialization: α(i,0) =  π(i) b(i,o(0)); 0<=i<N
// 2. Induction:      α(j,t+1) =  sum_{i=0}^{N-1}[α(i,t)a(i,j)] b(j,o(t+1)); 0<=t<T-1; 0<=j<N
// 3. Termination:    P(O/Φ) = sum_{i=0}^{N-1} α(i,T-1)
// For scaling details see Rabiner/Juang and
// http://courses.media.mit.edu/2010fall/mas622j/ProblemSets/ps4/tutorial.pdf
func (hmm *HMM) alpha(observations *matrix.Dense) (α *matrix.Dense, logProb float64, e error) {

	// Num states.
	N := hmm.nstates

	// expected num rows: numElements
	// expected num cols: T
	ne, T := observations.Dims()

	if ne != hmm.numElements {
		e = fmt.Errorf("Mismatch in num elements in observations [%d] expected [%d].", ne, hmm.numElements)
		return
	}

	// Allocate log-alpha matrix.
	// TODO: use a reusable data structure to minimize garbage.
	α = matrix.MustDense(matrix.ZeroDense(N, T))

	// 1. Initialization. Add in the log doman.
	for i := 0; i < N; i++ {
		α.Set(i, 0, hmm.logInitProbs.At(i, 0)+hmm.obsModels[i].LogProb(ColumnAt(observations, 0)))
	}

	// 2. Induction.
	var sumAlphas, sum float64
	for t := 0; t < T-1; t++ {
		sumAlphas = 0
		for j := 0; j < N; j++ {

			sum = 0
			for i := 0; i < N; i++ {
				sum += math.Exp(α.At(i, t) + hmm.logTransProbs.At(i, j))
			}
			v := math.Log(sum) + hmm.obsModels[j].LogProb(ColumnAt(observations, t+1))
			α.Set(j, t+1, v)

			sumAlphas += math.Exp(v)
			if glog.V(4) {
				glog.Infof("t: %4d | j: %2d | logAlpha: %5e | sumAlphas: %5e", t, j, v, sumAlphas)
			}
		}
		// Applied scale for t independent of j.
		logSumAlphas := math.Log(sumAlphas)
		for j := 0; j < N; j++ {
			v := α.At(j, t+1) - logSumAlphas
			α.Set(j, t+1, v)
		}
		logProb += logSumAlphas
	}

	return
}

// Compute betas. Indices are: β(state, time)
//
// β = | β(0,0),   β(0,1)   ... β(0,T-1)   |
//     | β(1,0),   β(1,1)   ... β(1,T-1)   |
//     ...
//     | β(N-1,0), β(N-1,1) ... β(N-1,T-1) |
//
//
// 1. Initialization: β(i,T-1) = 1;  0<=i<N
// 2. Induction:      β(i,t) =  sum_{j=0}^{N-1} a(i,j) b(j,o(t+1)) β(j,t+1); t=T-2,T-3,...,0; 0<=i<N
func (hmm *HMM) beta(observations *matrix.Dense) (β *matrix.Dense, e error) {

	// Num states.
	N := hmm.nstates

	// expected num rows: numElements
	// expected num cols: T
	ne, T := observations.Dims()

	if ne != hmm.numElements {
		e = fmt.Errorf("Mismatch in num elements in observations [%d] expected [%d].", ne, hmm.numElements)
		return
	}

	// Allocate log-beta matrix.
	// TODO: use a reusable data structure to minimize garbage.
	β = matrix.MustDense(matrix.ZeroDense(N, T))

	// 1. Initialization. Add in the log doman.
	for i := 0; i < N; i++ {
		β.Set(i, T-1, 1.0)
	}

	// 2. Induction.
	var sumBetas float64
	for t := T - 2; t >= 0; t-- {
		sumBetas = 0
		for i := 0; i < N; i++ {

			var sum float64
			for j := 0; j < N; j++ {

				sum += math.Exp(hmm.logTransProbs.At(i, j) + // a(i,j)
					hmm.obsModels[j].LogProb(ColumnAt(observations, t+1)) + // b(j,o(t+1))
					β.At(j, t+1)) // β(j,t+1)
			}
			β.Set(i, t, math.Log(sum))
			sumBetas += sum
		}
		// Applied scale for t independent of i.
		logSumBetas := math.Log(sumBetas)
		for i := 0; i < N; i++ {
			v := β.At(i, t) - logSumBetas
			β.Set(i, t, v)
		}
	}

	return
}

// Compute gammas. Indices are: γ(state, time)
//
// γ(i,t) =  α(i,t)β(i,t) / sum_{j=0}^{N-1} α(j,t)β(j,t);  0<=j<N
func (hmm *HMM) gamma(α, β *matrix.Dense) (γ *matrix.Dense, e error) {

	αr, αc := α.Dims()
	βr, βc := β.Dims()

	if αr != βr || αc != βc {
		e = fmt.Errorf("Shape mismatch: alpha[%d,%d] beta[%d,%d]", αr, αc, βr, βc)
		return
	}

	T := αc
	N := hmm.nstates
	if αr != N {
		e = fmt.Errorf("Num rows [%d] doesn't match num states [%d].", αr, N)
	}

	// Allocate log-gamma matrix.
	// TODO: use a reusable data structure to minimize garbage.
	γ = matrix.MustDense(matrix.ZeroDense(N, T))

	for t := 0; t < T; t++ {
		var sum float64 = 0.0
		for i := 0; i < N; i++ {
			x := α.At(i, t) + β.At(i, t)
			γ.Set(i, t, x)
			sum += math.Exp(x)
		}

		// Normalize.
		for i := 0; i < N; i++ {
			x := γ.At(i, t) - math.Log(sum)
			γ.Set(i, t, x)
		}
	}
	return
}

/*
   Compute xi. Indices are: ζ(from, to, time)

                         α(i,t) a(i,j) b(j,o(t+1)) β(j,t+1)
 ζ(i,j,t) = ------------------------------------------------------------------
            sum_{i=0}^{N-1} sum_{j=0}^{N-1} α(i,t) a(i,j) b(j,o(t+1)) β(j,t+1)

*/
func (hmm *HMM) xi(observations, α, β *matrix.Dense) (ζ [][][]float64, e error) {

	a := hmm.logTransProbs
	αr, αc := α.Dims()
	βr, βc := β.Dims()
	or, oc := observations.Dims()

	if αr != βr || αc != βc {
		e = fmt.Errorf("Shape mismatch: alpha[%d,%d] beta[%d,%d]", αr, αc, βr, βc)
		return
	}

	T := αc
	N := hmm.nstates
	if or != hmm.numElements {
		e = fmt.Errorf("Mismatch in num elements in observations [%d] expected [%d].",
			or, hmm.numElements)
		return
	}
	if oc != T {
		e = fmt.Errorf("Mismatch in T observations has [%d], expected [%d].", oc, T)
		return
	}
	if αr != N {
		e = fmt.Errorf("Num rows [%d] doesn't match num states [%d].", αr, N)
	}

	// Allocate log-xi matrix.
	// TODO: use a reusable data structure to minimize garbage.
	ζ = make([][][]float64, N)
	for i := 0; i < N; i++ {
		ζ[i] = make([][]float64, N)
		for j := 0; j < N; j++ {
			for t := 0; t < N; t++ {
				ζ[i][j] = make([]float64, T)
			}
		}
	}

	for t := 0; t < T-1; t++ {
		var sum float64 = 0.0
		for j := 0; j < N; j++ {
			b := hmm.obsModels[j].LogProb(ColumnAt(observations, t+1))
			for i := 0; i < N; i++ {
				x := α.At(i, t) + a.At(i, j) + b + β.At(j, t+1)
				ζ[i][j][t] = x
				sum += math.Exp(x)
			}
		}
		// Normalize.
		for i := 0; i < N; i++ {
			for j := 0; j < N; j++ {
				ζ[i][j][t] -= math.Log(sum)
			}
		}
	}
	return
}

// Update model statistics.
// sequence is a matrix
func (hmm *HMM) Update(observations *matrix.Dense) (e error) {

	var α, β, γ *matrix.Dense
	var ζ [][][]float64
	var logProb float64

	// Compute  α, β, γ, ζ
	// TODO: compute α, β, concurrently using go routines.
	α, logProb, e = hmm.alpha(observations)
	if e != nil {
		return
	}
	β, e = hmm.beta(observations)
	if e != nil {
		return
	}
	// TODO: compute γ, ζ concurrently using go routines.
	// Can we compute ζ more efficiently using γ?
	γ, e = hmm.gamma(α, β)
	if e != nil {
		return
	}
	ζ, e = hmm.xi(observations, α, β)
	if e != nil {
		return
	}

	fmt.Println(α, β, γ, ζ, logProb)
	return
}

// ColumnAt returns a *matrix.Dense column that is a copy of the values at column c of the matrix.
// Column will panic with ErrIndexOutOfRange is c is not a valid column index.
func ColumnAt(d *matrix.Dense, c int) *matrix.Dense {

	sl := d.Column(c)
	nrows := len(sl)

	col := matrix.MustDense(matrix.ZeroDense(nrows, 1))

	for i := 0; i < nrows; i++ {
		col.Set(i, 0, sl[i])
	}

	return col
}

func (hmm *HMM) NumElements() int {
	return hmm.numElements
}
<|endoftext|>


<commit_before>package model

import (
	"errors"
	"fmt"
)

const defaultServiceID = "workflow"

var emptyRequest = map[string]interface{}{}

//ServiceRequest represent an action request
type ServiceRequest struct {
	Service     string      `description:"service ID"`
	Action      string      `description:"service's'action "`
	Request     interface{} `description:"service request"`
}

//Init initialises action
func (r *ServiceRequest) Init() *ServiceRequest {
	if r == nil {
		r = &ServiceRequest{}
	}
	if r.Service == "" {
		r.Service = defaultServiceID
	}
	if r.Request == nil {
		r.Request = emptyRequest
	}
	return r
}

//Validate check is action request is valid
func (r *ServiceRequest) Validate() error {
	if r.Service == "" {
		return errors.New("service  was empty")
	}
	if r.Action == "" {
		return errors.New("action was empty")
	}
	if r.Request == nil {
		return fmt.Errorf("request was nil for %v.%v", r.Service, r.Action)
	}
	return nil
}

func (r *ServiceRequest) NewAction() *Action {
	var repeater = &Repeater{}
	return &Action{
		AbstractNode: &AbstractNode{
		},
		ServiceRequest: r,
		MetaTag:        &MetaTag{},
		Repeater:       repeater.Init(),
	}
}
<commit_msg>removed description from service request<commit_after>package model

import (
	"errors"
	"fmt"
)

const defaultServiceID = "workflow"

var emptyRequest = map[string]interface{}{}

//ServiceRequest represent an action request
type ServiceRequest struct {
	Service string      `description:"service ID"`
	Action  string      `description:"service's'action "`
	Request interface{} `description:"service request"`
}

//Init initialises action
func (r *ServiceRequest) Init() *ServiceRequest {
	if r == nil {
		r = &ServiceRequest{}
	}
	if r.Service == "" {
		r.Service = defaultServiceID
	}
	if r.Request == nil {
		r.Request = emptyRequest
	}
	return r
}

//Validate check is action request is valid
func (r *ServiceRequest) Validate() error {
	if r.Service == "" {
		return errors.New("service  was empty")
	}
	if r.Action == "" {
		return errors.New("action was empty")
	}
	if r.Request == nil {
		return fmt.Errorf("request was nil for %v.%v", r.Service, r.Action)
	}
	return nil
}

func (r *ServiceRequest) NewAction() *Action {
	var repeater = &Repeater{}
	return &Action{
		AbstractNode:   &AbstractNode{},
		ServiceRequest: r,
		MetaTag:        &MetaTag{},
		Repeater:       repeater.Init(),
	}
}
<|endoftext|>


<commit_before>package ethutil

import (
	"bytes"
	"fmt"
	"math/big"
	"reflect"
)

// Data values are returned by the rlp decoder. The data values represents
// one item within the rlp data structure. It's responsible for all the casting
// It always returns something valid
type Value struct {
	Val  interface{}
	kind reflect.Value
}

func (val *Value) String() string {
	return fmt.Sprintf("%q", val.Val)
}

func NewValue(val interface{}) *Value {
	return &Value{Val: val}
}

func (val *Value) Type() reflect.Kind {
	return reflect.TypeOf(val.Val).Kind()
}

func (val *Value) IsNil() bool {
	return val.Val == nil
}

func (val *Value) Len() int {
	//return val.kind.Len()
	if data, ok := val.Val.([]interface{}); ok {
		return len(data)
	} else if data, ok := val.Val.([]byte); ok {
		// FIXME
		return len(data)
	}

	return 0
}

func (val *Value) Raw() interface{} {
	return val.Val
}

func (val *Value) Interface() interface{} {
	return val.Val
}

func (val *Value) Uint() uint64 {
	if Val, ok := val.Val.(uint8); ok {
		return uint64(Val)
	} else if Val, ok := val.Val.(uint16); ok {
		return uint64(Val)
	} else if Val, ok := val.Val.(uint32); ok {
		return uint64(Val)
	} else if Val, ok := val.Val.(uint64); ok {
		return Val
	} else if Val, ok := val.Val.(int); ok {
		return uint64(Val)
	} else if Val, ok := val.Val.(uint); ok {
		return uint64(Val)
	} else if Val, ok := val.Val.([]byte); ok {
		return ReadVarint(bytes.NewReader(Val))
	}

	return 0
}

func (val *Value) Byte() byte {
	if Val, ok := val.Val.(byte); ok {
		return Val
	}

	return 0x0
}

func (val *Value) BigInt() *big.Int {
	if a, ok := val.Val.([]byte); ok {
		b := new(big.Int).SetBytes(a)

		return b
	} else if a, ok := val.Val.(*big.Int); ok {
		return a
	} else {
		return big.NewInt(int64(val.Uint()))
	}

	return big.NewInt(0)
}

func (val *Value) Str() string {
	if a, ok := val.Val.([]byte); ok {
		return string(a)
	} else if a, ok := val.Val.(string); ok {
		return a
	} else if a, ok := val.Val.(byte); ok {
		return string(a)
	}

	return ""
}

func (val *Value) Bytes() []byte {
	if a, ok := val.Val.([]byte); ok {
		return a
	}

	return []byte{}
}

func (val *Value) Slice() []interface{} {
	if d, ok := val.Val.([]interface{}); ok {
		return d
	}

	return []interface{}{}
}

func (val *Value) SliceFrom(from int) *Value {
	slice := val.Slice()

	return NewValue(slice[from:])
}

func (val *Value) SliceTo(to int) *Value {
	slice := val.Slice()

	return NewValue(slice[:to])
}

func (val *Value) SliceFromTo(from, to int) *Value {
	slice := val.Slice()

	return NewValue(slice[from:to])
}

// Threat the value as a slice
func (val *Value) Get(idx int) *Value {
	if d, ok := val.Val.([]interface{}); ok {
		// Guard for oob
		if len(d) <= idx {
			return NewValue(nil)
		}

		if idx < 0 {
			panic("negative idx for Value Get")
		}

		return NewValue(d[idx])
	}

	// If this wasn't a slice you probably shouldn't be using this function
	return NewValue(nil)
}

func (val *Value) Cmp(o *Value) bool {
	return reflect.DeepEqual(val.Val, o.Val)
}

func (val *Value) Encode() []byte {
	return Encode(val.Val)
}

func NewValueFromBytes(data []byte) *Value {
	if len(data) != 0 {
		data, _ := Decode(data, 0)
		return NewValue(data)
	}

	return NewValue(nil)
}

// Value setters
func NewSliceValue(s interface{}) *Value {
	list := EmptyValue()

	if s != nil {
		if slice, ok := s.([]interface{}); ok {
			for _, val := range slice {
				list.Append(val)
			}
		} else if slice, ok := s.([]string); ok {
			for _, val := range slice {
				list.Append(val)
			}
		}
	}

	return list
}

func EmptyValue() *Value {
	return NewValue([]interface{}{})
}

func (val *Value) AppendList() *Value {
	list := EmptyValue()
	val.Val = append(val.Slice(), list)

	return list
}

func (val *Value) Append(v interface{}) *Value {
	val.Val = append(val.Slice(), v)

	return val
}
<commit_msg>Type checking<commit_after>package ethutil

import (
	"bytes"
	"fmt"
	"math/big"
	"reflect"
)

// Data values are returned by the rlp decoder. The data values represents
// one item within the rlp data structure. It's responsible for all the casting
// It always returns something valid
type Value struct {
	Val  interface{}
	kind reflect.Value
}

func (val *Value) String() string {
	return fmt.Sprintf("%q", val.Val)
}

func NewValue(val interface{}) *Value {
	return &Value{Val: val}
}

func (val *Value) Type() reflect.Kind {
	return reflect.TypeOf(val.Val).Kind()
}

func (val *Value) IsNil() bool {
	return val.Val == nil
}

func (val *Value) Len() int {
	//return val.kind.Len()
	if data, ok := val.Val.([]interface{}); ok {
		return len(data)
	} else if data, ok := val.Val.([]byte); ok {
		return len(data)
	} else if data, ok := val.Val.(string); ok {
		return len(data)
	}

	return 0
}

func (val *Value) Raw() interface{} {
	return val.Val
}

func (val *Value) Interface() interface{} {
	return val.Val
}

func (val *Value) Uint() uint64 {
	if Val, ok := val.Val.(uint8); ok {
		return uint64(Val)
	} else if Val, ok := val.Val.(uint16); ok {
		return uint64(Val)
	} else if Val, ok := val.Val.(uint32); ok {
		return uint64(Val)
	} else if Val, ok := val.Val.(uint64); ok {
		return Val
	} else if Val, ok := val.Val.(int); ok {
		return uint64(Val)
	} else if Val, ok := val.Val.(uint); ok {
		return uint64(Val)
	} else if Val, ok := val.Val.([]byte); ok {
		return ReadVarint(bytes.NewReader(Val))
	}

	return 0
}

func (val *Value) Byte() byte {
	if Val, ok := val.Val.(byte); ok {
		return Val
	}

	return 0x0
}

func (val *Value) BigInt() *big.Int {
	if a, ok := val.Val.([]byte); ok {
		b := new(big.Int).SetBytes(a)

		return b
	} else if a, ok := val.Val.(*big.Int); ok {
		return a
	} else {
		return big.NewInt(int64(val.Uint()))
	}

	return big.NewInt(0)
}

func (val *Value) Str() string {
	if a, ok := val.Val.([]byte); ok {
		return string(a)
	} else if a, ok := val.Val.(string); ok {
		return a
	} else if a, ok := val.Val.(byte); ok {
		return string(a)
	}

	return ""
}

func (val *Value) Bytes() []byte {
	if a, ok := val.Val.([]byte); ok {
		return a
	}

	return []byte{}
}

func (val *Value) Slice() []interface{} {
	if d, ok := val.Val.([]interface{}); ok {
		return d
	}

	return []interface{}{}
}

func (val *Value) SliceFrom(from int) *Value {
	slice := val.Slice()

	return NewValue(slice[from:])
}

func (val *Value) SliceTo(to int) *Value {
	slice := val.Slice()

	return NewValue(slice[:to])
}

func (val *Value) SliceFromTo(from, to int) *Value {
	slice := val.Slice()

	return NewValue(slice[from:to])
}

// TODO More type checking methods
func (val *Value) IsSlice() bool {
	return val.Type() == reflect.Slice
}

func (val *Value) IsStr() bool {
	return val.Type() == reflect.String
}

func (val *Value) IsEmpty() bool {
	return (val.IsSlice() || val.IsStr()) && val.Len() == 0
}

// Threat the value as a slice
func (val *Value) Get(idx int) *Value {
	if d, ok := val.Val.([]interface{}); ok {
		// Guard for oob
		if len(d) <= idx {
			return NewValue(nil)
		}

		if idx < 0 {
			panic("negative idx for Value Get")
		}

		return NewValue(d[idx])
	}

	// If this wasn't a slice you probably shouldn't be using this function
	return NewValue(nil)
}

func (val *Value) Cmp(o *Value) bool {
	return reflect.DeepEqual(val.Val, o.Val)
}

func (val *Value) Encode() []byte {
	return Encode(val.Val)
}

func NewValueFromBytes(data []byte) *Value {
	if len(data) != 0 {
		data, _ := Decode(data, 0)
		return NewValue(data)
	}

	return NewValue(nil)
}

// Value setters
func NewSliceValue(s interface{}) *Value {
	list := EmptyValue()

	if s != nil {
		if slice, ok := s.([]interface{}); ok {
			for _, val := range slice {
				list.Append(val)
			}
		} else if slice, ok := s.([]string); ok {
			for _, val := range slice {
				list.Append(val)
			}
		}
	}

	return list
}

func EmptyValue() *Value {
	return NewValue([]interface{}{})
}

func (val *Value) AppendList() *Value {
	list := EmptyValue()
	val.Val = append(val.Slice(), list)

	return list
}

func (val *Value) Append(v interface{}) *Value {
	val.Val = append(val.Slice(), v)

	return val
}
<|endoftext|>


<commit_before>package fmom

import (
	"math"
)

type IPtCotThPhiM [4]float64

func NewIPtCotThPhiM(pt, eta, phi, m float64) IPtCotThPhiM {
	return IPtCotThPhiM([4]float64{pt, eta, phi, m})
}

func (p4 *IPtCotThPhiM) Clone() P4 {
	pp := *p4
	return &pp
}

func (p4 *IPtCotThPhiM) IPt() float64 {
	return p4[0]
}

func (p4 *IPtCotThPhiM) CotTh() float64 {
	return p4[1]
}

func (p4 *IPtCotThPhiM) Phi() float64 {
	return p4[2]
}

func (p4 *IPtCotThPhiM) M() float64 {
	return p4[3]
}

func (p4 *IPtCotThPhiM) Pt() float64 {
	ipt := p4.IPt()
	return 1 / ipt
}

func (p4 *IPtCotThPhiM) P() float64 {
	cotth := p4.CotTh()
	ipt := p4.IPt()
	return math.Sqrt(1+cotth*cotth) / ipt
}

func (p4 *IPtCotThPhiM) P2() float64 {
	cotth := p4.CotTh()
	ipt := p4.IPt()
	return (1 + cotth*cotth) / (ipt * ipt)
}

func (p4 *IPtCotThPhiM) M2() float64 {
	m := p4.M()
	return m * m
}

func (p4 *IPtCotThPhiM) TanTh() float64 {
	cotth := p4.CotTh()
	return 1 / cotth
}

func (p4 *IPtCotThPhiM) SinTh() float64 {
	cotth := p4.CotTh()
	return 1 / math.Sqrt(1+cotth*cotth)
}

func (p4 *IPtCotThPhiM) CosTh() float64 {
	cotth := p4.CotTh()
	cotth2 := cotth * cotth
	costh := math.Sqrt(cotth2 / (1 + cotth2))
	sign := 1.0
	if cotth < 0 {
		sign = -1.0
	}
	return sign * costh
}

func (p4 *IPtCotThPhiM) E() float64 {
	m := p4.M()
	p := p4.P()
	if m == 0 {
		return p
	}
	return math.Sqrt(p*p + m*m)
}

func (p4 *IPtCotThPhiM) Et() float64 {
	cotth := p4.CotTh()
	e := p4.E()
	return e / math.Sqrt(1+cotth*cotth)
}

func (p4 *IPtCotThPhiM) Eta() float64 {
	cotth := p4.CotTh()
	aux := math.Sqrt(1 + cotth*cotth)
	return -0.5 * math.Log((aux-cotth)/(aux+cotth))
}

func (p4 *IPtCotThPhiM) Rapidity() float64 {
	e := p4.E()
	pz := p4.Pz()
	return 0.5 * math.Log((e+pz)/(e-pz))
}

func (p4 *IPtCotThPhiM) Px() float64 {
	cosphi := p4.CosPhi()
	ipt := p4.IPt()
	return cosphi / ipt
}

func (p4 *IPtCotThPhiM) Py() float64 {
	sinphi := p4.SinPhi()
	ipt := p4.IPt()
	return sinphi / ipt
}

func (p4 *IPtCotThPhiM) Pz() float64 {
	cotth := p4.CotTh()
	ipt := p4.IPt()
	return cotth / ipt
}

func (p4 *IPtCotThPhiM) CosPhi() float64 {
	phi := p4.Phi()
	return math.Cos(phi)
}

func (p4 *IPtCotThPhiM) SinPhi() float64 {
	phi := p4.Phi()
	return math.Sin(phi)
}

func (p4 *IPtCotThPhiM) Set(p P4) {
	p4[0] = p.IPt()
	p4[1] = p.CotTh()
	p4[2] = p.Phi()
	p4[3] = p.M()
}
<commit_msg>iptcotthphim: cosmetics<commit_after>package fmom

import (
	"math"
)

type IPtCotThPhiM [4]float64

func NewIPtCotThPhiM(pt, eta, phi, m float64) IPtCotThPhiM {
	return IPtCotThPhiM([4]float64{pt, eta, phi, m})
}

func (p4 *IPtCotThPhiM) Clone() P4 {
	pp := *p4
	return &pp
}

func (p4 *IPtCotThPhiM) IPt() float64 {
	return p4[0]
}

func (p4 *IPtCotThPhiM) CotTh() float64 {
	return p4[1]
}

func (p4 *IPtCotThPhiM) Phi() float64 {
	return p4[2]
}

func (p4 *IPtCotThPhiM) M() float64 {
	return p4[3]
}

func (p4 *IPtCotThPhiM) Pt() float64 {
	ipt := p4.IPt()
	return 1 / ipt
}

func (p4 *IPtCotThPhiM) P() float64 {
	cotth := p4.CotTh()
	ipt := p4.IPt()
	return math.Sqrt(1+cotth*cotth) / ipt
}

func (p4 *IPtCotThPhiM) P2() float64 {
	cotth := p4.CotTh()
	ipt := p4.IPt()
	return (1 + cotth*cotth) / (ipt * ipt)
}

func (p4 *IPtCotThPhiM) M2() float64 {
	m := p4.M()
	return m * m
}

func (p4 *IPtCotThPhiM) TanTh() float64 {
	cotth := p4.CotTh()
	return 1 / cotth
}

func (p4 *IPtCotThPhiM) SinTh() float64 {
	cotth := p4.CotTh()
	return 1 / math.Sqrt(1+cotth*cotth)
}

func (p4 *IPtCotThPhiM) CosTh() float64 {
	cotth := p4.CotTh()
	cotth2 := cotth * cotth
	costh := math.Sqrt(cotth2 / (1 + cotth2))
	sign := 1.0
	if cotth < 0 {
		sign = -1.0
	}
	return sign * costh
}

func (p4 *IPtCotThPhiM) E() float64 {
	m := p4.M()
	p := p4.P()
	if m == 0 {
		return p
	}
	return math.Sqrt(p*p + m*m)
}

func (p4 *IPtCotThPhiM) Et() float64 {
	cotth := p4.CotTh()
	e := p4.E()
	return e / math.Sqrt(1+cotth*cotth)
}

func (p4 *IPtCotThPhiM) Eta() float64 {
	cotth := p4.CotTh()
	aux := math.Sqrt(1 + cotth*cotth)
	return -0.5 * math.Log((aux-cotth)/(aux+cotth))
}

func (p4 *IPtCotThPhiM) Rapidity() float64 {
	e := p4.E()
	pz := p4.Pz()
	return 0.5 * math.Log((e+pz)/(e-pz))
}

func (p4 *IPtCotThPhiM) Px() float64 {
	cosphi := p4.CosPhi()
	ipt := p4.IPt()
	pt := 1 / ipt
	return pt * cosphi
}

func (p4 *IPtCotThPhiM) Py() float64 {
	sinphi := p4.SinPhi()
	ipt := p4.IPt()
	pt := 1 / ipt
	return pt * sinphi
}

func (p4 *IPtCotThPhiM) Pz() float64 {
	cotth := p4.CotTh()
	ipt := p4.IPt()
	pt := 1 / ipt
	return pt * cotth
}

func (p4 *IPtCotThPhiM) CosPhi() float64 {
	phi := p4.Phi()
	return math.Cos(phi)
}

func (p4 *IPtCotThPhiM) SinPhi() float64 {
	phi := p4.Phi()
	return math.Sin(phi)
}

func (p4 *IPtCotThPhiM) Set(p P4) {
	p4[0] = p.IPt()
	p4[1] = p.CotTh()
	p4[2] = p.Phi()
	p4[3] = p.M()
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package provision provides interfaces that need to be satisfied in order to
// implement a new provisioner on tsuru.
package provision

import (
	"fmt"
	"io"
)

type Status string

func (s Status) String() string {
	return string(s)
}

const (
	StatusStarted    = Status("started")
	StatusPending    = Status("pending")
	StatusDown       = Status("down")
	StatusError      = Status("error")
	StatusInstalling = Status("installing")
	StatusCreating   = Status("creating")
)

// Unit represents a provision unit. Can be a machine, container or anything
// IP-addressable.
type Unit struct {
	Name       string
	AppName    string
	Type       string
	InstanceId string
	Machine    int
	Ip         string
	Status     Status
}

// AppUnit represents a unit in an app.
type AppUnit interface {
	// Returns the name of the unit.
	GetName() string

	// Returns the number of the unit.
	GetMachine() int

	// Returns the status of the unit.
	GetStatus() Status

	// Returns the IP of the unit.
	GetIp() string
}

// App represents a tsuru app.
//
// It contains only relevant information for provisioning.
type App interface {
	// Log should be used to log messages in the app.
	Log(message, source string) error

	// GetName returns the name of the app.
	GetName() string

	// GetFramework returns the framework (type) of the app. It is equivalent
	// to the Unit `Type` field.
	GetFramework() string

	// GetUnits returns all units of the app, in a slice.
	ProvisionUnits() []AppUnit
}

// Provisioner is the basic interface of this package.
//
// Any tsuru provisioner must implement this interface in order to provision
// tsuru apps.
//
// Tsuru comes with a default provisioner: juju. One can add other provisioners
// by satisfying this interface and registering it using the function Register.
type Provisioner interface {
	// Provision is called when tsuru is creating the app.
	Provision(App) error

	// Destroy is called when tsuru is destroying the app.
	Destroy(App) error

	// AddUnits adds units to an app. The first parameter is the app, the
	// second is the number of units to add.
	//
	// It returns a slice containing all added units
	AddUnits(App, uint) ([]Unit, error)

	// RemoveUnit removes a unit from the app. It receives the app and the name
	// of the unit to be removed.
	RemoveUnit(App, string) error

	// RemoveUnits removes multiple units from an app. The first parameter it
	// the app, the second is the number of units to remove.
	//
	// It returns a slice containing indices of all removed units (the index
	// must match the slice returned by App.ProvisionUnits). The list of
	// indices must be returned sorted.
	RemoveUnits(App, uint) ([]int, error)

	// ExecuteCommand runs a command in all units of the app.
	ExecuteCommand(stdout, stderr io.Writer, app App, cmd string, args ...string) error

	// CollectStatus returns information about all provisioned units. It's used
	// by tsuru collector when updating the status of apps in the database.
	CollectStatus() ([]Unit, error)

	// Addr returns the address for an app. It will probably be a DNS name
	// or IP address.
	//
	// Tsuru will use this method to get the IP (althought it might not be
	// an actual IP, collector calls it "IP") of the app from the
	// provisioner.
	Addr(App) (string, error)
}

var provisioners = make(map[string]Provisioner)

// Register registers a new provisioner in the Provisioner registry.
func Register(name string, p Provisioner) {
	provisioners[name] = p
}

// Get gets the named provisioner from the registry.
func Get(name string) (Provisioner, error) {
	p, ok := provisioners[name]
	if !ok {
		return nil, fmt.Errorf("Unknown provisioner: %q.", name)
	}
	return p, nil
}

type Error struct {
	Reason string
	Err    error
}

func (e *Error) Error() string {
	var err string
	if e.Err != nil {
		err = e.Err.Error() + ": " + e.Reason
	} else {
		err = e.Reason
	}
	return err
}
<commit_msg>provision: smaller interfaces<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package provision provides interfaces that need to be satisfied in order to
// implement a new provisioner on tsuru.
package provision

import (
	"fmt"
	"io"
)

type Status string

func (s Status) String() string {
	return string(s)
}

const (
	StatusStarted    = Status("started")
	StatusPending    = Status("pending")
	StatusDown       = Status("down")
	StatusError      = Status("error")
	StatusInstalling = Status("installing")
	StatusCreating   = Status("creating")
)

// Unit represents a provision unit. Can be a machine, container or anything
// IP-addressable.
type Unit struct {
	Name       string
	AppName    string
	Type       string
	InstanceId string
	Machine    int
	Ip         string
	Status     Status
}

// Named is something that has a name, providing the GetName method.
type Named interface {
	GetName() string
}

// AppUnit represents a unit in an app.
type AppUnit interface {
	Named

	// Returns the number of the unit.
	GetMachine() int

	// Returns the status of the unit.
	GetStatus() Status

	// Returns the IP of the unit.
	GetIp() string
}

// App represents a tsuru app.
//
// It contains only relevant information for provisioning.
type App interface {
	Named

	// Log should be used to log messages in the app.
	Log(message, source string) error

	// GetFramework returns the framework (type) of the app. It is equivalent
	// to the Unit `Type` field.
	GetFramework() string

	// GetUnits returns all units of the app, in a slice.
	ProvisionUnits() []AppUnit
}

// Provisioner is the basic interface of this package.
//
// Any tsuru provisioner must implement this interface in order to provision
// tsuru apps.
//
// Tsuru comes with a default provisioner: juju. One can add other provisioners
// by satisfying this interface and registering it using the function Register.
type Provisioner interface {
	// Provision is called when tsuru is creating the app.
	Provision(App) error

	// Destroy is called when tsuru is destroying the app.
	Destroy(App) error

	// AddUnits adds units to an app. The first parameter is the app, the
	// second is the number of units to add.
	//
	// It returns a slice containing all added units
	AddUnits(App, uint) ([]Unit, error)

	// RemoveUnit removes a unit from the app. It receives the app and the name
	// of the unit to be removed.
	RemoveUnit(App, string) error

	// RemoveUnits removes multiple units from an app. The first parameter it
	// the app, the second is the number of units to remove.
	//
	// It returns a slice containing indices of all removed units (the index
	// must match the slice returned by App.ProvisionUnits). The list of
	// indices must be returned sorted.
	RemoveUnits(App, uint) ([]int, error)

	// ExecuteCommand runs a command in all units of the app.
	ExecuteCommand(stdout, stderr io.Writer, app App, cmd string, args ...string) error

	// CollectStatus returns information about all provisioned units. It's used
	// by tsuru collector when updating the status of apps in the database.
	CollectStatus() ([]Unit, error)

	// Addr returns the address for an app. It will probably be a DNS name
	// or IP address.
	//
	// Tsuru will use this method to get the IP (althought it might not be
	// an actual IP, collector calls it "IP") of the app from the
	// provisioner.
	Addr(App) (string, error)
}

var provisioners = make(map[string]Provisioner)

// Register registers a new provisioner in the Provisioner registry.
func Register(name string, p Provisioner) {
	provisioners[name] = p
}

// Get gets the named provisioner from the registry.
func Get(name string) (Provisioner, error) {
	p, ok := provisioners[name]
	if !ok {
		return nil, fmt.Errorf("Unknown provisioner: %q.", name)
	}
	return p, nil
}

type Error struct {
	Reason string
	Err    error
}

func (e *Error) Error() string {
	var err string
	if e.Err != nil {
		err = e.Err.Error() + ": " + e.Reason
	} else {
		err = e.Reason
	}
	return err
}
<|endoftext|>


<commit_before>// Copyright 2014 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package provision provides interfaces that need to be satisfied in order to
// implement a new provisioner on tsuru.
package provision

import (
	"errors"
	"fmt"
	"io"

	"github.com/tsuru/tsuru/action"
	"github.com/tsuru/tsuru/app/bind"
)

var ErrInvalidStatus = errors.New("invalid status")

var ErrEmptyApp = errors.New("no units for this app")

// Status represents the status of a unit in tsuru.
type Status string

func (s Status) String() string {
	return string(s)
}

func ParseStatus(status string) (Status, error) {
	switch status {
	case "created":
		return StatusCreated, nil
	case "building":
		return StatusBuilding, nil
	case "error":
		return StatusError, nil
	case "started":
		return StatusStarted, nil
	case "starting":
		return StatusStarting, nil
	case "stopped":
		return StatusStopped, nil
	}
	return Status(""), ErrInvalidStatus
}

// Flow:
//
// +----------+                           Start          +---------+
// | Building |                   +---------------------+| Stopped |
// +----------+                   |                      +---------+
//       ^                        |                           ^
//       |                        |                           |
//  deploy unit                   |                         Stop
//       |                        |                           |
//       +                        v       RegisterUnit        +
//  +---------+  app unit   +----------+  SetUnitStatus  +---------+
//  | Created | +---------> | Starting | +-------------> | Started |
//  +---------+             +----------+                 +---------+
//                                +                         ^ +
//                                |                         | |
//                          SetUnitStatus                   | |
//                                |                         | |
//                                v                         | |
//                            +-------+     SetUnitStatus   | |
//                            | Error | +-------------------+ |
//                            +-------+ <---------------------+
const (
	// StatusCreated is the initial status of a unit in the database,
	// it should transition shortly to a more specific status
	StatusCreated = Status("created")

	// StatusBuilding is the status for units being provisioned by the
	// provisioner, like in the deployment.
	StatusBuilding = Status("building")

	// StatusError is the status for units that failed to start, because of
	// an application error.
	StatusError = Status("error")

	// StatusStarting is set when the container is started in docker.
	StatusStarting = Status("starting")

	// StatusStarted is for cases where the unit is up and running, and bound
	// to the proper status, it's set by RegisterUnit and SetUnitStatus.
	StatusStarted = Status("started")

	// StatusStopped is for cases where the unit has been stopped.
	StatusStopped = Status("stopped")
)

// Unit represents a provision unit. Can be a machine, container or anything
// IP-addressable.
type Unit struct {
	Name    string
	AppName string
	Type    string
	Ip      string
	Status  Status
}

// GetIp returns the Unit.IP.
func (u *Unit) GetIp() string {
	return u.Ip
}

// Available returns true if the unit is available. It will return true
// whenever the unit itself is available, even when the application process is
// not.
func (u *Unit) Available() bool {
	return u.Status == StatusStarted ||
		u.Status == StatusStarting ||
		u.Status == StatusError
}

// Named is something that has a name, providing the GetName method.
type Named interface {
	GetName() string
}

// App represents a tsuru app.
//
// It contains only relevant information for provisioning.
type App interface {
	Named

	BindUnit(*Unit) error
	UnbindUnit(*Unit) error

	// Log should be used to log messages in the app.
	Log(message, source, unit string) error

	// GetPlatform returns the platform (type) of the app. It is equivalent
	// to the Unit `Type` field.
	GetPlatform() string

	// GetDeploy returns the deploys that an app has.
	GetDeploys() uint

	Units() []Unit

	// Run executes the command in app units. Commands executed with this
	// method should have access to environment variables defined in the
	// app.
	Run(cmd string, w io.Writer, once bool) error

	Restart(io.Writer) error

	Envs() map[string]bind.EnvVar

	// Ready marks the app as ready for deployment.
	Ready() error

	GetMemory() int64
	GetSwap() int64
	GetCpuShare() int

	GetUpdatePlatform() bool

	GetRouter() (string, error)
}

// CNameManager represents a provisioner that supports cname on applications.
type CNameManager interface {
	SetCName(app App, cname string) error
	UnsetCName(app App, cname string) error
}

// ArchiveDeployer is a provisioner that can deploy archives.
type ArchiveDeployer interface {
	ArchiveDeploy(app App, archiveURL string, w io.Writer) error
}

// GitDeployer is a provisioner that can deploy the application from a Git
// repository.
type GitDeployer interface {
	GitDeploy(app App, version string, w io.Writer) error
}

// UploadDeployer is a provisioner that can deploy the application from an
// uploaded file.
type UploadDeployer interface {
	UploadDeploy(app App, file io.ReadCloser, w io.Writer) error
}

// Provisioner is the basic interface of this package.
//
// Any tsuru provisioner must implement this interface in order to provision
// tsuru apps.
type Provisioner interface {
	// Provision is called when tsuru is creating the app.
	Provision(App) error

	// Destroy is called when tsuru is destroying the app.
	Destroy(App) error

	// AddUnits adds units to an app. The first parameter is the app, the
	// second is the number of units to be added.
	//
	// It returns a slice containing all added units
	AddUnits(App, uint, io.Writer) ([]Unit, error)

	// RemoveUnits "undoes" AddUnits, removing the given number of units
	// from the app.
	RemoveUnits(App, uint) error

	// RemoveUnit removes a unit from the app. It receives the unit to be
	// removed.
	RemoveUnit(Unit) error

	// SetUnitStatus changes the status of a unit.
	SetUnitStatus(Unit, Status) error

	// ExecuteCommand runs a command in all units of the app.
	ExecuteCommand(stdout, stderr io.Writer, app App, cmd string, args ...string) error

	// ExecuteCommandOnce runs a command in one unit of the app.
	ExecuteCommandOnce(stdout, stderr io.Writer, app App, cmd string, args ...string) error

	Restart(App, io.Writer) error
	Stop(App) error

	// Start start the app units.
	Start(App) error

	// Addr returns the address for an app.
	//
	// tsuru will use this method to get the IP (although it might not be
	// an actual IP, collector calls it "IP") of the app from the
	// provisioner.
	Addr(App) (string, error)

	// Swap change the router between two apps.
	Swap(App, App) error

	// Units returns information about units by App.
	Units(App) []Unit

	// Register a unit after the container has been created or restarted.
	RegisterUnit(Unit) error
}

// InitializableProvisioner is a provisioner that provides an initialization
// method that should be called when the app is started
type InitializableProvisioner interface {
	Initialize() error
}

// CustomizedDeployPipelineProvisioner is a provisioner with a customized
// deploy pipeline.
type CustomizedDeployPipelineProvisioner interface {
	DeployPipeline() *action.Pipeline
}

// ExtensibleProvisioner is a provisioner where administrators can manage
// platforms (automatically adding, removing and updating platforms).
type ExtensibleProvisioner interface {
	PlatformAdd(name string, args map[string]string, w io.Writer) error
	PlatformUpdate(name string, args map[string]string, w io.Writer) error
	PlatformRemove(name string) error
}

var provisioners = make(map[string]Provisioner)

// Register registers a new provisioner in the Provisioner registry.
func Register(name string, p Provisioner) {
	provisioners[name] = p
}

// Get gets the named provisioner from the registry.
func Get(name string) (Provisioner, error) {
	p, ok := provisioners[name]
	if !ok {
		return nil, fmt.Errorf("unknown provisioner: %q", name)
	}
	return p, nil
}

// Registry returns the list of registered provisioners.
func Registry() []Provisioner {
	registry := make([]Provisioner, 0, len(provisioners))
	for _, p := range provisioners {
		registry = append(registry, p)
	}
	return registry
}

// Error represents a provisioning error. It encapsulates further errors.
type Error struct {
	Reason string
	Err    error
}

// Error is the string representation of a provisioning error.
func (e *Error) Error() string {
	var err string
	if e.Err != nil {
		err = e.Err.Error() + ": " + e.Reason
	} else {
		err = e.Reason
	}
	return err
}
<commit_msg>provison: fixed godoc format.<commit_after>// Copyright 2014 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package provision provides interfaces that need to be satisfied in order to
// implement a new provisioner on tsuru.
package provision

import (
	"errors"
	"fmt"
	"io"

	"github.com/tsuru/tsuru/action"
	"github.com/tsuru/tsuru/app/bind"
)

var ErrInvalidStatus = errors.New("invalid status")

var ErrEmptyApp = errors.New("no units for this app")

// Status represents the status of a unit in tsuru.
type Status string

func (s Status) String() string {
	return string(s)
}

func ParseStatus(status string) (Status, error) {
	switch status {
	case "created":
		return StatusCreated, nil
	case "building":
		return StatusBuilding, nil
	case "error":
		return StatusError, nil
	case "started":
		return StatusStarted, nil
	case "starting":
		return StatusStarting, nil
	case "stopped":
		return StatusStopped, nil
	}
	return Status(""), ErrInvalidStatus
}

//     Flow:
//
//     +----------+                           Start          +---------+
//     | Building |                   +---------------------+| Stopped |
//     +----------+                   |                      +---------+
//           ^                        |                           ^
//           |                        |                           |
//      deploy unit                   |                         Stop
//           |                        |                           |
//           +                        v       RegisterUnit        +
//      +---------+  app unit   +----------+  SetUnitStatus  +---------+
//      | Created | +---------> | Starting | +-------------> | Started |
//      +---------+             +----------+                 +---------+
//                                    +                         ^ +
//                                    |                         | |
//                              SetUnitStatus                   | |
//                                    |                         | |
//                                    v                         | |
//                                +-------+     SetUnitStatus   | |
//                                | Error | +-------------------+ |
//                                +-------+ <---------------------+
const (
	// StatusCreated is the initial status of a unit in the database,
	// it should transition shortly to a more specific status
	StatusCreated = Status("created")

	// StatusBuilding is the status for units being provisioned by the
	// provisioner, like in the deployment.
	StatusBuilding = Status("building")

	// StatusError is the status for units that failed to start, because of
	// an application error.
	StatusError = Status("error")

	// StatusStarting is set when the container is started in docker.
	StatusStarting = Status("starting")

	// StatusStarted is for cases where the unit is up and running, and bound
	// to the proper status, it's set by RegisterUnit and SetUnitStatus.
	StatusStarted = Status("started")

	// StatusStopped is for cases where the unit has been stopped.
	StatusStopped = Status("stopped")
)

// Unit represents a provision unit. Can be a machine, container or anything
// IP-addressable.
type Unit struct {
	Name    string
	AppName string
	Type    string
	Ip      string
	Status  Status
}

// GetIp returns the Unit.IP.
func (u *Unit) GetIp() string {
	return u.Ip
}

// Available returns true if the unit is available. It will return true
// whenever the unit itself is available, even when the application process is
// not.
func (u *Unit) Available() bool {
	return u.Status == StatusStarted ||
		u.Status == StatusStarting ||
		u.Status == StatusError
}

// Named is something that has a name, providing the GetName method.
type Named interface {
	GetName() string
}

// App represents a tsuru app.
//
// It contains only relevant information for provisioning.
type App interface {
	Named

	BindUnit(*Unit) error
	UnbindUnit(*Unit) error

	// Log should be used to log messages in the app.
	Log(message, source, unit string) error

	// GetPlatform returns the platform (type) of the app. It is equivalent
	// to the Unit `Type` field.
	GetPlatform() string

	// GetDeploy returns the deploys that an app has.
	GetDeploys() uint

	Units() []Unit

	// Run executes the command in app units. Commands executed with this
	// method should have access to environment variables defined in the
	// app.
	Run(cmd string, w io.Writer, once bool) error

	Restart(io.Writer) error

	Envs() map[string]bind.EnvVar

	// Ready marks the app as ready for deployment.
	Ready() error

	GetMemory() int64
	GetSwap() int64
	GetCpuShare() int

	GetUpdatePlatform() bool

	GetRouter() (string, error)
}

// CNameManager represents a provisioner that supports cname on applications.
type CNameManager interface {
	SetCName(app App, cname string) error
	UnsetCName(app App, cname string) error
}

// ArchiveDeployer is a provisioner that can deploy archives.
type ArchiveDeployer interface {
	ArchiveDeploy(app App, archiveURL string, w io.Writer) error
}

// GitDeployer is a provisioner that can deploy the application from a Git
// repository.
type GitDeployer interface {
	GitDeploy(app App, version string, w io.Writer) error
}

// UploadDeployer is a provisioner that can deploy the application from an
// uploaded file.
type UploadDeployer interface {
	UploadDeploy(app App, file io.ReadCloser, w io.Writer) error
}

// Provisioner is the basic interface of this package.
//
// Any tsuru provisioner must implement this interface in order to provision
// tsuru apps.
type Provisioner interface {
	// Provision is called when tsuru is creating the app.
	Provision(App) error

	// Destroy is called when tsuru is destroying the app.
	Destroy(App) error

	// AddUnits adds units to an app. The first parameter is the app, the
	// second is the number of units to be added.
	//
	// It returns a slice containing all added units
	AddUnits(App, uint, io.Writer) ([]Unit, error)

	// RemoveUnits "undoes" AddUnits, removing the given number of units
	// from the app.
	RemoveUnits(App, uint) error

	// RemoveUnit removes a unit from the app. It receives the unit to be
	// removed.
	RemoveUnit(Unit) error

	// SetUnitStatus changes the status of a unit.
	SetUnitStatus(Unit, Status) error

	// ExecuteCommand runs a command in all units of the app.
	ExecuteCommand(stdout, stderr io.Writer, app App, cmd string, args ...string) error

	// ExecuteCommandOnce runs a command in one unit of the app.
	ExecuteCommandOnce(stdout, stderr io.Writer, app App, cmd string, args ...string) error

	Restart(App, io.Writer) error
	Stop(App) error

	// Start start the app units.
	Start(App) error

	// Addr returns the address for an app.
	//
	// tsuru will use this method to get the IP (although it might not be
	// an actual IP, collector calls it "IP") of the app from the
	// provisioner.
	Addr(App) (string, error)

	// Swap change the router between two apps.
	Swap(App, App) error

	// Units returns information about units by App.
	Units(App) []Unit

	// Register a unit after the container has been created or restarted.
	RegisterUnit(Unit) error
}

// InitializableProvisioner is a provisioner that provides an initialization
// method that should be called when the app is started
type InitializableProvisioner interface {
	Initialize() error
}

// CustomizedDeployPipelineProvisioner is a provisioner with a customized
// deploy pipeline.
type CustomizedDeployPipelineProvisioner interface {
	DeployPipeline() *action.Pipeline
}

// ExtensibleProvisioner is a provisioner where administrators can manage
// platforms (automatically adding, removing and updating platforms).
type ExtensibleProvisioner interface {
	PlatformAdd(name string, args map[string]string, w io.Writer) error
	PlatformUpdate(name string, args map[string]string, w io.Writer) error
	PlatformRemove(name string) error
}

var provisioners = make(map[string]Provisioner)

// Register registers a new provisioner in the Provisioner registry.
func Register(name string, p Provisioner) {
	provisioners[name] = p
}

// Get gets the named provisioner from the registry.
func Get(name string) (Provisioner, error) {
	p, ok := provisioners[name]
	if !ok {
		return nil, fmt.Errorf("unknown provisioner: %q", name)
	}
	return p, nil
}

// Registry returns the list of registered provisioners.
func Registry() []Provisioner {
	registry := make([]Provisioner, 0, len(provisioners))
	for _, p := range provisioners {
		registry = append(registry, p)
	}
	return registry
}

// Error represents a provisioning error. It encapsulates further errors.
type Error struct {
	Reason string
	Err    error
}

// Error is the string representation of a provisioning error.
func (e *Error) Error() string {
	var err string
	if e.Err != nil {
		err = e.Err.Error() + ": " + e.Reason
	} else {
		err = e.Reason
	}
	return err
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"reflect"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/glue"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
	"github.com/hashicorp/terraform-plugin-sdk/v2/terraform"
	tfglue "github.com/terraform-providers/terraform-provider-aws/aws/internal/service/glue"
)

func TestAccAWSGluePartition_basic(t *testing.T) {
	rName := acctest.RandomWithPrefix("tf-acc-test")
	parValue := acctest.RandString(10)
	resourceName := "aws_glue_partition.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckGluePartitionDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccGluePartitionConfig(rName, parValue),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckGluePartitionExists(resourceName),
					resource.TestCheckResourceAttr(resourceName, "database_name", rName),
					resource.TestCheckResourceAttr(resourceName, "partition_values.#", "1"),
					resource.TestCheckResourceAttrSet(resourceName, "creation_time"),
				),
			},
			{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
			},
		},
	})
}

func TestAccAWSGluePartition_disappears(t *testing.T) {
	rName := acctest.RandomWithPrefix("tf-acc-test")
	parValue := acctest.RandString(10)
	resourceName := "aws_glue_partition.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckGluePartitionDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccGluePartitionConfig(rName, parValue),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckGluePartitionExists(resourceName),
					testAccCheckResourceDisappears(testAccProvider, resourceAwsGluePartition(), resourceName),
				),
				ExpectNonEmptyPlan: true,
			},
		},
	})
}

func testAccCheckGluePartitionDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).glueconn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_glue_partition" {
			continue
		}

		catalogID, dbName, tableName, values, err := tfglue.ReadAwsGluePartitionID(rs.Primary.ID)
		if err != nil {
			return err
		}

		input := &glue.GetPartitionInput{
			DatabaseName:    aws.String(dbName),
			CatalogId:       aws.String(catalogID),
			TableName:       aws.String(tableName),
			PartitionValues: aws.StringSlice(values),
		}
		if _, err := conn.GetPartition(input); err != nil {
			if isAWSErr(err, glue.ErrCodeEntityNotFoundException, "") {
				continue
			}

			return err
		}
		return fmt.Errorf("still exists")
	}
	return nil
}

func testAccCheckGluePartitionExists(name string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[name]
		if !ok {
			return fmt.Errorf("Not found: %s", name)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No ID is set")
		}

		catalogID, dbName, tableName, values, err := tfglue.ReadAwsGluePartitionID(rs.Primary.ID)
		if err != nil {
			return err
		}

		conn := testAccProvider.Meta().(*AWSClient).glueconn
		out, err := conn.GetPartition(&glue.GetPartitionInput{
			DatabaseName:    aws.String(dbName),
			CatalogId:       aws.String(catalogID),
			TableName:       aws.String(tableName),
			PartitionValues: aws.StringSlice(values),
		})

		if err != nil {
			return err
		}

		if out.Partition == nil {
			return fmt.Errorf("No Glue Partition Found")
		}

		if !reflect.DeepEqual(aws.StringValueSlice(out.Partition.Values), values) {
			return fmt.Errorf("Glue Partition Mismatch")
		}

		return nil
	}
}

func testAccGluePartitionConfigBase(rName string) string {
	return fmt.Sprintf(`
resource "aws_glue_catalog_database" "test" {
  name = %[1]q
}

resource "aws_glue_catalog_table" "test" {
  name          = %[1]q
  database_name = aws_glue_catalog_database.test.name

  storage_descriptor {
    bucket_columns            = ["bucket_column_1"]
    compressed                = false
    input_format              = "SequenceFileInputFormat"
    location                  = "my_location"
    number_of_buckets         = 1
    output_format             = "SequenceFileInputFormat"
    stored_as_sub_directories = false

    parameters = {
      param1 = "param1_val"
    }

    columns {
      name    = "my_column_1"
      type    = "int"
      comment = "my_column1_comment"
    }

    columns {
      name    = "my_column_2"
      type    = "string"
      comment = "my_column2_comment"
    }

    ser_de_info {
      name = "ser_de_name"

      parameters = {
        param1 = "param_val_1"
      }

      serialization_library = "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe"
    }

    sort_columns {
      column     = "my_column_1"
      sort_order = 1
    }

    skewed_info {
      skewed_column_names = [
        "my_column_1",
      ]

      skewed_column_value_location_maps = {
        my_column_1 = "my_column_1_val_loc_map"
      }

      skewed_column_values = [
        "skewed_val_1",
      ]
    }
  }

  partition_keys {
    name    = "my_column_12"
    type    = "date"
    comment = "my_column_1_comment2"
  }
}
`, rName)
}

func testAccGluePartitionConfig(rName string, parValue string) string {
	return testAccGluePartitionConfigBase(rName) +
		fmt.Sprintf(`
resource "aws_glue_partition" "test" {
  database_name    = aws_glue_catalog_database.test.name
  table_name       = aws_glue_catalog_table.test.name
  partition_values = ["%[1]s"]
}
`, parValue)
}
<commit_msg>multi part and parameters test<commit_after>package aws

import (
	"fmt"
	"reflect"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/glue"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
	"github.com/hashicorp/terraform-plugin-sdk/v2/terraform"
	tfglue "github.com/terraform-providers/terraform-provider-aws/aws/internal/service/glue"
	"github.com/terraform-providers/terraform-provider-aws/aws/internal/tfawsresource"
)

func TestAccAWSGluePartition_basic(t *testing.T) {
	rName := acctest.RandomWithPrefix("tf-acc-test")
	parValue := acctest.RandString(10)
	resourceName := "aws_glue_partition.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckGluePartitionDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccGluePartitionBasicConfig(rName, parValue),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckGluePartitionExists(resourceName),
					testAccCheckResourceAttrAccountID(resourceName, "catalog_id"),
					resource.TestCheckResourceAttr(resourceName, "database_name", rName),
					resource.TestCheckResourceAttr(resourceName, "partition_values.#", "1"),
					tfawsresource.TestCheckTypeSetElemAttr(resourceName, "partition_values.*", parValue),
					resource.TestCheckResourceAttr(resourceName, "parameters.%", "0"),
					resource.TestCheckResourceAttrSet(resourceName, "creation_time"),
				),
			},
			{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
			},
		},
	})
}

func TestAccAWSGluePartition_multipleValues(t *testing.T) {
	rName := acctest.RandomWithPrefix("tf-acc-test")
	parValue := acctest.RandString(10)
	parValue2 := acctest.RandString(11)
	resourceName := "aws_glue_partition.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckGluePartitionDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccGluePartitionMultiplePartValueConfig(rName, parValue, parValue2),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckGluePartitionExists(resourceName),
					resource.TestCheckResourceAttr(resourceName, "partition_values.#", "2"),
					tfawsresource.TestCheckTypeSetElemAttr(resourceName, "partition_values.*", parValue),
					tfawsresource.TestCheckTypeSetElemAttr(resourceName, "partition_values.*", parValue2),
				),
			},
			{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
			},
		},
	})
}

func TestAccAWSGluePartition_parameters(t *testing.T) {
	rName := acctest.RandomWithPrefix("tf-acc-test")
	parValue := acctest.RandString(10)
	resourceName := "aws_glue_partition.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckGluePartitionDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccGluePartitionParametersConfig1(rName, parValue, "key1", "value1"),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckGluePartitionExists(resourceName),
					resource.TestCheckResourceAttr(resourceName, "parameters.%", "1"),
					resource.TestCheckResourceAttr(resourceName, "parameters.key1", "value1"),
				),
			},
			{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
			},
			{
				Config: testAccGluePartitionParametersConfig2(rName, parValue, "key1", "valueUpdated1", "key2", "value2"),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckGluePartitionExists(resourceName),
					resource.TestCheckResourceAttr(resourceName, "parameters.%", "2"),
					resource.TestCheckResourceAttr(resourceName, "parameters.key1", "valueUpdated1"),
					resource.TestCheckResourceAttr(resourceName, "parameters.key2", "value2"),
				),
			},
			{
				Config: testAccGluePartitionParametersConfig1(rName, parValue, "key2", "value2"),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckGluePartitionExists(resourceName),
					resource.TestCheckResourceAttr(resourceName, "parameters.%", "1"),
					resource.TestCheckResourceAttr(resourceName, "parameters.key2", "value2"),
				),
			},
		},
	})
}

func TestAccAWSGluePartition_disappears(t *testing.T) {
	rName := acctest.RandomWithPrefix("tf-acc-test")
	parValue := acctest.RandString(10)
	resourceName := "aws_glue_partition.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckGluePartitionDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccGluePartitionBasicConfig(rName, parValue),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckGluePartitionExists(resourceName),
					testAccCheckResourceDisappears(testAccProvider, resourceAwsGluePartition(), resourceName),
				),
				ExpectNonEmptyPlan: true,
			},
		},
	})
}

func testAccCheckGluePartitionDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).glueconn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_glue_partition" {
			continue
		}

		catalogID, dbName, tableName, values, err := tfglue.ReadAwsGluePartitionID(rs.Primary.ID)
		if err != nil {
			return err
		}

		input := &glue.GetPartitionInput{
			DatabaseName:    aws.String(dbName),
			CatalogId:       aws.String(catalogID),
			TableName:       aws.String(tableName),
			PartitionValues: aws.StringSlice(values),
		}
		if _, err := conn.GetPartition(input); err != nil {
			if isAWSErr(err, glue.ErrCodeEntityNotFoundException, "") {
				continue
			}

			return err
		}
		return fmt.Errorf("still exists")
	}
	return nil
}

func testAccCheckGluePartitionExists(name string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[name]
		if !ok {
			return fmt.Errorf("Not found: %s", name)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No ID is set")
		}

		catalogID, dbName, tableName, values, err := tfglue.ReadAwsGluePartitionID(rs.Primary.ID)
		if err != nil {
			return err
		}

		conn := testAccProvider.Meta().(*AWSClient).glueconn
		out, err := conn.GetPartition(&glue.GetPartitionInput{
			DatabaseName:    aws.String(dbName),
			CatalogId:       aws.String(catalogID),
			TableName:       aws.String(tableName),
			PartitionValues: aws.StringSlice(values),
		})

		if err != nil {
			return err
		}

		if out.Partition == nil {
			return fmt.Errorf("No Glue Partition Found")
		}

		if !reflect.DeepEqual(aws.StringValueSlice(out.Partition.Values), values) {
			return fmt.Errorf("Glue Partition Mismatch")
		}

		return nil
	}
}

func testAccGluePartitionConfigBase(rName string) string {
	return fmt.Sprintf(`
resource "aws_glue_catalog_database" "test" {
  name = %[1]q
}

resource "aws_glue_catalog_table" "test" {
  name          = %[1]q
  database_name = aws_glue_catalog_database.test.name

  storage_descriptor {
    bucket_columns            = ["bucket_column_1"]
    compressed                = false
    input_format              = "SequenceFileInputFormat"
    location                  = "my_location"
    number_of_buckets         = 1
    output_format             = "SequenceFileInputFormat"
    stored_as_sub_directories = false

    parameters = {
      param1 = "param1_val"
    }

    columns {
      name    = "my_column_1"
      type    = "int"
      comment = "my_column1_comment"
    }

    columns {
      name    = "my_column_2"
      type    = "string"
      comment = "my_column2_comment"
    }

    ser_de_info {
      name = "ser_de_name"

      parameters = {
        param1 = "param_val_1"
      }

      serialization_library = "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe"
    }

    sort_columns {
      column     = "my_column_1"
      sort_order = 1
    }

    skewed_info {
      skewed_column_names = [
        "my_column_1",
      ]

      skewed_column_value_location_maps = {
        my_column_1 = "my_column_1_val_loc_map"
      }

      skewed_column_values = [
        "skewed_val_1",
      ]
    }
  }

  partition_keys {
    name    = "my_column_12"
    type    = "date"
    comment = "my_column_1_comment2"
  }
}
`, rName)
}

func testAccGluePartitionBasicConfig(rName, parValue string) string {
	return testAccGluePartitionConfigBase(rName) +
		fmt.Sprintf(`
resource "aws_glue_partition" "test" {
  database_name    = aws_glue_catalog_database.test.name
  table_name       = aws_glue_catalog_table.test.name
  partition_values = ["%[1]s"]
}
`, parValue)
}

func testAccGluePartitionParametersConfig1(rName, parValue, key1, value1 string) string {
	return testAccGluePartitionConfigBase(rName) +
		fmt.Sprintf(`
resource "aws_glue_partition" "test" {
  database_name    = aws_glue_catalog_database.test.name
  table_name       = aws_glue_catalog_table.test.name
  partition_values = ["%[1]s"]

  parameters {
    %[2]q = %[3]q
  }
}
`, parValue, key1, value1)
}

func testAccGluePartitionParametersConfig2(rName, parValue, key1, value1, key2, value2 string) string {
	return testAccGluePartitionConfigBase(rName) +
		fmt.Sprintf(`
resource "aws_glue_partition" "test" {
  database_name    = aws_glue_catalog_database.test.name
  table_name       = aws_glue_catalog_table.test.name
  partition_values = ["%[1]s"]

  parameters {
    %[2]q = %[3]q
    %[4]q = %[5]q
  }
}
`, parValue, key1, value1, key2, value2)
}

func testAccGluePartitionMultiplePartValueConfig(rName, parValue, parValue2 string) string {
	return fmt.Sprintf(`
resource "aws_glue_catalog_table" "test" {
  name          = %[1]q
  database_name = aws_glue_catalog_database.test.name

  storage_descriptor {
    bucket_columns            = ["bucket_column_1"]
    compressed                = false
    input_format              = "SequenceFileInputFormat"
    location                  = "my_location"
    number_of_buckets         = 1
    output_format             = "SequenceFileInputFormat"
    stored_as_sub_directories = false

    parameters = {
      param1 = "param1_val"
    }

    columns {
      name    = "my_column_1"
      type    = "int"
      comment = "my_column1_comment"
    }

    columns {
      name    = "my_column_2"
      type    = "string"
      comment = "my_column2_comment"
    }

    ser_de_info {
      name = "ser_de_name"

      parameters = {
        param1 = "param_val_1"
      }

      serialization_library = "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe"
    }

    sort_columns {
      column     = "my_column_1"
      sort_order = 1
    }

    skewed_info {
      skewed_column_names = [
        "my_column_1",
      ]

      skewed_column_value_location_maps = {
        my_column_1 = "my_column_1_val_loc_map"
      }

      skewed_column_values = [
        "skewed_val_1",
      ]
    }
  }

  partition_keys {
    name    = "my_column_12"
    type    = "date"
    comment = "my_column_1_comment2"
  }

  partition_keys {
    name    = "my_column_11"
    type    = "date"
    comment = "my_column_1_comment2"
  }
}

resource "aws_glue_partition" "test" {
  database_name    = aws_glue_catalog_database.test.name
  table_name       = aws_glue_catalog_table.test.name
  partition_values = ["%[2]s", "%[3]s"]
}
`, rName, parValue, parValue2)
}
<|endoftext|>


<commit_before>// Copyright 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A VMDK Docker Data Volume plugin for Windows
package main

import (
    "github.com/kardianos/service"
    log "github.com/Sirupsen/logrus"
)

// pluginService is a wrapper for vDVS plugin server so that the plugin can
// be run as a service on Windows. Windows controls services by setting up
// callbacks that is non-trivial. This wrapper utilizes a vendor library to
// handle this.
type pluginService struct{}

func (p *pluginService) Start(s service.Service) error {
    // Start should not block. Do the actual work async.
    go p.run()
    return nil
}

func (p *pluginService) run() {
    // Do work here
    startPluginServer()
}

func (p *pluginService) Stop(s service.Service) error {
    // Stop should not block. Return directly.
    return nil
}

// startDaemon starts vDVS plugin daemon on Windows
func startDaemon() {
    svcConfig := &service.Config {
        Name:        "vdvs",
        DisplayName: "vSphere Docker Volume Service",
        Description: "Enables user to run stateful containerized applications on top of VMware vSphere.",
    }

    ps := &pluginService{}
    svc, err := service.New(ps, svcConfig)
    if err != nil {
        log.Fatal("Failed to create the service: ", err)
    }

    logger, err := svc.Logger(nil)
    if err != nil {
        log.Fatal("Failed to get service logger: ", err)
    }

    err = svc.Run()
    if err != nil {
        log.Fatal("Failed to run the service: ", err)
        logger.Error(err)
    }
}
<commit_msg>Trivial format fixes [SKIP CI] (#1630)<commit_after>// Copyright 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A VMDK Docker Data Volume plugin for Windows
package main

import (
	log "github.com/Sirupsen/logrus"
	"github.com/kardianos/service"
)

// pluginService is a wrapper for vDVS plugin server so that the plugin can
// be run as a service on Windows. Windows controls services by setting up
// callbacks that is non-trivial. This wrapper utilizes a vendor library to
// handle this.
type pluginService struct{}

func (p *pluginService) Start(s service.Service) error {
	// Start should not block. Do the actual work async.
	go p.run()
	return nil
}

func (p *pluginService) run() {
	// Do work here
	startPluginServer()
}

func (p *pluginService) Stop(s service.Service) error {
	// Stop should not block. Return directly.
	return nil
}

// startDaemon starts vDVS plugin daemon on Windows
func startDaemon() {
	svcConfig := &service.Config{
		Name:        "vdvs",
		DisplayName: "vSphere Docker Volume Service",
		Description: "Enables user to run stateful containerized applications on top of VMware vSphere.",
	}

	ps := &pluginService{}
	svc, err := service.New(ps, svcConfig)
	if err != nil {
		log.Fatal("Failed to create the service: ", err)
	}

	logger, err := svc.Logger(nil)
	if err != nil {
		log.Fatal("Failed to get service logger: ", err)
	}

	err = svc.Run()
	if err != nil {
		log.Fatal("Failed to run the service: ", err)
		logger.Error(err)
	}
}
<|endoftext|>