<commit_before>package cmd

import (
	"encoding/json"
	"fmt"
	"io"
	"regexp"
	"sort"
	"strings"

	"github.com/pkg/errors"
	"github.com/skatteetaten/ao/pkg/client"
	"github.com/skatteetaten/ao/pkg/config"
	"github.com/skatteetaten/ao/pkg/fuzzy"
	"github.com/skatteetaten/ao/pkg/prompt"
	"github.com/spf13/cobra"
)

type deploymentUnitID struct {
	envName, clusterName string
}

type deploymentUnit struct {
	id             *deploymentUnitID
	deploySpecList []client.DeploySpec
	cluster        *config.Cluster
	auroraConfig   string
	overrideToken  string
}

var (
	flagAffiliation string
	flagOverrides   []string
	flagNoPrompt    bool
	flagVersion     string
	flagCluster     string
	flagExcludes    []string
)

const deployLong = `Deploys applications from the current AuroraConfig.
For use in CI environments use --no-prompt to disable interactivity.
`

const exampleDeploy = `  Given the following AuroraConfig:
    - about.json
    - foobar.json
    - bar.json
    - foo/about.json
    - foo/bar.json
		- foo/foobar.json
		- ref/about.json
    - ref/bar.json

  # Fuzzy matching: deploy foo/bar and foo/foobar
  ao deploy fo/ba

  # Exact matching: deploy foo/bar
  ao deploy foo/bar

  # Deploy an application with override for application file
  ao deploy foo/bar -o 'foo/bar.json:{"pause": true}'

  # Exclude application(s) from foo environment (regexp)
  ao deploy foo -e .*/bar -e .*/baz

  # Exclude environment(s) when deploying an application across environments (regexp)
  ao deploy bar -e ref/.*
`

var deployCmd = &cobra.Command{
	Aliases:     []string{"setup", "apply"},
	Use:         "deploy <applicationId>",
	Short:       "Deploy one or more ApplicationId (environment/application) to one or more clusters",
	Long:        deployLong,
	Example:     exampleDeploy,
	Annotations: map[string]string{"type": "actions"},
	RunE:        deploy,
}

func newDeploymentUnitID(clusterName, envName string) *deploymentUnitID {
	return &deploymentUnitID{
		clusterName: clusterName,
		envName:     envName,
	}
}

func newDeploymentUnit(unitID *deploymentUnitID, deploySpecs []client.DeploySpec, cluster *config.Cluster, auroraConfig string, overrideToken string) *deploymentUnit {
	return &deploymentUnit{
		id:             unitID,
		deploySpecList: deploySpecs,
		cluster:        cluster,
		auroraConfig:   auroraConfig,
		overrideToken:  overrideToken,
	}
}

func init() {
	RootCmd.AddCommand(deployCmd)

	deployCmd.Flags().StringVarP(&flagAffiliation, "auroraconfig", "a", "", "Overrides the logged in AuroraConfig")
	deployCmd.Flags().StringVarP(&flagCluster, "cluster", "c", "", "Limit deploy to given cluster name")
	deployCmd.Flags().BoolVarP(&flagNoPrompt, "no-prompt", "", false, "Suppress prompts")
	deployCmd.Flags().StringArrayVarP(&flagOverrides, "overrides", "o", []string{}, "Override in the form '[env/]file:{<json override>}'")
	deployCmd.Flags().StringArrayVarP(&flagExcludes, "exclude", "e", []string{}, "Select applications or environments to exclude from deploy")
	deployCmd.Flags().StringVarP(&flagVersion, "version", "v", "", "Set the given version in AuroraConfig before deploy")

	deployCmd.Flags().BoolVarP(&flagNoPrompt, "force", "f", false, "Suppress prompts")
	deployCmd.Flags().MarkHidden("force")
	deployCmd.Flags().StringVarP(&flagAffiliation, "affiliation", "", "", "Overrides the logged in affiliation")
	deployCmd.Flags().MarkHidden("affiliation")
}

func deploy(cmd *cobra.Command, args []string) error {

	if len(args) > 2 || len(args) < 1 {
		return cmd.Usage()
	}

	err := validateParams()
	if err != nil {
		return err
	}

	search := args[0]
	if len(args) == 2 {
		search = fmt.Sprintf("%s/%s", args[0], args[1])
	}

	auroraConfig := AO.Affiliation
	if flagAffiliation != "" {
		auroraConfig = flagAffiliation
	}

	apiClient, err := getAPIClient(auroraConfig, pFlagToken, flagCluster)
	if err != nil {
		return err
	}

	applications, err := getApplications(apiClient, search, flagVersion, flagExcludes, cmd.OutOrStdout())
	if err != nil {
		return err
	} else if len(applications) == 0 {
		return errors.New("No applications to deploy")
	}

	filteredDeploymentSpecs, err := getFilteredDeploymentSpecs(apiClient, applications, flagCluster)
	if err != nil {
		return err
	}

	overrideConfig, err := parseOverride()
	if err != nil {
		return err
	}

	if !userConfirmation(filteredDeploymentSpecs, cmd.OutOrStdout()) {
		return errors.New("No applications to deploy")
	}

	deploymentUnits := createDeploymentUnits(auroraConfig, pFlagToken, AO.Clusters, filteredDeploymentSpecs)

	result, err := deployToReachableClusters(getDeployClient, deploymentUnits, overrideConfig)
	if err != nil {
		return err
	}

	printDeployResult(result, cmd.OutOrStdout())

	return nil
}

func validateParams() error {

	if flagCluster != "" {
		if _, exists := AO.Clusters[flagCluster]; !exists {
			return errors.New(fmt.Sprintf("No such cluster %s", flagCluster))
		}
	}

	return nil
}

func getApplications(apiClient client.AuroraConfigClient, search, version string, excludes []string, out io.Writer) ([]string, error) {
	files, err := apiClient.GetFileNames()
	if err != nil {
		return nil, err
	}

	possibleDeploys := files.GetApplicationIds()
	applications := fuzzy.SearchForApplications(search, possibleDeploys)

	applications, err = filterExcludes(excludes, applications)
	if err != nil {
		return nil, err
	}

	if version != "" {
		if len(applications) > 1 {
			return nil, errors.New("Deploy with version does only support one application")
		}

		fileName, err := files.Find(applications[0])
		if err != nil {
			return nil, err
		}

		err = updateVersion(apiClient, version, fileName, out)
		if err != nil {
			return nil, err
		}
	}

	return applications, nil
}

func updateVersion(apiClient client.AuroraConfigClient, version, fileName string, out io.Writer) error {
	path, value := "/version", version

	fileName, err := SetValue(apiClient, fileName, path, value)
	if err != nil {
		return err
	}

	fmt.Fprintf(out, "%s has been updated with %s %s\n", fileName, path, value)

	return nil
}

func getFilteredDeploymentSpecs(apiClient client.DeploySpecClient, applications []string, overrideCluster string) ([]client.DeploySpec, error) {
	deploySpecs, err := apiClient.GetAuroraDeploySpec(applications, true)
	if err != nil {
		return nil, err
	}
	var filteredDeploymentSpecs []client.DeploySpec
	if overrideCluster != "" {
		for _, spec := range deploySpecs {
			if spec.Value("/cluster").(string) == overrideCluster {
				filteredDeploymentSpecs = append(filteredDeploymentSpecs, spec)
			}
		}
	} else {
		filteredDeploymentSpecs = deploySpecs
	}

	return filteredDeploymentSpecs, nil
}

func filterExcludes(expressions, applications []string) ([]string, error) {
	apps := make([]string, len(applications))
	copy(apps, applications)
	for _, expr := range expressions {
		r, err := regexp.Compile(expr)
		if err != nil {
			return nil, err
		}
		tmp := apps[:0]
		for _, app := range apps {
			match := r.MatchString(app)
			if !match {
				tmp = append(tmp, app)
			}
		}
		apps = tmp
	}

	return apps, nil
}

func createDeploymentUnits(auroraConfig, overrideToken string, clusters map[string]*config.Cluster, deploymentSpecs []client.DeploySpec) map[deploymentUnitID]*deploymentUnit {
	unitsMap := make(map[deploymentUnitID]*deploymentUnit)

	for _, spec := range deploymentSpecs {
		clusterName := spec.Value("cluster").(string)
		envName := spec.Value("envName").(string)

		unitID := newDeploymentUnitID(clusterName, envName)

		if _, exists := unitsMap[*unitID]; !exists {
			cluster := clusters[clusterName]
			unit := newDeploymentUnit(unitID, []client.DeploySpec{}, cluster, auroraConfig, overrideToken)
			unitsMap[*unitID] = unit
		}

		unitsMap[*unitID].deploySpecList = append(unitsMap[*unitID].deploySpecList, spec)
	}

	return unitsMap
}

func deployToReachableClusters(getClient func(unit *deploymentUnit) client.DeployClient, deploymentUnits map[deploymentUnitID]*deploymentUnit, overrideConfig map[string]string) ([]*client.DeployResults, error) {
	deployResult := make(chan *client.DeployResults)

	for _, unit := range deploymentUnits {
		go deployUnit(getClient(unit), unit, overrideConfig, deployResult)
	}

	var allResults []*client.DeployResults
	for i := 0; i < len(deploymentUnits); i++ {
		allResults = append(allResults, <-deployResult)
	}

	return allResults, nil
}

func deployUnit(deployClient client.DeployClient, unit *deploymentUnit, overrideConfig map[string]string, deployResults chan<- *client.DeployResults) {
	if !unit.cluster.Reachable {
		deployResults <- errorDeployResults("Cluster is not reachable", unit)
		return
	}

	var applicationList []string
	for _, spec := range unit.deploySpecList {
		applicationList = append(applicationList, spec.Value("applicationId").(string))
	}

	payload := client.NewDeployPayload(applicationList, overrideConfig)

	result, err := deployClient.Deploy(payload)
	if err != nil {
		deployResults <- errorDeployResults(err.Error(), unit)
	} else {
		deployResults <- result
	}
}

func errorDeployResults(reason string, unit *deploymentUnit) *client.DeployResults {
	var applicationResults []client.DeployResult

	for _, spec := range unit.deploySpecList {
		affiliation := spec.Value("affiliation").(string)
		applicationID := client.NewApplicationId(spec.Value("applicationId").(string))

		result := new(client.DeployResult)
		result.DeployId = "-"
		result.Ignored = false
		result.Success = false
		result.Reason = reason
		result.ADS.Cluster = unit.cluster.Name
		result.ADS.Name = applicationID.Application
		result.ADS.Deploy.Version = "-"
		result.ADS.Environment.Namespace = affiliation + "-" + applicationID.Environment

		applicationResults = append(applicationResults, *result)
	}

	return &client.DeployResults{
		Message: reason,
		Success: false,
		Results: applicationResults,
	}
}

func parseOverride() (map[string]string, error) {
	returnMap := make(map[string]string)
	for i := 0; i < len(flagOverrides); i++ {
		indexByte := strings.IndexByte(flagOverrides[i], ':')
		filename := flagOverrides[i][:indexByte]
		jsonOverride := flagOverrides[i][indexByte+1:]

		if !json.Valid([]byte(jsonOverride)) {
			msg := fmt.Sprintf("%s is not a valid json", jsonOverride)
			return nil, errors.New(msg)
		}

		returnMap[filename] = jsonOverride
	}
	return returnMap, nil
}

func userConfirmation(filteredDeploymentSpecs []client.DeploySpec, out io.Writer) bool {
	header, rows := GetDeploySpecTable(filteredDeploymentSpecs)
	DefaultTablePrinter(header, rows, out)

	var filteredApplications []string
	for _, spec := range filteredDeploymentSpecs {
		appID := spec.Value("applicationId").(string)
		filteredApplications = append(filteredApplications, appID)
	}

	shouldDeploy := true
	if !flagNoPrompt {
		defaultAnswer := len(filteredApplications) == 1
		message := fmt.Sprintf("Do you want to deploy %d application(s)?", len(filteredApplications))
		shouldDeploy = prompt.Confirm(message, defaultAnswer)
	}

	return shouldDeploy
}

func printDeployResult(result []*client.DeployResults, out io.Writer) error {
	var results []client.DeployResult
	for _, r := range result {
		results = append(results, r.Results...)
	}

	if len(results) == 0 {
		return errors.New("No deploys were made")
	}

	sort.Slice(results, func(i, j int) bool {
		return strings.Compare(results[i].ADS.Name, results[j].ADS.Name) < 1
	})

	header, rows := getDeployResultTable(results)
	if len(rows) == 0 {
		return nil
	}

	DefaultTablePrinter(header, rows, out)
	for _, deploy := range results {
		if !deploy.Success {
			return errors.New("One or more deploys failed")
		}
	}

	return nil
}

func getDeployResultTable(deploys []client.DeployResult) (string, []string) {
	var rows []string
	for _, item := range deploys {
		if item.Ignored {
			continue
		}
		ads := item.ADS
		pattern := "%s\t%s\t%s\t%s\t%s\t%s\t%s"
		status := "\x1b[32mDeployed\x1b[0m"
		if !item.Success {
			status = "\x1b[31mFailed\x1b[0m"
		}
		result := fmt.Sprintf(pattern, status, ads.Cluster, ads.Environment.Namespace, ads.Name, ads.Deploy.Version, item.DeployId, item.Reason)
		rows = append(rows, result)
	}

	header := "\x1b[00mSTATUS\x1b[0m\tCLUSTER\tENVIRONMENT\tAPPLICATION\tVERSION\tDEPLOY_ID\tMESSAGE"
	return header, rows
}

func getAPIClient(auroraConfig, overrideToken, overrideCluster string) (*client.ApiClient, error) {
	api := DefaultApiClient
	api.Affiliation = auroraConfig

	if overrideCluster != "" && !AO.Localhost {
		c := AO.Clusters[overrideCluster]
		if !c.Reachable {
			return nil, errors.Errorf("%s cluster is not reachable", overrideCluster)
		}

		api.Host = c.BooberUrl
		api.Token = c.Token
		if overrideToken != "" {
			api.Token = overrideToken
		}
	}

	return api, nil
}

func getDeployClient(unit *deploymentUnit) client.DeployClient {
	var deployClient *client.ApiClient
	if AO.Localhost {
		deployClient = DefaultApiClient
		deployClient.Affiliation = unit.auroraConfig
	} else {
		token := unit.cluster.Token
		if unit.overrideToken != "" {
			token = unit.overrideToken
		}
		deployClient = client.NewApiClient(unit.cluster.BooberUrl, token, unit.auroraConfig, AO.RefName)
	}

	return deployClient
}
<commit_msg>AOS-3030 Verify that clusters specified in deploy spec are properly configured<commit_after>package cmd

import (
	"encoding/json"
	"fmt"
	"io"
	"regexp"
	"sort"
	"strings"

	"github.com/pkg/errors"
	"github.com/skatteetaten/ao/pkg/client"
	"github.com/skatteetaten/ao/pkg/config"
	"github.com/skatteetaten/ao/pkg/fuzzy"
	"github.com/skatteetaten/ao/pkg/prompt"
	"github.com/spf13/cobra"
)

type deploymentUnitID struct {
	envName, clusterName string
}

type deploymentUnit struct {
	id             *deploymentUnitID
	deploySpecList []client.DeploySpec
	cluster        *config.Cluster
	auroraConfig   string
	overrideToken  string
}

var (
	flagAffiliation string
	flagOverrides   []string
	flagNoPrompt    bool
	flagVersion     string
	flagCluster     string
	flagExcludes    []string
)

const deployLong = `Deploys applications from the current AuroraConfig.
For use in CI environments use --no-prompt to disable interactivity.
`

const exampleDeploy = `  Given the following AuroraConfig:
    - about.json
    - foobar.json
    - bar.json
    - foo/about.json
    - foo/bar.json
		- foo/foobar.json
		- ref/about.json
    - ref/bar.json

  # Fuzzy matching: deploy foo/bar and foo/foobar
  ao deploy fo/ba

  # Exact matching: deploy foo/bar
  ao deploy foo/bar

  # Deploy an application with override for application file
  ao deploy foo/bar -o 'foo/bar.json:{"pause": true}'

  # Exclude application(s) from foo environment (regexp)
  ao deploy foo -e .*/bar -e .*/baz

  # Exclude environment(s) when deploying an application across environments (regexp)
  ao deploy bar -e ref/.*
`

var deployCmd = &cobra.Command{
	Aliases:     []string{"setup", "apply"},
	Use:         "deploy <applicationId>",
	Short:       "Deploy one or more ApplicationId (environment/application) to one or more clusters",
	Long:        deployLong,
	Example:     exampleDeploy,
	Annotations: map[string]string{"type": "actions"},
	RunE:        deploy,
}

func newDeploymentUnitID(clusterName, envName string) *deploymentUnitID {
	return &deploymentUnitID{
		clusterName: clusterName,
		envName:     envName,
	}
}

func newDeploymentUnit(unitID *deploymentUnitID, deploySpecs []client.DeploySpec, cluster *config.Cluster, auroraConfig string, overrideToken string) *deploymentUnit {
	return &deploymentUnit{
		id:             unitID,
		deploySpecList: deploySpecs,
		cluster:        cluster,
		auroraConfig:   auroraConfig,
		overrideToken:  overrideToken,
	}
}

func init() {
	RootCmd.AddCommand(deployCmd)

	deployCmd.Flags().StringVarP(&flagAffiliation, "auroraconfig", "a", "", "Overrides the logged in AuroraConfig")
	deployCmd.Flags().StringVarP(&flagCluster, "cluster", "c", "", "Limit deploy to given cluster name")
	deployCmd.Flags().BoolVarP(&flagNoPrompt, "no-prompt", "", false, "Suppress prompts")
	deployCmd.Flags().StringArrayVarP(&flagOverrides, "overrides", "o", []string{}, "Override in the form '[env/]file:{<json override>}'")
	deployCmd.Flags().StringArrayVarP(&flagExcludes, "exclude", "e", []string{}, "Select applications or environments to exclude from deploy")
	deployCmd.Flags().StringVarP(&flagVersion, "version", "v", "", "Set the given version in AuroraConfig before deploy")

	deployCmd.Flags().BoolVarP(&flagNoPrompt, "force", "f", false, "Suppress prompts")
	deployCmd.Flags().MarkHidden("force")
	deployCmd.Flags().StringVarP(&flagAffiliation, "affiliation", "", "", "Overrides the logged in affiliation")
	deployCmd.Flags().MarkHidden("affiliation")
}

func deploy(cmd *cobra.Command, args []string) error {

	if len(args) > 2 || len(args) < 1 {
		return cmd.Usage()
	}

	err := validateParams()
	if err != nil {
		return err
	}

	search := args[0]
	if len(args) == 2 {
		search = fmt.Sprintf("%s/%s", args[0], args[1])
	}

	auroraConfig := AO.Affiliation
	if flagAffiliation != "" {
		auroraConfig = flagAffiliation
	}

	apiClient, err := getAPIClient(auroraConfig, pFlagToken, flagCluster)
	if err != nil {
		return err
	}

	applications, err := getApplications(apiClient, search, flagVersion, flagExcludes, cmd.OutOrStdout())
	if err != nil {
		return err
	} else if len(applications) == 0 {
		return errors.New("No applications to deploy")
	}

	filteredDeploymentSpecs, err := getFilteredDeploymentSpecs(apiClient, applications, flagCluster)
	if err != nil {
		return err
	}

	overrideConfig, err := parseOverride()
	if err != nil {
		return err
	}

	deploymentUnits, err := createDeploymentUnits(auroraConfig, pFlagToken, AO.Clusters, filteredDeploymentSpecs)
	if err != nil {
		return err
	}

	if !userConfirmation(filteredDeploymentSpecs, cmd.OutOrStdout()) {
		return errors.New("No applications to deploy")
	}

	result, err := deployToReachableClusters(getDeployClient, deploymentUnits, overrideConfig)
	if err != nil {
		return err
	}

	printDeployResult(result, cmd.OutOrStdout())

	return nil
}

func validateParams() error {

	if flagCluster != "" {
		if _, exists := AO.Clusters[flagCluster]; !exists {
			return errors.New(fmt.Sprintf("No such cluster %s", flagCluster))
		}
	}

	return nil
}

func getApplications(apiClient client.AuroraConfigClient, search, version string, excludes []string, out io.Writer) ([]string, error) {
	files, err := apiClient.GetFileNames()
	if err != nil {
		return nil, err
	}

	possibleDeploys := files.GetApplicationIds()
	applications := fuzzy.SearchForApplications(search, possibleDeploys)

	applications, err = filterExcludes(excludes, applications)
	if err != nil {
		return nil, err
	}

	if version != "" {
		if len(applications) > 1 {
			return nil, errors.New("Deploy with version does only support one application")
		}

		fileName, err := files.Find(applications[0])
		if err != nil {
			return nil, err
		}

		err = updateVersion(apiClient, version, fileName, out)
		if err != nil {
			return nil, err
		}
	}

	return applications, nil
}

func updateVersion(apiClient client.AuroraConfigClient, version, fileName string, out io.Writer) error {
	path, value := "/version", version

	fileName, err := SetValue(apiClient, fileName, path, value)
	if err != nil {
		return err
	}

	fmt.Fprintf(out, "%s has been updated with %s %s\n", fileName, path, value)

	return nil
}

func getFilteredDeploymentSpecs(apiClient client.DeploySpecClient, applications []string, overrideCluster string) ([]client.DeploySpec, error) {
	deploySpecs, err := apiClient.GetAuroraDeploySpec(applications, true)
	if err != nil {
		return nil, err
	}
	var filteredDeploymentSpecs []client.DeploySpec
	if overrideCluster != "" {
		for _, spec := range deploySpecs {
			if spec.Value("/cluster").(string) == overrideCluster {
				filteredDeploymentSpecs = append(filteredDeploymentSpecs, spec)
			}
		}
	} else {
		filteredDeploymentSpecs = deploySpecs
	}

	return filteredDeploymentSpecs, nil
}

func filterExcludes(expressions, applications []string) ([]string, error) {
	apps := make([]string, len(applications))
	copy(apps, applications)
	for _, expr := range expressions {
		r, err := regexp.Compile(expr)
		if err != nil {
			return nil, err
		}
		tmp := apps[:0]
		for _, app := range apps {
			match := r.MatchString(app)
			if !match {
				tmp = append(tmp, app)
			}
		}
		apps = tmp
	}

	return apps, nil
}

func createDeploymentUnits(auroraConfig, overrideToken string, clusters map[string]*config.Cluster, deploymentSpecs []client.DeploySpec) (map[deploymentUnitID]*deploymentUnit, error) {
	unitsMap := make(map[deploymentUnitID]*deploymentUnit)

	for _, spec := range deploymentSpecs {
		clusterName := spec.Value("cluster").(string)
		envName := spec.Value("envName").(string)

		unitID := newDeploymentUnitID(clusterName, envName)

		if _, exists := unitsMap[*unitID]; !exists {
			if _, exists := clusters[clusterName]; !exists {
				return nil, errors.New(fmt.Sprintf("No such cluster %s", clusterName))
			}
			cluster := clusters[clusterName]
			unit := newDeploymentUnit(unitID, []client.DeploySpec{}, cluster, auroraConfig, overrideToken)
			unitsMap[*unitID] = unit
		}

		unitsMap[*unitID].deploySpecList = append(unitsMap[*unitID].deploySpecList, spec)
	}

	return unitsMap, nil
}

func deployToReachableClusters(getClient func(unit *deploymentUnit) client.DeployClient, deploymentUnits map[deploymentUnitID]*deploymentUnit, overrideConfig map[string]string) ([]*client.DeployResults, error) {
	deployResult := make(chan *client.DeployResults)

	for _, unit := range deploymentUnits {
		go deployUnit(getClient(unit), unit, overrideConfig, deployResult)
	}

	var allResults []*client.DeployResults
	for i := 0; i < len(deploymentUnits); i++ {
		allResults = append(allResults, <-deployResult)
	}

	return allResults, nil
}

func deployUnit(deployClient client.DeployClient, unit *deploymentUnit, overrideConfig map[string]string, deployResults chan<- *client.DeployResults) {
	if !unit.cluster.Reachable {
		deployResults <- errorDeployResults("Cluster is not reachable", unit)
		return
	}

	var applicationList []string
	for _, spec := range unit.deploySpecList {
		applicationList = append(applicationList, spec.Value("applicationId").(string))
	}

	payload := client.NewDeployPayload(applicationList, overrideConfig)

	result, err := deployClient.Deploy(payload)
	if err != nil {
		deployResults <- errorDeployResults(err.Error(), unit)
	} else {
		deployResults <- result
	}
}

func errorDeployResults(reason string, unit *deploymentUnit) *client.DeployResults {
	var applicationResults []client.DeployResult

	for _, spec := range unit.deploySpecList {
		affiliation := spec.Value("affiliation").(string)
		applicationID := client.NewApplicationId(spec.Value("applicationId").(string))

		result := new(client.DeployResult)
		result.DeployId = "-"
		result.Ignored = false
		result.Success = false
		result.Reason = reason
		result.ADS.Cluster = unit.cluster.Name
		result.ADS.Name = applicationID.Application
		result.ADS.Deploy.Version = "-"
		result.ADS.Environment.Namespace = affiliation + "-" + applicationID.Environment

		applicationResults = append(applicationResults, *result)
	}

	return &client.DeployResults{
		Message: reason,
		Success: false,
		Results: applicationResults,
	}
}

func parseOverride() (map[string]string, error) {
	returnMap := make(map[string]string)
	for i := 0; i < len(flagOverrides); i++ {
		indexByte := strings.IndexByte(flagOverrides[i], ':')
		filename := flagOverrides[i][:indexByte]
		jsonOverride := flagOverrides[i][indexByte+1:]

		if !json.Valid([]byte(jsonOverride)) {
			msg := fmt.Sprintf("%s is not a valid json", jsonOverride)
			return nil, errors.New(msg)
		}

		returnMap[filename] = jsonOverride
	}
	return returnMap, nil
}

func userConfirmation(filteredDeploymentSpecs []client.DeploySpec, out io.Writer) bool {
	header, rows := GetDeploySpecTable(filteredDeploymentSpecs)
	DefaultTablePrinter(header, rows, out)

	var filteredApplications []string
	for _, spec := range filteredDeploymentSpecs {
		appID := spec.Value("applicationId").(string)
		filteredApplications = append(filteredApplications, appID)
	}

	shouldDeploy := true
	if !flagNoPrompt {
		defaultAnswer := len(filteredApplications) == 1
		message := fmt.Sprintf("Do you want to deploy %d application(s)?", len(filteredApplications))
		shouldDeploy = prompt.Confirm(message, defaultAnswer)
	}

	return shouldDeploy
}

func printDeployResult(result []*client.DeployResults, out io.Writer) error {
	var results []client.DeployResult
	for _, r := range result {
		results = append(results, r.Results...)
	}

	if len(results) == 0 {
		return errors.New("No deploys were made")
	}

	sort.Slice(results, func(i, j int) bool {
		return strings.Compare(results[i].ADS.Name, results[j].ADS.Name) < 1
	})

	header, rows := getDeployResultTable(results)
	if len(rows) == 0 {
		return nil
	}

	DefaultTablePrinter(header, rows, out)
	for _, deploy := range results {
		if !deploy.Success {
			return errors.New("One or more deploys failed")
		}
	}

	return nil
}

func getDeployResultTable(deploys []client.DeployResult) (string, []string) {
	var rows []string
	for _, item := range deploys {
		if item.Ignored {
			continue
		}
		ads := item.ADS
		pattern := "%s\t%s\t%s\t%s\t%s\t%s\t%s"
		status := "\x1b[32mDeployed\x1b[0m"
		if !item.Success {
			status = "\x1b[31mFailed\x1b[0m"
		}
		result := fmt.Sprintf(pattern, status, ads.Cluster, ads.Environment.Namespace, ads.Name, ads.Deploy.Version, item.DeployId, item.Reason)
		rows = append(rows, result)
	}

	header := "\x1b[00mSTATUS\x1b[0m\tCLUSTER\tENVIRONMENT\tAPPLICATION\tVERSION\tDEPLOY_ID\tMESSAGE"
	return header, rows
}

func getAPIClient(auroraConfig, overrideToken, overrideCluster string) (*client.ApiClient, error) {
	api := DefaultApiClient
	api.Affiliation = auroraConfig

	if overrideCluster != "" && !AO.Localhost {
		c := AO.Clusters[overrideCluster]
		if !c.Reachable {
			return nil, errors.Errorf("%s cluster is not reachable", overrideCluster)
		}

		api.Host = c.BooberUrl
		api.Token = c.Token
		if overrideToken != "" {
			api.Token = overrideToken
		}
	}

	return api, nil
}

func getDeployClient(unit *deploymentUnit) client.DeployClient {
	var deployClient *client.ApiClient
	if AO.Localhost {
		deployClient = DefaultApiClient
		deployClient.Affiliation = unit.auroraConfig
	} else {
		token := unit.cluster.Token
		if unit.overrideToken != "" {
			token = unit.overrideToken
		}
		deployClient = client.NewApiClient(unit.cluster.BooberUrl, token, unit.auroraConfig, AO.RefName)
	}

	return deployClient
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"net"
	"os"
	"os/signal"
	"runtime"
	"syscall"

	"golang.org/x/net/internal/nettest"
	"golang.org/x/net/ipv4"

	"github.com/msgboxio/context"
	"github.com/msgboxio/ike"
	"github.com/msgboxio/ike/platform"
	"github.com/msgboxio/ike/protocol"
	"github.com/msgboxio/log"
	"github.com/msgboxio/packets"
)

func waitForSignal(cancel context.CancelFunc) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	sig := <-c
	// sig is a ^C, handle it
	cancel(errors.New("received signal: " + sig.String()))
}

func listen(localString string) (p *ipv4.PacketConn, err error) {
	udp, err := net.ListenPacket("udp4", localString)
	if err != nil {
		return
	}
	p = ipv4.NewPacketConn(udp)

	cf := ipv4.FlagTTL | ipv4.FlagSrc | ipv4.FlagDst | ipv4.FlagInterface
	if err := p.SetControlMessage(cf, true); err != nil { // probe before test
		if nettest.ProtocolNotSupported(err) {
			log.Warningf("not supported on %s", runtime.GOOS)
		} else {
			p.Close()
			return nil, err
		}
	}
	return
}

func read(p *ipv4.PacketConn) (b []byte, remoteAddr net.Addr, localIP net.IP, err error) {
	b = make([]byte, 1500)
	n, cm, remoteAddr, err := p.ReadFrom(b)
	if err == nil {
		b = b[:n]
		localIP = cm.Dst
	}
	log.V(1).Infof("%d from %v", n, remoteAddr)
	return
}

func write(p *ipv4.PacketConn, reply []byte, remoteAddr net.Addr) error {
	n, err := p.WriteTo(reply, nil, remoteAddr)
	if err != nil {
		return err
	} else if n != len(reply) {
		return fmt.Errorf("short write: %v of %v", n, len(reply))
	}
	log.V(1).Infof("%d to %v", n, remoteAddr)
	return nil
}

func decode(b []byte) (msg *ike.Message, err error) {
	msg = &ike.Message{}
	if err = msg.DecodeHeader(b); err != nil {
		return
	}
	if len(b) < int(msg.IkeHeader.MsgLength) {
		err = fmt.Errorf("short packet: %v vs %v", len(b), msg.IkeHeader.MsgLength)
		return
	}
	// further decode
	if err = msg.DecodePayloads(b[protocol.IKE_HEADER_LEN:msg.IkeHeader.MsgLength], msg.IkeHeader.NextPayload); err != nil {
		// o.Notify(ERR_INVALID_SYNTAX)
		return
	}
	// decrypt later
	msg.Data = b
	return
}

func main() {
	var localString, remoteString string
	flag.StringVar(&localString, "local", "0.0.0.0:5000", "address to bind to")
	flag.StringVar(&remoteString, "remote", "", "address to connect to")

	var isTunnelMode bool
	flag.BoolVar(&isTunnelMode, "tunnel", false, "use tunnel mode?")

	flag.Set("logtostderr", "true")
	flag.Parse()

	cxt, cancel := context.WithCancel(context.Background())
	go waitForSignal(cancel)

	ids := ike.PskIdentities{
		Primary: "ak@msgbox.io",
		Ids:     map[string][]byte{"ak@msgbox.io": []byte("foo")},
	}

	p, err := listen(localString)
	if err != nil {
		log.Fatal(err)
	}
	log.Infof("socket listening: %s", p.Conn.LocalAddr())

	// this should load the xfrm modules
	// requires root
	if xfrm := platform.ListenForEvents(cxt); xfrm != nil {
		go func() {
			<-xfrm.Done()
			if err := xfrm.Err(); err != context.Canceled {
				log.Fatal(err)
				cancel(err)
			}
			xfrm.Close()
		}()
	}

	// requires root
	if err := platform.SetSocketBypas(p.Conn, syscall.AF_INET); err != nil {
		log.Fatal(err)
	}

	sessions := make(map[uint64]*ike.Session)

	runSession := func(spi uint64, session *ike.Session, to net.Addr) {
		sessions[spi] = session
		for {
			select {
			case reply, ok := <-session.Replies():
				if !ok {
					break
				}
				if err = write(p, reply, to); err != nil {
					session.Close(err)
					break
				}
			case <-session.Done():
				delete(sessions, spi)
				log.Infof("Finished SA 0x%x", spi)
				return
			}
		}
	}

	if remoteString != "" {
		remoteAddr, _ := net.ResolveUDPAddr("udp4", remoteString)
		config := ike.DefaultConfig()
		if !isTunnelMode {
			config.IsTransportMode = true
		}
		initiator := ike.NewInitiator(context.Background(), ids, ike.AddrToIp(remoteAddr).To4(), config)
		spi, _ := packets.ReadB64(initiator.IkeSpiI, 0)
		go runSession(spi, &initiator.Session, remoteAddr)
	}

	go func() {
		// wait for app shutdown
		<-cxt.Done()
		// shutdown sessions
		for _, session := range sessions {
			// reply on this to drain replies
			session.Close(cxt.Err())
			// wait until client is done
			<-session.Done()
		}
		p.Close()
	}()

	for {
		b, remoteAddr, localIP, err := read(p)
		if err != nil {
			log.Fatal(err)
			// ask app to close
			cancel(err)
			break
		}
		msg, err := decode(b)
		if err != nil {
			log.Error(err)
			continue
		}
		msg.LocalIp = localIP
		msg.RemoteIp = ike.AddrToIp(remoteAddr)
		// convert spi to uint64 for map lookup
		spi, _ := packets.ReadB64(msg.IkeHeader.SpiI, 0)
		// check if a session exists
		session, found := sessions[spi]
		if !found {
			// create and run session
			responder, err := ike.NewResponder(context.Background(), ids, msg)
			if err != nil {
				log.Error(err)
				continue
			}
			session = &responder.Session
			go runSession(spi, session, remoteAddr)
		}
		session.HandleMessage(msg)
	}

	<-cxt.Done()
	fmt.Printf("shutdown: %v\n", cxt.Err())
}
<commit_msg>go 1.7 fix for internal package use<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"net"
	"os"
	"os/signal"
	"runtime"
	"syscall"

	"golang.org/x/net/ipv4"

	"github.com/msgboxio/context"
	"github.com/msgboxio/ike"
	"github.com/msgboxio/ike/platform"
	"github.com/msgboxio/ike/protocol"
	"github.com/msgboxio/log"
	"github.com/msgboxio/packets"
)

func waitForSignal(cancel context.CancelFunc) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	sig := <-c
	// sig is a ^C, handle it
	cancel(errors.New("received signal: " + sig.String()))
}

// copied from golang.org/x/net/internal/nettest
func protocolNotSupported(err error) bool {
	switch err := err.(type) {
	case syscall.Errno:
		switch err {
		case syscall.EPROTONOSUPPORT, syscall.ENOPROTOOPT:
			return true
		}
	case *os.SyscallError:
		switch err := err.Err.(type) {
		case syscall.Errno:
			switch err {
			case syscall.EPROTONOSUPPORT, syscall.ENOPROTOOPT:
				return true
			}
		}
	}
	return false
}

func listen(localString string) (p *ipv4.PacketConn, err error) {
	udp, err := net.ListenPacket("udp4", localString)
	if err != nil {
		return
	}
	p = ipv4.NewPacketConn(udp)

	cf := ipv4.FlagTTL | ipv4.FlagSrc | ipv4.FlagDst | ipv4.FlagInterface
	if err := p.SetControlMessage(cf, true); err != nil { // probe before test
		if protocolNotSupported(err) {
			log.Warningf("udp source address detection not supported on %s", runtime.GOOS)
		} else {
			p.Close()
			return nil, err
		}
	}
	return
}

func read(p *ipv4.PacketConn) (b []byte, remoteAddr net.Addr, localIP net.IP, err error) {
	b = make([]byte, 1500)
	n, cm, remoteAddr, err := p.ReadFrom(b)
	if err == nil {
		b = b[:n]
		localIP = cm.Dst
	}
	log.V(1).Infof("%d from %v", n, remoteAddr)
	return
}

func write(p *ipv4.PacketConn, reply []byte, remoteAddr net.Addr) error {
	n, err := p.WriteTo(reply, nil, remoteAddr)
	if err != nil {
		return err
	} else if n != len(reply) {
		return fmt.Errorf("short write: %v of %v", n, len(reply))
	}
	log.V(1).Infof("%d to %v", n, remoteAddr)
	return nil
}

func decode(b []byte) (msg *ike.Message, err error) {
	msg = &ike.Message{}
	if err = msg.DecodeHeader(b); err != nil {
		return
	}
	if len(b) < int(msg.IkeHeader.MsgLength) {
		err = fmt.Errorf("short packet: %v vs %v", len(b), msg.IkeHeader.MsgLength)
		return
	}
	// further decode
	if err = msg.DecodePayloads(b[protocol.IKE_HEADER_LEN:msg.IkeHeader.MsgLength], msg.IkeHeader.NextPayload); err != nil {
		// o.Notify(ERR_INVALID_SYNTAX)
		return
	}
	// decrypt later
	msg.Data = b
	return
}

func main() {
	var localString, remoteString string
	flag.StringVar(&localString, "local", "0.0.0.0:5000", "address to bind to")
	flag.StringVar(&remoteString, "remote", "", "address to connect to")

	var isTunnelMode bool
	flag.BoolVar(&isTunnelMode, "tunnel", false, "use tunnel mode?")

	flag.Set("logtostderr", "true")
	flag.Parse()

	cxt, cancel := context.WithCancel(context.Background())
	go waitForSignal(cancel)

	ids := ike.PskIdentities{
		Primary: "ak@msgbox.io",
		Ids:     map[string][]byte{"ak@msgbox.io": []byte("foo")},
	}

	p, err := listen(localString)
	if err != nil {
		log.Fatal(err)
	}
	log.Infof("socket listening: %s", p.Conn.LocalAddr())

	// this should load the xfrm modules
	// requires root
	if xfrm := platform.ListenForEvents(cxt); xfrm != nil {
		go func() {
			<-xfrm.Done()
			if err := xfrm.Err(); err != context.Canceled {
				log.Fatal(err)
				cancel(err)
			}
			xfrm.Close()
		}()
	}

	// requires root
	if err := platform.SetSocketBypas(p.Conn, syscall.AF_INET); err != nil {
		log.Fatal(err)
	}

	sessions := make(map[uint64]*ike.Session)

	runSession := func(spi uint64, session *ike.Session, to net.Addr) {
		sessions[spi] = session
		for {
			select {
			case reply, ok := <-session.Replies():
				if !ok {
					break
				}
				if err = write(p, reply, to); err != nil {
					session.Close(err)
					break
				}
			case <-session.Done():
				delete(sessions, spi)
				log.Infof("Finished SA 0x%x", spi)
				return
			}
		}
	}

	if remoteString != "" {
		remoteAddr, _ := net.ResolveUDPAddr("udp4", remoteString)
		config := ike.DefaultConfig()
		if !isTunnelMode {
			config.IsTransportMode = true
		}
		initiator := ike.NewInitiator(context.Background(), ids, ike.AddrToIp(remoteAddr).To4(), config)
		spi, _ := packets.ReadB64(initiator.IkeSpiI, 0)
		go runSession(spi, &initiator.Session, remoteAddr)
	}

	go func() {
		// wait for app shutdown
		<-cxt.Done()
		// shutdown sessions
		for _, session := range sessions {
			// reply on this to drain replies
			session.Close(cxt.Err())
			// wait until client is done
			<-session.Done()
		}
		p.Close()
	}()

	for {
		b, remoteAddr, localIP, err := read(p)
		if err != nil {
			log.Fatal(err)
			// ask app to close
			cancel(err)
			break
		}
		msg, err := decode(b)
		if err != nil {
			log.Error(err)
			continue
		}
		msg.LocalIp = localIP
		msg.RemoteIp = ike.AddrToIp(remoteAddr)
		// convert spi to uint64 for map lookup
		spi, _ := packets.ReadB64(msg.IkeHeader.SpiI, 0)
		// check if a session exists
		session, found := sessions[spi]
		if !found {
			// create and run session
			responder, err := ike.NewResponder(context.Background(), ids, msg)
			if err != nil {
				log.Error(err)
				continue
			}
			session = &responder.Session
			go runSession(spi, session, remoteAddr)
		}
		session.HandleMessage(msg)
	}

	<-cxt.Done()
	fmt.Printf("shutdown: %v\n", cxt.Err())
}
<|endoftext|>


<commit_before>package cmd

import (
	"github.com/spf13/cobra"

	"github.com/Shopify/themekit/kit"
)

var updateCmd = &cobra.Command{
	Use:   "update",
	Short: "Update Theme kit to the newest verion.",
	Long: `Update will check for a new release, then
if there is an applicable update it will download it and apply it.

For more documentation please see http://shopify.github.io/themekit/commands/#update
`,
	RunE: func(cmd *cobra.Command, args []string) error {
		return kit.InstallThemeKitVersion(updateVersion)
	},
}
<commit_msg>fixing typo in update command<commit_after>package cmd

import (
	"github.com/spf13/cobra"

	"github.com/Shopify/themekit/kit"
)

var updateCmd = &cobra.Command{
	Use:   "update",
	Short: "Update Theme kit to the newest version.",
	Long: `Update will check for a new release, then
if there is an applicable update it will download it and apply it.

For more documentation please see http://shopify.github.io/themekit/commands/#update
`,
	RunE: func(cmd *cobra.Command, args []string) error {
		return kit.InstallThemeKitVersion(updateVersion)
	},
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"

	"github.com/coreos/kube-aws/cluster"
	"github.com/coreos/kube-aws/config"
	"github.com/spf13/cobra"
)

var (
	cmdUpdate = &cobra.Command{
		Use:          "update",
		Short:        "Update an existing Kubernetes cluster",
		Long:         ``,
		RunE:         runCmdUpdate,
		SilenceUsage: true,
	}

	updateOpts = struct {
		awsDebug bool
		s3URI    string
	}{}
)

func init() {
	RootCmd.AddCommand(cmdUpdate)
	cmdUpdate.Flags().BoolVar(&updateOpts.awsDebug, "aws-debug", false, "Log debug information from aws-sdk-go library")
	cmdUpdate.Flags().StringVar(&updateOpts.s3URI, "s3-uri", "", "When your template is bigger than the cloudformation limit of 51200 bytes, upload the template to the specified location in S3. S3 location expressed as s3://<bucket>/path/to/dir")
}

func runCmdUpdate(cmd *cobra.Command, args []string) error {
	conf, err := config.ClusterFromFile(configPath)
	if err != nil {
		return fmt.Errorf("Failed to read cluster config: %v", err)
	}

	if err := conf.ValidateUserData(stackTemplateOptions); err != nil {
		return err
	}

	data, err := conf.RenderStackTemplate(stackTemplateOptions)
	if err != nil {
		return fmt.Errorf("Failed to render stack template: %v", err)
	}

	cluster := cluster.New(conf, updateOpts.awsDebug)

	report, err := cluster.Update(string(data), updateOpts.s3URI)
	if err != nil {
		return fmt.Errorf("Error updating cluster: %v", err)
	}
	if report != "" {
		fmt.Printf("Update stack: %s\n", report)
	}

	info, err := cluster.Info()
	if err != nil {
		return fmt.Errorf("Failed fetching cluster info: %v", err)
	}

	successMsg :=
		`Success! Your AWS resources are being updated:
`
	fmt.Printf(successMsg, info.String())

	return nil
}
<commit_msg>fix: `kube-aws update` success message formatting issue<commit_after>package cmd

import (
	"fmt"

	"github.com/coreos/kube-aws/cluster"
	"github.com/coreos/kube-aws/config"
	"github.com/spf13/cobra"
)

var (
	cmdUpdate = &cobra.Command{
		Use:          "update",
		Short:        "Update an existing Kubernetes cluster",
		Long:         ``,
		RunE:         runCmdUpdate,
		SilenceUsage: true,
	}

	updateOpts = struct {
		awsDebug bool
		s3URI    string
	}{}
)

func init() {
	RootCmd.AddCommand(cmdUpdate)
	cmdUpdate.Flags().BoolVar(&updateOpts.awsDebug, "aws-debug", false, "Log debug information from aws-sdk-go library")
	cmdUpdate.Flags().StringVar(&updateOpts.s3URI, "s3-uri", "", "When your template is bigger than the cloudformation limit of 51200 bytes, upload the template to the specified location in S3. S3 location expressed as s3://<bucket>/path/to/dir")
}

func runCmdUpdate(cmd *cobra.Command, args []string) error {
	conf, err := config.ClusterFromFile(configPath)
	if err != nil {
		return fmt.Errorf("Failed to read cluster config: %v", err)
	}

	if err := conf.ValidateUserData(stackTemplateOptions); err != nil {
		return err
	}

	data, err := conf.RenderStackTemplate(stackTemplateOptions)
	if err != nil {
		return fmt.Errorf("Failed to render stack template: %v", err)
	}

	cluster := cluster.New(conf, updateOpts.awsDebug)

	report, err := cluster.Update(string(data), updateOpts.s3URI)
	if err != nil {
		return fmt.Errorf("Error updating cluster: %v", err)
	}
	if report != "" {
		fmt.Printf("Update stack: %s\n", report)
	}

	info, err := cluster.Info()
	if err != nil {
		return fmt.Errorf("Failed fetching cluster info: %v", err)
	}

	successMsg :=
		`Success! Your AWS resources are being updated:
%s
`
	fmt.Printf(successMsg, info.String())

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"os"

	"github.com/urfave/cli/v2"
)

func doCreate(c *cli.Context) error {
	var (
		name = c.Args().First()
		vcs  = c.String("vcs")
		w    = c.App.Writer
	)
	u, err := newURL(name, false, true)
	if err != nil {
		return err
	}

	localRepo, err := LocalRepositoryFromURL(u)
	if err != nil {
		return err
	}

	p := localRepo.FullPath
	ok, err := isNotExistOrEmpty(p)
	if err != nil {
		return err
	}
	if !ok {
		return fmt.Errorf("directory %q already exists and not empty", p)
	}
	if err := os.MkdirAll(p, 0755); err != nil {
		return err
	}

	remoteRepo, err := NewRemoteRepository(u)
	if err != nil {
		return nil
	}

	vcsBackend, ok := vcsRegistry[vcs]
	if !ok {
		vcsBackend, u = remoteRepo.VCS()
	}
	initFunc := vcsBackend.Init
	if initFunc == nil {
		return fmt.Errorf("failed to init: unsupported VCS")
	}
	if err := initFunc(p); err != nil {
		return err
	}
	_, err = fmt.Fprintln(w, p)
	return err
}

func isNotExistOrEmpty(name string) (bool, error) {
	f, err := os.Open(name)
	if err != nil {
		if os.IsNotExist(err) {
			return true, nil
		}
		return false, err
	}
	defer f.Close()

	_, err = f.Readdirnames(1)
	if err == io.EOF {
		return true, nil
	}
	return false, err
}
<commit_msg>Fix panic when creating repo with unsupoprted VCS<commit_after>package main

import (
	"fmt"
	"io"
	"os"

	"github.com/urfave/cli/v2"
)

func doCreate(c *cli.Context) error {
	var (
		name = c.Args().First()
		vcs  = c.String("vcs")
		w    = c.App.Writer
	)
	u, err := newURL(name, false, true)
	if err != nil {
		return err
	}

	localRepo, err := LocalRepositoryFromURL(u)
	if err != nil {
		return err
	}

	p := localRepo.FullPath
	ok, err := isNotExistOrEmpty(p)
	if err != nil {
		return err
	}
	if !ok {
		return fmt.Errorf("directory %q already exists and not empty", p)
	}

	remoteRepo, err := NewRemoteRepository(u)
	if err != nil {
		return nil
	}

	vcsBackend, ok := vcsRegistry[vcs]
	if !ok {
		vcsBackend, u = remoteRepo.VCS()
	}
	if vcsBackend == nil {
		return fmt.Errorf("failed to init: unsupported VCS")
	}

	if err := os.MkdirAll(p, 0755); err != nil {
		return err
	}

	initFunc := vcsBackend.Init
	if initFunc == nil {
		return fmt.Errorf("failed to init: unsupported VCS")
	}
	if err := initFunc(p); err != nil {
		return err
	}
	_, err = fmt.Fprintln(w, p)
	return err
}

func isNotExistOrEmpty(name string) (bool, error) {
	f, err := os.Open(name)
	if err != nil {
		if os.IsNotExist(err) {
			return true, nil
		}
		return false, err
	}
	defer f.Close()

	_, err = f.Readdirnames(1)
	if err == io.EOF {
		return true, nil
	}
	return false, err
}
<|endoftext|>


<commit_before>package cluster

import (
	"time"

	lxd "github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/endpoints"
	"github.com/lxc/lxd/lxd/task"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/logger"
	"golang.org/x/net/context"
)

// Events starts a task that continuously monitors the list of cluster nodes and
// maintains a pool of websocket connections against all of them, in order to
// get notified about events.
//
// Whenever an event is received the given callback is invoked.
func Events(endpoints *endpoints.Endpoints, cluster *db.Cluster, f func(int64, interface{})) (task.Func, task.Schedule) {
	listeners := map[int64]*lxd.EventListener{}

	// Update our pool of event listeners.
	update := func(ctx context.Context) {
		// Get the current cluster nodes.
		var nodes []db.NodeInfo
		var offlineThreshold time.Duration

		err := cluster.Transaction(func(tx *db.ClusterTx) error {
			var err error

			nodes, err = tx.Nodes()
			if err != nil {
				return err
			}

			offlineThreshold, err = tx.NodeOfflineThreshold()
			if err != nil {
				return err
			}

			return nil
		})
		if err != nil {
			logger.Warnf("Failed to get current cluster nodes: %v", err)
			return
		}
		if len(nodes) == 1 {
			return // Either we're not clustered or this is a single-node cluster
		}

		address := endpoints.NetworkAddress()

		ids := make([]int, len(nodes))
		for i, node := range nodes {
			ids[i] = int(node.ID)

			// Don't bother trying to connect to offline nodes, or to ourselves.
			if node.IsOffline(offlineThreshold) || node.Address == address {
				continue
			}

			_, ok := listeners[node.ID]

			// The node has already a listener associated to it.
			if ok {
				// Double check that the listener is still
				// connected. If it is, just move on, other
				// we'll try to connect again.
				if listeners[node.ID].Active() {
					continue
				}
				delete(listeners, node.ID)
			}

			listener, err := eventsConnect(node.Address, endpoints.NetworkCert())
			if err != nil {
				logger.Warnf("Failed to get events from node %s: %v", node.Address, err)
				continue
			}
			logger.Debugf("Listening for events on node %s", node.Address)
			listener.AddHandler(nil, func(event interface{}) { f(node.ID, event) })
			listeners[node.ID] = listener
		}
		for id, listener := range listeners {
			if !shared.IntInSlice(int(id), ids) {
				listener.Disconnect()
				delete(listeners, id)
			}
		}
	}

	schedule := task.Every(time.Second)

	return update, schedule
}

// Establish a client connection to get events from the given node.
func eventsConnect(address string, cert *shared.CertInfo) (*lxd.EventListener, error) {
	client, err := Connect(address, cert, true)
	if err != nil {
		return nil, err
	}
	return client.GetEvents()
}
<commit_msg>Make the cluster.Events task non-blocking when it stops<commit_after>package cluster

import (
	"time"

	lxd "github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/endpoints"
	"github.com/lxc/lxd/lxd/task"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/logger"
	"golang.org/x/net/context"
)

// Events starts a task that continuously monitors the list of cluster nodes and
// maintains a pool of websocket connections against all of them, in order to
// get notified about events.
//
// Whenever an event is received the given callback is invoked.
func Events(endpoints *endpoints.Endpoints, cluster *db.Cluster, f func(int64, interface{})) (task.Func, task.Schedule) {
	listeners := map[int64]*lxd.EventListener{}

	// Update our pool of event listeners. Since database queries are
	// blocking, we spawn the actual logic in a goroutine, to abort
	// immediately when we receive the stop signal.
	update := func(ctx context.Context) {
		ch := make(chan struct{})
		go func() {
			eventsUpdateListeners(endpoints, cluster, listeners, f)
			ch <- struct{}{}
		}()
		select {
		case <-ch:
		case <-ctx.Done():
		}

	}

	schedule := task.Every(time.Second)

	return update, schedule
}

func eventsUpdateListeners(endpoints *endpoints.Endpoints, cluster *db.Cluster, listeners map[int64]*lxd.EventListener, f func(int64, interface{})) {
	// Get the current cluster nodes.
	var nodes []db.NodeInfo
	var offlineThreshold time.Duration

	err := cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error

		nodes, err = tx.Nodes()
		if err != nil {
			return err
		}

		offlineThreshold, err = tx.NodeOfflineThreshold()
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		logger.Warnf("Failed to get current cluster nodes: %v", err)
		return
	}
	if len(nodes) == 1 {
		return // Either we're not clustered or this is a single-node cluster
	}

	address := endpoints.NetworkAddress()

	ids := make([]int, len(nodes))
	for i, node := range nodes {
		ids[i] = int(node.ID)

		// Don't bother trying to connect to offline nodes, or to ourselves.
		if node.IsOffline(offlineThreshold) || node.Address == address {
			continue
		}

		_, ok := listeners[node.ID]

		// The node has already a listener associated to it.
		if ok {
			// Double check that the listener is still
			// connected. If it is, just move on, other
			// we'll try to connect again.
			if listeners[node.ID].Active() {
				continue
			}
			delete(listeners, node.ID)
		}

		listener, err := eventsConnect(node.Address, endpoints.NetworkCert())
		if err != nil {
			logger.Warnf("Failed to get events from node %s: %v", node.Address, err)
			continue
		}
		logger.Debugf("Listening for events on node %s", node.Address)
		listener.AddHandler(nil, func(event interface{}) { f(node.ID, event) })
		listeners[node.ID] = listener
	}
	for id, listener := range listeners {
		if !shared.IntInSlice(int(id), ids) {
			listener.Disconnect()
			delete(listeners, id)
		}
	}
}

// Establish a client connection to get events from the given node.
func eventsConnect(address string, cert *shared.CertInfo) (*lxd.EventListener, error) {
	client, err := Connect(address, cert, true)
	if err != nil {
		return nil, err
	}
	return client.GetEvents()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"log"
	"os"
	"strings"

	"github.com/dustin/go-nntp/client"
)

const examplepost = `From: <nobody@example.com>
Newsgroups: misc.test
Subject: Code test
Organization: spy internetworking

This is a test post.
`

func maybefatal(s string, e error) {
	if e != nil {
		log.Fatalf("Error in %s: %v", s, e)
	}
}

func main() {
	server, user, pass := os.Args[1], os.Args[2], os.Args[3]
	c, err := nntpclient.New("tcp", server)
	maybefatal("connecting", err)
	log.Printf("Got banner:  %v", c.Banner)

	// Authenticate
	msg, err := c.Authenticate(user, pass)
	maybefatal("authenticating", err)
	log.Printf("Post authentication message:  %v", msg)

	// Set the reader mode
	_, _, err = c.Command("mode reader", 2)
	maybefatal("setting reader mode", err)

	g, err := c.Group("misc.test")
	maybefatal("grouping", err)
	log.Printf("Got %#v", g)

	n, id, r, err := c.Head(fmt.Sprintf("%v", g.High-1))
	maybefatal("getting head", err)
	log.Printf("msg %d has id %v and the following headers", n, id)
	io.Copy(os.Stdout, r)

	n, id, r, err = c.Body(fmt.Sprintf("%v", n))
	maybefatal("getting body", err)
	log.Printf("Body of message %v", id)
	io.Copy(os.Stdout, r)

	n, id, r, err = c.Article(fmt.Sprintf("%v", n))
	maybefatal("getting the whole thing", err)
	log.Printf("Full message %v", id)
	io.Copy(os.Stdout, r)

	err = c.Post(strings.NewReader(examplepost))
	maybefatal("posting", err)
	log.Printf("Posted!")
}
<commit_msg>Use fmt instead of strconv for simple int stringing<commit_after>package main

import (
	"io"
	"log"
	"os"
	"strconv"
	"strings"

	"github.com/dustin/go-nntp/client"
)

const examplepost = `From: <nobody@example.com>
Newsgroups: misc.test
Subject: Code test
Organization: spy internetworking

This is a test post.
`

func maybefatal(s string, e error) {
	if e != nil {
		log.Fatalf("Error in %s: %v", s, e)
	}
}

func main() {
	server, user, pass := os.Args[1], os.Args[2], os.Args[3]
	c, err := nntpclient.New("tcp", server)
	maybefatal("connecting", err)
	log.Printf("Got banner:  %v", c.Banner)

	// Authenticate
	msg, err := c.Authenticate(user, pass)
	maybefatal("authenticating", err)
	log.Printf("Post authentication message:  %v", msg)

	// Set the reader mode
	_, _, err = c.Command("mode reader", 2)
	maybefatal("setting reader mode", err)

	g, err := c.Group("misc.test")
	maybefatal("grouping", err)
	log.Printf("Got %#v", g)

	n, id, r, err := c.Head(strconv.FormatInt(g.High-1, 10))
	maybefatal("getting head", err)
	log.Printf("msg %d has id %v and the following headers", n, id)
	io.Copy(os.Stdout, r)

	n, id, r, err = c.Body(strconv.FormatInt(n, 10))
	maybefatal("getting body", err)
	log.Printf("Body of message %v", id)
	io.Copy(os.Stdout, r)

	n, id, r, err = c.Article(strconv.FormatInt(n, 10))
	maybefatal("getting the whole thing", err)
	log.Printf("Full message %v", id)
	io.Copy(os.Stdout, r)

	err = c.Post(strings.NewReader(examplepost))
	maybefatal("posting", err)
	log.Printf("Posted!")
}
<|endoftext|>


<commit_before>package statistics

import (
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = FDescribe("AggregatorSnapShot", func() {

	var (
		targetSnapShot  *AggregatorSnapShot
		subjectSnapShot *AggregatorSnapShot
	)

	BeforeEach(func() {
		targetSnapShot = NewAggregatorSnapShot()
		subjectSnapShot = NewAggregatorSnapShot()
	})

	It("Updates an empty snap shot counters", func() {

		key := "counter:1"
		value := int64(101)

		timeStamp := time.Now().UnixNano()
		subjectSnapShot.updateCounter(key, value)
		subjectSnapShot.updateTime(timeStamp)

		targetSnapShot.Update(*subjectSnapShot)

		Expect(len(targetSnapShot.Times)).To(Equal(1))
		Expect(targetSnapShot.Times[0]).To(Equal(subjectSnapShot.Times[0]))
		Expect(targetSnapShot.Counters[key]).ToNot(BeNil())
		Expect(targetSnapShot.Counters[key]).To(Equal(subjectSnapShot.Counters[key]))
	})

	It("Updates an empty snap shot gauges", func() {
		key := "gauge:1"
		value := float64(102.1)

		timeStamp := time.Now().UnixNano()
		subjectSnapShot.updateGauge(key, value)
		subjectSnapShot.updateTime(timeStamp)

		targetSnapShot.Update(*subjectSnapShot)

		Expect(len(targetSnapShot.Times)).To(Equal(1))
		Expect(targetSnapShot.Times[0]).To(Equal(subjectSnapShot.Times[0]))
		Expect(targetSnapShot.Gauges[key]).ToNot(BeNil())
		Expect(targetSnapShot.Gauges[key]).To(Equal(subjectSnapShot.Gauges[key]))
	})

	It("Updates an empty snap shot histograms", func() {
		key := "h:key"
		subKey := "h:subkey"
		value := float64(103.1)

		timeStamp := time.Now().UnixNano()
		subjectSnapShot.updateHistogram(key, subKey, value)
		subjectSnapShot.updateTime(timeStamp)

		targetSnapShot.Update(*subjectSnapShot)

		Expect(len(targetSnapShot.Times)).To(Equal(1))
		Expect(targetSnapShot.Times[0]).To(Equal(subjectSnapShot.Times[0]))
		Expect(targetSnapShot.Histograms[key]).ToNot(BeNil())
		Expect(targetSnapShot.Histograms[key][subKey]).ToNot(BeNil())
		Expect(targetSnapShot.Histograms[key][subKey]).To(Equal(subjectSnapShot.Histograms[key][subKey]))
	})

	It("Updates an empty snap shot meters", func() {
		key := "m:key"
		subKey := "m:subkey"
		value := float64(104.1)

		timeStamp := time.Now().UnixNano()
		subjectSnapShot.updateMeter(key, subKey, value)
		subjectSnapShot.updateTime(timeStamp)

		targetSnapShot.Update(*subjectSnapShot)

		Expect(len(targetSnapShot.Times)).To(Equal(1))
		Expect(targetSnapShot.Times[0]).To(Equal(subjectSnapShot.Times[0]))
		Expect(targetSnapShot.Meters[key]).ToNot(BeNil())
		Expect(targetSnapShot.Meters[key][subKey]).ToNot(BeNil())
		Expect(targetSnapShot.Meters[key][subKey]).To(Equal(subjectSnapShot.Meters[key][subKey]))
	})

	It("Updates an empty snap shot timers", func() {
		key := "t:key"
		subKey := "t:subkey"
		value := float64(105.1)

		timeStamp := time.Now().UnixNano()
		subjectSnapShot.updateTimer(key, subKey, value)
		subjectSnapShot.updateTime(timeStamp)

		targetSnapShot.Update(*subjectSnapShot)

		Expect(len(targetSnapShot.Times)).To(Equal(1))
		Expect(targetSnapShot.Times[0]).To(Equal(subjectSnapShot.Times[0]))
		Expect(targetSnapShot.Timers[key]).ToNot(BeNil())
		Expect(targetSnapShot.Timers[key][subKey]).ToNot(BeNil())
		Expect(targetSnapShot.Timers[key][subKey]).To(Equal(subjectSnapShot.Timers[key][subKey]))
	})

	PIt("Only updates with the last values of the subject snap shot")

})
<commit_msg>Only updates with the last values of the subject snapshot for Counters<commit_after>package statistics

import (
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = FDescribe("AggregatorSnapShot", func() {

	var (
		targetSnapShot  *AggregatorSnapShot
		subjectSnapShot *AggregatorSnapShot
	)

	BeforeEach(func() {
		targetSnapShot = NewAggregatorSnapShot()
		subjectSnapShot = NewAggregatorSnapShot()
	})

	It("Updates an empty snap shot counters", func() {

		key := "counter:1"
		value := int64(101)

		timeStamp := time.Now().UnixNano()
		subjectSnapShot.updateCounter(key, value)
		subjectSnapShot.updateTime(timeStamp)

		targetSnapShot.Update(*subjectSnapShot)

		Expect(len(targetSnapShot.Times)).To(Equal(1))
		Expect(targetSnapShot.Times[0]).To(Equal(subjectSnapShot.Times[0]))
		Expect(targetSnapShot.Counters[key]).ToNot(BeNil())
		Expect(targetSnapShot.Counters[key]).To(Equal(subjectSnapShot.Counters[key]))
	})

	It("Updates an empty snap shot gauges", func() {
		key := "gauge:1"
		value := float64(102.1)

		timeStamp := time.Now().UnixNano()
		subjectSnapShot.updateGauge(key, value)
		subjectSnapShot.updateTime(timeStamp)

		targetSnapShot.Update(*subjectSnapShot)

		Expect(len(targetSnapShot.Times)).To(Equal(1))
		Expect(targetSnapShot.Times[0]).To(Equal(subjectSnapShot.Times[0]))
		Expect(targetSnapShot.Gauges[key]).ToNot(BeNil())
		Expect(targetSnapShot.Gauges[key]).To(Equal(subjectSnapShot.Gauges[key]))
	})

	It("Updates an empty snap shot histograms", func() {
		key := "h:key"
		subKey := "h:subkey"
		value := float64(103.1)

		timeStamp := time.Now().UnixNano()
		subjectSnapShot.updateHistogram(key, subKey, value)
		subjectSnapShot.updateTime(timeStamp)

		targetSnapShot.Update(*subjectSnapShot)

		Expect(len(targetSnapShot.Times)).To(Equal(1))
		Expect(targetSnapShot.Times[0]).To(Equal(subjectSnapShot.Times[0]))
		Expect(targetSnapShot.Histograms[key]).ToNot(BeNil())
		Expect(targetSnapShot.Histograms[key][subKey]).ToNot(BeNil())
		Expect(targetSnapShot.Histograms[key][subKey]).To(Equal(subjectSnapShot.Histograms[key][subKey]))
	})

	It("Updates an empty snap shot meters", func() {
		key := "m:key"
		subKey := "m:subkey"
		value := float64(104.1)

		timeStamp := time.Now().UnixNano()
		subjectSnapShot.updateMeter(key, subKey, value)
		subjectSnapShot.updateTime(timeStamp)

		targetSnapShot.Update(*subjectSnapShot)

		Expect(len(targetSnapShot.Times)).To(Equal(1))
		Expect(targetSnapShot.Times[0]).To(Equal(subjectSnapShot.Times[0]))
		Expect(targetSnapShot.Meters[key]).ToNot(BeNil())
		Expect(targetSnapShot.Meters[key][subKey]).ToNot(BeNil())
		Expect(targetSnapShot.Meters[key][subKey]).To(Equal(subjectSnapShot.Meters[key][subKey]))
	})

	It("Updates an empty snap shot timers", func() {
		key := "t:key"
		subKey := "t:subkey"
		value := float64(105.1)

		timeStamp := time.Now().UnixNano()
		subjectSnapShot.updateTimer(key, subKey, value)
		subjectSnapShot.updateTime(timeStamp)

		targetSnapShot.Update(*subjectSnapShot)

		Expect(len(targetSnapShot.Times)).To(Equal(1))
		Expect(targetSnapShot.Times[0]).To(Equal(subjectSnapShot.Times[0]))
		Expect(targetSnapShot.Timers[key]).ToNot(BeNil())
		Expect(targetSnapShot.Timers[key][subKey]).ToNot(BeNil())
		Expect(targetSnapShot.Timers[key][subKey]).To(Equal(subjectSnapShot.Timers[key][subKey]))
	})

	Context("Only updates with the last values of the subject snap shot for", func() {

		It("Counters", func() {
			count := 10
			key := "counter:1"
			targetSnapShot.updateCounter(key, int64(0))
			targetSnapShot.updateTime(time.Now().UnixNano())
			for i := 0; i < count; i++ {
				subjectSnapShot.updateCounter(key, int64(i+1))
				subjectSnapShot.updateTime(time.Now().UnixNano())
			}
			targetSnapShot.Update(*subjectSnapShot)

			Expect(len(targetSnapShot.Times)).To(Equal(2))
			Expect(targetSnapShot.Counters[key][1]).To(Equal(subjectSnapShot.Counters[key][count-1]))
		})

	})

})
<|endoftext|>


<commit_before>package ocrworker

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"

	"github.com/couchbaselabs/logg"
)

type ConvertPdf struct {
}

func (c ConvertPdf) preprocess(ocrRequest *OcrRequest) error {

	// write bytes to a temp file

	tmpFileNameInput, err := createTempFileName()
	tmpFileNameInput = fmt.Sprintf("%s.pdf", tmpFileNameInput)
	if err != nil {
		return err
	}
	defer os.Remove(tmpFileNameInput)

	tmpFileNameOutput, err := createTempFileName()
	tmpFileNameOutput = fmt.Sprintf("%s.png", tmpFileNameOutput)
	if err != nil {
		return err
	}
	defer os.Remove(tmpFileNameOutput)

	err = saveBytesToFileName(ocrRequest.ImgBytes, tmpFileNameInput)
	if err != nil {
		return err
	}

	logg.LogTo(
		"PREPROCESSOR_WORKER",
		"Convert PDF on %s -> %s",
		tmpFileNameInput,
		tmpFileNameOutput,
	)

	tmpDir, err_tmpdir := createTempDir()
	if err_tmpdir != nil {
		logg.LogFatal("Error running command: %s.", err_tmpdir)
		return err_tmpdir
	}
	defer os.RemoveAll(tmpDir)

	tmpDirFiles := fmt.Sprintf("%s/%s_%s.pdf", tmpDir, ocrRequest.Name, "%03d")
	logg.LogTo(
		"PREPROCESSOR_WORKER",
		"Temp dir is: %s",
		tmpDir,
	)
	logg.LogTo(
		"PREPROCESSOR_WORKER",
		"Temp dir files is: %s",
		tmpDirFiles,
	)

	out_pdftk, err_pdftk := exec.Command(
		"pdftk",
		tmpFileNameInput,
		"burst",
		"dont_ask",
		"output",
		tmpDirFiles,
	).CombinedOutput()
	if err_pdftk != nil {
		logg.LogFatal("Error running command: %s.  out: %s", err_pdftk, out_pdftk)
	}
	logg.LogTo("PREPROCESSOR_WORKER", "output: %v", string(out_pdftk))

	err_walk := filepath.Walk(tmpDir + "/", func(path string, f os.FileInfo, err error) error {
		if err != nil {
			logg.LogFatal("Error running command: %s.", err)
		}
		logg.LogTo("PREPROCESSOR_WORKER", "Path is: %s. Name is: %s.", path, f.Name())
		matched, _ := regexp.MatchString("^.*?_[0-9]{3}\\.pdf", f.Name())
		if matched {
			out_imagemagick, err_imagemagick := exec.Command(
				"convert",
				"-density",
				"300",
				"-depth",
				"8",
				"-alpha",
				"Off",
				path,
				tmpFileNameOutput,
			).CombinedOutput()
			logg.LogTo("PREPROCESSOR_WORKER", "output: %v, error: %v. ", string(out_imagemagick), err_imagemagick)
		}
		return nil
	})
	if err_walk != nil {
		logg.LogFatal("Error running command: %s.", err_walk)
	}

	// read bytes from output file into ocrRequest.ImgBytes
	resultBytes, err := ioutil.ReadFile(tmpFileNameOutput)
	if err != nil {
		return err
	}

	ocrRequest.ImgBytes = resultBytes

	return nil
}
<commit_msg>deliver multiple images files<commit_after>package ocrworker

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"

	"github.com/couchbaselabs/logg"
)

type ConvertPdf struct {
}

func (c ConvertPdf) preprocess(ocrRequest *OcrRequest) error {

	// write bytes to a temp file

	tmpFileNameInput, err := createTempFileName()
	tmpFileNameInput = fmt.Sprintf("%s.pdf", tmpFileNameInput)
	if err != nil {
		return err
	}
	defer os.Remove(tmpFileNameInput)

	tmpFileNameOutput, err := createTempFileName()
	tmpFileNameOutput = fmt.Sprintf("%s.png", tmpFileNameOutput)
	if err != nil {
		return err
	}
	defer os.Remove(tmpFileNameOutput)

	err = saveBytesToFileName(ocrRequest.ImgBytes, tmpFileNameInput)
	if err != nil {
		return err
	}

	logg.LogTo(
		"PREPROCESSOR_WORKER",
		"Convert PDF on %s -> %s",
		tmpFileNameInput,
		tmpFileNameOutput,
	)

	tmpDir, err_tmpdir := createTempDir()
	if err_tmpdir != nil {
		logg.LogFatal("Error running command: %s.", err_tmpdir)
	}
	defer os.RemoveAll(tmpDir)

	tmpDirFiles := fmt.Sprintf("%s/%s_%s.pdf", tmpDir, ocrRequest.Name, "%03d")
	logg.LogTo(
		"PREPROCESSOR_WORKER",
		"Temp dir is: %s",
		tmpDir,
	)
	logg.LogTo(
		"PREPROCESSOR_WORKER",
		"Temp dir files is: %s",
		tmpDirFiles,
	)

	out_pdftk, err_pdftk := exec.Command(
		"pdftk",
		tmpFileNameInput,
		"burst",
		"dont_ask",
		"output",
		tmpDirFiles,
	).CombinedOutput()
	if err_pdftk != nil {
		logg.LogFatal("Error running command: %s.  out: %s", err_pdftk, out_pdftk)
	}
	logg.LogTo("PREPROCESSOR_WORKER", "output: %v", string(out_pdftk))

	err_walk := filepath.Walk(tmpDir + "/", func(path string, f os.FileInfo, err error) error {
		if err != nil {
			logg.LogFatal("Error running command: %s.", err)
		}
		logg.LogTo("PREPROCESSOR_WORKER", "Path is: %s. Name is: %s.", path, f.Name())
		matched, _ := regexp.MatchString("^.*?_[0-9]{3}\\.pdf", f.Name())
		if matched {
			reg := regexp.MustCompile("(^.*_|\\.pdf)")
			i, _ := strconv.Atoi(reg.ReplaceAllString(f.Name(), ""))
			out_imagemagick, err_imagemagick := exec.Command(
				"convert",
				"-density",
				"300",
				"-depth",
				"8",
				"-alpha",
				"Off",
				path,
				tmpFileNameOutput,
			).CombinedOutput()
			logg.LogTo("PREPROCESSOR_WORKER", "output: %v, error: %v. ", string(out_imagemagick), err_imagemagick)
			// read bytes from output file into ocrRequest.ImgBytes
			resultBytes, err := ioutil.ReadFile(tmpFileNameOutput)
			if err != nil {
				return err
			}

			ocrRequest.ImgFiles[i] = resultBytes
		}
		return nil
	})
	if err_walk != nil {
		logg.LogFatal("Error running command: %s.", err_walk)
	}

	logg.LogTo("PREPROCESSOR_WORKER", "length: %v", len(ocrRequest.ImgFiles))

	return nil
}
<|endoftext|>


<commit_before>package flags

import (
	"fmt"
	"path/filepath"
	"reflect"
	"sort"
	"strings"
	"unicode/utf8"
)

// Completion is a type containing information of a completion.
type Completion struct {
	// The completed item
	Item string

	// A description of the completed item (optional)
	Description string
}

type completions []Completion

func (c completions) Len() int {
	return len(c)
}

func (c completions) Less(i, j int) bool {
	return c[i].Item < c[j].Item
}

func (c completions) Swap(i, j int) {
	c[i], c[j] = c[j], c[i]
}

// Completer is an interface which can be implemented by types
// to provide custom command line argument completion.
type Completer interface {
	// Complete receives a prefix representing a (partial) value
	// for its type and should provide a list of possible valid
	// completions.
	Complete(match string) []Completion
}

type completion struct {
	parser *Parser

	ShowDescriptions bool `short:"d" long:"show-descriptions" description:"Show descriptions next to completion items"`
}

// Filename is a string alias which provides filename completion.
type Filename string

func completionsWithoutDescriptions(items []string) []Completion {
	ret := make([]Completion, len(items))

	for i, v := range items {
		ret[i].Item = v
	}

	return ret
}

// Complete returns a list of existing files with the given
// prefix.
func (f *Filename) Complete(match string) []Completion {
	ret, _ := filepath.Glob(match + "*")
	return completionsWithoutDescriptions(ret)
}

func (c *completion) skipPositional(s *parseState, n int) {
	if n >= len(s.positional) {
		s.positional = nil
	} else {
		s.positional = s.positional[n:]
	}
}

func (c *completion) completeOptionNames(names map[string]*Option, prefix string, match string) []Completion {
	n := make([]Completion, 0, len(names))

	for k, opt := range names {
		if strings.HasPrefix(k, match) {
			n = append(n, Completion{
				Item:        prefix + k,
				Description: opt.Description,
			})
		}
	}

	return n
}

func (c *completion) completeLongNames(s *parseState, prefix string, match string) []Completion {
	return c.completeOptionNames(s.lookup.longNames, prefix, match)
}

func (c *completion) completeShortNames(s *parseState, prefix string, match string) []Completion {
	if len(match) != 0 {
		return []Completion{
			Completion{
				Item: prefix + match,
			},
		}
	}

	return c.completeOptionNames(s.lookup.shortNames, prefix, match)
}

func (c *completion) completeCommands(s *parseState, match string) []Completion {
	n := make([]Completion, 0, len(s.command.commands))

	for _, cmd := range s.command.commands {
		if cmd.data != c && strings.HasPrefix(cmd.Name, match) {
			n = append(n, Completion{
				Item:        cmd.Name,
				Description: cmd.ShortDescription,
			})
		}
	}

	return n
}

func (c *completion) completeValue(value reflect.Value, prefix string, match string, isRemaining bool) []Completion {
	// For remaining positional args (that are parsed into a slice), complete
	// based on the element type.
	if isRemaining {
		value = reflect.New(value.Type().Elem())
	}

	i := value.Interface()

	var ret []Completion

	if cmp, ok := i.(Completer); ok {
		ret = cmp.Complete(match)
	} else if value.CanAddr() {
		if cmp, ok = value.Addr().Interface().(Completer); ok {
			ret = cmp.Complete(match)
		}
	}

	for i, v := range ret {
		ret[i].Item = prefix + v.Item
	}

	return ret
}

func (c *completion) complete(args []string) []Completion {
	if len(args) == 0 {
		args = []string{""}
	}

	s := &parseState{
		args: args,
	}

	c.parser.fillParseState(s)

	var opt *Option

	for len(s.args) > 1 {
		arg := s.pop()

		if (c.parser.Options&PassDoubleDash) != None && arg == "--" {
			opt = nil
			c.skipPositional(s, len(s.args)-1)

			break
		}

		if argumentIsOption(arg) {
			prefix, optname, islong := stripOptionPrefix(arg)
			optname, _, argument := splitOption(prefix, optname, islong)

			if argument == nil {
				var o *Option
				canarg := true

				if islong {
					o = s.lookup.longNames[optname]
				} else {
					for i, r := range optname {
						sname := string(r)
						o = s.lookup.shortNames[sname]

						if o == nil {
							break
						}

						if i == 0 && o.canArgument() && len(optname) != len(sname) {
							canarg = false
							break
						}
					}
				}

				if o == nil && (c.parser.Options&PassAfterNonOption) != None {
					opt = nil
					c.skipPositional(s, len(s.args)-1)

					break
				} else if o != nil && o.canArgument() && !o.OptionalArgument && canarg {
					if len(s.args) > 1 {
						s.pop()
					} else {
						opt = o
					}
				}
			}
		} else {
			if len(s.positional) > 0 {
				if !s.positional[0].isRemaining() {
					// Don't advance beyond a remaining positional arg (because
					// it consumes all subsequent args).
					s.positional = s.positional[1:]
				}
			} else if cmd, ok := s.lookup.commands[arg]; ok {
				cmd.fillParseState(s)
			}

			opt = nil
		}
	}

	lastarg := s.args[len(s.args)-1]
	var ret []Completion

	if opt != nil {
		// Completion for the argument of 'opt'
		ret = c.completeValue(opt.value, "", lastarg, false)
	} else if argumentIsOption(lastarg) {
		// Complete the option
		prefix, optname, islong := stripOptionPrefix(lastarg)
		optname, split, argument := splitOption(prefix, optname, islong)

		if argument == nil && !islong {
			rname, n := utf8.DecodeRuneInString(optname)
			sname := string(rname)

			if opt := s.lookup.shortNames[sname]; opt != nil && opt.canArgument() {
				ret = c.completeValue(opt.value, prefix+sname, optname[n:], false)
			} else {
				ret = c.completeShortNames(s, prefix, optname)
			}
		} else if argument != nil {
			if islong {
				opt = s.lookup.longNames[optname]
			} else {
				opt = s.lookup.shortNames[optname]
			}

			if opt != nil {
				ret = c.completeValue(opt.value, prefix+optname+split, *argument, false)
			}
		} else if islong {
			ret = c.completeLongNames(s, prefix, optname)
		} else {
			ret = c.completeShortNames(s, prefix, optname)
		}
	} else if len(s.positional) > 0 {
		// Complete for positional argument
		ret = c.completeValue(s.positional[0].value, "", lastarg, s.positional[0].isRemaining())
	} else if len(s.command.commands) > 0 {
		// Complete for command
		ret = c.completeCommands(s, lastarg)
	}

	sort.Sort(completions(ret))
	return ret
}

func (c *completion) Execute(args []string) error {
	ret := c.complete(args)

	if c.ShowDescriptions && len(ret) > 1 {
		maxl := 0

		for _, v := range ret {
			if len(v.Item) > maxl {
				maxl = len(v.Item)
			}
		}

		for _, v := range ret {
			fmt.Printf("%s", v.Item)

			if len(v.Description) > 0 {
				fmt.Printf("%s  # %s", strings.Repeat(" ", maxl-len(v.Item)), v.Description)
			}

			fmt.Printf("\n")
		}
	} else {
		for _, v := range ret {
			fmt.Println(v.Item)
		}
	}

	return nil
}
<commit_msg>Factor out completeArg<commit_after>package flags

import (
	"fmt"
	"path/filepath"
	"reflect"
	"sort"
	"strings"
	"unicode/utf8"
)

// Completion is a type containing information of a completion.
type Completion struct {
	// The completed item
	Item string

	// A description of the completed item (optional)
	Description string
}

type completions []Completion

func (c completions) Len() int {
	return len(c)
}

func (c completions) Less(i, j int) bool {
	return c[i].Item < c[j].Item
}

func (c completions) Swap(i, j int) {
	c[i], c[j] = c[j], c[i]
}

// Completer is an interface which can be implemented by types
// to provide custom command line argument completion.
type Completer interface {
	// Complete receives a prefix representing a (partial) value
	// for its type and should provide a list of possible valid
	// completions.
	Complete(match string) []Completion
}

type completion struct {
	parser *Parser

	ShowDescriptions bool `short:"d" long:"show-descriptions" description:"Show descriptions next to completion items"`
}

// Filename is a string alias which provides filename completion.
type Filename string

func completionsWithoutDescriptions(items []string) []Completion {
	ret := make([]Completion, len(items))

	for i, v := range items {
		ret[i].Item = v
	}

	return ret
}

// Complete returns a list of existing files with the given
// prefix.
func (f *Filename) Complete(match string) []Completion {
	ret, _ := filepath.Glob(match + "*")
	return completionsWithoutDescriptions(ret)
}

func (c *completion) skipPositional(s *parseState, n int) {
	if n >= len(s.positional) {
		s.positional = nil
	} else {
		s.positional = s.positional[n:]
	}
}

func (c *completion) completeOptionNames(names map[string]*Option, prefix string, match string) []Completion {
	n := make([]Completion, 0, len(names))

	for k, opt := range names {
		if strings.HasPrefix(k, match) {
			n = append(n, Completion{
				Item:        prefix + k,
				Description: opt.Description,
			})
		}
	}

	return n
}

func (c *completion) completeLongNames(s *parseState, prefix string, match string) []Completion {
	return c.completeOptionNames(s.lookup.longNames, prefix, match)
}

func (c *completion) completeShortNames(s *parseState, prefix string, match string) []Completion {
	if len(match) != 0 {
		return []Completion{
			Completion{
				Item: prefix + match,
			},
		}
	}

	return c.completeOptionNames(s.lookup.shortNames, prefix, match)
}

func (c *completion) completeCommands(s *parseState, match string) []Completion {
	n := make([]Completion, 0, len(s.command.commands))

	for _, cmd := range s.command.commands {
		if cmd.data != c && strings.HasPrefix(cmd.Name, match) {
			n = append(n, Completion{
				Item:        cmd.Name,
				Description: cmd.ShortDescription,
			})
		}
	}

	return n
}

func (c *completion) completeValue(value reflect.Value, prefix string, match string) []Completion {
	i := value.Interface()

	var ret []Completion

	if cmp, ok := i.(Completer); ok {
		ret = cmp.Complete(match)
	} else if value.CanAddr() {
		if cmp, ok = value.Addr().Interface().(Completer); ok {
			ret = cmp.Complete(match)
		}
	}

	for i, v := range ret {
		ret[i].Item = prefix + v.Item
	}

	return ret
}

func (c *completion) completeArg(arg *Arg, prefix string, match string) []Completion {
	if arg.isRemaining() {
		// For remaining positional args (that are parsed into a slice), complete
		// based on the element type.
		return c.completeValue(reflect.New(arg.value.Type().Elem()), prefix, match)
	}

	return c.completeValue(arg.value, prefix, match)
}

func (c *completion) complete(args []string) []Completion {
	if len(args) == 0 {
		args = []string{""}
	}

	s := &parseState{
		args: args,
	}

	c.parser.fillParseState(s)

	var opt *Option

	for len(s.args) > 1 {
		arg := s.pop()

		if (c.parser.Options&PassDoubleDash) != None && arg == "--" {
			opt = nil
			c.skipPositional(s, len(s.args)-1)

			break
		}

		if argumentIsOption(arg) {
			prefix, optname, islong := stripOptionPrefix(arg)
			optname, _, argument := splitOption(prefix, optname, islong)

			if argument == nil {
				var o *Option
				canarg := true

				if islong {
					o = s.lookup.longNames[optname]
				} else {
					for i, r := range optname {
						sname := string(r)
						o = s.lookup.shortNames[sname]

						if o == nil {
							break
						}

						if i == 0 && o.canArgument() && len(optname) != len(sname) {
							canarg = false
							break
						}
					}
				}

				if o == nil && (c.parser.Options&PassAfterNonOption) != None {
					opt = nil
					c.skipPositional(s, len(s.args)-1)

					break
				} else if o != nil && o.canArgument() && !o.OptionalArgument && canarg {
					if len(s.args) > 1 {
						s.pop()
					} else {
						opt = o
					}
				}
			}
		} else {
			if len(s.positional) > 0 {
				if !s.positional[0].isRemaining() {
					// Don't advance beyond a remaining positional arg (because
					// it consumes all subsequent args).
					s.positional = s.positional[1:]
				}
			} else if cmd, ok := s.lookup.commands[arg]; ok {
				cmd.fillParseState(s)
			}

			opt = nil
		}
	}

	lastarg := s.args[len(s.args)-1]
	var ret []Completion

	if opt != nil {
		// Completion for the argument of 'opt'
		ret = c.completeValue(opt.value, "", lastarg)
	} else if argumentIsOption(lastarg) {
		// Complete the option
		prefix, optname, islong := stripOptionPrefix(lastarg)
		optname, split, argument := splitOption(prefix, optname, islong)

		if argument == nil && !islong {
			rname, n := utf8.DecodeRuneInString(optname)
			sname := string(rname)

			if opt := s.lookup.shortNames[sname]; opt != nil && opt.canArgument() {
				ret = c.completeValue(opt.value, prefix+sname, optname[n:])
			} else {
				ret = c.completeShortNames(s, prefix, optname)
			}
		} else if argument != nil {
			if islong {
				opt = s.lookup.longNames[optname]
			} else {
				opt = s.lookup.shortNames[optname]
			}

			if opt != nil {
				ret = c.completeValue(opt.value, prefix+optname+split, *argument)
			}
		} else if islong {
			ret = c.completeLongNames(s, prefix, optname)
		} else {
			ret = c.completeShortNames(s, prefix, optname)
		}
	} else if len(s.positional) > 0 {
		// Complete for positional argument
		ret = c.completeArg(s.positional[0], "", lastarg)
	} else if len(s.command.commands) > 0 {
		// Complete for command
		ret = c.completeCommands(s, lastarg)
	}

	sort.Sort(completions(ret))
	return ret
}

func (c *completion) Execute(args []string) error {
	ret := c.complete(args)

	if c.ShowDescriptions && len(ret) > 1 {
		maxl := 0

		for _, v := range ret {
			if len(v.Item) > maxl {
				maxl = len(v.Item)
			}
		}

		for _, v := range ret {
			fmt.Printf("%s", v.Item)

			if len(v.Description) > 0 {
				fmt.Printf("%s  # %s", strings.Repeat(" ", maxl-len(v.Item)), v.Description)
			}

			fmt.Printf("\n")
		}
	} else {
		for _, v := range ret {
			fmt.Println(v.Item)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Compose, an IBM Company
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// The composeapi package provides an idiomatic Go wrapper around the Compose
// API for database platform for deployment, management and monitoring.
package composeapi

import (
	"encoding/json"
	"fmt"
	"log"

	"github.com/parnurzeal/gorequest"
)

var ()

const (
	apibase = "https://api.compose.io/2016-07/"
)

// Client is a structure that holds session information for the API
type Client struct {
	apiToken string
}

// NewClient returns a Client for further interaction with the API
func NewClient(apiToken string) (*Client, error) {
	return &Client{
		apiToken: apiToken,
	}, nil
}

// Link structure for JSON+HAL links
type Link struct {
	HREF      string `json:"href"`
	Templated bool   `json:"templated"`
}

//Errors struct for parsing error returns
type Errors struct {
	Error map[string][]string `json:"errors,omitempty"`
}

func printJSON(jsontext string) {
	var tempholder map[string]interface{}

	if err := json.Unmarshal([]byte(jsontext), &tempholder); err != nil {
		log.Fatal(err)
	}
	indentedjson, err := json.MarshalIndent(tempholder, "", "  ")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(string(indentedjson))
}

//SetAPIToken overrides the API token
func (c *Client) SetAPIToken(newtoken string) {
	c.apiToken = newtoken
}

//GetJSON Gets JSON string of content at an endpoint
func (c *Client) getJSON(endpoint string) (string, []error) {
	response, body, errs := gorequest.New().Get(apibase+endpoint).
		Set("Authorization", "Bearer "+c.apiToken).
		Set("Content-type", "json").
		End()
	if response.StatusCode != 200 {
		myerrors := Errors{}
		err := json.Unmarshal([]byte(body), &myerrors)
		if err != nil {
			errs = append(errs, fmt.Errorf("Unable to parse error - status code %d - body %s", response.StatusCode, response.Body))
		} else {
			errs = append(errs, fmt.Errorf("%v", myerrors.Error))
		}
	}
	return body, errs
}
<commit_msg>Changed comment for compatibility<commit_after>// Copyright 2016 Compose, an IBM Company
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Package composeapi provides an idiomatic Go wrapper around the Compose
// API for database platform for deployment, management and monitoring.
package composeapi

import (
	"encoding/json"
	"fmt"
	"log"

	"github.com/parnurzeal/gorequest"
)

var ()

const (
	apibase = "https://api.compose.io/2016-07/"
)

// Client is a structure that holds session information for the API
type Client struct {
	apiToken string
}

// NewClient returns a Client for further interaction with the API
func NewClient(apiToken string) (*Client, error) {
	return &Client{
		apiToken: apiToken,
	}, nil
}

// Link structure for JSON+HAL links
type Link struct {
	HREF      string `json:"href"`
	Templated bool   `json:"templated"`
}

//Errors struct for parsing error returns
type Errors struct {
	Error map[string][]string `json:"errors,omitempty"`
}

func printJSON(jsontext string) {
	var tempholder map[string]interface{}

	if err := json.Unmarshal([]byte(jsontext), &tempholder); err != nil {
		log.Fatal(err)
	}
	indentedjson, err := json.MarshalIndent(tempholder, "", "  ")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(string(indentedjson))
}

//SetAPIToken overrides the API token
func (c *Client) SetAPIToken(newtoken string) {
	c.apiToken = newtoken
}

//GetJSON Gets JSON string of content at an endpoint
func (c *Client) getJSON(endpoint string) (string, []error) {
	response, body, errs := gorequest.New().Get(apibase+endpoint).
		Set("Authorization", "Bearer "+c.apiToken).
		Set("Content-type", "json").
		End()
	if response.StatusCode != 200 {
		myerrors := Errors{}
		err := json.Unmarshal([]byte(body), &myerrors)
		if err != nil {
			errs = append(errs, fmt.Errorf("Unable to parse error - status code %d - body %s", response.StatusCode, response.Body))
		} else {
			errs = append(errs, fmt.Errorf("%v", myerrors.Error))
		}
	}
	return body, errs
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"log"
	"net/http"
	_ "net/http/pprof"
	"path"
	"time"

	"go.pedge.io/lion"

	"golang.org/x/sync/errgroup"

	etcd "github.com/coreos/etcd/clientv3"
	"github.com/gogo/protobuf/proto"
	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/server/pkg/cmdutil"
	"github.com/pachyderm/pachyderm/src/server/pkg/worker"
	"google.golang.org/grpc"
)

// appEnv stores the environment variables that this worker needs
type appEnv struct {
	// Address of etcd, so that worker can write its own IP there for discoverh
	EtcdAddress string `env:"ETCD_PORT_2379_TCP_ADDR,required"`

	// Address for connecting to pachd (so this can download input data)
	PachdAddress string `env:"PACHD_PORT_650_TCP_ADDR,required"`

	// Prefix in etcd for all pachd-related records
	PPSPrefix string `env:"PPS_ETCD_PREFIX,required"`

	// worker gets its own IP here, via the k8s downward API. It then writes that
	// IP back to etcd so that pachd can discover it
	PPSWorkerIP string `env:"PPS_WORKER_IP,required"`

	// The name of the pipeline that this worker belongs to
	PPSPipelineName string `env:"PPS_PIPELINE_NAME"`

	// The name of this pod
	PodName string `env:"PPS_POD_NAME,required"`

	// The namespace in which Pachyderm is deployed
	Namespace string `env:"PPS_NAMESPACE"`
}

func main() {
	cmdutil.Main(do, &appEnv{})
}

func validateEnv(appEnv *appEnv) error {
	if appEnv.PPSPipelineName == "" {
		return fmt.Errorf("worker must recieve pipeline name")
	}
	return nil
}

// getPipelineInfo gets the PipelineInfo proto describing the pipeline that this
// worker is part of
func getPipelineInfo(etcdClient *etcd.Client, appEnv *appEnv) (*pps.PipelineInfo, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()
	resp, err := etcdClient.Get(ctx, path.Join(appEnv.PPSPrefix, "pipelines", appEnv.PPSPipelineName))
	if err != nil {
		return nil, err
	}
	if len(resp.Kvs) != 1 {
		return nil, fmt.Errorf("expected to find 1 pipeline, got %d: %v", len(resp.Kvs), resp)
	}
	pipelineInfo := new(pps.PipelineInfo)
	if err := proto.UnmarshalText(string(resp.Kvs[0].Value), pipelineInfo); err != nil {
		return nil, err
	}
	return pipelineInfo, nil
}

func do(appEnvObj interface{}) error {
	go func() {
		lion.Println(http.ListenAndServe(":652", nil))
	}()

	log.SetFlags(log.LstdFlags | log.Lshortfile)
	appEnv := appEnvObj.(*appEnv)
	if err := validateEnv(appEnv); err != nil {
		return fmt.Errorf("error validating env: %v", err)
	}

	// get pachd client, so we can upload output data from the user binary
	pachClient, err := client.NewFromAddress("localhost:650")
	if err != nil {
		return fmt.Errorf("error constructing pachClient: %v", err)
	}
	go pachClient.KeepConnected(make(chan bool)) // we never cancel the connection

	// Get etcd client, so we can register our IP (so pachd can discover us)
	etcdClient, err := etcd.New(etcd.Config{
		Endpoints:   []string{fmt.Sprintf("%s:2379", appEnv.EtcdAddress)},
		DialOptions: client.EtcdDialOptions(),
	})
	if err != nil {
		return fmt.Errorf("error constructing etcdClient: %v", err)
	}

	// Construct worker API server.
	pipelineInfo, err := getPipelineInfo(etcdClient, appEnv)
	if err != nil {
		return fmt.Errorf("error getting pipelineInfo: %v", err)
	}
	workerRcName := pps.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version)
	apiServer, err := worker.NewAPIServer(pachClient, etcdClient, appEnv.PPSPrefix, pipelineInfo, appEnv.PodName, appEnv.Namespace)
	if err != nil {
		return err
	}

	// Start worker api server
	eg := errgroup.Group{}
	ready := make(chan error)
	eg.Go(func() error {
		return grpcutil.Serve(
			func(s *grpc.Server) {
				worker.RegisterWorkerServer(s, apiServer)
				close(ready)
			},
			grpcutil.ServeOptions{
				Version:    version.Version,
				MaxMsgSize: grpcutil.MaxMsgSize,
			},
			grpcutil.ServeEnv{
				GRPCPort: client.PPSWorkerPort,
			},
		)
	})

	// Wait until server is ready, then put our IP address into etcd, so pachd can
	// discover us
	<-ready
	key := path.Join(appEnv.PPSPrefix, "workers", workerRcName, appEnv.PPSWorkerIP)

	// Prepare to write "key" into etcd by creating lease -- if worker dies, our
	// IP will be removed from etcd
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	resp, err := etcdClient.Grant(ctx, 10 /* seconds */)
	if err != nil {
		return fmt.Errorf("error granting lease: %v", err)
	}

	// keepalive forever
	if _, err := etcdClient.KeepAlive(context.Background(), resp.ID); err != nil {
		return fmt.Errorf("error with KeepAlive: %v", err)
	}

	// Actually write "key" into etcd
	ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) // new ctx
	defer cancel()
	if _, err := etcdClient.Put(ctx, key, "", etcd.WithLease(resp.ID)); err != nil {
		return fmt.Errorf("error putting IP address: %v", err)
	}

	// If server ever exits, return error
	if err := eg.Wait(); err != nil {
		return err
	}
	return nil
}
<commit_msg>Fix an issue where the worker cannot use PPS API<commit_after>package main

import (
	"context"
	"fmt"
	"log"
	"net/http"
	_ "net/http/pprof"
	"path"
	"time"

	"go.pedge.io/lion"

	"golang.org/x/sync/errgroup"

	etcd "github.com/coreos/etcd/clientv3"
	"github.com/gogo/protobuf/proto"
	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/server/pkg/cmdutil"
	"github.com/pachyderm/pachyderm/src/server/pkg/worker"
	"google.golang.org/grpc"
)

// appEnv stores the environment variables that this worker needs
type appEnv struct {
	// Address of etcd, so that worker can write its own IP there for discoverh
	EtcdAddress string `env:"ETCD_PORT_2379_TCP_ADDR,required"`

	// Address for connecting to pachd (so this can download input data)
	PachdAddress string `env:"PACHD_PORT_650_TCP_ADDR,required"`

	// Prefix in etcd for all pachd-related records
	PPSPrefix string `env:"PPS_ETCD_PREFIX,required"`

	// worker gets its own IP here, via the k8s downward API. It then writes that
	// IP back to etcd so that pachd can discover it
	PPSWorkerIP string `env:"PPS_WORKER_IP,required"`

	// The name of the pipeline that this worker belongs to
	PPSPipelineName string `env:"PPS_PIPELINE_NAME"`

	// The name of this pod
	PodName string `env:"PPS_POD_NAME,required"`

	// The namespace in which Pachyderm is deployed
	Namespace string `env:"PPS_NAMESPACE,required"`
}

func main() {
	cmdutil.Main(do, &appEnv{})
}

func validateEnv(appEnv *appEnv) error {
	if appEnv.PPSPipelineName == "" {
		return fmt.Errorf("worker must recieve pipeline name")
	}
	return nil
}

// getPipelineInfo gets the PipelineInfo proto describing the pipeline that this
// worker is part of
func getPipelineInfo(etcdClient *etcd.Client, appEnv *appEnv) (*pps.PipelineInfo, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()
	resp, err := etcdClient.Get(ctx, path.Join(appEnv.PPSPrefix, "pipelines", appEnv.PPSPipelineName))
	if err != nil {
		return nil, err
	}
	if len(resp.Kvs) != 1 {
		return nil, fmt.Errorf("expected to find 1 pipeline, got %d: %v", len(resp.Kvs), resp)
	}
	pipelineInfo := new(pps.PipelineInfo)
	if err := proto.UnmarshalText(string(resp.Kvs[0].Value), pipelineInfo); err != nil {
		return nil, err
	}
	return pipelineInfo, nil
}

func do(appEnvObj interface{}) error {
	go func() {
		lion.Println(http.ListenAndServe(":652", nil))
	}()

	log.SetFlags(log.LstdFlags | log.Lshortfile)
	appEnv := appEnvObj.(*appEnv)
	if err := validateEnv(appEnv); err != nil {
		return fmt.Errorf("error validating env: %v", err)
	}

	// Construct a client that connects to the sidecar.
	pachClient, err := client.NewFromAddress("localhost:650")
	if err != nil {
		return fmt.Errorf("error constructing pachClient: %v", err)
	}
	go pachClient.KeepConnected(make(chan bool)) // we never cancel the connection

	// Construct a client that connects to real pachd nodes
	_ppsClient, err := client.NewFromAddress(fmt.Sprintf("%v:650", appEnv.PachdAddress))
	if err != nil {
		return err
	}
	go _ppsClient.KeepConnected(make(chan bool)) // we never cancel the connection
	// This is hacky but it makes the code simpler.  Basically any PFS calls
	// will go to the sidecar, whereas PPS calls go to real pachd nodes.
	pachClient.PpsAPIClient = _ppsClient.PpsAPIClient

	// Get etcd client, so we can register our IP (so pachd can discover us)
	etcdClient, err := etcd.New(etcd.Config{
		Endpoints:   []string{fmt.Sprintf("%s:2379", appEnv.EtcdAddress)},
		DialOptions: client.EtcdDialOptions(),
	})
	if err != nil {
		return fmt.Errorf("error constructing etcdClient: %v", err)
	}

	// Construct worker API server.
	pipelineInfo, err := getPipelineInfo(etcdClient, appEnv)
	if err != nil {
		return fmt.Errorf("error getting pipelineInfo: %v", err)
	}
	workerRcName := pps.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version)
	apiServer, err := worker.NewAPIServer(pachClient, etcdClient, appEnv.PPSPrefix, pipelineInfo, appEnv.PodName, appEnv.Namespace)
	if err != nil {
		return err
	}

	// Start worker api server
	eg := errgroup.Group{}
	ready := make(chan error)
	eg.Go(func() error {
		return grpcutil.Serve(
			func(s *grpc.Server) {
				worker.RegisterWorkerServer(s, apiServer)
				close(ready)
			},
			grpcutil.ServeOptions{
				Version:    version.Version,
				MaxMsgSize: grpcutil.MaxMsgSize,
			},
			grpcutil.ServeEnv{
				GRPCPort: client.PPSWorkerPort,
			},
		)
	})

	// Wait until server is ready, then put our IP address into etcd, so pachd can
	// discover us
	<-ready
	key := path.Join(appEnv.PPSPrefix, "workers", workerRcName, appEnv.PPSWorkerIP)

	// Prepare to write "key" into etcd by creating lease -- if worker dies, our
	// IP will be removed from etcd
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	resp, err := etcdClient.Grant(ctx, 10 /* seconds */)
	if err != nil {
		return fmt.Errorf("error granting lease: %v", err)
	}

	// keepalive forever
	if _, err := etcdClient.KeepAlive(context.Background(), resp.ID); err != nil {
		return fmt.Errorf("error with KeepAlive: %v", err)
	}

	// Actually write "key" into etcd
	ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) // new ctx
	defer cancel()
	if _, err := etcdClient.Put(ctx, key, "", etcd.WithLease(resp.ID)); err != nil {
		return fmt.Errorf("error putting IP address: %v", err)
	}

	// If server ever exits, return error
	if err := eg.Wait(); err != nil {
		return err
	}
	return nil
}
<|endoftext|>