<commit_before>// Copyright 2018 The Terraformer Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package providerwrapper //nolint

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/terraformer/terraformutils/terraformerstring"

	"github.com/zclconf/go-cty/cty"

	"github.com/hashicorp/go-hclog"
	"github.com/hashicorp/go-plugin"
	"github.com/hashicorp/terraform/configs/configschema"
	tfplugin "github.com/hashicorp/terraform/plugin"
	"github.com/hashicorp/terraform/providers"
	"github.com/hashicorp/terraform/terraform"
	"github.com/hashicorp/terraform/version"
)

// DefaultDataDir is the default directory for storing local data.
const DefaultDataDir = ".terraform"

// DefaultPluginVendorDir is the location in the config directory to look for
// user-added plugin binaries. Terraform only reads from this path if it
// exists, it is never created by terraform.
const DefaultPluginVendorDirV12 = "terraform.d/plugins/" + pluginMachineName

// pluginMachineName is the directory name used in new plugin paths.
const pluginMachineName = runtime.GOOS + "_" + runtime.GOARCH

type ProviderWrapper struct {
	Provider     *tfplugin.GRPCProvider
	client       *plugin.Client
	rpcClient    plugin.ClientProtocol
	providerName string
	config       cty.Value
	schema       *providers.GetSchemaResponse
	retryCount   int
	retrySleepMs int
}

func NewProviderWrapper(providerName string, providerConfig cty.Value, verbose bool, options ...map[string]int) (*ProviderWrapper, error) {
	p := &ProviderWrapper{retryCount: 5, retrySleepMs: 300}
	p.providerName = providerName
	p.config = providerConfig

	if len(options) > 0 {
		retryCount, hasOption := options[0]["retryCount"]
		if hasOption {
			p.retryCount = retryCount
		}
		retrySleepMs, hasOption := options[0]["retrySleepMs"]
		if hasOption {
			p.retrySleepMs = retrySleepMs
		}
	}

	err := p.initProvider(verbose)

	return p, err
}

func (p *ProviderWrapper) Kill() {
	p.client.Kill()
}

func (p *ProviderWrapper) GetSchema() *providers.GetSchemaResponse {
	if p.schema == nil {
		r := p.Provider.GetSchema()
		p.schema = &r
	}
	return p.schema
}

func (p *ProviderWrapper) GetReadOnlyAttributes(resourceTypes []string) (map[string][]string, error) {
	r := p.GetSchema()

	if r.Diagnostics.HasErrors() {
		return nil, r.Diagnostics.Err()
	}
	readOnlyAttributes := map[string][]string{}
	for resourceName, obj := range r.ResourceTypes {
		if terraformerstring.ContainsString(resourceTypes, resourceName) {
			readOnlyAttributes[resourceName] = append(readOnlyAttributes[resourceName], "^id$")
			for k, v := range obj.Block.Attributes {
				if !v.Optional && !v.Required {
					if v.Type.IsListType() || v.Type.IsSetType() {
						readOnlyAttributes[resourceName] = append(readOnlyAttributes[resourceName], "^"+k+"\\.(.*)")
					} else {
						readOnlyAttributes[resourceName] = append(readOnlyAttributes[resourceName], "^"+k+"$")
					}
				}
			}
			readOnlyAttributes[resourceName] = p.readObjBlocks(obj.Block.BlockTypes, readOnlyAttributes[resourceName], "-1")
		}
	}
	return readOnlyAttributes, nil
}

func (p *ProviderWrapper) readObjBlocks(block map[string]*configschema.NestedBlock, readOnlyAttributes []string, parent string) []string {
	for k, v := range block {
		if len(v.BlockTypes) > 0 {
			if parent == "-1" {
				readOnlyAttributes = p.readObjBlocks(v.BlockTypes, readOnlyAttributes, k)
			} else {
				readOnlyAttributes = p.readObjBlocks(v.BlockTypes, readOnlyAttributes, parent+"\\.[0-9]+\\."+k)
			}
		}
		fieldCount := 0
		for key, l := range v.Attributes {
			if !l.Optional && !l.Required {
				fieldCount++
				switch v.Nesting {
				case configschema.NestingList:
					if parent == "-1" {
						readOnlyAttributes = append(readOnlyAttributes, "^"+k+"\\.[0-9]+\\."+key+"($|\\.[0-9]+|\\.#)")
					} else {
						readOnlyAttributes = append(readOnlyAttributes, "^"+parent+"\\.(.*)\\."+key+"$")
					}
				case configschema.NestingSet:
					if parent == "-1" {
						readOnlyAttributes = append(readOnlyAttributes, "^"+k+"\\.[0-9]+\\."+key+"$")
					} else {
						readOnlyAttributes = append(readOnlyAttributes, "^"+parent+"\\.(.*)\\."+key+"($|\\.(.*))")
					}
				case configschema.NestingMap:
					readOnlyAttributes = append(readOnlyAttributes, parent+"\\."+key)
				default:
					readOnlyAttributes = append(readOnlyAttributes, parent+"\\."+key+"$")
				}
			}
		}
		if fieldCount == len(v.Block.Attributes) && fieldCount > 0 {
			readOnlyAttributes = append(readOnlyAttributes, "^"+k)
		}
	}
	return readOnlyAttributes
}

func (p *ProviderWrapper) Refresh(info *terraform.InstanceInfo, state *terraform.InstanceState) (*terraform.InstanceState, error) {
	schema := p.GetSchema()
	impliedType := schema.ResourceTypes[info.Type].Block.ImpliedType()
	priorState, err := state.AttrsAsObjectValue(impliedType)
	if err != nil {
		return nil, err
	}
	successReadResource := false
	resp := providers.ReadResourceResponse{}
	for i := 0; i < p.retryCount; i++ {
		resp = p.Provider.ReadResource(providers.ReadResourceRequest{
			TypeName:   info.Type,
			PriorState: priorState,
			Private:    []byte{},
		})
		if resp.Diagnostics.HasErrors() {
			log.Println(resp.Diagnostics.Err())
			log.Printf("WARN: Fail read resource from provider, wait %dms before retry\n", p.retrySleepMs)
			time.Sleep(time.Duration(p.retrySleepMs) * time.Millisecond)
			continue
		} else {
			successReadResource = true
			break
		}
	}

	if !successReadResource {
		log.Println("Fail read resource from provider, trying import command")
		// retry with regular import command - without resource attributes
		importResponse := p.Provider.ImportResourceState(providers.ImportResourceStateRequest{
			TypeName: info.Type,
			ID:       state.ID,
		})
		if importResponse.Diagnostics.HasErrors() {
			return nil, resp.Diagnostics.Err()
		}
		if len(importResponse.ImportedResources) == 0 {
			return nil, errors.New("not able to import resource for a given ID")
		}
		return terraform.NewInstanceStateShimmedFromValue(importResponse.ImportedResources[0].State, int(schema.ResourceTypes[info.Type].Version)), nil
	}

	if resp.NewState.IsNull() {
		msg := fmt.Sprintf("ERROR: Read resource response is null for resource %s", info.Id)
		return nil, errors.New(msg)
	}

	return terraform.NewInstanceStateShimmedFromValue(resp.NewState, int(schema.ResourceTypes[info.Type].Version)), nil
}

func (p *ProviderWrapper) initProvider(verbose bool) error {
	providerFilePath, err := getProviderFileName(p.providerName)
	if err != nil {
		return err
	}
	options := hclog.LoggerOptions{
		Name:   "plugin",
		Level:  hclog.Error,
		Output: os.Stdout,
	}
	if verbose {
		options.Level = hclog.Trace
	}
	logger := hclog.New(&options)
	p.client = plugin.NewClient(
		&plugin.ClientConfig{
			Cmd:              exec.Command(providerFilePath),
			HandshakeConfig:  tfplugin.Handshake,
			VersionedPlugins: tfplugin.VersionedPlugins,
			Managed:          true,
			Logger:           logger,
			AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC},
			AutoMTLS:         true,
		})
	p.rpcClient, err = p.client.Client()
	if err != nil {
		return err
	}
	raw, err := p.rpcClient.Dispense(tfplugin.ProviderPluginName)
	if err != nil {
		return err
	}

	p.Provider = raw.(*tfplugin.GRPCProvider)

	config, err := p.GetSchema().Provider.Block.CoerceValue(p.config)
	if err != nil {
		return err
	}
	p.Provider.Configure(providers.ConfigureRequest{
		TerraformVersion: version.Version,
		Config:           config,
	})

	return nil
}

func getProviderFileName(providerName string) (string, error) {
	defaultDataDir := os.Getenv("TF_DATA_DIR")
	if defaultDataDir == "" {
		defaultDataDir = DefaultDataDir
	}
	providerFilePath, err := getProviderFileNameV13andV14(defaultDataDir, providerName)
	if err != nil || providerFilePath == "" {
		providerFilePath, err = getProviderFileNameV13andV14(os.Getenv("HOME")+string(os.PathSeparator)+
			".terraform.d", providerName)
	}
	if err != nil || providerFilePath == "" {
		return getProviderFileNameV12(providerName)
	}
	return providerFilePath, nil
}

func getProviderFileNameV13andV14(prefix, providerName string) (string, error) {
	// Read terraform v14 file path
	registryDir := prefix + string(os.PathSeparator) + "providers" + string(os.PathSeparator) +
		"registry.terraform.io"
	providerDirs, err := ioutil.ReadDir(registryDir)
	if err != nil {
		// Read terraform v13 file path
		registryDir = prefix + string(os.PathSeparator) + "plugins" + string(os.PathSeparator) +
			"registry.terraform.io"
		providerDirs, err = ioutil.ReadDir(registryDir)
		if err != nil {
			return "", err
		}
	}
	providerFilePath := ""
	for _, providerDir := range providerDirs {
		pluginPath := registryDir + string(os.PathSeparator) + providerDir.Name() +
			string(os.PathSeparator) + providerName
		dirs, err := ioutil.ReadDir(pluginPath)
		if err != nil {
			continue
		}
		for _, dir := range dirs {
			if !dir.IsDir() {
				continue
			}
			for _, dir := range dirs {
				fullPluginPath := pluginPath + string(os.PathSeparator) + dir.Name() +
					string(os.PathSeparator) + runtime.GOOS + "_" + runtime.GOARCH
				files, err := ioutil.ReadDir(fullPluginPath)
				if err == nil {
					for _, file := range files {
						if strings.HasPrefix(file.Name(), "terraform-provider-"+providerName) {
							providerFilePath = fullPluginPath + string(os.PathSeparator) + file.Name()
						}
					}
				}
			}
		}
	}
	return providerFilePath, nil
}

func getProviderFileNameV12(providerName string) (string, error) {
	defaultDataDir := os.Getenv("TF_DATA_DIR")
	if defaultDataDir == "" {
		defaultDataDir = DefaultDataDir
	}
	pluginPath := defaultDataDir + string(os.PathSeparator) + "plugins" + string(os.PathSeparator) + runtime.GOOS + "_" + runtime.GOARCH
	files, err := ioutil.ReadDir(pluginPath)
	if err != nil {
		pluginPath = os.Getenv("HOME") + string(os.PathSeparator) + "." + DefaultPluginVendorDirV12
		files, err = ioutil.ReadDir(pluginPath)
		if err != nil {
			return "", err
		}
	}
	providerFilePath := ""
	for _, file := range files {
		if file.IsDir() {
			continue
		}
		if strings.HasPrefix(file.Name(), "terraform-provider-"+providerName) {
			providerFilePath = pluginPath + string(os.PathSeparator) + file.Name()
		}
	}
	return providerFilePath, nil
}

func GetProviderVersion(providerName string) string {
	providerFilePath, err := getProviderFileName(providerName)
	if err != nil {
		log.Println("Can't find provider file path. Ensure that you are following https://www.terraform.io/docs/configuration/providers.html#third-party-plugins.")
		return ""
	}
	t := strings.Split(providerFilePath, string(os.PathSeparator))
	providerFileName := t[len(t)-1]
	providerFileNameParts := strings.Split(providerFileName, "_")
	if len(providerFileNameParts) < 2 {
		log.Println("Can't find provider version. Ensure that you are following https://www.terraform.io/docs/configuration/providers.html#plugin-names-and-versions.")
		return ""
	}
	providerVersion := providerFileNameParts[1]
	return "~> " + strings.TrimPrefix(providerVersion, "v")
}
<commit_msg>Fix ignored blocks when all attributes are ignored although it includes nested blocks (#985)<commit_after>// Copyright 2018 The Terraformer Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package providerwrapper //nolint

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/terraformer/terraformutils/terraformerstring"

	"github.com/zclconf/go-cty/cty"

	"github.com/hashicorp/go-hclog"
	"github.com/hashicorp/go-plugin"
	"github.com/hashicorp/terraform/configs/configschema"
	tfplugin "github.com/hashicorp/terraform/plugin"
	"github.com/hashicorp/terraform/providers"
	"github.com/hashicorp/terraform/terraform"
	"github.com/hashicorp/terraform/version"
)

// DefaultDataDir is the default directory for storing local data.
const DefaultDataDir = ".terraform"

// DefaultPluginVendorDir is the location in the config directory to look for
// user-added plugin binaries. Terraform only reads from this path if it
// exists, it is never created by terraform.
const DefaultPluginVendorDirV12 = "terraform.d/plugins/" + pluginMachineName

// pluginMachineName is the directory name used in new plugin paths.
const pluginMachineName = runtime.GOOS + "_" + runtime.GOARCH

type ProviderWrapper struct {
	Provider     *tfplugin.GRPCProvider
	client       *plugin.Client
	rpcClient    plugin.ClientProtocol
	providerName string
	config       cty.Value
	schema       *providers.GetSchemaResponse
	retryCount   int
	retrySleepMs int
}

func NewProviderWrapper(providerName string, providerConfig cty.Value, verbose bool, options ...map[string]int) (*ProviderWrapper, error) {
	p := &ProviderWrapper{retryCount: 5, retrySleepMs: 300}
	p.providerName = providerName
	p.config = providerConfig

	if len(options) > 0 {
		retryCount, hasOption := options[0]["retryCount"]
		if hasOption {
			p.retryCount = retryCount
		}
		retrySleepMs, hasOption := options[0]["retrySleepMs"]
		if hasOption {
			p.retrySleepMs = retrySleepMs
		}
	}

	err := p.initProvider(verbose)

	return p, err
}

func (p *ProviderWrapper) Kill() {
	p.client.Kill()
}

func (p *ProviderWrapper) GetSchema() *providers.GetSchemaResponse {
	if p.schema == nil {
		r := p.Provider.GetSchema()
		p.schema = &r
	}
	return p.schema
}

func (p *ProviderWrapper) GetReadOnlyAttributes(resourceTypes []string) (map[string][]string, error) {
	r := p.GetSchema()

	if r.Diagnostics.HasErrors() {
		return nil, r.Diagnostics.Err()
	}
	readOnlyAttributes := map[string][]string{}
	for resourceName, obj := range r.ResourceTypes {
		if terraformerstring.ContainsString(resourceTypes, resourceName) {
			readOnlyAttributes[resourceName] = append(readOnlyAttributes[resourceName], "^id$")
			for k, v := range obj.Block.Attributes {
				if !v.Optional && !v.Required {
					if v.Type.IsListType() || v.Type.IsSetType() {
						readOnlyAttributes[resourceName] = append(readOnlyAttributes[resourceName], "^"+k+"\\.(.*)")
					} else {
						readOnlyAttributes[resourceName] = append(readOnlyAttributes[resourceName], "^"+k+"$")
					}
				}
			}
			readOnlyAttributes[resourceName] = p.readObjBlocks(obj.Block.BlockTypes, readOnlyAttributes[resourceName], "-1")
		}
	}
	return readOnlyAttributes, nil
}

func (p *ProviderWrapper) readObjBlocks(block map[string]*configschema.NestedBlock, readOnlyAttributes []string, parent string) []string {
	for k, v := range block {
		if len(v.BlockTypes) > 0 {
			if parent == "-1" {
				readOnlyAttributes = p.readObjBlocks(v.BlockTypes, readOnlyAttributes, k)
			} else {
				readOnlyAttributes = p.readObjBlocks(v.BlockTypes, readOnlyAttributes, parent+"\\.[0-9]+\\."+k)
			}
		}
		fieldCount := 0
		for key, l := range v.Attributes {
			if !l.Optional && !l.Required {
				fieldCount++
				switch v.Nesting {
				case configschema.NestingList:
					if parent == "-1" {
						readOnlyAttributes = append(readOnlyAttributes, "^"+k+"\\.[0-9]+\\."+key+"($|\\.[0-9]+|\\.#)")
					} else {
						readOnlyAttributes = append(readOnlyAttributes, "^"+parent+"\\.(.*)\\."+key+"$")
					}
				case configschema.NestingSet:
					if parent == "-1" {
						readOnlyAttributes = append(readOnlyAttributes, "^"+k+"\\.[0-9]+\\."+key+"$")
					} else {
						readOnlyAttributes = append(readOnlyAttributes, "^"+parent+"\\.(.*)\\."+key+"($|\\.(.*))")
					}
				case configschema.NestingMap:
					readOnlyAttributes = append(readOnlyAttributes, parent+"\\."+key)
				default:
					readOnlyAttributes = append(readOnlyAttributes, parent+"\\."+key+"$")
				}
			}
		}
		if fieldCount == len(v.Block.Attributes) && fieldCount > 0 && len(v.BlockTypes) == 0 {
			readOnlyAttributes = append(readOnlyAttributes, "^"+k)
		}
	}
	return readOnlyAttributes
}

func (p *ProviderWrapper) Refresh(info *terraform.InstanceInfo, state *terraform.InstanceState) (*terraform.InstanceState, error) {
	schema := p.GetSchema()
	impliedType := schema.ResourceTypes[info.Type].Block.ImpliedType()
	priorState, err := state.AttrsAsObjectValue(impliedType)
	if err != nil {
		return nil, err
	}
	successReadResource := false
	resp := providers.ReadResourceResponse{}
	for i := 0; i < p.retryCount; i++ {
		resp = p.Provider.ReadResource(providers.ReadResourceRequest{
			TypeName:   info.Type,
			PriorState: priorState,
			Private:    []byte{},
		})
		if resp.Diagnostics.HasErrors() {
			log.Println(resp.Diagnostics.Err())
			log.Printf("WARN: Fail read resource from provider, wait %dms before retry\n", p.retrySleepMs)
			time.Sleep(time.Duration(p.retrySleepMs) * time.Millisecond)
			continue
		} else {
			successReadResource = true
			break
		}
	}

	if !successReadResource {
		log.Println("Fail read resource from provider, trying import command")
		// retry with regular import command - without resource attributes
		importResponse := p.Provider.ImportResourceState(providers.ImportResourceStateRequest{
			TypeName: info.Type,
			ID:       state.ID,
		})
		if importResponse.Diagnostics.HasErrors() {
			return nil, resp.Diagnostics.Err()
		}
		if len(importResponse.ImportedResources) == 0 {
			return nil, errors.New("not able to import resource for a given ID")
		}
		return terraform.NewInstanceStateShimmedFromValue(importResponse.ImportedResources[0].State, int(schema.ResourceTypes[info.Type].Version)), nil
	}

	if resp.NewState.IsNull() {
		msg := fmt.Sprintf("ERROR: Read resource response is null for resource %s", info.Id)
		return nil, errors.New(msg)
	}

	return terraform.NewInstanceStateShimmedFromValue(resp.NewState, int(schema.ResourceTypes[info.Type].Version)), nil
}

func (p *ProviderWrapper) initProvider(verbose bool) error {
	providerFilePath, err := getProviderFileName(p.providerName)
	if err != nil {
		return err
	}
	options := hclog.LoggerOptions{
		Name:   "plugin",
		Level:  hclog.Error,
		Output: os.Stdout,
	}
	if verbose {
		options.Level = hclog.Trace
	}
	logger := hclog.New(&options)
	p.client = plugin.NewClient(
		&plugin.ClientConfig{
			Cmd:              exec.Command(providerFilePath),
			HandshakeConfig:  tfplugin.Handshake,
			VersionedPlugins: tfplugin.VersionedPlugins,
			Managed:          true,
			Logger:           logger,
			AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC},
			AutoMTLS:         true,
		})
	p.rpcClient, err = p.client.Client()
	if err != nil {
		return err
	}
	raw, err := p.rpcClient.Dispense(tfplugin.ProviderPluginName)
	if err != nil {
		return err
	}

	p.Provider = raw.(*tfplugin.GRPCProvider)

	config, err := p.GetSchema().Provider.Block.CoerceValue(p.config)
	if err != nil {
		return err
	}
	p.Provider.Configure(providers.ConfigureRequest{
		TerraformVersion: version.Version,
		Config:           config,
	})

	return nil
}

func getProviderFileName(providerName string) (string, error) {
	defaultDataDir := os.Getenv("TF_DATA_DIR")
	if defaultDataDir == "" {
		defaultDataDir = DefaultDataDir
	}
	providerFilePath, err := getProviderFileNameV13andV14(defaultDataDir, providerName)
	if err != nil || providerFilePath == "" {
		providerFilePath, err = getProviderFileNameV13andV14(os.Getenv("HOME")+string(os.PathSeparator)+
			".terraform.d", providerName)
	}
	if err != nil || providerFilePath == "" {
		return getProviderFileNameV12(providerName)
	}
	return providerFilePath, nil
}

func getProviderFileNameV13andV14(prefix, providerName string) (string, error) {
	// Read terraform v14 file path
	registryDir := prefix + string(os.PathSeparator) + "providers" + string(os.PathSeparator) +
		"registry.terraform.io"
	providerDirs, err := ioutil.ReadDir(registryDir)
	if err != nil {
		// Read terraform v13 file path
		registryDir = prefix + string(os.PathSeparator) + "plugins" + string(os.PathSeparator) +
			"registry.terraform.io"
		providerDirs, err = ioutil.ReadDir(registryDir)
		if err != nil {
			return "", err
		}
	}
	providerFilePath := ""
	for _, providerDir := range providerDirs {
		pluginPath := registryDir + string(os.PathSeparator) + providerDir.Name() +
			string(os.PathSeparator) + providerName
		dirs, err := ioutil.ReadDir(pluginPath)
		if err != nil {
			continue
		}
		for _, dir := range dirs {
			if !dir.IsDir() {
				continue
			}
			for _, dir := range dirs {
				fullPluginPath := pluginPath + string(os.PathSeparator) + dir.Name() +
					string(os.PathSeparator) + runtime.GOOS + "_" + runtime.GOARCH
				files, err := ioutil.ReadDir(fullPluginPath)
				if err == nil {
					for _, file := range files {
						if strings.HasPrefix(file.Name(), "terraform-provider-"+providerName) {
							providerFilePath = fullPluginPath + string(os.PathSeparator) + file.Name()
						}
					}
				}
			}
		}
	}
	return providerFilePath, nil
}

func getProviderFileNameV12(providerName string) (string, error) {
	defaultDataDir := os.Getenv("TF_DATA_DIR")
	if defaultDataDir == "" {
		defaultDataDir = DefaultDataDir
	}
	pluginPath := defaultDataDir + string(os.PathSeparator) + "plugins" + string(os.PathSeparator) + runtime.GOOS + "_" + runtime.GOARCH
	files, err := ioutil.ReadDir(pluginPath)
	if err != nil {
		pluginPath = os.Getenv("HOME") + string(os.PathSeparator) + "." + DefaultPluginVendorDirV12
		files, err = ioutil.ReadDir(pluginPath)
		if err != nil {
			return "", err
		}
	}
	providerFilePath := ""
	for _, file := range files {
		if file.IsDir() {
			continue
		}
		if strings.HasPrefix(file.Name(), "terraform-provider-"+providerName) {
			providerFilePath = pluginPath + string(os.PathSeparator) + file.Name()
		}
	}
	return providerFilePath, nil
}

func GetProviderVersion(providerName string) string {
	providerFilePath, err := getProviderFileName(providerName)
	if err != nil {
		log.Println("Can't find provider file path. Ensure that you are following https://www.terraform.io/docs/configuration/providers.html#third-party-plugins.")
		return ""
	}
	t := strings.Split(providerFilePath, string(os.PathSeparator))
	providerFileName := t[len(t)-1]
	providerFileNameParts := strings.Split(providerFileName, "_")
	if len(providerFileNameParts) < 2 {
		log.Println("Can't find provider version. Ensure that you are following https://www.terraform.io/docs/configuration/providers.html#plugin-names-and-versions.")
		return ""
	}
	providerVersion := providerFileNameParts[1]
	return "~> " + strings.TrimPrefix(providerVersion, "v")
}
<|endoftext|>


<commit_before>package gucumber

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	"github.com/lsegal/gucumber/gherkin"
	"github.com/shiena/ansicolor"
)

const (
	clrRed    = "31"
	clrGreen  = "32"
	clrYellow = "33"

	txtUnmatchInt   = `(\d+)`
	txtUnmatchFloat = `(-?\d+(?:\.\d+)?)`
	txtUnmatchStr   = `"(.+?)"`
)

var (
	reUnmatchInt   = regexp.MustCompile(txtUnmatchInt)
	reUnmatchFloat = regexp.MustCompile(txtUnmatchFloat)
	reUnmatchStr   = regexp.MustCompile(`(<|").+?("|>)`)
	reOutlineVal   = regexp.MustCompile(`<(.+?)>`)
)

type Runner struct {
	*Context
	Features  []*gherkin.Feature
	Results   []RunnerResult
	Unmatched []*gherkin.Step
	FailCount int
	SkipCount int
}

type RunnerResult struct {
	*TestingT
	*gherkin.Feature
	*gherkin.Scenario
}

func (c *Context) RunDir(dir string) (*Runner, error) {
	g, _ := filepath.Glob(filepath.Join(dir, "*.feature"))
	g2, _ := filepath.Glob(filepath.Join(dir, "**", "*.feature"))
	g = append(g, g2...)

	runner, err := c.RunFiles(g)
	if err != nil {
		panic(err)
	}

	if len(runner.Unmatched) > 0 {
		fmt.Println("Some steps were missing, you can add them by using the following step definition stubs: ")
		fmt.Println("")
		fmt.Print(runner.MissingMatcherStubs())
	}

	return runner, err
}

func (c *Context) RunFiles(featureFiles []string) (*Runner, error) {
	r := Runner{
		Context:   c,
		Features:  []*gherkin.Feature{},
		Results:   []RunnerResult{},
		Unmatched: []*gherkin.Step{},
	}

	for _, file := range featureFiles {
		fd, err := os.Open(file)
		if err != nil {
			return nil, err
		}
		defer fd.Close()

		b, err := ioutil.ReadAll(fd)
		if err != nil {
			return nil, err
		}

		fs, err := gherkin.ParseFilename(string(b), file)
		if err != nil {
			return nil, err
		}

		for _, f := range fs {
			r.Features = append(r.Features, &f)
		}
	}

	r.run()
	return &r, nil
}

func (c *Runner) MissingMatcherStubs() string {
	var buf bytes.Buffer
	matches := map[string]bool{}

	buf.WriteString(`import . "github.com/lsegal/gucumber"` + "\n\n")
	buf.WriteString("func init() {\n")

	for _, m := range c.Unmatched {
		numInts, numFloats, numStrs := 1, 1, 1
		str, args := m.Text, []string{}
		str = reUnmatchInt.ReplaceAllStringFunc(str, func(s string) string {
			args = append(args, fmt.Sprintf("i%d int", numInts))
			numInts++
			return txtUnmatchInt
		})
		str = reUnmatchFloat.ReplaceAllStringFunc(str, func(s string) string {
			args = append(args, fmt.Sprintf("s%d float64", numFloats))
			numFloats++
			return txtUnmatchFloat
		})
		str = reUnmatchStr.ReplaceAllStringFunc(str, func(s string) string {
			args = append(args, fmt.Sprintf("s%d string", numStrs))
			numStrs++
			return txtUnmatchStr
		})

		switch m.Argument.(type) {
		case gherkin.TabularData:
			args = append(args, "table [][]string")
		case gherkin.StringData:
			args = append(args, "data string")
		}

		// Don't duplicate matchers. This is mostly for scenario outlines.
		if matches[str] {
			continue
		}
		matches[str] = true

		fmt.Fprintf(&buf, "\t%s(`^%s$`, func(%s) {\n\t\tT.Skip() // pending\n\t})\n\n",
			m.Type, str, strings.Join(args, ", "))
	}

	buf.WriteString("}\n")
	return buf.String()
}

func (c *Runner) run() {
	for _, f := range c.Features {
		c.runFeature(f)
	}

	c.line("0;1", "finished (%d passed, %d failed, %d skipped).\n",
		len(c.Results)-c.FailCount-c.SkipCount, c.FailCount, c.SkipCount)
}

func (c *Runner) runFeature(f *gherkin.Feature) {
	c.line("0;1", "Feature: %s", f.Title)

	if f.Background.Steps != nil {
		c.runScenario("Background", f, &f.Background)
	}

	for _, s := range f.Scenarios {
		c.runScenario("Scenario", f, &s)
	}
}

func (c *Runner) runScenario(title string, f *gherkin.Feature, s *gherkin.Scenario) {
	if len(s.Examples) > 1 { // run scenario outline data
		for i, rows := 0, s.Examples.NumRows(); i < rows; i++ {
			other := gherkin.Scenario{
				Filename: s.Filename,
				Line:     s.Line,
				Title:    s.Title,
				Examples: gherkin.TabularDataMap{},
				Steps:    []gherkin.Step{},
			}

			for _, step := range s.Steps {
				step.Text = reOutlineVal.ReplaceAllStringFunc(step.Text, func(t string) string {
					return s.Examples[t[1:len(t)-1]][i]
				})
				other.Steps = append(other.Steps, step)
			}
			c.runScenario(title, f, &other)
		}
		return
	}

	t := &TestingT{}
	skipping := false
	clr := clrGreen

	c.fileLine("0;1", "  %s: %s", s.Filename, s.Line, s.LongestLine(),
		title, s.Title)

	for _, step := range s.Steps {
		errCount := len(t.errors)
		found := false
		if !skipping {
			done := make(chan bool)
			go func() {
				defer func() {
					c.Results = append(c.Results, RunnerResult{t, f, s})

					if t.Skipped() {
						c.SkipCount++
						skipping = true
						clr = clrYellow
					} else if t.Failed() {
						c.FailCount++
						clr = clrRed
					}

					done <- true
				}()

				f, err := c.Execute(t, step.Text, step.Argument)
				if err != nil {
					t.Error(err)
				}
				found = f

				if !f {
					t.Skip("no match function for step")
				}
			}()
			<-done
		}

		if skipping && !found {
			cstep := step
			c.Unmatched = append(c.Unmatched, &cstep)
		}

		c.fileLine(clr, "    %s %s", step.Filename, step.Line,
			s.LongestLine(), step.Type, step.Text)

		if len(t.errors) > errCount {
			c.line(clrRed, "\n"+t.errors[len(t.errors)-1].message)
		}
	}
	c.line("0", "")
}

var writer = ansicolor.NewAnsiColorWriter(os.Stdout)

func (c *Runner) line(clr, text string, args ...interface{}) {
	fmt.Fprintf(writer, "\033[%sm%s\033[0;0m\n", clr, fmt.Sprintf(text, args...))
}

func (c *Runner) fileLine(clr, text, filename string, line int, max int, args ...interface{}) {
	space, str := "", fmt.Sprintf(text, args...)
	if l := max + 5 - len(str); l > 0 {
		space = strings.Repeat(" ", l)
	}
	comment := fmt.Sprintf("%s \033[39;0m# %s:%d", space, filename, line)
	c.line(clr, "%s%s", str, comment)
}

type Tester interface {
	Errorf(format string, args ...interface{})
}

type TestingT struct {
	skipped bool
	errors  []TestError
}

type TestError struct {
	message string
	stack   []byte
}

func (t *TestingT) Errorf(format string, args ...interface{}) {
	var buf bytes.Buffer

	str := fmt.Sprintf(format, args...)
	sbuf := make([]byte, 8192)
	for {
		size := runtime.Stack(sbuf, false)
		if size < len(sbuf) {
			break
		}
		buf.Write(sbuf[0:size])
	}

	t.errors = append(t.errors, TestError{message: str, stack: buf.Bytes()})
}

func (t *TestingT) Skip(args ...interface{}) {
	t.skipped = true
}

func (t *TestingT) Skipped() bool {
	return t.skipped
}

func (t *TestingT) Failed() bool {
	return len(t.errors) > 0
}

func (t *TestingT) Error(err error) {
	t.errors = append(t.errors, TestError{message: err.Error()})
}
<commit_msg>Set exit code at the end of the runner<commit_after>package gucumber

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	"github.com/lsegal/gucumber/gherkin"
	"github.com/shiena/ansicolor"
)

const (
	clrRed    = "31"
	clrGreen  = "32"
	clrYellow = "33"

	txtUnmatchInt   = `(\d+)`
	txtUnmatchFloat = `(-?\d+(?:\.\d+)?)`
	txtUnmatchStr   = `"(.+?)"`
)

var (
	reUnmatchInt   = regexp.MustCompile(txtUnmatchInt)
	reUnmatchFloat = regexp.MustCompile(txtUnmatchFloat)
	reUnmatchStr   = regexp.MustCompile(`(<|").+?("|>)`)
	reOutlineVal   = regexp.MustCompile(`<(.+?)>`)
)

type Runner struct {
	*Context
	Features  []*gherkin.Feature
	Results   []RunnerResult
	Unmatched []*gherkin.Step
	FailCount int
	SkipCount int
}

type RunnerResult struct {
	*TestingT
	*gherkin.Feature
	*gherkin.Scenario
}

func (c *Context) RunDir(dir string) (*Runner, error) {
	g, _ := filepath.Glob(filepath.Join(dir, "*.feature"))
	g2, _ := filepath.Glob(filepath.Join(dir, "**", "*.feature"))
	g = append(g, g2...)

	runner, err := c.RunFiles(g)
	if err != nil {
		panic(err)
	}

	if len(runner.Unmatched) > 0 {
		fmt.Println("Some steps were missing, you can add them by using the following step definition stubs: ")
		fmt.Println("")
		fmt.Print(runner.MissingMatcherStubs())
	}

	return runner, err
}

func (c *Context) RunFiles(featureFiles []string) (*Runner, error) {
	r := Runner{
		Context:   c,
		Features:  []*gherkin.Feature{},
		Results:   []RunnerResult{},
		Unmatched: []*gherkin.Step{},
	}

	for _, file := range featureFiles {
		fd, err := os.Open(file)
		if err != nil {
			return nil, err
		}
		defer fd.Close()

		b, err := ioutil.ReadAll(fd)
		if err != nil {
			return nil, err
		}

		fs, err := gherkin.ParseFilename(string(b), file)
		if err != nil {
			return nil, err
		}

		for _, f := range fs {
			r.Features = append(r.Features, &f)
		}
	}

	r.run()
	return &r, nil
}

func (c *Runner) MissingMatcherStubs() string {
	var buf bytes.Buffer
	matches := map[string]bool{}

	buf.WriteString(`import . "github.com/lsegal/gucumber"` + "\n\n")
	buf.WriteString("func init() {\n")

	for _, m := range c.Unmatched {
		numInts, numFloats, numStrs := 1, 1, 1
		str, args := m.Text, []string{}
		str = reUnmatchInt.ReplaceAllStringFunc(str, func(s string) string {
			args = append(args, fmt.Sprintf("i%d int", numInts))
			numInts++
			return txtUnmatchInt
		})
		str = reUnmatchFloat.ReplaceAllStringFunc(str, func(s string) string {
			args = append(args, fmt.Sprintf("s%d float64", numFloats))
			numFloats++
			return txtUnmatchFloat
		})
		str = reUnmatchStr.ReplaceAllStringFunc(str, func(s string) string {
			args = append(args, fmt.Sprintf("s%d string", numStrs))
			numStrs++
			return txtUnmatchStr
		})

		switch m.Argument.(type) {
		case gherkin.TabularData:
			args = append(args, "table [][]string")
		case gherkin.StringData:
			args = append(args, "data string")
		}

		// Don't duplicate matchers. This is mostly for scenario outlines.
		if matches[str] {
			continue
		}
		matches[str] = true

		fmt.Fprintf(&buf, "\t%s(`^%s$`, func(%s) {\n\t\tT.Skip() // pending\n\t})\n\n",
			m.Type, str, strings.Join(args, ", "))
	}

	buf.WriteString("}\n")
	return buf.String()
}

func (c *Runner) run() {
	for _, f := range c.Features {
		c.runFeature(f)
	}

	c.line("0;1", "finished (%d passed, %d failed, %d skipped).\n",
		len(c.Results)-c.FailCount-c.SkipCount, c.FailCount, c.SkipCount)

	os.Exit(c.FailCount)
}

func (c *Runner) runFeature(f *gherkin.Feature) {
	c.line("0;1", "Feature: %s", f.Title)

	if f.Background.Steps != nil {
		c.runScenario("Background", f, &f.Background)
	}

	for _, s := range f.Scenarios {
		c.runScenario("Scenario", f, &s)
	}
}

func (c *Runner) runScenario(title string, f *gherkin.Feature, s *gherkin.Scenario) {
	if len(s.Examples) > 1 { // run scenario outline data
		for i, rows := 0, s.Examples.NumRows(); i < rows; i++ {
			other := gherkin.Scenario{
				Filename: s.Filename,
				Line:     s.Line,
				Title:    s.Title,
				Examples: gherkin.TabularDataMap{},
				Steps:    []gherkin.Step{},
			}

			for _, step := range s.Steps {
				step.Text = reOutlineVal.ReplaceAllStringFunc(step.Text, func(t string) string {
					return s.Examples[t[1:len(t)-1]][i]
				})
				other.Steps = append(other.Steps, step)
			}
			c.runScenario(title, f, &other)
		}
		return
	}

	t := &TestingT{}
	skipping := false
	clr := clrGreen

	c.fileLine("0;1", "  %s: %s", s.Filename, s.Line, s.LongestLine(),
		title, s.Title)

	for _, step := range s.Steps {
		errCount := len(t.errors)
		found := false
		if !skipping {
			done := make(chan bool)
			go func() {
				defer func() {
					c.Results = append(c.Results, RunnerResult{t, f, s})

					if t.Skipped() {
						c.SkipCount++
						skipping = true
						clr = clrYellow
					} else if t.Failed() {
						c.FailCount++
						clr = clrRed
					}

					done <- true
				}()

				f, err := c.Execute(t, step.Text, step.Argument)
				if err != nil {
					t.Error(err)
				}
				found = f

				if !f {
					t.Skip("no match function for step")
				}
			}()
			<-done
		}

		if skipping && !found {
			cstep := step
			c.Unmatched = append(c.Unmatched, &cstep)
		}

		c.fileLine(clr, "    %s %s", step.Filename, step.Line,
			s.LongestLine(), step.Type, step.Text)

		if len(t.errors) > errCount {
			c.line(clrRed, "\n"+t.errors[len(t.errors)-1].message)
		}
	}
	c.line("0", "")
}

var writer = ansicolor.NewAnsiColorWriter(os.Stdout)

func (c *Runner) line(clr, text string, args ...interface{}) {
	fmt.Fprintf(writer, "\033[%sm%s\033[0;0m\n", clr, fmt.Sprintf(text, args...))
}

func (c *Runner) fileLine(clr, text, filename string, line int, max int, args ...interface{}) {
	space, str := "", fmt.Sprintf(text, args...)
	if l := max + 5 - len(str); l > 0 {
		space = strings.Repeat(" ", l)
	}
	comment := fmt.Sprintf("%s \033[39;0m# %s:%d", space, filename, line)
	c.line(clr, "%s%s", str, comment)
}

type Tester interface {
	Errorf(format string, args ...interface{})
}

type TestingT struct {
	skipped bool
	errors  []TestError
}

type TestError struct {
	message string
	stack   []byte
}

func (t *TestingT) Errorf(format string, args ...interface{}) {
	var buf bytes.Buffer

	str := fmt.Sprintf(format, args...)
	sbuf := make([]byte, 8192)
	for {
		size := runtime.Stack(sbuf, false)
		if size < len(sbuf) {
			break
		}
		buf.Write(sbuf[0:size])
	}

	t.errors = append(t.errors, TestError{message: str, stack: buf.Bytes()})
}

func (t *TestingT) Skip(args ...interface{}) {
	t.skipped = true
}

func (t *TestingT) Skipped() bool {
	return t.skipped
}

func (t *TestingT) Failed() bool {
	return len(t.errors) > 0
}

func (t *TestingT) Error(err error) {
	t.errors = append(t.errors, TestError{message: err.Error()})
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"context"
	"encoding/binary"
	"fmt"
	"math"
	"time"

	"github.com/yobert/alsa"
)

func main() {

	cards, err := alsa.OpenCards()
	if err != nil {
		fmt.Println(err)
		return
	}
	defer alsa.CloseCards(cards)

	for _, card := range cards {
		fmt.Println(card)

		if err := beepCard(card); err != nil {
			fmt.Printf("error when beeping card: %v\n", err)
		}
	}
}

func beepCard(card *alsa.Card) error {
	devices, err := card.Devices()
	if err != nil {
		return err
	}
	for _, device := range devices {
		if device.Type != alsa.PCM || !device.Play {
			continue
		}
		fmt.Println("───", device)

		if err := beepDevice(device); err != nil {
			return err
		}
	}
	return nil
}

func beepDevice(device *alsa.Device) error {
	var err error

	if err = device.Open(); err != nil {
		return err
	}

	// Cleanup device when done or force cleanup after 4 seconds.
	childCtx, cancel := context.WithDeadline(context.Background(), time.Now().Add(4*time.Second))
	defer cancel()
	go func(ctx context.Context) {
		defer device.Close()
		<-ctx.Done()
		fmt.Println("Closing device.")
	}(childCtx)

	channels, err := device.NegotiateChannels(1, 2)
	if err != nil {
		return err
	}

	rate, err := device.NegotiateRate(44100)
	if err != nil {
		return err
	}

	format, err := device.NegotiateFormat(alsa.S16_LE, alsa.S32_LE)
	if err != nil {
		return err
	}

	// Prefer larger buffer to avoid under-run.
	bufferSize, err := device.NegotiateBufferSize(16384, 8192)
	if err != nil {
		return err
	}

	if err = device.Prepare(); err != nil {
		return err
	}

	fmt.Printf("Negotiated parameters: %d channels, %d hz, %v, %d frame buffer\n",
		channels, rate, format, bufferSize)

	// Play 2 seconds of beep.
	for t := 0.; t < 2; {
		var buf bytes.Buffer

		for i := 0; i < bufferSize; i++ {
			v := math.Sin(t * 2 * math.Pi * 440) // A4
			v *= 0.1                             // make a little quieter

			switch format {
			case alsa.S16_LE:
				sample := int16(v * math.MaxInt16)

				for c := 0; c < channels; c++ {
					binary.Write(&buf, binary.LittleEndian, sample)
				}

			case alsa.S32_LE:
				sample := int32(v * math.MaxInt32)

				for c := 0; c < channels; c++ {
					binary.Write(&buf, binary.LittleEndian, sample)
				}

			default:
				return fmt.Errorf("Unhandled sample format: %v", format)
			}

			t += 1 / float64(rate)
		}

		if err := device.Write(buf.Bytes(), bufferSize); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>cmd/beep: address buffer underruns by configuring 50 ms period size<commit_after>package main

import (
	"bytes"
	"context"
	"encoding/binary"
	"fmt"
	"math"
	"sync"
	"time"

	"github.com/yobert/alsa"
)

func main() {

	cards, err := alsa.OpenCards()
	if err != nil {
		fmt.Println(err)
		return
	}
	defer alsa.CloseCards(cards)

	for _, card := range cards {
		fmt.Println(card)

		if err := beepCard(card); err != nil {
			fmt.Printf("error when beeping card: %v\n", err)
		}
	}
}

func beepCard(card *alsa.Card) error {
	devices, err := card.Devices()
	if err != nil {
		return err
	}
	for _, device := range devices {
		if device.Type != alsa.PCM || !device.Play {
			continue
		}
		fmt.Println("───", device)

		if err := beepDevice(device); err != nil {
			fmt.Printf("error when beeping device: %v\n", err)
		}
	}
	return nil
}

func beepDevice(device *alsa.Device) error {
	var err error

	if err = device.Open(); err != nil {
		return err
	}

	// Cleanup device when done or force cleanup after 3 seconds.
	wg := sync.WaitGroup{}
	wg.Add(1)
	defer wg.Wait()
	childCtx, cancel := context.WithDeadline(context.Background(), time.Now().Add(3*time.Second))
	defer cancel()
	go func(ctx context.Context) {
		defer device.Close()
		<-ctx.Done()
		fmt.Println("Closing device.")
		wg.Done()
	}(childCtx)

	channels, err := device.NegotiateChannels(1, 2)
	if err != nil {
		return err
	}

	rate, err := device.NegotiateRate(44100)
	if err != nil {
		return err
	}

	format, err := device.NegotiateFormat(alsa.S16_LE, alsa.S32_LE)
	if err != nil {
		return err
	}

	// A 50ms period is a sensible value to test low-ish latency.
	// We adjust the buffer so it's of minimal size (period * 2) since it appear ALSA won't
	// start playback until the buffer has been filled to a certain degree and the automatic
	// buffer size can be quite large.
	wantPeriodSize := 44100 / 20

	periodSize, err := device.NegotiatePeriodSize(wantPeriodSize)
	if err != nil {
		return err
	}

	bufferSize, err := device.NegotiateBufferSize(wantPeriodSize * 2)
	if err != nil {
		return err
	}

	if err = device.Prepare(); err != nil {
		return err
	}

	fmt.Printf("Negotiated parameters: %d channels, %d hz, %v, %d period size, %d buffer size\n",
		channels, rate, format, periodSize, bufferSize)

	// Play 2 seconds of beep.
	duration := 2 * time.Second
	t := time.NewTimer(duration)
	for t := 0.; t < duration.Seconds(); {
		var buf bytes.Buffer

		for i := 0; i < periodSize; i++ {
			v := math.Sin(t * 2 * math.Pi * 440) // A4
			v *= 0.1                             // make a little quieter

			switch format {
			case alsa.S16_LE:
				sample := int16(v * math.MaxInt16)

				for c := 0; c < channels; c++ {
					binary.Write(&buf, binary.LittleEndian, sample)
				}

			case alsa.S32_LE:
				sample := int32(v * math.MaxInt32)

				for c := 0; c < channels; c++ {
					binary.Write(&buf, binary.LittleEndian, sample)
				}

			default:
				return fmt.Errorf("Unhandled sample format: %v", format)
			}

			t += 1 / float64(rate)
		}

		if err := device.Write(buf.Bytes(), periodSize); err != nil {
			return err
		}
	}
	// Wait for playback to complete.
	<-t.C
	fmt.Printf("Playback should be complete now.\n")
	time.Sleep(1 * time.Second) // To allow a human to compare real playback end with supposed.

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path"

	context "context"
	assets "github.com/ipfs/go-ipfs/assets"
	cmds "github.com/ipfs/go-ipfs/commands"
	core "github.com/ipfs/go-ipfs/core"
	namesys "github.com/ipfs/go-ipfs/namesys"
	config "github.com/ipfs/go-ipfs/repo/config"
	fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
)

const (
	nBitsForKeypairDefault = 2048
)

var initCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Initializes ipfs config file.",
		ShortDescription: `
Initializes ipfs configuration files and generates a new keypair.

ipfs uses a repository in the local file system. By default, the repo is
located at ~/.ipfs. To change the repo location, set the $IPFS_PATH
environment variable:

    export IPFS_PATH=/path/to/ipfsrepo
`,
	},
	Arguments: []cmds.Argument{
		cmds.FileArg("default-config", false, false, "Initialize with the given configuration.").EnableStdin(),
	},
	Options: []cmds.Option{
		cmds.IntOption("bits", "b", "Number of bits to use in the generated RSA private key.").Default(nBitsForKeypairDefault),
		cmds.BoolOption("empty-repo", "e", "Don't add and pin help files to the local storage.").Default(false),

		// TODO need to decide whether to expose the override as a file or a
		// directory. That is: should we allow the user to also specify the
		// name of the file?
		// TODO cmds.StringOption("event-logs", "l", "Location for machine-readable event logs."),
	},
	PreRun: func(req cmds.Request) error {
		daemonLocked, err := fsrepo.LockedByOtherProcess(req.InvocContext().ConfigRoot)
		if err != nil {
			return err
		}

		log.Info("checking if daemon is running...")
		if daemonLocked {
			log.Debug("ipfs daemon is running")
			e := "ipfs daemon is running. please stop it to run this command"
			return cmds.ClientError(e)
		}

		return nil
	},
	Run: func(req cmds.Request, res cmds.Response) {
		if req.InvocContext().Online {
			res.SetError(errors.New("init must be run offline only!"), cmds.ErrNormal)
			return
		}

		empty, _, err := req.Option("e").Bool()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		nBitsForKeypair, _, err := req.Option("b").Int()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		var conf *config.Config

		f := req.Files()
		if f != nil {
			confFile, err := f.NextFile()
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}

			conf = &config.Config{}
			if err := json.NewDecoder(confFile).Decode(conf); err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}
		}

		if err := doInit(os.Stdout, req.InvocContext().ConfigRoot, empty, nBitsForKeypair, conf); err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
	},
}

var errRepoExists = errors.New(`ipfs configuration file already exists!
Reinitializing would overwrite your keys.
`)

func initWithDefaults(out io.Writer, repoRoot string) error {
	return doInit(out, repoRoot, false, nBitsForKeypairDefault, nil)
}

func doInit(out io.Writer, repoRoot string, empty bool, nBitsForKeypair int, conf *config.Config) error {
	if _, err := fmt.Fprintf(out, "initializing IPFS node at %s\n", repoRoot); err != nil {
		return err
	}

	if err := checkWriteable(repoRoot); err != nil {
		return err
	}

	if fsrepo.IsInitialized(repoRoot) {
		return errRepoExists
	}

	if conf == nil {
		var err error
		conf, err = config.Init(out, nBitsForKeypair)
		if err != nil {
			return err
		}
	}

	if err := fsrepo.Init(repoRoot, conf); err != nil {
		return err
	}

	if !empty {
		if err := addDefaultAssets(out, repoRoot); err != nil {
			return err
		}
	}

	return initializeIpnsKeyspace(repoRoot)
}

func checkWriteable(dir string) error {
	_, err := os.Stat(dir)
	if err == nil {
		// dir exists, make sure we can write to it
		testfile := path.Join(dir, "test")
		fi, err := os.Create(testfile)
		if err != nil {
			if os.IsPermission(err) {
				return fmt.Errorf("%s is not writeable by the current user", dir)
			}
			return fmt.Errorf("unexpected error while checking writeablility of repo root: %s", err)
		}
		fi.Close()
		return os.Remove(testfile)
	}

	if os.IsNotExist(err) {
		// dir doesnt exist, check that we can create it
		return os.Mkdir(dir, 0775)
	}

	if os.IsPermission(err) {
		return fmt.Errorf("cannot write to %s, incorrect permissions", err)
	}

	return err
}

func addDefaultAssets(out io.Writer, repoRoot string) error {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	r, err := fsrepo.Open(repoRoot)
	if err != nil { // NB: repo is owned by the node
		return err
	}

	nd, err := core.NewNode(ctx, &core.BuildCfg{Repo: r})
	if err != nil {
		return err
	}
	defer nd.Close()

	dkey, err := assets.SeedInitDocs(nd)
	if err != nil {
		return fmt.Errorf("init: seeding init docs failed: %s", err)
	}
	log.Debugf("init: seeded init docs %s", dkey)

	if _, err = fmt.Fprintf(out, "to get started, enter:\n"); err != nil {
		return err
	}

	_, err = fmt.Fprintf(out, "\n\tipfs cat /ipfs/%s/readme\n\n", dkey)
	return err
}

func initializeIpnsKeyspace(repoRoot string) error {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	r, err := fsrepo.Open(repoRoot)
	if err != nil { // NB: repo is owned by the node
		return err
	}

	nd, err := core.NewNode(ctx, &core.BuildCfg{Repo: r})
	if err != nil {
		return err
	}
	defer nd.Close()

	err = nd.SetupOfflineRouting()
	if err != nil {
		return err
	}

	return namesys.InitializeKeyspace(ctx, nd.DAG, nd.Namesys, nd.Pinning, nd.PrivateKey)
}
<commit_msg>Implement ipfs init --profile<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"strings"

	context "context"
	assets "github.com/ipfs/go-ipfs/assets"
	cmds "github.com/ipfs/go-ipfs/commands"
	core "github.com/ipfs/go-ipfs/core"
	namesys "github.com/ipfs/go-ipfs/namesys"
	config "github.com/ipfs/go-ipfs/repo/config"
	fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
)

const (
	nBitsForKeypairDefault = 2048
)

// TODO: move this out(where?)
// ConfigProfiles is a map holding configuration transformers
var ConfigProfiles = map[string]func(*config.Config) error{
	"server": func(c *config.Config) error {

		// defaultServerFilters has a list of non-routable IPv4 prefixes
		// according to http://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml
		defaultServerFilters := []string{
			"/ip4/10.0.0.0/ipcidr/8",
			"/ip4/100.64.0.0/ipcidr/10",
			"/ip4/169.254.0.0/ipcidr/16",
			"/ip4/172.16.0.0/ipcidr/12",
			"/ip4/192.0.0.0/ipcidr/24",
			"/ip4/192.0.0.0/ipcidr/29",
			"/ip4/192.0.0.8/ipcidr/32",
			"/ip4/192.0.0.170/ipcidr/32",
			"/ip4/192.0.0.171/ipcidr/32",
			"/ip4/192.0.2.0/ipcidr/24",
			"/ip4/192.168.0.0/ipcidr/16",
			"/ip4/198.18.0.0/ipcidr/15",
			"/ip4/198.51.100.0/ipcidr/24",
			"/ip4/203.0.113.0/ipcidr/24",
			"/ip4/240.0.0.0/ipcidr/4",
		}

		c.Swarm.AddrFilters = append(c.Swarm.AddrFilters, defaultServerFilters...)
		return nil
	},
}

var initCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Initializes ipfs config file.",
		ShortDescription: `
Initializes ipfs configuration files and generates a new keypair.

ipfs uses a repository in the local file system. By default, the repo is
located at ~/.ipfs. To change the repo location, set the $IPFS_PATH
environment variable:

    export IPFS_PATH=/path/to/ipfsrepo
`,
	},
	Arguments: []cmds.Argument{
		cmds.FileArg("default-config", false, false, "Initialize with the given configuration.").EnableStdin(),
	},
	Options: []cmds.Option{
		cmds.IntOption("bits", "b", "Number of bits to use in the generated RSA private key.").Default(nBitsForKeypairDefault),
		cmds.BoolOption("empty-repo", "e", "Don't add and pin help files to the local storage.").Default(false),
		cmds.StringOption("profile", "p", "Apply profile settings to config"),

		// TODO need to decide whether to expose the override as a file or a
		// directory. That is: should we allow the user to also specify the
		// name of the file?
		// TODO cmds.StringOption("event-logs", "l", "Location for machine-readable event logs."),
	},
	PreRun: func(req cmds.Request) error {
		daemonLocked, err := fsrepo.LockedByOtherProcess(req.InvocContext().ConfigRoot)
		if err != nil {
			return err
		}

		log.Info("checking if daemon is running...")
		if daemonLocked {
			log.Debug("ipfs daemon is running")
			e := "ipfs daemon is running. please stop it to run this command"
			return cmds.ClientError(e)
		}

		return nil
	},
	Run: func(req cmds.Request, res cmds.Response) {
		if req.InvocContext().Online {
			res.SetError(errors.New("init must be run offline only!"), cmds.ErrNormal)
			return
		}

		empty, _, err := req.Option("e").Bool()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		nBitsForKeypair, _, err := req.Option("b").Int()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		var conf *config.Config

		f := req.Files()
		if f != nil {
			confFile, err := f.NextFile()
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}

			conf = &config.Config{}
			if err := json.NewDecoder(confFile).Decode(conf); err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}
		}

		profile, _, err := req.Option("profile").String()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		var profiles []string
		if profile != "" {
			profiles = strings.Split(profile, ",")
		}

		if err := doInit(os.Stdout, req.InvocContext().ConfigRoot, empty, nBitsForKeypair, profiles, conf); err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
	},
}

var errRepoExists = errors.New(`ipfs configuration file already exists!
Reinitializing would overwrite your keys.
`)

func initWithDefaults(out io.Writer, repoRoot string) error {
	return doInit(out, repoRoot, false, nBitsForKeypairDefault, nil, nil)
}

func doInit(out io.Writer, repoRoot string, empty bool, nBitsForKeypair int, confProfiles []string, conf *config.Config) error {
	if _, err := fmt.Fprintf(out, "initializing IPFS node at %s\n", repoRoot); err != nil {
		return err
	}

	if err := checkWriteable(repoRoot); err != nil {
		return err
	}

	if fsrepo.IsInitialized(repoRoot) {
		return errRepoExists
	}

	if conf == nil {
		var err error
		conf, err = config.Init(out, nBitsForKeypair)
		if err != nil {
			return err
		}
	}

	for _, profile := range confProfiles {
		transformer := ConfigProfiles[profile]
		if transformer == nil {
			return fmt.Errorf("invalid configuration profile: %s", profile)
		}

		if err := transformer(conf); err != nil {
			return err
		}
	}

	if err := fsrepo.Init(repoRoot, conf); err != nil {
		return err
	}

	if !empty {
		if err := addDefaultAssets(out, repoRoot); err != nil {
			return err
		}
	}

	return initializeIpnsKeyspace(repoRoot)
}

func checkWriteable(dir string) error {
	_, err := os.Stat(dir)
	if err == nil {
		// dir exists, make sure we can write to it
		testfile := path.Join(dir, "test")
		fi, err := os.Create(testfile)
		if err != nil {
			if os.IsPermission(err) {
				return fmt.Errorf("%s is not writeable by the current user", dir)
			}
			return fmt.Errorf("unexpected error while checking writeablility of repo root: %s", err)
		}
		fi.Close()
		return os.Remove(testfile)
	}

	if os.IsNotExist(err) {
		// dir doesnt exist, check that we can create it
		return os.Mkdir(dir, 0775)
	}

	if os.IsPermission(err) {
		return fmt.Errorf("cannot write to %s, incorrect permissions", err)
	}

	return err
}

func addDefaultAssets(out io.Writer, repoRoot string) error {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	r, err := fsrepo.Open(repoRoot)
	if err != nil { // NB: repo is owned by the node
		return err
	}

	nd, err := core.NewNode(ctx, &core.BuildCfg{Repo: r})
	if err != nil {
		return err
	}
	defer nd.Close()

	dkey, err := assets.SeedInitDocs(nd)
	if err != nil {
		return fmt.Errorf("init: seeding init docs failed: %s", err)
	}
	log.Debugf("init: seeded init docs %s", dkey)

	if _, err = fmt.Fprintf(out, "to get started, enter:\n"); err != nil {
		return err
	}

	_, err = fmt.Fprintf(out, "\n\tipfs cat /ipfs/%s/readme\n\n", dkey)
	return err
}

func initializeIpnsKeyspace(repoRoot string) error {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	r, err := fsrepo.Open(repoRoot)
	if err != nil { // NB: repo is owned by the node
		return err
	}

	nd, err := core.NewNode(ctx, &core.BuildCfg{Repo: r})
	if err != nil {
		return err
	}
	defer nd.Close()

	err = nd.SetupOfflineRouting()
	if err != nil {
		return err
	}

	return namesys.InitializeKeyspace(ctx, nd.DAG, nd.Namesys, nd.Pinning, nd.PrivateKey)
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"os/signal"
	"path/filepath"
	"time"

	"context"

	netContext "golang.org/x/net/context"
	kingpin "gopkg.in/alecthomas/kingpin.v2"

	"github.com/keel-hq/keel/approvals"
	"github.com/keel-hq/keel/bot"
	// "github.com/keel-hq/keel/cache/memory"
	"github.com/keel-hq/keel/pkg/http"
	"github.com/keel-hq/keel/pkg/store"
	"github.com/keel-hq/keel/pkg/store/sql"

	"github.com/keel-hq/keel/constants"
	"github.com/keel-hq/keel/extension/credentialshelper"
	"github.com/keel-hq/keel/extension/notification"
	"github.com/keel-hq/keel/internal/k8s"
	"github.com/keel-hq/keel/internal/workgroup"
	"github.com/keel-hq/keel/provider"
	"github.com/keel-hq/keel/provider/helm"
	"github.com/keel-hq/keel/provider/kubernetes"
	"github.com/keel-hq/keel/registry"
	"github.com/keel-hq/keel/secrets"
	"github.com/keel-hq/keel/trigger/poll"
	"github.com/keel-hq/keel/trigger/pubsub"
	"github.com/keel-hq/keel/types"
	"github.com/keel-hq/keel/version"

	// notification extensions
	"github.com/keel-hq/keel/extension/notification/auditor"
	_ "github.com/keel-hq/keel/extension/notification/hipchat"
	_ "github.com/keel-hq/keel/extension/notification/mattermost"
	_ "github.com/keel-hq/keel/extension/notification/slack"
	_ "github.com/keel-hq/keel/extension/notification/webhook"

	// credentials helpers
	_ "github.com/keel-hq/keel/extension/credentialshelper/aws"
	secretsCredentialsHelper "github.com/keel-hq/keel/extension/credentialshelper/secrets"

	// bots
	_ "github.com/keel-hq/keel/bot/hipchat"
	_ "github.com/keel-hq/keel/bot/slack"

	log "github.com/sirupsen/logrus"
)

// gcloud pubsub related config
const (
	EnvTriggerPubSub = "PUBSUB" // set to 1 or something to enable pub/sub trigger
	EnvTriggerPoll   = "POLL"   // set to 1 or something to enable poll trigger
	EnvProjectID     = "PROJECT_ID"
	EnvClusterName   = "CLUSTER_NAME"

	EnvNamespace = "NAMESPACE" // Keel's namespace

	EnvHelmProvider      = "HELM_PROVIDER"  // helm provider
	EnvHelmTillerAddress = "TILLER_ADDRESS" // helm provider

	// EnvDefaultDockerRegistryCfg - default registry configuration that can be passed into
	// keel for polling trigger
	EnvDefaultDockerRegistryCfg = "DOCKER_REGISTRY_CFG"
)

// kubernetes config, if empty - will default to InCluster
const (
	EnvKubernetesConfig = "KUBERNETES_CONFIG"
)

// EnvDebug - set to 1 or anything else to enable debug logging
const EnvDebug = "DEBUG"

func main() {
	ver := version.GetKeelVersion()

	inCluster := kingpin.Flag("incluster", "use in cluster configuration (defaults to 'true'), use '--no-incluster' if running outside of the cluster").Default("true").Bool()
	kubeconfig := kingpin.Flag("kubeconfig", "path to kubeconfig (if not in running inside a cluster)").Default(filepath.Join(os.Getenv("HOME"), ".kube", "config")).String()

	kingpin.UsageTemplate(kingpin.CompactUsageTemplate).Version(ver.Version)
	kingpin.CommandLine.Help = "Automated Kubernetes deployment updates. Learn more on https://keel.sh."
	kingpin.Parse()

	log.WithFields(log.Fields{
		"os":         ver.OS,
		"build_date": ver.BuildDate,
		"revision":   ver.Revision,
		"version":    ver.Version,
		"go_version": ver.GoVersion,
		"arch":       ver.Arch,
	}).Info("keel starting...")

	if os.Getenv(EnvDebug) != "" {
		log.SetLevel(log.DebugLevel)
	}

	sqlStore, err := sql.New(sql.Opts{
		DatabaseType: "sqlite3",
		URI:          "keel.db",
	})
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
		}).Error("failed to initialize database")
		os.Exit(1)
	}

	// registering auditor to log events
	auditLogger := auditor.New(sqlStore)
	notification.RegisterSender("auditor", auditLogger)

	// setting up triggers
	ctx, cancel := netContext.WithCancel(context.Background())
	defer cancel()

	notificationLevel := types.LevelInfo
	if os.Getenv(constants.EnvNotificationLevel) != "" {
		parsedLevel, err := types.ParseLevel(os.Getenv(constants.EnvNotificationLevel))
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
			}).Errorf("main: got error while parsing notification level, defaulting to: %s", notificationLevel)
		} else {
			notificationLevel = parsedLevel
		}
	}

	notifCfg := &notification.Config{
		Attempts: 10,
		Level:    notificationLevel,
	}
	sender := notification.New(ctx)

	_, err = sender.Configure(notifCfg)
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
		}).Fatal("main: failed to configure notification sender manager")
	}

	// getting k8s provider
	k8sCfg := &kubernetes.Opts{
		ConfigPath: *kubeconfig,
	}

	if os.Getenv(EnvKubernetesConfig) != "" {
		k8sCfg.ConfigPath = os.Getenv(EnvKubernetesConfig)
	}

	k8sCfg.InCluster = *inCluster

	implementer, err := kubernetes.NewKubernetesImplementer(k8sCfg)
	if err != nil {
		log.WithFields(log.Fields{
			"error":  err,
			"config": k8sCfg,
		}).Fatal("main: failed to create kubernetes implementer")
	}

	var g workgroup.Group

	t := &k8s.Translator{
		FieldLogger: log.WithField("context", "translator"),
	}

	buf := k8s.NewBuffer(&g, t, log.StandardLogger(), 128)
	wl := log.WithField("context", "watch")
	k8s.WatchDeployments(&g, implementer.Client(), wl, buf)
	k8s.WatchStatefulSets(&g, implementer.Client(), wl, buf)
	k8s.WatchDaemonSets(&g, implementer.Client(), wl, buf)
	k8s.WatchCronJobs(&g, implementer.Client(), wl, buf)

	// approvalsCache := memory.NewMemoryCache()
	approvalsManager := approvals.New(&approvals.Opts{
		// Cache: approvalsCache,
		Store: sqlStore,
	})

	go approvalsManager.StartExpiryService(ctx)

	// setting up providers
	providers := setupProviders(&ProviderOpts{
		k8sImplementer:   implementer,
		sender:           sender,
		approvalsManager: approvalsManager,
		grc:              &t.GenericResourceCache,
		store:            sqlStore,
	})

	// registering secrets based credentials helper
	dockerConfig := make(secrets.DockerCfg)
	if os.Getenv(EnvDefaultDockerRegistryCfg) != "" {
		dockerConfigStr := os.Getenv(EnvDefaultDockerRegistryCfg)
		dockerConfig, err = secrets.DecodeDockerCfgJson([]byte(dockerConfigStr))
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
			}).Fatalf("failed to decode secret provided in %s env variable", EnvDefaultDockerRegistryCfg)
		}
	}
	secretsGetter := secrets.NewGetter(implementer, dockerConfig)

	ch := secretsCredentialsHelper.New(secretsGetter)
	credentialshelper.RegisterCredentialsHelper("secrets", ch)

	// trigger setup
	// teardownTriggers := setupTriggers(ctx, providers, approvalsManager, &t.GenericResourceCache, implementer)
	teardownTriggers := setupTriggers(ctx, &TriggerOpts{
		providers:        providers,
		approvalsManager: approvalsManager,
		grc:              &t.GenericResourceCache,
		k8sClient:        implementer,
		store:            sqlStore,
	})

	bot.Run(implementer, approvalsManager)

	signalChan := make(chan os.Signal, 1)
	cleanupDone := make(chan bool)
	signal.Notify(signalChan, os.Interrupt)
	g.Add(func(stop <-chan struct{}) {
		go func() {
			for range signalChan {
				log.Info("received an interrupt, shutting down...")
				go func() {
					select {
					case <-time.After(10 * time.Second):
						log.Info("connection shutdown took too long, exiting... ")
						close(cleanupDone)
						return
					case <-cleanupDone:
						return
					}
				}()
				providers.Stop()
				teardownTriggers()
				bot.Stop()

				cleanupDone <- true
			}
		}()
		<-cleanupDone
	})
	g.Run()
}

type ProviderOpts struct {
	k8sImplementer   kubernetes.Implementer
	sender           notification.Sender
	approvalsManager approvals.Manager
	grc              *k8s.GenericResourceCache
	store            store.Store
}

// setupProviders - setting up available providers. New providers should be initialised here and added to
// provider map
func setupProviders(opts *ProviderOpts) (providers provider.Providers) {
	var enabledProviders []provider.Provider

	k8sProvider, err := kubernetes.NewProvider(opts.k8sImplementer, opts.sender, opts.approvalsManager, opts.grc)
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
		}).Fatal("main.setupProviders: failed to create kubernetes provider")
	}
	go func() {
		err := k8sProvider.Start()
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
			}).Fatal("kubernetes provider stopped with an error")
		}
	}()

	enabledProviders = append(enabledProviders, k8sProvider)

	if os.Getenv(EnvHelmProvider) == "1" {
		tillerAddr := os.Getenv(EnvHelmTillerAddress)
		helmImplementer := helm.NewHelmImplementer(tillerAddr)
		helmProvider := helm.NewProvider(helmImplementer, opts.sender, opts.approvalsManager)

		go func() {
			err := helmProvider.Start()
			if err != nil {
				log.WithFields(log.Fields{
					"error": err,
				}).Fatal("helm provider stopped with an error")
			}
		}()

		enabledProviders = append(enabledProviders, helmProvider)
	}

	providers = provider.New(enabledProviders, opts.approvalsManager)

	return providers
}

type TriggerOpts struct {
	providers        provider.Providers
	approvalsManager approvals.Manager
	grc              *k8s.GenericResourceCache
	k8sClient        kubernetes.Implementer
	store            store.Store
}

// setupTriggers - setting up triggers. New triggers should be added to this function. Each trigger
// should go through all providers (or not if there is a reason) and submit events)
// func setupTriggers(ctx context.Context, providers provider.Providers, approvalsManager approvals.Manager, grc *k8s.GenericResourceCache, k8sClient kubernetes.Implementer) (teardown func()) {
func setupTriggers(ctx context.Context, opts *TriggerOpts) (teardown func()) {

	// setting up generic http webhook server
	whs := http.NewTriggerServer(&http.Opts{
		Port:             types.KeelDefaultPort,
		GRC:              opts.grc,
		KubernetesClient: opts.k8sClient,
		Providers:        opts.providers,
		ApprovalManager:  opts.approvalsManager,
		Store:            opts.store,
		Username:         os.Getenv(constants.EnvBasicAuthUser),
		Password:         os.Getenv(constants.EnvBasicAuthPassword),
	})

	go func() {
		err := whs.Start()
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
				"port":  types.KeelDefaultPort,
			}).Fatal("trigger server stopped")
		}
	}()

	// checking whether pubsub (GCR) trigger is enabled
	if os.Getenv(EnvTriggerPubSub) != "" {
		projectID := os.Getenv(EnvProjectID)
		if projectID == "" {
			log.Fatalf("main.setupTriggers: project ID env variable not set")
			return
		}

		ps, err := pubsub.NewPubsubSubscriber(&pubsub.Opts{
			ProjectID: projectID,
			Providers: opts.providers,
		})
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
			}).Fatal("main.setupTriggers: failed to create gcloud pubsub subscriber")
			return
		}

		subManager := pubsub.NewDefaultManager(os.Getenv(EnvClusterName), projectID, opts.providers, ps)
		go subManager.Start(ctx)
	}

	if os.Getenv(EnvTriggerPoll) != "0" {

		registryClient := registry.New()
		watcher := poll.NewRepositoryWatcher(opts.providers, registryClient)
		pollManager := poll.NewPollManager(opts.providers, watcher)

		// start poll manager, will finish with ctx
		go watcher.Start(ctx)
		go pollManager.Start(ctx)
	}

	teardown = func() {
		whs.Stop()
	}

	return teardown
}
<commit_msg>configurable data dir path<commit_after>package main

import (
	"os"
	"os/signal"
	"path/filepath"
	"time"

	"context"

	netContext "golang.org/x/net/context"
	kingpin "gopkg.in/alecthomas/kingpin.v2"

	"github.com/keel-hq/keel/approvals"
	"github.com/keel-hq/keel/bot"

	// "github.com/keel-hq/keel/cache/memory"
	"github.com/keel-hq/keel/pkg/http"
	"github.com/keel-hq/keel/pkg/store"
	"github.com/keel-hq/keel/pkg/store/sql"

	"github.com/keel-hq/keel/constants"
	"github.com/keel-hq/keel/extension/credentialshelper"
	"github.com/keel-hq/keel/extension/notification"
	"github.com/keel-hq/keel/internal/k8s"
	"github.com/keel-hq/keel/internal/workgroup"
	"github.com/keel-hq/keel/provider"
	"github.com/keel-hq/keel/provider/helm"
	"github.com/keel-hq/keel/provider/kubernetes"
	"github.com/keel-hq/keel/registry"
	"github.com/keel-hq/keel/secrets"
	"github.com/keel-hq/keel/trigger/poll"
	"github.com/keel-hq/keel/trigger/pubsub"
	"github.com/keel-hq/keel/types"
	"github.com/keel-hq/keel/version"

	// notification extensions
	"github.com/keel-hq/keel/extension/notification/auditor"
	_ "github.com/keel-hq/keel/extension/notification/hipchat"
	_ "github.com/keel-hq/keel/extension/notification/mattermost"
	_ "github.com/keel-hq/keel/extension/notification/slack"
	_ "github.com/keel-hq/keel/extension/notification/webhook"

	// credentials helpers
	_ "github.com/keel-hq/keel/extension/credentialshelper/aws"
	secretsCredentialsHelper "github.com/keel-hq/keel/extension/credentialshelper/secrets"

	// bots
	_ "github.com/keel-hq/keel/bot/hipchat"
	_ "github.com/keel-hq/keel/bot/slack"

	log "github.com/sirupsen/logrus"
)

// gcloud pubsub related config
const (
	EnvTriggerPubSub     = "PUBSUB" // set to 1 or something to enable pub/sub trigger
	EnvTriggerPoll       = "POLL"   // set to 0 to disable poll trigger
	EnvProjectID         = "PROJECT_ID"
	EnvClusterName       = "CLUSTER_NAME"
	EnvDataDir           = "DATA_DIR"
	EnvHelmProvider      = "HELM_PROVIDER"  // helm provider
	EnvHelmTillerAddress = "TILLER_ADDRESS" // helm provider

	// EnvDefaultDockerRegistryCfg - default registry configuration that can be passed into
	// keel for polling trigger
	EnvDefaultDockerRegistryCfg = "DOCKER_REGISTRY_CFG"
)

// kubernetes config, if empty - will default to InCluster
const (
	EnvKubernetesConfig = "KUBERNETES_CONFIG"
)

// EnvDebug - set to 1 or anything else to enable debug logging
const EnvDebug = "DEBUG"

func main() {
	ver := version.GetKeelVersion()

	inCluster := kingpin.Flag("incluster", "use in cluster configuration (defaults to 'true'), use '--no-incluster' if running outside of the cluster").Default("true").Bool()
	kubeconfig := kingpin.Flag("kubeconfig", "path to kubeconfig (if not in running inside a cluster)").Default(filepath.Join(os.Getenv("HOME"), ".kube", "config")).String()

	kingpin.UsageTemplate(kingpin.CompactUsageTemplate).Version(ver.Version)
	kingpin.CommandLine.Help = "Automated Kubernetes deployment updates. Learn more on https://keel.sh."
	kingpin.Parse()

	log.WithFields(log.Fields{
		"os":         ver.OS,
		"build_date": ver.BuildDate,
		"revision":   ver.Revision,
		"version":    ver.Version,
		"go_version": ver.GoVersion,
		"arch":       ver.Arch,
	}).Info("keel starting...")

	if os.Getenv(EnvDebug) != "" {
		log.SetLevel(log.DebugLevel)
	}

	dataDir := "/data"
	if os.Getenv(EnvDataDir) != "" {
		dataDir = os.Getenv(EnvDataDir)
	}

	sqlStore, err := sql.New(sql.Opts{
		DatabaseType: "sqlite3",
		URI:          filepath.Join(dataDir, "keel.db"),
	})
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
		}).Error("failed to initialize database")
		os.Exit(1)
	}
	log.WithFields(log.Fields{
		"database_path": filepath.Join(dataDir, "keel.db"),
		"type":          "sqlite3",
	}).Info("initializing database")

	// registering auditor to log events
	auditLogger := auditor.New(sqlStore)
	notification.RegisterSender("auditor", auditLogger)

	// setting up triggers
	ctx, cancel := netContext.WithCancel(context.Background())
	defer cancel()

	notificationLevel := types.LevelInfo
	if os.Getenv(constants.EnvNotificationLevel) != "" {
		parsedLevel, err := types.ParseLevel(os.Getenv(constants.EnvNotificationLevel))
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
			}).Errorf("main: got error while parsing notification level, defaulting to: %s", notificationLevel)
		} else {
			notificationLevel = parsedLevel
		}
	}

	notifCfg := &notification.Config{
		Attempts: 10,
		Level:    notificationLevel,
	}
	sender := notification.New(ctx)

	_, err = sender.Configure(notifCfg)
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
		}).Fatal("main: failed to configure notification sender manager")
	}

	// getting k8s provider
	k8sCfg := &kubernetes.Opts{
		ConfigPath: *kubeconfig,
	}

	if os.Getenv(EnvKubernetesConfig) != "" {
		k8sCfg.ConfigPath = os.Getenv(EnvKubernetesConfig)
	}

	k8sCfg.InCluster = *inCluster

	implementer, err := kubernetes.NewKubernetesImplementer(k8sCfg)
	if err != nil {
		log.WithFields(log.Fields{
			"error":  err,
			"config": k8sCfg,
		}).Fatal("main: failed to create kubernetes implementer")
	}

	var g workgroup.Group

	t := &k8s.Translator{
		FieldLogger: log.WithField("context", "translator"),
	}

	buf := k8s.NewBuffer(&g, t, log.StandardLogger(), 128)
	wl := log.WithField("context", "watch")
	k8s.WatchDeployments(&g, implementer.Client(), wl, buf)
	k8s.WatchStatefulSets(&g, implementer.Client(), wl, buf)
	k8s.WatchDaemonSets(&g, implementer.Client(), wl, buf)
	k8s.WatchCronJobs(&g, implementer.Client(), wl, buf)

	// approvalsCache := memory.NewMemoryCache()
	approvalsManager := approvals.New(&approvals.Opts{
		// Cache: approvalsCache,
		Store: sqlStore,
	})

	go approvalsManager.StartExpiryService(ctx)

	// setting up providers
	providers := setupProviders(&ProviderOpts{
		k8sImplementer:   implementer,
		sender:           sender,
		approvalsManager: approvalsManager,
		grc:              &t.GenericResourceCache,
		store:            sqlStore,
	})

	// registering secrets based credentials helper
	dockerConfig := make(secrets.DockerCfg)
	if os.Getenv(EnvDefaultDockerRegistryCfg) != "" {
		dockerConfigStr := os.Getenv(EnvDefaultDockerRegistryCfg)
		dockerConfig, err = secrets.DecodeDockerCfgJson([]byte(dockerConfigStr))
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
			}).Fatalf("failed to decode secret provided in %s env variable", EnvDefaultDockerRegistryCfg)
		}
	}
	secretsGetter := secrets.NewGetter(implementer, dockerConfig)

	ch := secretsCredentialsHelper.New(secretsGetter)
	credentialshelper.RegisterCredentialsHelper("secrets", ch)

	// trigger setup
	// teardownTriggers := setupTriggers(ctx, providers, approvalsManager, &t.GenericResourceCache, implementer)
	teardownTriggers := setupTriggers(ctx, &TriggerOpts{
		providers:        providers,
		approvalsManager: approvalsManager,
		grc:              &t.GenericResourceCache,
		k8sClient:        implementer,
		store:            sqlStore,
	})

	bot.Run(implementer, approvalsManager)

	signalChan := make(chan os.Signal, 1)
	cleanupDone := make(chan bool)
	signal.Notify(signalChan, os.Interrupt)
	g.Add(func(stop <-chan struct{}) {
		go func() {
			for range signalChan {
				log.Info("received an interrupt, shutting down...")
				go func() {
					select {
					case <-time.After(10 * time.Second):
						log.Info("connection shutdown took too long, exiting... ")
						close(cleanupDone)
						return
					case <-cleanupDone:
						return
					}
				}()
				providers.Stop()
				teardownTriggers()
				bot.Stop()

				cleanupDone <- true
			}
		}()
		<-cleanupDone
	})
	g.Run()
}

type ProviderOpts struct {
	k8sImplementer   kubernetes.Implementer
	sender           notification.Sender
	approvalsManager approvals.Manager
	grc              *k8s.GenericResourceCache
	store            store.Store
}

// setupProviders - setting up available providers. New providers should be initialised here and added to
// provider map
func setupProviders(opts *ProviderOpts) (providers provider.Providers) {
	var enabledProviders []provider.Provider

	k8sProvider, err := kubernetes.NewProvider(opts.k8sImplementer, opts.sender, opts.approvalsManager, opts.grc)
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
		}).Fatal("main.setupProviders: failed to create kubernetes provider")
	}
	go func() {
		err := k8sProvider.Start()
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
			}).Fatal("kubernetes provider stopped with an error")
		}
	}()

	enabledProviders = append(enabledProviders, k8sProvider)

	if os.Getenv(EnvHelmProvider) == "1" {
		tillerAddr := os.Getenv(EnvHelmTillerAddress)
		helmImplementer := helm.NewHelmImplementer(tillerAddr)
		helmProvider := helm.NewProvider(helmImplementer, opts.sender, opts.approvalsManager)

		go func() {
			err := helmProvider.Start()
			if err != nil {
				log.WithFields(log.Fields{
					"error": err,
				}).Fatal("helm provider stopped with an error")
			}
		}()

		enabledProviders = append(enabledProviders, helmProvider)
	}

	providers = provider.New(enabledProviders, opts.approvalsManager)

	return providers
}

type TriggerOpts struct {
	providers        provider.Providers
	approvalsManager approvals.Manager
	grc              *k8s.GenericResourceCache
	k8sClient        kubernetes.Implementer
	store            store.Store
}

// setupTriggers - setting up triggers. New triggers should be added to this function. Each trigger
// should go through all providers (or not if there is a reason) and submit events)
// func setupTriggers(ctx context.Context, providers provider.Providers, approvalsManager approvals.Manager, grc *k8s.GenericResourceCache, k8sClient kubernetes.Implementer) (teardown func()) {
func setupTriggers(ctx context.Context, opts *TriggerOpts) (teardown func()) {

	// setting up generic http webhook server
	whs := http.NewTriggerServer(&http.Opts{
		Port:             types.KeelDefaultPort,
		GRC:              opts.grc,
		KubernetesClient: opts.k8sClient,
		Providers:        opts.providers,
		ApprovalManager:  opts.approvalsManager,
		Store:            opts.store,
		Username:         os.Getenv(constants.EnvBasicAuthUser),
		Password:         os.Getenv(constants.EnvBasicAuthPassword),
	})

	go func() {
		err := whs.Start()
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
				"port":  types.KeelDefaultPort,
			}).Fatal("trigger server stopped")
		}
	}()

	// checking whether pubsub (GCR) trigger is enabled
	if os.Getenv(EnvTriggerPubSub) != "" {
		projectID := os.Getenv(EnvProjectID)
		if projectID == "" {
			log.Fatalf("main.setupTriggers: project ID env variable not set")
			return
		}

		ps, err := pubsub.NewPubsubSubscriber(&pubsub.Opts{
			ProjectID: projectID,
			Providers: opts.providers,
		})
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
			}).Fatal("main.setupTriggers: failed to create gcloud pubsub subscriber")
			return
		}

		subManager := pubsub.NewDefaultManager(os.Getenv(EnvClusterName), projectID, opts.providers, ps)
		go subManager.Start(ctx)
	}

	if os.Getenv(EnvTriggerPoll) != "0" {

		registryClient := registry.New()
		watcher := poll.NewRepositoryWatcher(opts.providers, registryClient)
		pollManager := poll.NewPollManager(opts.providers, watcher)

		// start poll manager, will finish with ctx
		go watcher.Start(ctx)
		go pollManager.Start(ctx)
	}

	teardown = func() {
		whs.Stop()
	}

	return teardown
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/dt/go-metrics-reporting"
	"github.com/dt/httpthrift"
	"github.com/foursquare/quiver/gen"
)

type Load struct {
	collection string
	sample     *int64
	keys       [][]byte

	server func() string

	diffing bool
	diff    func() string

	work chan bool

	rtt     string
	diffRtt string

	queueSize report.Guage
	dropped   report.Meter

	// for atomic keyset swaps in setKeys.
	sync.RWMutex
}

func GetQuiverClient(url func() string) *gen.HFileServiceClient {
	recv, send := httpthrift.NewDynamicClientProts(url)
	return gen.NewHFileServiceClientProtocol(nil, recv, send)
}

// Feeds the work channel at requested qps.
func (l *Load) generator(qps int) {
	pause := time.Duration(time.Second.Nanoseconds() / int64(qps))

	for _ = range time.Tick(pause) {
		l.queueSize.Update(int64(len(l.work)))
		select {
		case l.work <- true:
		default:
			l.dropped.Mark(1)
		}
	}
}

// given a string like testing=fsan44:20202, return (http://fsan44:20202/rpc/HFileService, testing).
func hfileUrlAndName(s string) (func() string, string) {
	name := strings.NewReplacer("http://", "", ".", "_", ":", "_", "/", "_").Replace(s)

	if parts := strings.Split(s, "="); len(parts) > 1 {
		s = parts[1]
		name = parts[0]
	}

	if !strings.Contains(s, "/") {
		fmt.Printf("'%s' doens't appear to specify a path. Appending /rpc/HFileService...\n", s)
		s = s + "/rpc/HFileService"
	}

	if !strings.HasPrefix(s, "http") {
		s = "http://" + s
	}
	return func() string { return s }, name
}

func main() {
	orig := flag.String("server", "localhost:9999", "URL of hfile server")
	rawDiff := flag.String("diff", "", "URL of second hfile server to compare")
	collection := flag.String("collection", "", "name of collection")
	graphite := report.Flag()
	workers := flag.Int("workers", 8, "worker pool size")
	qps := flag.Int("qps", 100, "qps to attempt")
	sample := flag.Int64("sampleSize", 1000, "number of random keys to use")

	flag.Parse()

	r := report.NewRecorder().
		MaybeReportTo(graphite).
		SetAsDefault()

	rttName := "rtt"
	server, name := hfileUrlAndName(*orig)

	if collection == nil || len(*collection) < 1 {
		fmt.Println("--collection is required")
		c := GetQuiverClient(server)
		r := &gen.InfoRequest{}

		if resp, err := c.GetInfo(r); err != nil {
			fmt.Println("tried to fetch possible collections but got an error:", err)
		} else {
			fmt.Println("possible --collection options:")
			for _, v := range resp {
				fmt.Println("\t", v.GetName())
			}
		}
		os.Exit(1)
	}

	diffing := false
	diffRtt := ""
	diffName := ""
	diff := func() string { return "" }

	if rawDiff != nil && len(*rawDiff) > 0 {
		diffing = true
		diff, diffName = hfileUrlAndName(*rawDiff)
		diffRtt = "rtt." + diffName
		rttName = "rtt." + name
	}

	l := &Load{
		collection: *collection,
		sample:     sample,
		server:     server,
		diffing:    diffing,
		diff:       diff,
		work:       make(chan bool, (*qps)*(*workers)),
		dropped:    r.GetMeter("dropped"),
		queueSize:  r.GetGuage("queue"),
		rtt:        rttName,
		diffRtt:    diffRtt,
	}

	if err := l.setKeys(); err != nil {
		fmt.Println("Failed to fetch testing keys:", err)
		os.Exit(1)
	}
	fmt.Printf("Sending %dqps to %s (%s), drawing from %d random keys...\n", *qps, name, server(), len(l.keys))
	if l.diff != nil {
		fmt.Printf("Diffing against %s (%s)\n", diffName, l.diff())
	}

	l.startWorkers(*workers)
	go l.startKeyFetcher(time.Minute)
	go l.generator(*qps)

	fmt.Print("Press enter for stats summary.\n")
	reader := bufio.NewReader(os.Stdin)

	for {
		reader.ReadString('\n')
		PrintSummary(rttName, diffRtt, l.diff != nil)
	}
}
<commit_msg>add service discovery support to load tester<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/dt/go-curator-discovery"
	"github.com/dt/go-metrics-reporting"
	"github.com/dt/httpthrift"
	"github.com/foursquare/quiver/gen"
)

type Load struct {
	collection string
	sample     *int64
	keys       [][]byte

	server func() string

	diffing bool
	diff    func() string

	work chan bool

	rtt     string
	diffRtt string

	queueSize report.Guage
	dropped   report.Meter

	// for atomic keyset swaps in setKeys.
	sync.RWMutex
}

func GetQuiverClient(url func() string) *gen.HFileServiceClient {
	recv, send := httpthrift.NewDynamicClientProts(url)
	return gen.NewHFileServiceClientProtocol(nil, recv, send)
}

// Feeds the work channel at requested qps.
func (l *Load) generator(qps int) {
	pause := time.Duration(time.Second.Nanoseconds() / int64(qps))

	for _ = range time.Tick(pause) {
		l.queueSize.Update(int64(len(l.work)))
		select {
		case l.work <- true:
		default:
			l.dropped.Mark(1)
		}
	}
}

// given a string like testing=fsan44:20202, return (http://fsan44:20202/rpc/HFileService, testing).
func hfileUrlAndName(s string) (func() string, string, discovery.Conn) {
	name := strings.NewReplacer("http://", "", ".", "_", ":", "_", "/", "_").Replace(s)

	if parts := strings.Split(s, "="); len(parts) > 1 {
		s = parts[1]
		name = parts[0]
	}

	if strings.HasPrefix(s, "zk:") {
		if len(zk) < 1 {
			log.Fatal("must specify --zk to use discovery")
		}
		s := s[len("zk:"):]
		shardAndPath := strings.Split(s, "@")
		if len(shardAndPath) != 2 {
			log.Fatal("format: zk:$SHARD@$PATH")
		}
		shard, path := shardAndPath[0], shardAndPath[1]

		disco, conn, err := discovery.NewServiceDiscoveryAndConn(zk, path)
		if err != nil {
			log.Fatal(err)
		} else {
			disco.Watch()
		}

		log.Printf("discovering instances of %s at %s\n", shard, path)
		provider := disco.Provider(shard)
		f := func() string {
			i, err := provider.GetInstance()
			if err != nil {
				log.Println("error discovering instance:", err)
			} else if i == nil {
				log.Println("no instances found")
			} else {
				return fmt.Sprintf("http://%s/rpc/HFileService", i.Spec())
			}
			return ""
		}

		return f, name, conn
	}

	if !strings.Contains(s, "/") {
		fmt.Printf("'%s' doens't appear to specify a path. Appending /rpc/HFileService...\n", s)
		s = s + "/rpc/HFileService"
	}

	if !strings.HasPrefix(s, "http") {
		s = "http://" + s
	}
	return func() string { return s }, name, nil
}

var zk = ""

func main() {
	orig := flag.String("server", "localhost:9999", "URL of hfile server")
	rawDiff := flag.String("diff", "", "URL of second hfile server to compare")
	collection := flag.String("collection", "", "name of collection")
	graphite := report.Flag()
	workers := flag.Int("workers", 8, "worker pool size")
	flag.StringVar(&zk, "zk", "", "zookeeper host")
	qps := flag.Int("qps", 100, "qps to attempt")
	sample := flag.Int64("sampleSize", 1000, "number of random keys to use")

	flag.Parse()

	r := report.NewRecorder().
		MaybeReportTo(graphite).
		SetAsDefault()

	rttName := "rtt"
	server, name, conn := hfileUrlAndName(*orig)
	if conn != nil {
		defer conn.Close()
	}

	if collection == nil || len(*collection) < 1 {
		fmt.Println("--collection is required")
		c := GetQuiverClient(server)
		r := &gen.InfoRequest{}

		if resp, err := c.GetInfo(r); err != nil {
			fmt.Println("tried to fetch possible collections but got an error:", err)
		} else {
			fmt.Println("possible --collection options:")
			for _, v := range resp {
				fmt.Println("\t", v.GetName())
			}
		}
		os.Exit(1)
	}

	diffing := false
	diffRtt := ""
	diffName := ""
	diff := func() string { return "" }

	if rawDiff != nil && len(*rawDiff) > 0 {
		diffing = true
		diff, diffName, conn = hfileUrlAndName(*rawDiff)
		if conn != nil {
			defer conn.Close()
		}
		diffRtt = "rtt." + diffName
		rttName = "rtt." + name
	}

	l := &Load{
		collection: *collection,
		sample:     sample,
		server:     server,
		diffing:    diffing,
		diff:       diff,
		work:       make(chan bool, (*qps)*(*workers)),
		dropped:    r.GetMeter("dropped"),
		queueSize:  r.GetGuage("queue"),
		rtt:        rttName,
		diffRtt:    diffRtt,
	}

	if err := l.setKeys(); err != nil {
		fmt.Println("Failed to fetch testing keys:", err)
		os.Exit(1)
	}
	fmt.Printf("Sending %dqps to %s (%s), drawing from %d random keys...\n", *qps, name, server(), len(l.keys))
	if l.diff != nil {
		fmt.Printf("Diffing against %s (%s)\n", diffName, l.diff())
	}

	l.startWorkers(*workers)
	go l.startKeyFetcher(time.Minute)
	go l.generator(*qps)

	fmt.Print("Press enter for stats summary.\n")
	reader := bufio.NewReader(os.Stdin)

	for {
		reader.ReadString('\n')
		PrintSummary(rttName, diffRtt, l.diff != nil)
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"os"
	"syscall"

	"github.com/minio/cli"
	"github.com/minio/mc/pkg/probe"
)

var (
	pipeFlags = []cli.Flag{
		cli.StringFlag{
			Name:  "encrypt",
			Usage: "encrypt objects (using server-side encryption with server managed keys)",
		},
		cli.StringFlag{
			Name:  "storage-class, sc",
			Usage: "set storage class for new object(s) on target",
		},
		cli.StringFlag{
			Name:  "attr",
			Usage: "add custom metadata for the object",
		},
		cli.StringFlag{
			Name:  "tags",
			Usage: "apply tags to the uploaded objects",
		},
	}
)

// Display contents of a file.
var pipeCmd = cli.Command{
	Name:         "pipe",
	Usage:        "stream STDIN to an object",
	Action:       mainPipe,
	OnUsageError: onUsageError,
	Before:       setGlobalsFromContext,
	Flags:        append(append(pipeFlags, ioFlags...), globalFlags...),
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} [FLAGS] [TARGET]
{{if .VisibleFlags}}
FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}{{end}}
ENVIRONMENT VARIABLES:
  MC_ENCRYPT:      list of comma delimited prefix values
  MC_ENCRYPT_KEY:  list of comma delimited prefix=secret values

EXAMPLES:
  1. Write contents of stdin to a file on local filesystem.
     {{.Prompt}} {{.HelpName}} /tmp/hello-world.go

  2. Write contents of stdin to an object on Amazon S3 cloud storage.
     {{.Prompt}} {{.HelpName}} s3/personalbuck/meeting-notes.txt

  3. Copy an ISO image to an object on Amazon S3 cloud storage.
     {{.Prompt}} cat debian-8.2.iso | {{.HelpName}} s3/opensource-isos/gnuos.iso

  4. Stream MySQL database dump to Amazon S3 directly.
     {{.Prompt}} mysqldump -u root -p ******* accountsdb | {{.HelpName}} s3/sql-backups/backups/accountsdb-oct-9-2015.sql

  5. Write contents of stdin to an object on Amazon S3 cloud storage and assign REDUCED_REDUNDANCY storage-class to the uploaded object.
     {{.Prompt}} {{.HelpName}} --storage-class REDUCED_REDUNDANCY s3/personalbuck/meeting-notes.txt

  6. Copy to MinIO cloud storage with specified metadata, separated by ";"
      {{.Prompt}} cat music.mp3 | {{.HelpName}} --attr "Cache-Control=max-age=90000,min-fresh=9000;Artist=Unknown" play/mybucket/music.mp3

  7. Set tags to the uploaded objects
      {{.Prompt}} tar cvf - . | {{.HelpName}} --tags "category=backup" play/mybucket/backup.tar
`,
}

func pipe(targetURL string, encKeyDB map[string][]prefixSSEPair, storageClass string, meta map[string]string) *probe.Error {
	if targetURL == "" {
		// When no target is specified, pipe cat's stdin to stdout.
		return catOut(os.Stdin, -1).Trace()
	}
	alias, _ := url2Alias(targetURL)
	sseKey := getSSE(targetURL, encKeyDB[alias])

	// Stream from stdin to multiple objects until EOF.
	// Ignore size, since os.Stat() would not return proper size all the time
	// for local filesystem for example /proc files.
	opts := PutOptions{
		sse:          sseKey,
		storageClass: storageClass,
		metadata:     meta,
	}
	_, err := putTargetStreamWithURL(targetURL, os.Stdin, -1, opts)
	// TODO: See if this check is necessary.
	switch e := err.ToGoError().(type) {
	case *os.PathError:
		if e.Err == syscall.EPIPE {
			// stdin closed by the user. Gracefully exit.
			return nil
		}
	}
	return err.Trace(targetURL)
}

// check pipe input arguments.
func checkPipeSyntax(ctx *cli.Context) {
	if len(ctx.Args()) > 1 {
		cli.ShowCommandHelpAndExit(ctx, "pipe", 1) // last argument is exit code.
	}
}

// mainPipe is the main entry point for pipe command.
func mainPipe(ctx *cli.Context) error {
	// Parse encryption keys per command.
	encKeyDB, err := getEncKeys(ctx)
	fatalIf(err, "Unable to parse encryption keys.")

	// validate pipe input arguments.
	checkPipeSyntax(ctx)

	meta, err := getMetaDataEntry(ctx.String("attr"))
	fatalIf(err.Trace(""), "Unable to parse --attr value")
	if tags := ctx.String("tags"); tags != "" {
		meta["X-Amz-Tagging"] = tags
	}
	if len(ctx.Args()) == 0 {
		err = pipe("", nil, ctx.String("storage-class"), meta)
		fatalIf(err.Trace("stdout"), "Unable to write to one or more targets.")
	} else {
		// extract URLs.
		URLs := ctx.Args()
		err = pipe(URLs[0], encKeyDB, ctx.String("storage-class"), meta)
		fatalIf(err.Trace(URLs[0]), "Unable to write to one or more targets.")
	}

	// Done.
	return nil
}
<commit_msg>attr is optional for mc pipe<commit_after>// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"os"
	"syscall"

	"github.com/minio/cli"
	"github.com/minio/mc/pkg/probe"
)

var (
	pipeFlags = []cli.Flag{
		cli.StringFlag{
			Name:  "encrypt",
			Usage: "encrypt objects (using server-side encryption with server managed keys)",
		},
		cli.StringFlag{
			Name:  "storage-class, sc",
			Usage: "set storage class for new object(s) on target",
		},
		cli.StringFlag{
			Name:  "attr",
			Usage: "add custom metadata for the object",
		},
		cli.StringFlag{
			Name:  "tags",
			Usage: "apply tags to the uploaded objects",
		},
	}
)

// Display contents of a file.
var pipeCmd = cli.Command{
	Name:         "pipe",
	Usage:        "stream STDIN to an object",
	Action:       mainPipe,
	OnUsageError: onUsageError,
	Before:       setGlobalsFromContext,
	Flags:        append(append(pipeFlags, ioFlags...), globalFlags...),
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} [FLAGS] [TARGET]
{{if .VisibleFlags}}
FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}{{end}}
ENVIRONMENT VARIABLES:
  MC_ENCRYPT:      list of comma delimited prefix values
  MC_ENCRYPT_KEY:  list of comma delimited prefix=secret values

EXAMPLES:
  1. Write contents of stdin to a file on local filesystem.
     {{.Prompt}} {{.HelpName}} /tmp/hello-world.go

  2. Write contents of stdin to an object on Amazon S3 cloud storage.
     {{.Prompt}} {{.HelpName}} s3/personalbuck/meeting-notes.txt

  3. Copy an ISO image to an object on Amazon S3 cloud storage.
     {{.Prompt}} cat debian-8.2.iso | {{.HelpName}} s3/opensource-isos/gnuos.iso

  4. Stream MySQL database dump to Amazon S3 directly.
     {{.Prompt}} mysqldump -u root -p ******* accountsdb | {{.HelpName}} s3/sql-backups/backups/accountsdb-oct-9-2015.sql

  5. Write contents of stdin to an object on Amazon S3 cloud storage and assign REDUCED_REDUNDANCY storage-class to the uploaded object.
     {{.Prompt}} {{.HelpName}} --storage-class REDUCED_REDUNDANCY s3/personalbuck/meeting-notes.txt

  6. Copy to MinIO cloud storage with specified metadata, separated by ";"
      {{.Prompt}} cat music.mp3 | {{.HelpName}} --attr "Cache-Control=max-age=90000,min-fresh=9000;Artist=Unknown" play/mybucket/music.mp3

  7. Set tags to the uploaded objects
      {{.Prompt}} tar cvf - . | {{.HelpName}} --tags "category=backup" play/mybucket/backup.tar
`,
}

func pipe(targetURL string, encKeyDB map[string][]prefixSSEPair, storageClass string, meta map[string]string) *probe.Error {
	if targetURL == "" {
		// When no target is specified, pipe cat's stdin to stdout.
		return catOut(os.Stdin, -1).Trace()
	}
	alias, _ := url2Alias(targetURL)
	sseKey := getSSE(targetURL, encKeyDB[alias])

	// Stream from stdin to multiple objects until EOF.
	// Ignore size, since os.Stat() would not return proper size all the time
	// for local filesystem for example /proc files.
	opts := PutOptions{
		sse:          sseKey,
		storageClass: storageClass,
		metadata:     meta,
	}
	_, err := putTargetStreamWithURL(targetURL, os.Stdin, -1, opts)
	// TODO: See if this check is necessary.
	switch e := err.ToGoError().(type) {
	case *os.PathError:
		if e.Err == syscall.EPIPE {
			// stdin closed by the user. Gracefully exit.
			return nil
		}
	}
	return err.Trace(targetURL)
}

// check pipe input arguments.
func checkPipeSyntax(ctx *cli.Context) {
	if len(ctx.Args()) > 1 {
		cli.ShowCommandHelpAndExit(ctx, "pipe", 1) // last argument is exit code.
	}
}

// mainPipe is the main entry point for pipe command.
func mainPipe(ctx *cli.Context) error {
	// Parse encryption keys per command.
	encKeyDB, err := getEncKeys(ctx)
	fatalIf(err, "Unable to parse encryption keys.")

	// validate pipe input arguments.
	checkPipeSyntax(ctx)

	var meta = map[string]string{}
	if attr := ctx.String("attr"); attr != "" {
		meta, err = getMetaDataEntry(attr)
		fatalIf(err.Trace(attr), "Unable to parse --attr value")
	}
	if tags := ctx.String("tags"); tags != "" {
		meta["X-Amz-Tagging"] = tags
	}
	if len(ctx.Args()) == 0 {
		err = pipe("", nil, ctx.String("storage-class"), meta)
		fatalIf(err.Trace("stdout"), "Unable to write to one or more targets.")
	} else {
		// extract URLs.
		URLs := ctx.Args()
		err = pipe(URLs[0], encKeyDB, ctx.String("storage-class"), meta)
		fatalIf(err.Trace(URLs[0]), "Unable to write to one or more targets.")
	}

	// Done.
	return nil
}
<|endoftext|>