<commit_before>// Copyright 2017-2021 The Usacloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/sacloud/usacloud/pkg/query"

	"github.com/sacloud/usacloud/pkg/validate"

	"github.com/sacloud/libsacloud/v2/sacloud"
	"github.com/sacloud/libsacloud/v2/sacloud/profile"
	"github.com/spf13/pflag"
)

// Config CLI全コマンドが利用するフラグ
type Config struct {
	profile.ConfigValue

	// Profile プロファイル名
	Profile string `json:"-"`

	// DefaultOutputType デフォルトアウトプットタイプ
	DefaultOutputType string

	// NoColor ANSIエスケープシーケンスによる色つけを無効化
	NoColor bool

	// ProcessTimeoutSec コマンド全体の実行タイムアウトまでの秒数
	ProcessTimeoutSec int

	// ArgumentMatchMode 引数でリソースを特定する際にリソース名と引数を比較する方法を指定
	// 有効な値:
	//   - `partial`(デフォルト): 部分一致
	//   - `exact`: 完全一致
	// Note: 引数はID or Name or Tagsと比較されるが、この項目はNameとの比較にのみ影響する。IDとTagsは常に完全一致となる。
	ArgumentMatchMode string

	// DefaultQueryDriver 各コマンドで--query-driverが省略された場合のデフォルト値
	// 有効な値:
	//   - `jmespath`(デフォルト): JMESPath
	//   - `jq` : gojq
	DefaultQueryDriver string
}

var DefaultProcessTimeoutSec = 60 * 60 * 2 // 2時間
var DefaultQueryDriver = query.DriverJMESPath

// LoadConfigValue 指定のフラグセットからフラグを読み取り*Flagsを組み立てて返す
func LoadConfigValue(flags *pflag.FlagSet, errW io.Writer, skipLoadingProfile bool) (*Config, error) {
	o := &Config{
		ConfigValue: profile.ConfigValue{
			Zones: append(sacloud.SakuraCloudZones, "all"),
		},
	}
	if skipLoadingProfile {
		return o, nil
	}

	o.loadConfig(flags, errW)
	return o, o.Validate(false)
}

func (o *Config) IsEmpty() bool {
	return o.AccessToken == "" &&
		o.AccessTokenSecret == "" &&
		o.Zone == "" && o.DefaultOutputType == ""
}

func (o *Config) loadConfig(flags *pflag.FlagSet, errW io.Writer) {
	// プロファイルだけ先に環境変数を読んでおく
	if o.Profile == "" {
		o.Profile = stringFromEnvMulti([]string{"SAKURACLOUD_PROFILE", "USACLOUD_PROFILE"}, "default")
	}

	o.loadFromProfile(flags, errW)
	o.loadFromEnv()
	o.loadFromFlags(flags, errW)
	o.fillDefaults()
}

func (o *Config) fillDefaults() {
	if len(o.Zones) == 0 {
		o.Zones = sacloud.SakuraCloudZones
	}
}

func (o *Config) loadFromEnv() {
	if o.AccessToken == "" {
		o.AccessToken = stringFromEnv("SAKURACLOUD_ACCESS_TOKEN", "")
	}
	if o.AccessTokenSecret == "" {
		o.AccessTokenSecret = stringFromEnv("SAKURACLOUD_ACCESS_TOKEN_SECRET", "")
	}
	if o.Zone == "" {
		o.Zone = stringFromEnv("SAKURACLOUD_ZONE", "")
	}
	if len(o.Zones) == 0 {
		o.Zones = stringSliceFromEnv("SAKURACLOUD_ZONES", append(sacloud.SakuraCloudZones, "all"))
	}
	if o.AcceptLanguage == "" {
		o.AcceptLanguage = stringFromEnv("SAKURACLOUD_ACCEPT_LANGUAGE", "")
	}
	if o.RetryMax <= 0 {
		o.RetryMax = intFromEnv("SAKURACLOUD_RETRY_MAX", sacloud.APIDefaultRetryMax)
	}
	if o.RetryWaitMax <= 0 {
		o.RetryWaitMax = intFromEnv("SAKURACLOUD_RETRY_WAIT_MAX", 64)
	}
	if o.RetryWaitMin <= 0 {
		o.RetryWaitMin = intFromEnv("SAKURACLOUD_RETRY_WAIT_MIN", 1)
	}
	if o.HTTPRequestTimeout <= 0 {
		o.HTTPRequestTimeout = intFromEnv("SAKURACLOUD_API_REQUEST_TIMEOUT", 300)
	}
	if o.HTTPRequestRateLimit <= 0 {
		o.HTTPRequestRateLimit = intFromEnv("SAKURACLOUD_API_REQUEST_RATE_LIMIT", 5) // デフォルト5ゾーン分(is1a/is1b/tk1a/tk1b/tk1v)
	}
	if o.APIRootURL == "" {
		o.APIRootURL = stringFromEnv("SAKURACLOUD_API_ROOT_URL", sacloud.SakuraCloudAPIRoot)
	}
	if o.DefaultZone == "" {
		o.DefaultZone = stringFromEnv("SAKURACLOUD_DEFAULT_ZONE", sacloud.APIDefaultZone)
	}
	if o.TraceMode == "" {
		o.TraceMode = stringFromEnv("SAKURACLOUD_TRACE", "")
	}
	if !o.FakeMode {
		o.FakeMode = os.Getenv("SAKURACLOUD_FAKE_MODE") != ""
	}
	if o.FakeStorePath == "" {
		o.FakeStorePath = stringFromEnv("SAKURACLOUD_FAKE_STORE_PATH", "")
	}
	if o.ProcessTimeoutSec <= 0 {
		o.ProcessTimeoutSec = intFromEnv("SAKURACLOUD_PROCESS_TIMEOUT_SEC", DefaultProcessTimeoutSec)
	}
	if o.ArgumentMatchMode == "" {
		o.ArgumentMatchMode = stringFromEnv("SAKURACLOUD_ARGUMENT_MATCH_MODE", "partial")
	}
	if o.DefaultOutputType == "" {
		o.DefaultOutputType = stringFromEnv("SAKURACLOUD_DEFAULT_OUTPUT_TYPE", "")
	}
	if o.DefaultQueryDriver == "" {
		o.DefaultQueryDriver = stringFromEnv("SAKURACLOUD_DEFAULT_QUERY_DRIVER", "")
	}
}

func (o *Config) loadFromFlags(flags *pflag.FlagSet, errW io.Writer) {
	if flags.Changed("token") {
		v, err := flags.GetString("token")
		if err != nil {
			fmt.Fprintf(errW, "[WARN] reading value of %q flag is failed: %s", "token", err) // nolint
			return
		}
		o.AccessToken = v
	}
	if flags.Changed("secret") {
		v, err := flags.GetString("secret")
		if err != nil {
			fmt.Fprintf(errW, "[WARN] reading value of %q flag is failed: %s", "secret", err) // nolint
			return
		}
		o.AccessTokenSecret = v
	}
	if flags.Changed("zones") {
		v, err := flags.GetStringSlice("zones")
		if err != nil {
			fmt.Fprintf(errW, "[WARN] reading value of %q flag is failed: %s", "zones", err) // nolint
			return
		}
		o.Zones = v
	}
	if flags.Changed("no-color") {
		v, err := flags.GetBool("no-color")
		if err != nil {
			fmt.Fprintf(errW, "[WARN] reading value of %q flag is failed: %s", "no-color", err) // nolint
			return
		}
		o.NoColor = v
	}
	if flags.Changed("trace") {
		v, err := flags.GetBool("trace")
		if err != nil {
			fmt.Fprintf(errW, "[WARN] reading value of %q flag is failed: %s", "trace", err) // nolint
			return
		}
		if v {
			o.TraceMode = "all"
		}
	}
	if flags.Changed("fake") {
		v, err := flags.GetBool("fake")
		if err != nil {
			fmt.Fprintf(errW, "[WARN] reading value of %q flag is failed: %s", "fake", err) // nolint
			return
		}
		o.FakeMode = v
	}
	if flags.Changed("fake-store") {
		v, err := flags.GetString("fake-store")
		if err != nil {
			fmt.Fprintf(errW, "[WARN] reading value of %q flag is failed: %s", "fake-store", err) // nolint
			return
		}
		o.FakeStorePath = v
	}
	if flags.Changed("process-timeout-sec") {
		v, err := flags.GetInt("process-timeout-sec")
		if err != nil {
			fmt.Fprintf(errW, "[WARN] reading value of %q flag is failed: %s", "process-timeout-sec", err) // nolint
			return
		}
		o.ProcessTimeoutSec = v
	}
	if flags.Changed("argument-match-mode") {
		v, err := flags.GetString("argument-match-mode")
		if err != nil {
			fmt.Fprintf(errW, "[WARN] reading value of %q flag is failed: %s", "argument-match-mode", err) // nolint
			return
		}
		o.ArgumentMatchMode = v
	}
}

func stringFromEnv(key, defaultValue string) string {
	v := os.Getenv(key)
	if v == "" {
		return defaultValue
	}
	return v
}

func stringFromEnvMulti(keys []string, defaultValue string) string {
	for _, key := range keys {
		v := os.Getenv(key)
		if v != "" {
			return v
		}
	}
	return defaultValue
}

func stringSliceFromEnv(key string, defaultValue []string) []string {
	v := os.Getenv(key)
	if v == "" {
		return defaultValue
	}
	values := strings.Split(v, ",")
	for i := range values {
		values[i] = strings.Trim(values[i], " ")
	}
	return values
}

func intFromEnv(key string, defaultValue int) int {
	v := os.Getenv(key)
	if v == "" {
		return defaultValue
	}
	i, err := strconv.ParseInt(v, 10, 64)
	if err != nil {
		return defaultValue
	}
	return int(i)
}

func (o *Config) Validate(skipCred bool) error {
	var errs []error

	if !skipCred {
		if o.AccessToken == "" {
			errs = append(errs, validate.NewFlagError("--token", "required"))
		}
		if o.AccessTokenSecret == "" {
			errs = append(errs, validate.NewFlagError("--secret", "required"))
		}
	}
	switch o.DefaultOutputType {
	case "", "table", "json", "yaml":
		// noop
	default:
		errs = append(errs, validate.NewFlagError("profile.DefaultOutputType", "must be one of [table/json/yaml]"))
	}

	switch o.ArgumentMatchMode {
	case "", "partial", "exact":
		// noop
	default:
		errs = append(errs, validate.NewFlagError("--argument-match-mode", "must be one of [partial/exact]"))
	}

	switch o.DefaultQueryDriver {
	case "", query.DriverJMESPath, query.DriverGoJQ:
		// noop
	default:
		errs = append(errs, validate.NewFlagError("profile.DefaultQueryDriver", "must be one of [jmespath/jq]"))
	}

	return validate.NewValidationError(errs...)
}

func (o *Config) ProcessTimeout() time.Duration {
	sec := o.ProcessTimeoutSec
	if sec <= 0 {
		sec = DefaultProcessTimeoutSec
	}
	return time.Duration(sec) * time.Second
}

func (o *Config) ArgumentMatchModeValue() string {
	if o.ArgumentMatchMode == "" {
		return "partial"
	}
	return o.ArgumentMatchMode
}
<commit_msg>Fix cannot load current config<commit_after>// Copyright 2017-2021 The Usacloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/sacloud/usacloud/pkg/query"

	"github.com/sacloud/usacloud/pkg/validate"

	"github.com/sacloud/libsacloud/v2/sacloud"
	"github.com/sacloud/libsacloud/v2/sacloud/profile"
	"github.com/spf13/pflag"
)

// Config CLI全コマンドが利用するフラグ
type Config struct {
	profile.ConfigValue

	// Profile プロファイル名
	Profile string `json:"-"`

	// DefaultOutputType デフォルトアウトプットタイプ
	DefaultOutputType string

	// NoColor ANSIエスケープシーケンスによる色つけを無効化
	NoColor bool

	// ProcessTimeoutSec コマンド全体の実行タイムアウトまでの秒数
	ProcessTimeoutSec int

	// ArgumentMatchMode 引数でリソースを特定する際にリソース名と引数を比較する方法を指定
	// 有効な値:
	//   - `partial`(デフォルト): 部分一致
	//   - `exact`: 完全一致
	// Note: 引数はID or Name or Tagsと比較されるが、この項目はNameとの比較にのみ影響する。IDとTagsは常に完全一致となる。
	ArgumentMatchMode string

	// DefaultQueryDriver 各コマンドで--query-driverが省略された場合のデフォルト値
	// 有効な値:
	//   - `jmespath`(デフォルト): JMESPath
	//   - `jq` : gojq
	DefaultQueryDriver string
}

var DefaultProcessTimeoutSec = 60 * 60 * 2 // 2時間
var DefaultQueryDriver = query.DriverJMESPath

// LoadConfigValue 指定のフラグセットからフラグを読み取り*Flagsを組み立てて返す
func LoadConfigValue(flags *pflag.FlagSet, errW io.Writer, skipLoadingProfile bool) (*Config, error) {
	o := &Config{
		ConfigValue: profile.ConfigValue{
			Zones: append(sacloud.SakuraCloudZones, "all"),
		},
	}
	if skipLoadingProfile {
		return o, nil
	}

	o.loadConfig(flags, errW)
	return o, o.Validate(false)
}

func (o *Config) IsEmpty() bool {
	return o.AccessToken == "" &&
		o.AccessTokenSecret == "" &&
		o.Zone == "" && o.DefaultOutputType == ""
}

func (o *Config) loadConfig(flags *pflag.FlagSet, errW io.Writer) {
	// プロファイルだけ先に環境変数を読んでおく
	if o.Profile == "" {
		o.Profile = stringFromEnvMulti([]string{"SAKURACLOUD_PROFILE", "USACLOUD_PROFILE"}, "")
	}

	o.loadFromProfile(flags, errW)
	o.loadFromEnv()
	o.loadFromFlags(flags, errW)
	o.fillDefaults()
}

func (o *Config) fillDefaults() {
	if len(o.Zones) == 0 {
		o.Zones = sacloud.SakuraCloudZones
	}
}

func (o *Config) loadFromEnv() {
	if o.AccessToken == "" {
		o.AccessToken = stringFromEnv("SAKURACLOUD_ACCESS_TOKEN", "")
	}
	if o.AccessTokenSecret == "" {
		o.AccessTokenSecret = stringFromEnv("SAKURACLOUD_ACCESS_TOKEN_SECRET", "")
	}
	if o.Zone == "" {
		o.Zone = stringFromEnv("SAKURACLOUD_ZONE", "")
	}
	if len(o.Zones) == 0 {
		o.Zones = stringSliceFromEnv("SAKURACLOUD_ZONES", append(sacloud.SakuraCloudZones, "all"))
	}
	if o.AcceptLanguage == "" {
		o.AcceptLanguage = stringFromEnv("SAKURACLOUD_ACCEPT_LANGUAGE", "")
	}
	if o.RetryMax <= 0 {
		o.RetryMax = intFromEnv("SAKURACLOUD_RETRY_MAX", sacloud.APIDefaultRetryMax)
	}
	if o.RetryWaitMax <= 0 {
		o.RetryWaitMax = intFromEnv("SAKURACLOUD_RETRY_WAIT_MAX", 64)
	}
	if o.RetryWaitMin <= 0 {
		o.RetryWaitMin = intFromEnv("SAKURACLOUD_RETRY_WAIT_MIN", 1)
	}
	if o.HTTPRequestTimeout <= 0 {
		o.HTTPRequestTimeout = intFromEnv("SAKURACLOUD_API_REQUEST_TIMEOUT", 300)
	}
	if o.HTTPRequestRateLimit <= 0 {
		o.HTTPRequestRateLimit = intFromEnv("SAKURACLOUD_API_REQUEST_RATE_LIMIT", 5) // デフォルト5ゾーン分(is1a/is1b/tk1a/tk1b/tk1v)
	}
	if o.APIRootURL == "" {
		o.APIRootURL = stringFromEnv("SAKURACLOUD_API_ROOT_URL", sacloud.SakuraCloudAPIRoot)
	}
	if o.DefaultZone == "" {
		o.DefaultZone = stringFromEnv("SAKURACLOUD_DEFAULT_ZONE", sacloud.APIDefaultZone)
	}
	if o.TraceMode == "" {
		o.TraceMode = stringFromEnv("SAKURACLOUD_TRACE", "")
	}
	if !o.FakeMode {
		o.FakeMode = os.Getenv("SAKURACLOUD_FAKE_MODE") != ""
	}
	if o.FakeStorePath == "" {
		o.FakeStorePath = stringFromEnv("SAKURACLOUD_FAKE_STORE_PATH", "")
	}
	if o.ProcessTimeoutSec <= 0 {
		o.ProcessTimeoutSec = intFromEnv("SAKURACLOUD_PROCESS_TIMEOUT_SEC", DefaultProcessTimeoutSec)
	}
	if o.ArgumentMatchMode == "" {
		o.ArgumentMatchMode = stringFromEnv("SAKURACLOUD_ARGUMENT_MATCH_MODE", "partial")
	}
	if o.DefaultOutputType == "" {
		o.DefaultOutputType = stringFromEnv("SAKURACLOUD_DEFAULT_OUTPUT_TYPE", "")
	}
	if o.DefaultQueryDriver == "" {
		o.DefaultQueryDriver = stringFromEnv("SAKURACLOUD_DEFAULT_QUERY_DRIVER", "")
	}
}

func (o *Config) loadFromFlags(flags *pflag.FlagSet, errW io.Writer) {
	if flags.Changed("token") {
		v, err := flags.GetString("token")
		if err != nil {
			fmt.Fprintf(errW, "[WARN] reading value of %q flag is failed: %s", "token", err) // nolint
			return
		}
		o.AccessToken = v
	}
	if flags.Changed("secret") {
		v, err := flags.GetString("secret")
		if err != nil {
			fmt.Fprintf(errW, "[WARN] reading value of %q flag is failed: %s", "secret", err) // nolint
			return
		}
		o.AccessTokenSecret = v
	}
	if flags.Changed("zones") {
		v, err := flags.GetStringSlice("zones")
		if err != nil {
			fmt.Fprintf(errW, "[WARN] reading value of %q flag is failed: %s", "zones", err) // nolint
			return
		}
		o.Zones = v
	}
	if flags.Changed("no-color") {
		v, err := flags.GetBool("no-color")
		if err != nil {
			fmt.Fprintf(errW, "[WARN] reading value of %q flag is failed: %s", "no-color", err) // nolint
			return
		}
		o.NoColor = v
	}
	if flags.Changed("trace") {
		v, err := flags.GetBool("trace")
		if err != nil {
			fmt.Fprintf(errW, "[WARN] reading value of %q flag is failed: %s", "trace", err) // nolint
			return
		}
		if v {
			o.TraceMode = "all"
		}
	}
	if flags.Changed("fake") {
		v, err := flags.GetBool("fake")
		if err != nil {
			fmt.Fprintf(errW, "[WARN] reading value of %q flag is failed: %s", "fake", err) // nolint
			return
		}
		o.FakeMode = v
	}
	if flags.Changed("fake-store") {
		v, err := flags.GetString("fake-store")
		if err != nil {
			fmt.Fprintf(errW, "[WARN] reading value of %q flag is failed: %s", "fake-store", err) // nolint
			return
		}
		o.FakeStorePath = v
	}
	if flags.Changed("process-timeout-sec") {
		v, err := flags.GetInt("process-timeout-sec")
		if err != nil {
			fmt.Fprintf(errW, "[WARN] reading value of %q flag is failed: %s", "process-timeout-sec", err) // nolint
			return
		}
		o.ProcessTimeoutSec = v
	}
	if flags.Changed("argument-match-mode") {
		v, err := flags.GetString("argument-match-mode")
		if err != nil {
			fmt.Fprintf(errW, "[WARN] reading value of %q flag is failed: %s", "argument-match-mode", err) // nolint
			return
		}
		o.ArgumentMatchMode = v
	}
}

func stringFromEnv(key, defaultValue string) string {
	v := os.Getenv(key)
	if v == "" {
		return defaultValue
	}
	return v
}

func stringFromEnvMulti(keys []string, defaultValue string) string {
	for _, key := range keys {
		v := os.Getenv(key)
		if v != "" {
			return v
		}
	}
	return defaultValue
}

func stringSliceFromEnv(key string, defaultValue []string) []string {
	v := os.Getenv(key)
	if v == "" {
		return defaultValue
	}
	values := strings.Split(v, ",")
	for i := range values {
		values[i] = strings.Trim(values[i], " ")
	}
	return values
}

func intFromEnv(key string, defaultValue int) int {
	v := os.Getenv(key)
	if v == "" {
		return defaultValue
	}
	i, err := strconv.ParseInt(v, 10, 64)
	if err != nil {
		return defaultValue
	}
	return int(i)
}

func (o *Config) Validate(skipCred bool) error {
	var errs []error

	if !skipCred {
		if o.AccessToken == "" {
			errs = append(errs, validate.NewFlagError("--token", "required"))
		}
		if o.AccessTokenSecret == "" {
			errs = append(errs, validate.NewFlagError("--secret", "required"))
		}
	}
	switch o.DefaultOutputType {
	case "", "table", "json", "yaml":
		// noop
	default:
		errs = append(errs, validate.NewFlagError("profile.DefaultOutputType", "must be one of [table/json/yaml]"))
	}

	switch o.ArgumentMatchMode {
	case "", "partial", "exact":
		// noop
	default:
		errs = append(errs, validate.NewFlagError("--argument-match-mode", "must be one of [partial/exact]"))
	}

	switch o.DefaultQueryDriver {
	case "", query.DriverJMESPath, query.DriverGoJQ:
		// noop
	default:
		errs = append(errs, validate.NewFlagError("profile.DefaultQueryDriver", "must be one of [jmespath/jq]"))
	}

	return validate.NewValidationError(errs...)
}

func (o *Config) ProcessTimeout() time.Duration {
	sec := o.ProcessTimeoutSec
	if sec <= 0 {
		sec = DefaultProcessTimeoutSec
	}
	return time.Duration(sec) * time.Second
}

func (o *Config) ArgumentMatchModeValue() string {
	if o.ArgumentMatchMode == "" {
		return "partial"
	}
	return o.ArgumentMatchMode
}
<|endoftext|>


<commit_before>package config

type RubyType interface {
	TagYAML() string
}

type Config struct {
	Dimg      []Dimg      `yaml:"_dimg,omitempty"`
	DimgGroup []DimgGroup `yaml:"_dimg_group,omitempty"`
}

func (cfg Config) TagYAML() string {
	return "!ruby/object:Dapp::Config::Config"
}

type DimgGroup struct {
	Dimg      []Dimg      `yaml:"_dimg,omitempty"`
	DimgGroup []DimgGroup `yaml:"_dimg_group,omitempty"`
}

func (cfg DimgGroup) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::DimgGroup"
}

type Dimg struct {
	Name        string           `yaml:"_name,omitempty"`
	Docker      DockerDimg       `yaml:"_docker,omitempty"`
	Builder     string           `yaml:"_builder,omitempty"`
	Shell       *ShellDimg       `yaml:"_shell,omitempty"`
	Chef        Chef             `yaml:"_chef,omitempty"`
	Artifact    []ArtifactExport `yaml:"_artifact,omitempty"`
	GitArtifact GitArtifact      `yaml:"_git_artifact,omitempty"`
	Mount       []Mount          `yaml:"_mount,omitempty"`
}

func (cfg Dimg) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Dimg"
}

type ArtifactDimg struct {
	Dimg   `yaml:",inline"`
	Docker DockerArtifact `yaml:"_docker,omitempty"`
	Shell  *ShellArtifact `yaml:"_shell,omitempty"`
}

func (cfg ArtifactDimg) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::ArtifactDimg"
}

type DockerDimg struct {
	DockerBase `yaml:",inline"`
	Volume     []string          `yaml:"_volume,omitempty"`
	Expose     []string          `yaml:"_expose,omitempty"`
	Env        map[string]string `yaml:"_env,omitempty"`
	Label      map[string]string `yaml:"_label,omitempty"`
	Cmd        []string          `yaml:"_cmd,omitempty"`
	Onbuild    []string          `yaml:"_onbuild,omitempty"`
	Workdir    string            `yaml:"_workdir,omitempty"`
	User       string            `yaml:"_user,omitempty"`
	Entrypoint []string          `yaml:"_entrypoint,omitempty"`
}

func (cfg DockerDimg) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Docker::Dimg"
}

type DockerArtifact struct {
	DockerBase `yaml:",inline"`
}

func (cfg DockerArtifact) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Docker::Artifact"
}

type DockerBase struct {
	From             string `yaml:"_from,omitempty"`
	FromCacheVersion string `yaml:"_from_cache_version,omitempty"`
}

type ShellDimg struct {
	Version       string       `yaml:"_version,omitempty"`
	BeforeInstall StageCommand `yaml:"_before_install,omitempty"`
	BeforeSetup   StageCommand `yaml:"_before_setup,omitempty"`
	Install       StageCommand `yaml:"_install,omitempty"`
	Setup         StageCommand `yaml:"_setup,omitempty"`
}

func (cfg ShellDimg) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Shell::Dimg"
}

type ShellArtifact struct {
	ShellDimg     `yaml:",inline"`
	BuildArtifact StageCommand `yaml:"_build_artifact,omitempty"`
}

func (cfg ShellArtifact) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Shell::Artifact"
}

type StageCommand struct {
	Version string   `yaml:"_version,omitempty"`
	Run     []string `yaml:"_run,omitempty"`
}

func (cfg StageCommand) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::Shell::Dimg::StageCommand"
}

type Chef struct {
	Dimod      []string       `yaml:"_dimod,omitempty"`
	Recipe     []string       `yaml:"_recipe,omitempty"`
	Attributes ChefAttributes `yaml:"_attributes,omitempty"`
}

func (cfg Chef) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Chef"
}

type ChefAttributes map[interface{}]interface{}

func (cfg ChefAttributes) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::Chef::Attributes"
}

type ArtifactExport struct {
	ArtifactBaseExport `yaml:",inline"`
	Config             ArtifactDimg `yaml:"_config,omitempty"`
	Before             string       `yaml:"_before,omitempty"`
	After              string       `yaml:"_after,omitempty"`
}

func (cfg ArtifactExport) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::Artifact::Export"
}

type GitArtifact struct {
	Local  []GitArtifactLocal  `yaml:"_local,omitempty"`
	Remote []GitArtifactRemote `yaml:"_remote,omitempty"`
}

func (cfg GitArtifact) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::Dimg::InstanceMethods::GitArtifact"
}

type GitArtifactLocal struct {
	Export []GitArtifactLocalExport `yaml:"_export,omitempty"`
}

func (cfg GitArtifactLocal) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::GitArtifactLocal"
}

type GitArtifactLocalExport struct {
	ArtifactBaseExport `yaml:",inline"`
	As                 string            `yaml:"_as,omitempty"`
	StageDependencies  StageDependencies `yaml:"_stage_dependencies,omitempty"`
}

func (cfg GitArtifactLocalExport) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::GitArtifactLocal::Export"
}

type StageDependencies struct {
	Install       []string `yaml:"_install,omitempty"`
	Setup         []string `yaml:"_setup,omitempty"`
	BeforeSetup   []string `yaml:"_before_setup,omitempty"`
	BuildArtifact []string `yaml:"_build_artifact,omitempty"`
}

func (cfg StageDependencies) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::GitArtifactLocal::Export::StageDependencies"
}

type GitArtifactRemote struct {
	Export []GitArtifactRemoteExport `yaml:"_export,omitempty"`
}

func (cfg GitArtifactRemote) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::GitArtifactRemote"
}

type GitArtifactRemoteExport struct {
	GitArtifactLocalExport `yaml:",inline"`
	Url                    string `yaml:"_url,omitempty"`
	Name                   string `yaml:"_name,omitempty"`
	Branch                 string `yaml:"_branch,omitempty"`
	Commit                 string `yaml:"_commit,omitempty"`
}

func (cfg GitArtifactRemoteExport) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::GitArtifactRemote::Export"
}

type ArtifactBaseExport struct {
	Cwd          string   `yaml:"_cwd,omitempty"`
	To           string   `yaml:"_to,omitempty"`
	IncludePaths []string `yaml:"_include_paths,omitempty"`
	ExcludePaths []string `yaml:"_exclude_paths,omitempty"`
	Owner        string   `yaml:"_owner,omitempty"`
	Group        string   `yaml:"_group,omitempty"`
}

type Mount struct {
	To   string `yaml:"_to,omitempty"`
	From string `yaml:"_from,omitempty"`
	Type string `yaml:"_type,omitempty"`
}

func (cfg Mount) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Mount"
}
<commit_msg>Config: ruby/hash -> ruby/object<commit_after>package config

type RubyType interface {
	TagYAML() string
}

type Config struct {
	Dimg      []Dimg      `yaml:"_dimg,omitempty"`
	DimgGroup []DimgGroup `yaml:"_dimg_group,omitempty"`
}

func (cfg Config) TagYAML() string {
	return "!ruby/object:Dapp::Config::Config"
}

type DimgGroup struct {
	Dimg      []Dimg      `yaml:"_dimg,omitempty"`
	DimgGroup []DimgGroup `yaml:"_dimg_group,omitempty"`
}

func (cfg DimgGroup) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::DimgGroup"
}

type Dimg struct {
	Name        string           `yaml:"_name,omitempty"`
	Docker      DockerDimg       `yaml:"_docker,omitempty"`
	Builder     string           `yaml:"_builder,omitempty"`
	Shell       *ShellDimg       `yaml:"_shell,omitempty"`
	Chef        Chef             `yaml:"_chef,omitempty"`
	Artifact    []ArtifactExport `yaml:"_artifact,omitempty"`
	GitArtifact GitArtifact      `yaml:"_git_artifact,omitempty"`
	Mount       []Mount          `yaml:"_mount,omitempty"`
}

func (cfg Dimg) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Dimg"
}

type ArtifactDimg struct {
	Dimg   `yaml:",inline"`
	Docker DockerArtifact `yaml:"_docker,omitempty"`
	Shell  *ShellArtifact `yaml:"_shell,omitempty"`
}

func (cfg ArtifactDimg) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::ArtifactDimg"
}

type DockerDimg struct {
	DockerBase `yaml:",inline"`
	Volume     []string          `yaml:"_volume,omitempty"`
	Expose     []string          `yaml:"_expose,omitempty"`
	Env        map[string]string `yaml:"_env,omitempty"`
	Label      map[string]string `yaml:"_label,omitempty"`
	Cmd        []string          `yaml:"_cmd,omitempty"`
	Onbuild    []string          `yaml:"_onbuild,omitempty"`
	Workdir    string            `yaml:"_workdir,omitempty"`
	User       string            `yaml:"_user,omitempty"`
	Entrypoint []string          `yaml:"_entrypoint,omitempty"`
}

func (cfg DockerDimg) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Docker::Dimg"
}

type DockerArtifact struct {
	DockerBase `yaml:",inline"`
}

func (cfg DockerArtifact) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Docker::Artifact"
}

type DockerBase struct {
	From             string `yaml:"_from,omitempty"`
	FromCacheVersion string `yaml:"_from_cache_version,omitempty"`
}

type ShellDimg struct {
	Version       string       `yaml:"_version,omitempty"`
	BeforeInstall StageCommand `yaml:"_before_install,omitempty"`
	BeforeSetup   StageCommand `yaml:"_before_setup,omitempty"`
	Install       StageCommand `yaml:"_install,omitempty"`
	Setup         StageCommand `yaml:"_setup,omitempty"`
}

func (cfg ShellDimg) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Shell::Dimg"
}

type ShellArtifact struct {
	ShellDimg     `yaml:",inline"`
	BuildArtifact StageCommand `yaml:"_build_artifact,omitempty"`
}

func (cfg ShellArtifact) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Shell::Artifact"
}

type StageCommand struct {
	Version string   `yaml:"_version,omitempty"`
	Run     []string `yaml:"_run,omitempty"`
}

func (cfg StageCommand) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Shell::Dimg::StageCommand"
}

type Chef struct {
	Dimod      []string       `yaml:"_dimod,omitempty"`
	Recipe     []string       `yaml:"_recipe,omitempty"`
	Attributes ChefAttributes `yaml:"_attributes,omitempty"`
}

func (cfg Chef) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Chef"
}

type ChefAttributes map[interface{}]interface{}

func (cfg ChefAttributes) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::Chef::Attributes"
}

type ArtifactExport struct {
	ArtifactBaseExport `yaml:",inline"`
	Config             ArtifactDimg `yaml:"_config,omitempty"`
	Before             string       `yaml:"_before,omitempty"`
	After              string       `yaml:"_after,omitempty"`
}

func (cfg ArtifactExport) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Artifact::Export"
}

type GitArtifact struct {
	Local  []GitArtifactLocal  `yaml:"_local,omitempty"`
	Remote []GitArtifactRemote `yaml:"_remote,omitempty"`
}

func (cfg GitArtifact) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Dimg::InstanceMethods::GitArtifact"
}

type GitArtifactLocal struct {
	Export []GitArtifactLocalExport `yaml:"_export,omitempty"`
}

func (cfg GitArtifactLocal) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::GitArtifactLocal"
}

type GitArtifactLocalExport struct {
	ArtifactBaseExport `yaml:",inline"`
	As                 string            `yaml:"_as,omitempty"`
	StageDependencies  StageDependencies `yaml:"_stage_dependencies,omitempty"`
}

func (cfg GitArtifactLocalExport) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::GitArtifactLocal::Export"
}

type StageDependencies struct {
	Install       []string `yaml:"_install,omitempty"`
	Setup         []string `yaml:"_setup,omitempty"`
	BeforeSetup   []string `yaml:"_before_setup,omitempty"`
	BuildArtifact []string `yaml:"_build_artifact,omitempty"`
}

func (cfg StageDependencies) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::GitArtifactLocal::Export::StageDependencies"
}

type GitArtifactRemote struct {
	Export []GitArtifactRemoteExport `yaml:"_export,omitempty"`
}

func (cfg GitArtifactRemote) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::GitArtifactRemote"
}

type GitArtifactRemoteExport struct {
	GitArtifactLocalExport `yaml:",inline"`
	Url                    string `yaml:"_url,omitempty"`
	Name                   string `yaml:"_name,omitempty"`
	Branch                 string `yaml:"_branch,omitempty"`
	Commit                 string `yaml:"_commit,omitempty"`
}

func (cfg GitArtifactRemoteExport) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::GitArtifactRemote::Export"
}

type ArtifactBaseExport struct {
	Cwd          string   `yaml:"_cwd,omitempty"`
	To           string   `yaml:"_to,omitempty"`
	IncludePaths []string `yaml:"_include_paths,omitempty"`
	ExcludePaths []string `yaml:"_exclude_paths,omitempty"`
	Owner        string   `yaml:"_owner,omitempty"`
	Group        string   `yaml:"_group,omitempty"`
}

type Mount struct {
	To   string `yaml:"_to,omitempty"`
	From string `yaml:"_from,omitempty"`
	Type string `yaml:"_type,omitempty"`
}

func (cfg Mount) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Mount"
}
<|endoftext|>


<commit_before>/*
Copyright 2018 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"os"

	"github.com/containernetworking/plugins/pkg/utils/sysctl"
	"github.com/coreos/go-iptables/iptables"
	"github.com/vishvananda/netlink"
)

var ipt *iptables.IPTables

type Config interface {
	Ensure() error
}

type SysctlConfig struct {
	Key, Value string
}

type IPRouteConfig struct {
	Route netlink.Route
}

type IPRuleConfig struct {
	Rule netlink.Rule
}

type IPTablesChainConfig struct {
	ChainName, TableName string
}

type IPTablesRuleConfig struct {
	ChainName, TableName string
	RuleSpec             []string
}

func init() {
	var err error
	if ipt, err = iptables.NewWithProtocol(iptables.ProtocolIPv4); err != nil {
	}
}

func (s *SysctlConfig) Ensure() error {
	_, err := sysctl.Sysctl(s.Key, s.Value)
	return err
}

func (r *IPRouteConfig) Ensure() error {
	return netlink.RouteAdd(&r.Route)
}

func (r *IPRuleConfig) Ensure() error {
	return netlink.RuleAdd(&r.Rule)
}

func (c *IPTablesChainConfig) Ensure() error {
	if err := ipt.NewChain(c.TableName, c.ChainName); err != nil && err != os.ErrExist {
		return err
	}
	return nil
}

func (r *IPTablesRuleConfig) Ensure() error {
	return ipt.AppendUnique(r.TableName, r.ChainName, r.RuleSpec...)
}
<commit_msg>Move package variable ipt close to init function, where it gets initialized.<commit_after>/*
Copyright 2018 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"os"

	"github.com/containernetworking/plugins/pkg/utils/sysctl"
	"github.com/coreos/go-iptables/iptables"
	"github.com/golang/glog"
	"github.com/vishvananda/netlink"
)

type Config interface {
	Ensure() error
}

type SysctlConfig struct {
	Key, Value string
}

type IPRouteConfig struct {
	Route netlink.Route
}

type IPRuleConfig struct {
	Rule netlink.Rule
}

type IPTablesChainConfig struct {
	ChainName, TableName string
}

type IPTablesRuleConfig struct {
	ChainName, TableName string
	RuleSpec             []string
}

var ipt *iptables.IPTables

func init() {
	var err error
	if ipt, err = iptables.NewWithProtocol(iptables.ProtocolIPv4); err != nil {
		glog.Errorf("failed to initialize iptables")
	}
}

func (s *SysctlConfig) Ensure() error {
	_, err := sysctl.Sysctl(s.Key, s.Value)
	return err
}

func (r *IPRouteConfig) Ensure() error {
	return netlink.RouteAdd(&r.Route)
}

func (r *IPRuleConfig) Ensure() error {
	return netlink.RuleAdd(&r.Rule)
}

func (c *IPTablesChainConfig) Ensure() error {
	if err := ipt.NewChain(c.TableName, c.ChainName); err != nil && err != os.ErrExist {
		return err
	}
	return nil
}

func (r *IPTablesRuleConfig) Ensure() error {
	return ipt.AppendUnique(r.TableName, r.ChainName, r.RuleSpec...)
}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package gcv provides a library and a RPC service for Forseti Config Validator.
package gcv

import (
	"context"
	"flag"
	"io/ioutil"
	"runtime"

	"github.com/forseti-security/config-validator/pkg/api/validator"
	asset2 "github.com/forseti-security/config-validator/pkg/asset"
	"github.com/forseti-security/config-validator/pkg/gcv/cf"
	"github.com/forseti-security/config-validator/pkg/gcv/configs"
	"github.com/forseti-security/config-validator/pkg/multierror"
	"github.com/golang/glog"
	"github.com/pkg/errors"
)

const logRequestsVerboseLevel = 2

var flags struct {
	workerCount int
}

func init() {
	flag.IntVar(
		&flags.workerCount,
		"workerCount",
		runtime.NumCPU(),
		"Number of workers that Validator will spawn to handle validate calls, this defaults to core count on the host")
}

// Validator checks GCP resource metadata for constraint violation.
//
// Expected usage pattern:
//   - call NewValidator to create a new Validator
//   - call AddData one or more times to add the GCP resource metadata to check
//   - call Audit to validate the GCP resource metadata that has been added so far
//   - call Reset to delete existing data
//   - call AddData to add a new set of GCP resource metadata to check
//   - call Reset to delete existing data
//
// Any data added in AddData stays in the underlying rule evaluation engine's memory.
// To avoid out of memory errors, callers can invoke Reset to delete existing data.
type Validator struct {
	// policyPath points to a directory where the constraints and constraint templates are stored as yaml files.
	policyPath string
	// policy dependencies directory points to rego files that provide supporting code for templates.
	// These rego dependencies should be packaged with the GCV deployment.
	// Right now expected to be set to point to "//policies/validator/lib" folder
	policyLibraryDir    string
	constraintFramework *cf.ConstraintFramework
	work                chan func()
}

func loadRegoFiles(dir string) (map[string]string, error) {
	loadedFiles := make(map[string]string)
	files, err := configs.ListRegoFiles(dir)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to list rego files from %s", dir)
	}
	for _, filePath := range files {
		glog.V(logRequestsVerboseLevel).Infof("Loading rego file: %s", filePath)
		if _, exists := loadedFiles[filePath]; exists {
			// This shouldn't happen
			return nil, errors.Errorf("unexpected file collision with file %s", filePath)
		}
		fileBytes, err := ioutil.ReadFile(filePath)
		if err != nil {
			return nil, errors.Wrapf(err, "unable to read file %s", filePath)
		}
		loadedFiles[filePath] = string(fileBytes)
	}
	return loadedFiles, nil
}

func loadYAMLFiles(dir string) ([]*configs.ConstraintTemplate, []*configs.Constraint, error) {
	var templates []*configs.ConstraintTemplate
	var constraints []*configs.Constraint
	files, err := configs.ListYAMLFiles(dir)
	if err != nil {
		return nil, nil, err
	}
	for _, filePath := range files {
		glog.V(logRequestsVerboseLevel).Infof("Loading yaml file: %s", filePath)
		fileContents, err := ioutil.ReadFile(filePath)
		if err != nil {
			return nil, nil, errors.Wrapf(err, "unable to read file %s", filePath)
		}
		categorizedData, err := configs.CategorizeYAMLFile(fileContents, filePath)
		if err != nil {
			glog.Infof("Unable to convert file %s, with error %v, assuming this file should be skipped and continuing", filePath, err)
			continue
		}
		switch data := categorizedData.(type) {
		case *configs.ConstraintTemplate:
			templates = append(templates, data)
		case *configs.Constraint:
			constraints = append(constraints, data)
		default:
			// Unexpected: CategorizeYAMLFile shouldn't return any types
			return nil, nil, errors.Errorf("CategorizeYAMLFile returned unexpected data type when converting file %s", filePath)
		}
	}
	return templates, constraints, nil
}

// NewValidator returns a new Validator.
// By default it will initialize the underlying query evaluation engine by loading supporting library, constraints, and constraint templates.
// We may want to make this initialization behavior configurable in the future.
func NewValidator(stopChannel <-chan struct{}, policyPath string, policyLibraryPath string) (*Validator, error) {
	if policyPath == "" {
		return nil, errors.Errorf("No policy path set, provide an option to set the policy path gcv.PolicyPath")
	}
	if policyLibraryPath == "" {
		return nil, errors.Errorf("No policy library set")
	}

	ret := &Validator{
		work: make(chan func(), flags.workerCount*2),
	}

	glog.V(logRequestsVerboseLevel).Infof("loading policy library dir: %s", ret.policyLibraryDir)
	regoLib, err := loadRegoFiles(policyLibraryPath)
	if err != nil {
		return nil, err
	}

	ret.constraintFramework, err = cf.New(regoLib)
	if err != nil {
		return nil, err
	}
	glog.V(logRequestsVerboseLevel).Infof("loading policy dir: %s", ret.policyPath)
	templates, constraints, err := loadYAMLFiles(policyPath)
	if err != nil {
		return nil, err
	}

	if err := ret.constraintFramework.Configure(templates, constraints); err != nil {
		return nil, err
	}

	go func() {
		<-stopChannel
		glog.Infof("validator stopchannel closed, closing work channel")
		close(ret.work)
	}()

	workerCount := flags.workerCount
	glog.Infof("starting %d workers", workerCount)
	for i := 0; i < workerCount; i++ {
		go ret.reviewWorker(i)
	}

	return ret, nil
}

func (v *Validator) reviewWorker(idx int) {
	glog.Infof("worker %d starting", idx)
	for f := range v.work {
		f()
	}
	glog.Infof("worker %d terminated", idx)
}

// AddData adds GCP resource metadata to be audited later.
func (v *Validator) AddData(request *validator.AddDataRequest) error {
	for i, asset := range request.Assets {
		if err := asset2.ValidateAsset(asset); err != nil {
			return errors.Wrapf(err, "index %d", i)
		}
		f, err := asset2.ConvertResourceViaJSONToInterface(asset)
		if err != nil {
			return errors.Wrapf(err, "index %d", i)
		}
		v.constraintFramework.AddData(f)
	}

	return nil
}

type assetResult struct {
	violations []*validator.Violation
	err        error
}

func (v *Validator) handleReview(ctx context.Context, idx int, asset *validator.Asset, resultChan chan<- *assetResult) func() {
	return func() {
		resultChan <- func() *assetResult {
			if err := asset2.ValidateAsset(asset); err != nil {
				return &assetResult{err: errors.Wrapf(err, "index %d", idx)}
			}

			assetInterface, err := asset2.ConvertResourceViaJSONToInterface(asset)
			if err != nil {
				return &assetResult{err: errors.Wrapf(err, "index %d", idx)}
			}

			violations, err := v.constraintFramework.Review(ctx, assetInterface)
			if err != nil {
				return &assetResult{err: errors.Wrapf(err, "index %d", idx)}
			}

			return &assetResult{violations: violations}
		}()
	}
}

// Review evaluates each asset in the review request in parallel and returns any
// violations found.
func (v *Validator) Review(ctx context.Context, request *validator.ReviewRequest) (*validator.ReviewResponse, error) {
	assetCount := len(request.Assets)
	resultChan := make(chan *assetResult, flags.workerCount*2)
	defer close(resultChan)

	go func() {
		for idx, asset := range request.Assets {
			v.work <- v.handleReview(ctx, idx, asset, resultChan)
		}
	}()

	response := &validator.ReviewResponse{}
	var errs multierror.Errors
	for i := 0; i < assetCount; i++ {
		result := <-resultChan
		if result.err != nil {
			errs.Add(result.err)
			continue
		}
		response.Violations = append(response.Violations, result.violations...)
	}

	if !errs.Empty() {
		return response, errs.ToError()
	}
	return response, nil
}

// Reset clears previously added data from the underlying query evaluation engine.
func (v *Validator) Reset(ctx context.Context) error {
	return v.constraintFramework.Reset(ctx)
}

// Audit checks the GCP resource metadata that has been added via AddData to determine if any of the constraint is violated.
func (v *Validator) Audit(ctx context.Context) (*validator.AuditResponse, error) {
	response, err := v.constraintFramework.Audit(ctx)
	return response, err
}
<commit_msg>Expose API for directly reviewing an asset.<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package gcv provides a library and a RPC service for Forseti Config Validator.
package gcv

import (
	"context"
	"flag"
	"io/ioutil"
	"runtime"

	"github.com/forseti-security/config-validator/pkg/api/validator"
	asset2 "github.com/forseti-security/config-validator/pkg/asset"
	"github.com/forseti-security/config-validator/pkg/gcv/cf"
	"github.com/forseti-security/config-validator/pkg/gcv/configs"
	"github.com/forseti-security/config-validator/pkg/multierror"
	"github.com/golang/glog"
	"github.com/pkg/errors"
)

const logRequestsVerboseLevel = 2

var flags struct {
	workerCount int
}

func init() {
	flag.IntVar(
		&flags.workerCount,
		"workerCount",
		runtime.NumCPU(),
		"Number of workers that Validator will spawn to handle validate calls, this defaults to core count on the host")
}

// Validator checks GCP resource metadata for constraint violation.
//
// Expected usage pattern:
//   - call NewValidator to create a new Validator
//   - call AddData one or more times to add the GCP resource metadata to check
//   - call Audit to validate the GCP resource metadata that has been added so far
//   - call Reset to delete existing data
//   - call AddData to add a new set of GCP resource metadata to check
//   - call Reset to delete existing data
//
// Any data added in AddData stays in the underlying rule evaluation engine's memory.
// To avoid out of memory errors, callers can invoke Reset to delete existing data.
type Validator struct {
	// policyPath points to a directory where the constraints and constraint templates are stored as yaml files.
	policyPath string
	// policy dependencies directory points to rego files that provide supporting code for templates.
	// These rego dependencies should be packaged with the GCV deployment.
	// Right now expected to be set to point to "//policies/validator/lib" folder
	policyLibraryDir    string
	constraintFramework *cf.ConstraintFramework
	work                chan func()
}

func loadRegoFiles(dir string) (map[string]string, error) {
	loadedFiles := make(map[string]string)
	files, err := configs.ListRegoFiles(dir)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to list rego files from %s", dir)
	}
	for _, filePath := range files {
		glog.V(logRequestsVerboseLevel).Infof("Loading rego file: %s", filePath)
		if _, exists := loadedFiles[filePath]; exists {
			// This shouldn't happen
			return nil, errors.Errorf("unexpected file collision with file %s", filePath)
		}
		fileBytes, err := ioutil.ReadFile(filePath)
		if err != nil {
			return nil, errors.Wrapf(err, "unable to read file %s", filePath)
		}
		loadedFiles[filePath] = string(fileBytes)
	}
	return loadedFiles, nil
}

func loadYAMLFiles(dir string) ([]*configs.ConstraintTemplate, []*configs.Constraint, error) {
	var templates []*configs.ConstraintTemplate
	var constraints []*configs.Constraint
	files, err := configs.ListYAMLFiles(dir)
	if err != nil {
		return nil, nil, err
	}
	for _, filePath := range files {
		glog.V(logRequestsVerboseLevel).Infof("Loading yaml file: %s", filePath)
		fileContents, err := ioutil.ReadFile(filePath)
		if err != nil {
			return nil, nil, errors.Wrapf(err, "unable to read file %s", filePath)
		}
		categorizedData, err := configs.CategorizeYAMLFile(fileContents, filePath)
		if err != nil {
			glog.Infof("Unable to convert file %s, with error %v, assuming this file should be skipped and continuing", filePath, err)
			continue
		}
		switch data := categorizedData.(type) {
		case *configs.ConstraintTemplate:
			templates = append(templates, data)
		case *configs.Constraint:
			constraints = append(constraints, data)
		default:
			// Unexpected: CategorizeYAMLFile shouldn't return any types
			return nil, nil, errors.Errorf("CategorizeYAMLFile returned unexpected data type when converting file %s", filePath)
		}
	}
	return templates, constraints, nil
}

// NewValidator returns a new Validator.
// By default it will initialize the underlying query evaluation engine by loading supporting library, constraints, and constraint templates.
// We may want to make this initialization behavior configurable in the future.
func NewValidator(stopChannel <-chan struct{}, policyPath string, policyLibraryPath string) (*Validator, error) {
	if policyPath == "" {
		return nil, errors.Errorf("No policy path set, provide an option to set the policy path gcv.PolicyPath")
	}
	if policyLibraryPath == "" {
		return nil, errors.Errorf("No policy library set")
	}

	ret := &Validator{
		work: make(chan func(), flags.workerCount*2),
	}

	glog.V(logRequestsVerboseLevel).Infof("loading policy library dir: %s", ret.policyLibraryDir)
	regoLib, err := loadRegoFiles(policyLibraryPath)
	if err != nil {
		return nil, err
	}

	ret.constraintFramework, err = cf.New(regoLib)
	if err != nil {
		return nil, err
	}
	glog.V(logRequestsVerboseLevel).Infof("loading policy dir: %s", ret.policyPath)
	templates, constraints, err := loadYAMLFiles(policyPath)
	if err != nil {
		return nil, err
	}

	if err := ret.constraintFramework.Configure(templates, constraints); err != nil {
		return nil, err
	}

	go func() {
		<-stopChannel
		glog.Infof("validator stopchannel closed, closing work channel")
		close(ret.work)
	}()

	workerCount := flags.workerCount
	glog.Infof("starting %d workers", workerCount)
	for i := 0; i < workerCount; i++ {
		go ret.reviewWorker(i)
	}

	return ret, nil
}

func (v *Validator) reviewWorker(idx int) {
	glog.Infof("worker %d starting", idx)
	for f := range v.work {
		f()
	}
	glog.Infof("worker %d terminated", idx)
}

// AddData adds GCP resource metadata to be audited later.
func (v *Validator) AddData(request *validator.AddDataRequest) error {
	for i, asset := range request.Assets {
		if err := asset2.ValidateAsset(asset); err != nil {
			return errors.Wrapf(err, "index %d", i)
		}
		f, err := asset2.ConvertResourceViaJSONToInterface(asset)
		if err != nil {
			return errors.Wrapf(err, "index %d", i)
		}
		v.constraintFramework.AddData(f)
	}

	return nil
}

type assetResult struct {
	violations []*validator.Violation
	err        error
}

func (v *Validator) handleReview(ctx context.Context, idx int, asset *validator.Asset, resultChan chan<- *assetResult) func() {
	return func() {
		resultChan <- func() *assetResult {
			if err := asset2.ValidateAsset(asset); err != nil {
				return &assetResult{err: errors.Wrapf(err, "index %d", idx)}
			}

			assetInterface, err := asset2.ConvertResourceViaJSONToInterface(asset)
			if err != nil {
				return &assetResult{err: errors.Wrapf(err, "index %d", idx)}
			}

			violations, err := v.constraintFramework.Review(ctx, assetInterface)
			if err != nil {
				return &assetResult{err: errors.Wrapf(err, "index %d", idx)}
			}

			return &assetResult{violations: violations}
		}()
	}
}

// ReviewJSON evaluates a single asset without any threading in the background.
func (v *Validator) ReviewJSON(ctx context.Context, asset interface{}) ([]*validator.Violation, error) {
	return v.constraintFramework.Review(ctx, asset)
}

// Review evaluates each asset in the review request in parallel and returns any
// violations found.
func (v *Validator) Review(ctx context.Context, request *validator.ReviewRequest) (*validator.ReviewResponse, error) {
	assetCount := len(request.Assets)
	resultChan := make(chan *assetResult, flags.workerCount*2)
	defer close(resultChan)

	go func() {
		for idx, asset := range request.Assets {
			v.work <- v.handleReview(ctx, idx, asset, resultChan)
		}
	}()

	response := &validator.ReviewResponse{}
	var errs multierror.Errors
	for i := 0; i < assetCount; i++ {
		result := <-resultChan
		if result.err != nil {
			errs.Add(result.err)
			continue
		}
		response.Violations = append(response.Violations, result.violations...)
	}

	if !errs.Empty() {
		return response, errs.ToError()
	}
	return response, nil
}

// Reset clears previously added data from the underlying query evaluation engine.
func (v *Validator) Reset(ctx context.Context) error {
	return v.constraintFramework.Reset(ctx)
}

// Audit checks the GCP resource metadata that has been added via AddData to determine if any of the constraint is violated.
func (v *Validator) Audit(ctx context.Context) (*validator.AuditResponse, error) {
	response, err := v.constraintFramework.Audit(ctx)
	return response, err
}
<|endoftext|>


<commit_before>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

//go:generate bash -c "echo -en '// AUTOGENERATED FILE\n\n' > generated.go"
//go:generate bash -c "echo -en 'package kernel\n\n' >> generated.go"
//go:generate bash -c "echo -en 'const createImageScript = `#!/bin/bash\n' >> generated.go"
//go:generate bash -c "cat ../../tools/create-gce-image.sh | grep -v '#' >> generated.go"
//go:generate bash -c "echo -en '`\n\n' >> generated.go"

// Package kernel contains helper functions for working with Linux kernel
// (building kernel/image).
package kernel

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/google/syzkaller/pkg/fileutil"
	"github.com/google/syzkaller/pkg/osutil"
)

func Build(dir, compiler, config string, fullConfig bool) error {
	const timeout = 10 * time.Minute // default timeout for command invocations
	if fullConfig {
		if err := ioutil.WriteFile(filepath.Join(dir, ".config"), []byte(config), 0600); err != nil {
			return fmt.Errorf("failed to write config file: %v", err)
		}
	} else {
		os.Remove(filepath.Join(dir, ".config"))
		configFile := filepath.Join(dir, "syz.config")
		if err := ioutil.WriteFile(configFile, []byte(config), 0600); err != nil {
			return fmt.Errorf("failed to write config file: %v", err)
		}
		defer os.Remove(configFile)
		if _, err := osutil.RunCmd(timeout, dir, "make", "defconfig"); err != nil {
			return err
		}
		if _, err := osutil.RunCmd(timeout, dir, "make", "kvmconfig"); err != nil {
			return err
		}
		if _, err := osutil.RunCmd(timeout, dir, "scripts/kconfig/merge_config.sh", "-n", ".config", configFile); err != nil {
			return err
		}
	}
	if _, err := osutil.RunCmd(timeout, dir, "make", "olddefconfig"); err != nil {
		return err
	}
	// We build only bzImage as we currently don't use modules.
	// Build of a large kernel can take a while on a 1 CPU VM.
	if _, err := osutil.RunCmd(3*time.Hour, dir, "make", "bzImage", "-j", strconv.Itoa(runtime.NumCPU()*2), "CC="+compiler); err != nil {
		return err
	}
	return nil
}

// CreateImage creates a disk image that is suitable for syzkaller.
// Kernel is taken from kernelDir, userspace system is taken from userspaceDir.
// Produces image and root ssh key in the specified files.
func CreateImage(kernelDir, userspaceDir, image, sshkey string) error {
	tempDir, err := ioutil.TempDir("", "syz-build")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tempDir)
	scriptFile := filepath.Join(tempDir, "create.sh")
	if err := ioutil.WriteFile(scriptFile, []byte(createImageScript), 0700); err != nil {
		return fmt.Errorf("failed to write script file: %v", err)
	}
	bzImage := filepath.Join(kernelDir, filepath.FromSlash("arch/x86/boot/bzImage"))
	if _, err := osutil.RunCmd(time.Hour, tempDir, scriptFile, userspaceDir, bzImage); err != nil {
		return fmt.Errorf("image build failed: %v", err)
	}
	if err := fileutil.CopyFile(filepath.Join(tempDir, "disk.raw"), image); err != nil {
		return err
	}
	if err := fileutil.CopyFile(filepath.Join(tempDir, "key"), sshkey); err != nil {
		return err
	}
	if err := os.Chmod(sshkey, 0600); err != nil {
		return err
	}
	return nil
}

func CompilerIdentity(compiler string) (string, error) {
	output, err := osutil.RunCmd(time.Minute, "", compiler, "--version")
	if err != nil {
		return "", err
	}
	if len(output) == 0 {
		return "", fmt.Errorf("no output from compiler --version")
	}
	return strings.Split(string(output), "\n")[0], nil
}
<commit_msg>pkg/kernel: reduce Build parallelism<commit_after>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

//go:generate bash -c "echo -en '// AUTOGENERATED FILE\n\n' > generated.go"
//go:generate bash -c "echo -en 'package kernel\n\n' >> generated.go"
//go:generate bash -c "echo -en 'const createImageScript = `#!/bin/bash\n' >> generated.go"
//go:generate bash -c "cat ../../tools/create-gce-image.sh | grep -v '#' >> generated.go"
//go:generate bash -c "echo -en '`\n\n' >> generated.go"

// Package kernel contains helper functions for working with Linux kernel
// (building kernel/image).
package kernel

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/google/syzkaller/pkg/fileutil"
	"github.com/google/syzkaller/pkg/osutil"
)

func Build(dir, compiler, config string, fullConfig bool) error {
	const timeout = 10 * time.Minute // default timeout for command invocations
	if fullConfig {
		if err := ioutil.WriteFile(filepath.Join(dir, ".config"), []byte(config), 0600); err != nil {
			return fmt.Errorf("failed to write config file: %v", err)
		}
	} else {
		os.Remove(filepath.Join(dir, ".config"))
		configFile := filepath.Join(dir, "syz.config")
		if err := ioutil.WriteFile(configFile, []byte(config), 0600); err != nil {
			return fmt.Errorf("failed to write config file: %v", err)
		}
		defer os.Remove(configFile)
		if _, err := osutil.RunCmd(timeout, dir, "make", "defconfig"); err != nil {
			return err
		}
		if _, err := osutil.RunCmd(timeout, dir, "make", "kvmconfig"); err != nil {
			return err
		}
		if _, err := osutil.RunCmd(timeout, dir, "scripts/kconfig/merge_config.sh", "-n", ".config", configFile); err != nil {
			return err
		}
	}
	if _, err := osutil.RunCmd(timeout, dir, "make", "olddefconfig"); err != nil {
		return err
	}
	// We build only bzImage as we currently don't use modules.
	// Build of a large kernel can take a while on a 1 CPU VM.
	if _, err := osutil.RunCmd(3*time.Hour, dir, "make", "bzImage", "-j", strconv.Itoa(runtime.NumCPU()), "CC="+compiler); err != nil {
		return err
	}
	return nil
}

// CreateImage creates a disk image that is suitable for syzkaller.
// Kernel is taken from kernelDir, userspace system is taken from userspaceDir.
// Produces image and root ssh key in the specified files.
func CreateImage(kernelDir, userspaceDir, image, sshkey string) error {
	tempDir, err := ioutil.TempDir("", "syz-build")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tempDir)
	scriptFile := filepath.Join(tempDir, "create.sh")
	if err := ioutil.WriteFile(scriptFile, []byte(createImageScript), 0700); err != nil {
		return fmt.Errorf("failed to write script file: %v", err)
	}
	bzImage := filepath.Join(kernelDir, filepath.FromSlash("arch/x86/boot/bzImage"))
	if _, err := osutil.RunCmd(time.Hour, tempDir, scriptFile, userspaceDir, bzImage); err != nil {
		return fmt.Errorf("image build failed: %v", err)
	}
	if err := fileutil.CopyFile(filepath.Join(tempDir, "disk.raw"), image); err != nil {
		return err
	}
	if err := fileutil.CopyFile(filepath.Join(tempDir, "key"), sshkey); err != nil {
		return err
	}
	if err := os.Chmod(sshkey, 0600); err != nil {
		return err
	}
	return nil
}

func CompilerIdentity(compiler string) (string, error) {
	output, err := osutil.RunCmd(time.Minute, "", compiler, "--version")
	if err != nil {
		return "", err
	}
	if len(output) == 0 {
		return "", fmt.Errorf("no output from compiler --version")
	}
	return strings.Split(string(output), "\n")[0], nil
}
<|endoftext|>


<commit_before>package paxos

import (
	"log"
	"os"

	"doozer/store"
	"doozer/util"
	"time"
)

const (
	fillDelay = 5e8 // 500ms
)

type instReq struct {
	seqn uint64
	ch   chan instance
}

type Manager struct {
	st        *store.Store
	ops       chan<- store.Op
	rg        *Registrar
	seqns     chan uint64
	fillUntil chan uint64
	reqs      chan instReq
	logger    *log.Logger
	Self      string
	alpha     int
	outs      PutterTo
}

// start is the seqn at which this member was defined.
// start+alpha is the first seqn this manager is expected to participate in.
func NewManager(self string, start uint64, alpha int, st *store.Store, ops chan<- store.Op, outs PutterTo) *Manager {
	m := &Manager{
		st:        st,
		ops:       ops,
		rg:        NewRegistrar(st, start, alpha),
		seqns:     make(chan uint64),
		fillUntil: make(chan uint64),
		reqs:      make(chan instReq),
		logger:    util.NewLogger("manager"),
		Self:      self,
		alpha:     alpha,
		outs:      outs,
	}

	go m.gen(start + uint64(alpha))
	go m.fill(start + uint64(alpha))
	go m.process()

	return m
}

func (m *Manager) Alpha() int {
	return m.alpha
}

func (m *Manager) cluster(seqn uint64) *cluster {
	members, cals := m.rg.setsForSeqn(seqn)
	return newCluster(m.Self, members, cals, putToWrapper{seqn, m.outs})
}

func (mg *Manager) gen(next uint64) {
	for {
		cx := mg.cluster(next)
		leader := int(next % uint64(cx.Len()))
		if leader == cx.SelfIndex() {
			mg.seqns <- next
		}
		next++
	}
}

func (mg *Manager) fill(seqn uint64) {
	for next := range mg.fillUntil {
		for seqn < next {
			go mg.fillOne(seqn)
			seqn++
		}
		seqn = next + 1 // no need to fill in our own seqn
	}
}

func (m *Manager) process() {
	instances := make(map[uint64]instance)
	for req := range m.reqs {
		inst, ok := instances[req.seqn]
		if !ok {
			inst = make(instance)
			instances[req.seqn] = inst
			go inst.process(req.seqn, m, m.ops)
		}
		req.ch <- inst
	}
}

func (m *Manager) getInstance(seqn uint64) instance {
	ch := make(chan instance)
	m.reqs <- instReq{seqn, ch}
	return <-ch
}

func (m *Manager) PutFrom(addr string, msg Msg) {
	if !msg.Ok() {
		return
	}
	it := m.getInstance(msg.Seqn())
	it.PutFrom(addr, msg)
}

func (m *Manager) proposeAt(seqn uint64, v string) {
	m.getInstance(seqn).Propose(v)
	m.logger.Printf("paxos propose -> %d %q", seqn, v)
}

func (m *Manager) Propose(v string) (seqn uint64, cas string, err os.Error) {
	var ev store.Event

	// If a competing proposal succeeded in the same seqn, we should try again.
	for v != ev.Mut {
		seqn = <-m.seqns
		ch := m.st.Wait(seqn)
		m.proposeAt(seqn, v)
		m.fillUntil <- seqn
		ev = <-ch
	}
	return seqn, ev.Cas, ev.Err
}

func (m *Manager) fillOne(seqn uint64) {
	time.Sleep(fillDelay)
	// yes, we'll act as coordinator for a seqn we don't "own"
	// this is intentional, since we want to exersize this code all the time,
	// not just during a failure.
	m.proposeAt(seqn, store.Nop)
}
<commit_msg>refactor<commit_after>package paxos

import (
	"log"
	"os"

	"doozer/store"
	"doozer/util"
	"time"
)

const (
	fillDelay = 5e8 // 500ms
)

type instReq struct {
	seqn uint64
	ch   chan instance
}

type Manager struct {
	st        *store.Store
	ops       chan<- store.Op
	rg        *Registrar
	seqns     chan uint64
	fillUntil chan uint64
	reqs      chan instReq
	logger    *log.Logger
	Self      string
	alpha     int
	outs      PutterTo
}

// start is the seqn at which this member was defined.
// start+alpha is the first seqn this manager is expected to participate in.
func NewManager(self string, start uint64, alpha int, st *store.Store, ops chan<- store.Op, outs PutterTo) *Manager {
	m := &Manager{
		st:        st,
		ops:       ops,
		rg:        NewRegistrar(st, start, alpha),
		seqns:     make(chan uint64),
		fillUntil: make(chan uint64),
		reqs:      make(chan instReq),
		logger:    util.NewLogger("manager"),
		Self:      self,
		alpha:     alpha,
		outs:      outs,
	}

	go m.gen(start + uint64(alpha))
	go m.fill(start + uint64(alpha))
	go m.process()

	return m
}

func (m *Manager) Alpha() int {
	return m.alpha
}

func (m *Manager) cluster(seqn uint64) *cluster {
	members, cals := m.rg.setsForSeqn(seqn)
	return newCluster(m.Self, members, cals, putToWrapper{seqn, m.outs})
}

func (mg *Manager) gen(next uint64) {
	for {
		cx := mg.cluster(next)
		leader := int(next % uint64(cx.Len()))
		if leader == cx.SelfIndex() {
			mg.seqns <- next
		}
		next++
	}
}

func (mg *Manager) fill(seqn uint64) {
	for next := range mg.fillUntil {
		for seqn < next {
			go mg.fillOne(seqn)
			seqn++
		}
		seqn = next + 1 // no need to fill in our own seqn
	}
}

func (m *Manager) process() {
	instances := make(map[uint64]instance)
	for req := range m.reqs {
		inst, ok := instances[req.seqn]
		if !ok {
			inst = make(instance)
			instances[req.seqn] = inst
			go inst.process(req.seqn, m, m.ops)
		}
		req.ch <- inst
	}
}

func (m *Manager) getInstance(seqn uint64) instance {
	ch := make(chan instance)
	m.reqs <- instReq{seqn, ch}
	return <-ch
}

func (m *Manager) PutFrom(addr string, msg Msg) {
	if !msg.Ok() {
		return
	}
	it := m.getInstance(msg.Seqn())
	it.PutFrom(addr, msg)
}

func (m *Manager) proposeAt(seqn uint64, v string) {
	m.getInstance(seqn).Propose(v)
	m.logger.Printf("paxos propose -> %d %q", seqn, v)
}

func (m *Manager) ProposeOnce(v string) store.Event {
	seqn := <-m.seqns
	ch := m.st.Wait(seqn)
	m.proposeAt(seqn, v)
	m.fillUntil <- seqn
	return <-ch
}

func (m *Manager) Propose(v string) (seqn uint64, cas string, err os.Error) {
	var ev store.Event

	// If a competing proposal succeeded in the same seqn, we should try again.
	for v != ev.Mut {
		ev = m.ProposeOnce(v)
	}
	return ev.Seqn, ev.Cas, ev.Err
}

func (m *Manager) fillOne(seqn uint64) {
	time.Sleep(fillDelay)
	// yes, we'll act as coordinator for a seqn we don't "own"
	// this is intentional, since we want to exersize this code all the time,
	// not just during a failure.
	m.proposeAt(seqn, store.Nop)
}
<|endoftext|>


<commit_before>/*******************************************************************************
*
* Copyright 2022 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package reports

import (
	"database/sql"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/sapcc/go-bits/sqlext"

	"github.com/sapcc/limes/pkg/core"
	"github.com/sapcc/limes/pkg/db"
)

var scrapeErrorsQuery = sqlext.SimplifyWhitespace(`
	SELECT d.uuid, d.name, p.uuid, p.name, ps.type, ps.checked_at, ps.scrape_error_message
	  FROM projects p
	  JOIN domains d ON d.id = p.domain_id
	  JOIN project_services ps ON ps.project_id = p.id
	WHERE %s AND ps.scrape_error_message != ''
	ORDER BY d.name, p.name, ps.type, ps.scrape_error_message
`)

type ScrapeError struct {
	Project struct {
		ID     string `json:"id"`
		Name   string `json:"name"`
		Domain struct {
			ID   string `json:"id"`
			Name string `json:"name"`
		} `json:"domain"`
	} `json:"project"`
	AffectedProjects int    `json:"affected_projects,omitempty"`
	ServiceType      string `json:"service_type"`
	CheckedAt        *int64 `json:"checked_at"`
	Message          string `json:"message"`
}

func GetScrapeErrors(cluster *core.Cluster, dbi db.Interface, filter Filter) ([]ScrapeError, error) {
	return getScrapeErrors(cluster, dbi, filter, scrapeErrorsQuery)
}

func GetRateScrapeErrors(cluster *core.Cluster, dbi db.Interface, filter Filter) ([]ScrapeError, error) {
	dbQuery := strings.ReplaceAll(scrapeErrorsQuery, "scrape_error_message", "rates_scrape_error_message")
	dbQuery = strings.ReplaceAll(dbQuery, "checked_at", "rates_checked_at")
	return getScrapeErrors(cluster, dbi, filter, dbQuery)
}

func getScrapeErrors(cluster *core.Cluster, dbi db.Interface, filter Filter, dbQuery string) ([]ScrapeError, error) {
	fields := map[string]interface{}{"d.cluster_id": cluster.ID}

	var result []ScrapeError
	queryStr, joinArgs := filter.PrepareQuery(dbQuery)
	whereStr, whereArgs := db.BuildSimpleWhereClause(fields, len(joinArgs))
	err := sqlext.ForeachRow(dbi, fmt.Sprintf(queryStr, whereStr), append(joinArgs, whereArgs...), func(rows *sql.Rows) error {
		var sErr ScrapeError
		var checkedAtAsTime *time.Time
		err := rows.Scan(
			&sErr.Project.Domain.ID, &sErr.Project.Domain.Name, &sErr.Project.ID,
			&sErr.Project.Name, &sErr.ServiceType, &checkedAtAsTime, &sErr.Message,
		)
		if err != nil {
			return err
		}

		if checkedAtAsTime != nil {
			v := checkedAtAsTime.Unix()
			sErr.CheckedAt = &v
		}
		result = append(result, sErr)

		return nil
	})
	if err != nil {
		return nil, err
	}

	if len(result) == 0 {
		//Ensure that empty list gets serialized as `[]` rather than as `null`.
		return []ScrapeError{}, nil
	}

	//To avoid excessively large responses, we group identical scrape errors for multiple
	//project services of the same type into one item.
	uniqueErrors := make(map[string]map[string]ScrapeError) //map[serviceType]map[Message]ScrapeError
	for _, v := range result {
		if vFromMap, found := uniqueErrors[v.ServiceType][v.Message]; found {
			//Use the value from map so we can preserve AffectedProject count.
			v = vFromMap
		}
		if _, ok := uniqueErrors[v.ServiceType]; !ok {
			uniqueErrors[v.ServiceType] = make(map[string]ScrapeError)
		}
		v.AffectedProjects++
		uniqueErrors[v.ServiceType][v.Message] = v
	}

	var newScrapeError []ScrapeError
	for _, errMsgs := range uniqueErrors {
		for _, sErr := range errMsgs {
			if sErr.AffectedProjects == 1 {
				//If only one project is affected then set to 0 so that this field can
				//omitted in JSON response.
				sErr.AffectedProjects = 0
			}
			newScrapeError = append(newScrapeError, sErr)
		}
	}

	//Deterministic ordering for unit tests
	sort.Slice(newScrapeError, func(i, j int) bool {
		srvType1 := newScrapeError[i].ServiceType
		srvType2 := newScrapeError[j].ServiceType
		if srvType1 != srvType2 {
			return srvType1 < srvType2
		}
		pID1 := newScrapeError[i].Project.ID
		pID2 := newScrapeError[j].Project.ID
		return pID1 < pID2
	})
	result = newScrapeError

	return result, nil
}
<commit_msg>use less clunky variable name<commit_after>/*******************************************************************************
*
* Copyright 2022 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package reports

import (
	"database/sql"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/sapcc/go-bits/sqlext"

	"github.com/sapcc/limes/pkg/core"
	"github.com/sapcc/limes/pkg/db"
)

var scrapeErrorsQuery = sqlext.SimplifyWhitespace(`
	SELECT d.uuid, d.name, p.uuid, p.name, ps.type, ps.checked_at, ps.scrape_error_message
	  FROM projects p
	  JOIN domains d ON d.id = p.domain_id
	  JOIN project_services ps ON ps.project_id = p.id
	WHERE %s AND ps.scrape_error_message != ''
	ORDER BY d.name, p.name, ps.type, ps.scrape_error_message
`)

type ScrapeError struct {
	Project struct {
		ID     string `json:"id"`
		Name   string `json:"name"`
		Domain struct {
			ID   string `json:"id"`
			Name string `json:"name"`
		} `json:"domain"`
	} `json:"project"`
	AffectedProjects int    `json:"affected_projects,omitempty"`
	ServiceType      string `json:"service_type"`
	CheckedAt        *int64 `json:"checked_at"`
	Message          string `json:"message"`
}

func GetScrapeErrors(cluster *core.Cluster, dbi db.Interface, filter Filter) ([]ScrapeError, error) {
	return getScrapeErrors(cluster, dbi, filter, scrapeErrorsQuery)
}

func GetRateScrapeErrors(cluster *core.Cluster, dbi db.Interface, filter Filter) ([]ScrapeError, error) {
	dbQuery := strings.ReplaceAll(scrapeErrorsQuery, "scrape_error_message", "rates_scrape_error_message")
	dbQuery = strings.ReplaceAll(dbQuery, "checked_at", "rates_checked_at")
	return getScrapeErrors(cluster, dbi, filter, dbQuery)
}

func getScrapeErrors(cluster *core.Cluster, dbi db.Interface, filter Filter, dbQuery string) ([]ScrapeError, error) {
	fields := map[string]interface{}{"d.cluster_id": cluster.ID}

	var result []ScrapeError
	queryStr, joinArgs := filter.PrepareQuery(dbQuery)
	whereStr, whereArgs := db.BuildSimpleWhereClause(fields, len(joinArgs))
	err := sqlext.ForeachRow(dbi, fmt.Sprintf(queryStr, whereStr), append(joinArgs, whereArgs...), func(rows *sql.Rows) error {
		var sErr ScrapeError
		var checkedAtAsTime *time.Time
		err := rows.Scan(
			&sErr.Project.Domain.ID, &sErr.Project.Domain.Name, &sErr.Project.ID,
			&sErr.Project.Name, &sErr.ServiceType, &checkedAtAsTime, &sErr.Message,
		)
		if err != nil {
			return err
		}

		if checkedAtAsTime != nil {
			v := checkedAtAsTime.Unix()
			sErr.CheckedAt = &v
		}
		result = append(result, sErr)

		return nil
	})
	if err != nil {
		return nil, err
	}

	if len(result) == 0 {
		//Ensure that empty list gets serialized as `[]` rather than as `null`.
		return []ScrapeError{}, nil
	}

	//To avoid excessively large responses, we group identical scrape errors for multiple
	//project services of the same type into one item.
	uniqueErrors := make(map[string]map[string]ScrapeError) //map[serviceType]map[Message]ScrapeError
	for _, v := range result {
		if vFromMap, found := uniqueErrors[v.ServiceType][v.Message]; found {
			//Use the value from map so we can preserve AffectedProject count.
			v = vFromMap
		}
		if _, ok := uniqueErrors[v.ServiceType]; !ok {
			uniqueErrors[v.ServiceType] = make(map[string]ScrapeError)
		}
		v.AffectedProjects++
		uniqueErrors[v.ServiceType][v.Message] = v
	}

	result = nil
	for _, errMsgs := range uniqueErrors {
		for _, sErr := range errMsgs {
			if sErr.AffectedProjects == 1 {
				//If only one project is affected then set to 0 so that this field can
				//omitted in JSON response.
				sErr.AffectedProjects = 0
			}
			result = append(result, sErr)
		}
	}

	//Deterministic ordering for unit tests
	sort.Slice(result, func(i, j int) bool {
		srvType1 := result[i].ServiceType
		srvType2 := result[j].ServiceType
		if srvType1 != srvType2 {
			return srvType1 < srvType2
		}
		pID1 := result[i].Project.ID
		pID2 := result[j].Project.ID
		return pID1 < pID2
	})

	return result, nil
}
<|endoftext|>


<commit_before>package server

import (
	"os"
	"net"

	"junta/util"
	"junta/paxos"
	"junta/proto"
	"junta/store"
	"strconv"
)

type conn struct {
	net.Conn
	s *Server
}

type Server struct {
	Addr string
	St *store.Store
	Mg *paxos.Manager
}

func (sv *Server) ListenAndServe() os.Error {
	logger := util.NewLogger("server %s", sv.Addr)

	logger.Log("binding")
	l, err := net.Listen("tcp", sv.Addr)
	if err != nil {
		logger.Log(err)
		return err
	}
	defer l.Close()
	logger.Log("listening")

	err = sv.Serve(l)
	if err != nil {
		logger.Logf("%s: %s", l, err)
	}
	return err
}

func (sv *Server) ListenAndServeUdp(outs chan paxos.Msg) os.Error {
	logger := util.NewLogger("udp server %s", sv.Addr)

	logger.Log("binding")
	u, err := net.ListenPacket("udp", sv.Addr)
	if err != nil {
		logger.Log(err)
		return err
	}
	defer u.Close()
	logger.Log("listening")

	err = sv.ServeUdp(u, outs)
	if err != nil {
		logger.Logf("%s: %s", u, err)
	}
	return err
}

func (sv *Server) ServeUdp(u net.PacketConn, outs chan paxos.Msg) os.Error {
	logger := util.NewLogger("udp server %s", u.LocalAddr())
	go func() {
		logger.Log("reading messages...")
		for {
			// TODO pull out this magic number into a const
			msg, addr, err := paxos.ReadMsg(u, 3000)
			if err != nil {
				logger.Log(err)
				continue
			}
			logger.Logf("read %v from %s", msg, addr)
			sv.Mg.PutFrom(addr, msg)
		}
	}()

	logger.Log("sending messages...")
	for msg := range outs {
		logger.Logf("sending %v", msg)
		for _, addr := range sv.Mg.AddrsFor(msg) {
			logger.Logf("sending to %s", addr)
			udpAddr, err := net.ResolveUDPAddr(addr)
			if err != nil {
				logger.Log(err)
				continue
			}
			_, err = u.WriteTo(msg.WireBytes(), udpAddr)
			if err != nil {
				logger.Log(err)
				continue
			}
		}
	}

	panic("not reached")
}

func (s *Server) Serve(l net.Listener) os.Error {
	for {
		rw, e := l.Accept()
		if e != nil {
			return e
		}
		c := &conn{rw, s}
		go c.serve()
	}

	panic("not reached")
}

func (sv *Server) Set(path, body, cas string) (uint64, os.Error) {
	mut, err := store.EncodeSet(path, body, cas)
	if err != nil {
		return 0, err
	}

	seqn, v, err := sv.Mg.Propose(mut)
	if err != nil {
		return 0, err
	}

	// We failed, but only because of a competing proposal. The client should
	// retry.
	if v != mut {
		return 0, os.EAGAIN
	}

	return seqn, nil
}

func (sv *Server) Del(path, cas string) (uint64, os.Error) {
	mut, err := store.EncodeDel(path, cas)
	if err != nil {
		return 0, err
	}

	seqn, v, err := sv.Mg.Propose(mut)
	if err != nil {
		return 0, err
	}

	// We failed, but only because of a competing proposal. The client should
	// retry.
	if v != mut {
		return 0, os.EAGAIN
	}

	return seqn, nil
}

func (c *conn) serve() {
	pc := proto.NewConn(c)
	logger := util.NewLogger("%v", c.RemoteAddr())
	logger.Log("accepted connection")
	for {
		rid, parts, err := pc.ReadRequest()
		if err != nil {
			if err == os.EOF {
				logger.Log("connection closed by peer")
			} else {
				logger.Log(err)
			}
			return
		}

		rlogger := util.NewLogger("%v - req [%d]", c.RemoteAddr(), rid)
		rlogger.Logf("received <%v>", parts)

		if len(parts) == 0 {
			rlogger.Log("zero parts supplied")
			pc.SendError(rid, proto.InvalidCommand)
			continue
		}

		switch parts[0] {
		default:
			rlogger.Logf("unknown command <%s>", parts[0])
			pc.SendError(rid, proto.InvalidCommand)
		case "set":
			if len(parts) != 4 {
				rlogger.Logf("invalid set command: %#v", parts)
				pc.SendError(rid, "wrong number of parts")
				break
			}
			rlogger.Logf("set %q=%q (cas %q)", parts[1], parts[2], parts[3])
			err := os.EAGAIN
			for err == os.EAGAIN {
				_, err = c.s.Set(parts[1], parts[2], parts[3])
			}
			if err != nil {
				rlogger.Logf("bad: %s", err)
				pc.SendError(rid, err.String())
			} else {
				rlogger.Logf("good")
				pc.SendResponse(rid, "true")
			}
		case "del":
			if len(parts) != 3 {
				rlogger.Logf("invalid del command: %v", parts)
				pc.SendError(rid, "wrong number of parts")
				break
			}
			rlogger.Logf("del %q (cas %q)", parts[1], parts[2])
			err := os.EAGAIN
			for err == os.EAGAIN {
				_, err = c.s.Del(parts[1], parts[2])
			}
			if err != nil {
				rlogger.Logf("bad: %s", err)
				pc.SendError(rid, err.String())
			} else {
				rlogger.Logf("good")
				pc.SendResponse(rid, "true")
			}
		case "join":
			// join abc123 1.2.3.4:999
			if len(parts) != 3 {
				rlogger.Logf("invalid join command: %v", parts)
				pc.SendError(rid, "wrong number of parts")
				break
			}

			who, addr := parts[1], parts[2]
			rlogger.Logf("membership requested for %s at %s", who, addr)

			key := "/j/junta/members/" + who

			var seqn uint64
			err := os.EAGAIN
			for err == os.EAGAIN {
				seqn, err = c.s.Set(key, addr, store.Missing)
			}
			if err != nil {
				rlogger.Logf("bad: %s", err)
				pc.SendError(rid, err.String())
			} else {
				rlogger.Logf("good")
				c.s.St.Sync(seqn + uint64(c.s.Mg.Alpha))
				seqn, snap := c.s.St.Snapshot()
				pc.SendResponse(rid, strconv.Uitoa64(seqn), snap)
			}
		}
	}
}
<commit_msg>refactor<commit_after>package server

import (
	"os"
	"net"

	"junta/util"
	"junta/paxos"
	"junta/proto"
	"junta/store"
	"strconv"
)

const packetSize = 3000

type conn struct {
	net.Conn
	s *Server
}

type Server struct {
	Addr string
	St *store.Store
	Mg *paxos.Manager
}

func (sv *Server) ListenAndServe() os.Error {
	logger := util.NewLogger("server %s", sv.Addr)

	logger.Log("binding")
	l, err := net.Listen("tcp", sv.Addr)
	if err != nil {
		logger.Log(err)
		return err
	}
	defer l.Close()
	logger.Log("listening")

	err = sv.Serve(l)
	if err != nil {
		logger.Logf("%s: %s", l, err)
	}
	return err
}

func (sv *Server) ListenAndServeUdp(outs chan paxos.Msg) os.Error {
	logger := util.NewLogger("udp server %s", sv.Addr)

	logger.Log("binding")
	u, err := net.ListenPacket("udp", sv.Addr)
	if err != nil {
		logger.Log(err)
		return err
	}
	defer u.Close()
	logger.Log("listening")

	err = sv.ServeUdp(u, outs)
	if err != nil {
		logger.Logf("%s: %s", u, err)
	}
	return err
}

func (sv *Server) ServeUdp(u net.PacketConn, outs chan paxos.Msg) os.Error {
	logger := util.NewLogger("udp server %s", u.LocalAddr())
	go func() {
		logger.Log("reading messages...")
		for {
			msg, addr, err := paxos.ReadMsg(u, packetSize)
			if err != nil {
				logger.Log(err)
				continue
			}
			logger.Logf("read %v from %s", msg, addr)
			sv.Mg.PutFrom(addr, msg)
		}
	}()

	logger.Log("sending messages...")
	for msg := range outs {
		logger.Logf("sending %v", msg)
		for _, addr := range sv.Mg.AddrsFor(msg) {
			logger.Logf("sending to %s", addr)
			udpAddr, err := net.ResolveUDPAddr(addr)
			if err != nil {
				logger.Log(err)
				continue
			}
			_, err = u.WriteTo(msg.WireBytes(), udpAddr)
			if err != nil {
				logger.Log(err)
				continue
			}
		}
	}

	panic("not reached")
}

func (s *Server) Serve(l net.Listener) os.Error {
	for {
		rw, e := l.Accept()
		if e != nil {
			return e
		}
		c := &conn{rw, s}
		go c.serve()
	}

	panic("not reached")
}

func (sv *Server) Set(path, body, cas string) (uint64, os.Error) {
	mut, err := store.EncodeSet(path, body, cas)
	if err != nil {
		return 0, err
	}

	seqn, v, err := sv.Mg.Propose(mut)
	if err != nil {
		return 0, err
	}

	// We failed, but only because of a competing proposal. The client should
	// retry.
	if v != mut {
		return 0, os.EAGAIN
	}

	return seqn, nil
}

func (sv *Server) Del(path, cas string) (uint64, os.Error) {
	mut, err := store.EncodeDel(path, cas)
	if err != nil {
		return 0, err
	}

	seqn, v, err := sv.Mg.Propose(mut)
	if err != nil {
		return 0, err
	}

	// We failed, but only because of a competing proposal. The client should
	// retry.
	if v != mut {
		return 0, os.EAGAIN
	}

	return seqn, nil
}

func (c *conn) serve() {
	pc := proto.NewConn(c)
	logger := util.NewLogger("%v", c.RemoteAddr())
	logger.Log("accepted connection")
	for {
		rid, parts, err := pc.ReadRequest()
		if err != nil {
			if err == os.EOF {
				logger.Log("connection closed by peer")
			} else {
				logger.Log(err)
			}
			return
		}

		rlogger := util.NewLogger("%v - req [%d]", c.RemoteAddr(), rid)
		rlogger.Logf("received <%v>", parts)

		if len(parts) == 0 {
			rlogger.Log("zero parts supplied")
			pc.SendError(rid, proto.InvalidCommand)
			continue
		}

		switch parts[0] {
		default:
			rlogger.Logf("unknown command <%s>", parts[0])
			pc.SendError(rid, proto.InvalidCommand)
		case "set":
			if len(parts) != 4 {
				rlogger.Logf("invalid set command: %#v", parts)
				pc.SendError(rid, "wrong number of parts")
				break
			}
			rlogger.Logf("set %q=%q (cas %q)", parts[1], parts[2], parts[3])
			err := os.EAGAIN
			for err == os.EAGAIN {
				_, err = c.s.Set(parts[1], parts[2], parts[3])
			}
			if err != nil {
				rlogger.Logf("bad: %s", err)
				pc.SendError(rid, err.String())
			} else {
				rlogger.Logf("good")
				pc.SendResponse(rid, "true")
			}
		case "del":
			if len(parts) != 3 {
				rlogger.Logf("invalid del command: %v", parts)
				pc.SendError(rid, "wrong number of parts")
				break
			}
			rlogger.Logf("del %q (cas %q)", parts[1], parts[2])
			err := os.EAGAIN
			for err == os.EAGAIN {
				_, err = c.s.Del(parts[1], parts[2])
			}
			if err != nil {
				rlogger.Logf("bad: %s", err)
				pc.SendError(rid, err.String())
			} else {
				rlogger.Logf("good")
				pc.SendResponse(rid, "true")
			}
		case "join":
			// join abc123 1.2.3.4:999
			if len(parts) != 3 {
				rlogger.Logf("invalid join command: %v", parts)
				pc.SendError(rid, "wrong number of parts")
				break
			}

			who, addr := parts[1], parts[2]
			rlogger.Logf("membership requested for %s at %s", who, addr)

			key := "/j/junta/members/" + who

			var seqn uint64
			err := os.EAGAIN
			for err == os.EAGAIN {
				seqn, err = c.s.Set(key, addr, store.Missing)
			}
			if err != nil {
				rlogger.Logf("bad: %s", err)
				pc.SendError(rid, err.String())
			} else {
				rlogger.Logf("good")
				c.s.St.Sync(seqn + uint64(c.s.Mg.Alpha))
				seqn, snap := c.s.St.Snapshot()
				pc.SendResponse(rid, strconv.Uitoa64(seqn), snap)
			}
		}
	}
}
<|endoftext|>


<commit_before>package db

import (
	. "github.com/Aptomi/aptomi/pkg/slinga/util"
	"os"
	"path/filepath"
)

// AptomiOject represents an aptomi entity, which gets stored in aptomi DB
type AptomiOject string

// AptomiCurrentRunDir is where results of the last run are stored
const AptomiCurrentRunDir = "last-run-results"

const (
	/*
		The following objects can be added to Aptomi
	*/

	// TypeCluster is k8s cluster or any other cluster
	TypeCluster AptomiOject = "cluster"

	// TypeService is service definitions
	TypeService AptomiOject = "service"

	// TypeContext is how service gets allocated
	TypeContext AptomiOject = "context"

	// TypeRules is global rules of the land
	TypeRules AptomiOject = "rules"

	// TypeDependencies is who requested what
	TypeDependencies AptomiOject = "dependencies"

	/*
		The following objects must be configured to point to external resources
	*/

	// TypeUsersFile is where users are stored (this is for file-based storage)
	TypeUsersFile AptomiOject = "users"

	// TypeUsersLDAP is where ldap configuration is stored\
	TypeUsersLDAP AptomiOject = "ldap"

	// TypeSecrets is where secret tokens are stored (later in Hashicorp Vault)
	TypeSecrets AptomiOject = "secrets"

	// TypeCharts is where binary charts/images are stored (later in external repo)
	TypeCharts AptomiOject = "charts"

	/*
		The following objects are generated by aptomi during or after dependency resolution via policy
	*/

	// TypeRevision holds revision number for the last successful aptomi run
	TypeRevision AptomiOject = "revision"

	// TypePolicyResolution holds usage data for components/dependencies
	TypePolicyResolution AptomiOject = "db"

	// TypeLogs contains debug logs
	TypeLogs AptomiOject = "logs"

	// TypeGraphics contains images generated by graphviz
	TypeGraphics AptomiOject = "graphics"
)

// Return aptomi DB directory
func getAptomiEnvVarAsDir(key string) string {
	value, ok := os.LookupEnv(key)
	if !ok {
		panic("Environment variable is not present. Must point to a directory")
	}
	if stat, err := os.Stat(value); err != nil || !stat.IsDir() {
		panic("Directory doesn't exist or error encountered")
	}
	return value
}

// GetAptomiBaseDir returns base directory, i.e. the value of APTOMI_DB environment variable
func GetAptomiBaseDir() string {
	return getAptomiEnvVarAsDir("APTOMI_DB")
}

// GetAptomiPolicyDir returns default aptomi policy dir
func GetAptomiPolicyDir() string {
	return filepath.Join(GetAptomiBaseDir(), "policy")
}

// GetAptomiDebugLogName returns filename for aptomi debug log (in db, but outside of current run)
func GetAptomiDebugLogName() string {
	dir := filepath.Join(GetAptomiBaseDir(), string(TypeLogs))
	if stat, err := os.Stat(dir); err != nil || !stat.IsDir() {
		err = os.MkdirAll(dir, 0755)
		if err != nil {
			panic("Directory can't be created or error encountered")
		}
	}
	if stat, err := os.Stat(dir); err != nil || !stat.IsDir() {
		panic("Directory can't be created or error encountered")
	}
	return filepath.Join(dir, "debug.log")
}

// GetAptomiObjectFilePatternYaml returns file pattern for aptomi objects (so they can be loaded from those files)
func GetAptomiObjectFilePatternYaml(baseDir string, aptomiObject AptomiOject) string {
	return filepath.Join(baseDir, "**", string(aptomiObject)+"*.yaml")
}

// GetAptomiObjectFilePatternTgz returns file pattern for tgz objects (so they can be loaded from those files)
func GetAptomiObjectFilePatternTgz(baseDir string, aptomiObject AptomiOject, chartName string) string {
	return filepath.Join(baseDir, "**", chartName+".tgz")
}

// GetAptomiObjectWriteFileGlobal returns file name for global aptomi objects (e.g. revision)
// It will place files into AptomiCurrentRunDir. It will create the corresponding directories if they don't exist
func GetAptomiObjectWriteFileGlobal(baseDir string, aptomiObject AptomiOject) string {
	return filepath.Join(baseDir, string(aptomiObject)+".yaml")
}

// GetAptomiObjectFileFromRun returns file name for aptomi objects (so they can be saved)
// It will place files into AptomiCurrentRunDir. It will create the corresponding directories if they don't exist
func GetAptomiObjectFileFromRun(baseDir string, runDir string, aptomiObject AptomiOject, fileName string) string {
	return filepath.Join(baseDir, runDir, string(aptomiObject), fileName)
}

// GetAptomiObjectWriteFileCurrentRun returns file name for aptomi objects (so they can be saved)
// It will place files into AptomiCurrentRunDir. It will create the corresponding directories if they don't exist
func GetAptomiObjectWriteFileCurrentRun(baseDir string, aptomiObject AptomiOject, fileName string) string {
	dir := filepath.Join(baseDir, AptomiCurrentRunDir, string(aptomiObject))
	if stat, err := os.Stat(dir); err != nil || !stat.IsDir() {
		err = os.MkdirAll(dir, 0755)
		if err != nil {
			panic("Directory can't be created or error encountered")
		}
	}
	if stat, err := os.Stat(dir); err != nil || !stat.IsDir() {
		panic("Directory can't be created or error encountered")
	}
	return filepath.Join(dir, fileName)
}

// CleanCurrentRunDirectory deletes contents of a "current run" directory
func CleanCurrentRunDirectory(baseDir string) {
	dir := filepath.Join(baseDir, AptomiCurrentRunDir)
	err := DeleteDirectoryContents(dir)
	if err != nil {
		panic("Directory contents can't be deleted")
	}
}
<commit_msg>comment typo fix<commit_after>package db

import (
	. "github.com/Aptomi/aptomi/pkg/slinga/util"
	"os"
	"path/filepath"
)

// AptomiOject represents an aptomi entity, which gets stored in aptomi DB
type AptomiOject string

// AptomiCurrentRunDir is where results of the last run are stored
const AptomiCurrentRunDir = "last-run-results"

const (
	/*
		The following objects can be added to Aptomi
	*/

	// TypeCluster is k8s cluster or any other cluster
	TypeCluster AptomiOject = "cluster"

	// TypeService is service definitions
	TypeService AptomiOject = "service"

	// TypeContext is how service gets allocated
	TypeContext AptomiOject = "context"

	// TypeRules is global rules of the land
	TypeRules AptomiOject = "rules"

	// TypeDependencies is who requested what
	TypeDependencies AptomiOject = "dependencies"

	/*
		The following objects must be configured to point to external resources
	*/

	// TypeUsersFile is where users are stored (this is for file-based storage)
	TypeUsersFile AptomiOject = "users"

	// TypeUsersLDAP is where ldap configuration is stored
	TypeUsersLDAP AptomiOject = "ldap"

	// TypeSecrets is where secret tokens are stored (later in Hashicorp Vault)
	TypeSecrets AptomiOject = "secrets"

	// TypeCharts is where binary charts/images are stored (later in external repo)
	TypeCharts AptomiOject = "charts"

	/*
		The following objects are generated by aptomi during or after dependency resolution via policy
	*/

	// TypeRevision holds revision number for the last successful aptomi run
	TypeRevision AptomiOject = "revision"

	// TypePolicyResolution holds usage data for components/dependencies
	TypePolicyResolution AptomiOject = "db"

	// TypeLogs contains debug logs
	TypeLogs AptomiOject = "logs"

	// TypeGraphics contains images generated by graphviz
	TypeGraphics AptomiOject = "graphics"
)

// Return aptomi DB directory
func getAptomiEnvVarAsDir(key string) string {
	value, ok := os.LookupEnv(key)
	if !ok {
		panic("Environment variable is not present. Must point to a directory")
	}
	if stat, err := os.Stat(value); err != nil || !stat.IsDir() {
		panic("Directory doesn't exist or error encountered")
	}
	return value
}

// GetAptomiBaseDir returns base directory, i.e. the value of APTOMI_DB environment variable
func GetAptomiBaseDir() string {
	return getAptomiEnvVarAsDir("APTOMI_DB")
}

// GetAptomiPolicyDir returns default aptomi policy dir
func GetAptomiPolicyDir() string {
	return filepath.Join(GetAptomiBaseDir(), "policy")
}

// GetAptomiDebugLogName returns filename for aptomi debug log (in db, but outside of current run)
func GetAptomiDebugLogName() string {
	dir := filepath.Join(GetAptomiBaseDir(), string(TypeLogs))
	if stat, err := os.Stat(dir); err != nil || !stat.IsDir() {
		err = os.MkdirAll(dir, 0755)
		if err != nil {
			panic("Directory can't be created or error encountered")
		}
	}
	if stat, err := os.Stat(dir); err != nil || !stat.IsDir() {
		panic("Directory can't be created or error encountered")
	}
	return filepath.Join(dir, "debug.log")
}

// GetAptomiObjectFilePatternYaml returns file pattern for aptomi objects (so they can be loaded from those files)
func GetAptomiObjectFilePatternYaml(baseDir string, aptomiObject AptomiOject) string {
	return filepath.Join(baseDir, "**", string(aptomiObject)+"*.yaml")
}

// GetAptomiObjectFilePatternTgz returns file pattern for tgz objects (so they can be loaded from those files)
func GetAptomiObjectFilePatternTgz(baseDir string, aptomiObject AptomiOject, chartName string) string {
	return filepath.Join(baseDir, "**", chartName+".tgz")
}

// GetAptomiObjectWriteFileGlobal returns file name for global aptomi objects (e.g. revision)
// It will place files into AptomiCurrentRunDir. It will create the corresponding directories if they don't exist
func GetAptomiObjectWriteFileGlobal(baseDir string, aptomiObject AptomiOject) string {
	return filepath.Join(baseDir, string(aptomiObject)+".yaml")
}

// GetAptomiObjectFileFromRun returns file name for aptomi objects (so they can be saved)
// It will place files into AptomiCurrentRunDir. It will create the corresponding directories if they don't exist
func GetAptomiObjectFileFromRun(baseDir string, runDir string, aptomiObject AptomiOject, fileName string) string {
	return filepath.Join(baseDir, runDir, string(aptomiObject), fileName)
}

// GetAptomiObjectWriteFileCurrentRun returns file name for aptomi objects (so they can be saved)
// It will place files into AptomiCurrentRunDir. It will create the corresponding directories if they don't exist
func GetAptomiObjectWriteFileCurrentRun(baseDir string, aptomiObject AptomiOject, fileName string) string {
	dir := filepath.Join(baseDir, AptomiCurrentRunDir, string(aptomiObject))
	if stat, err := os.Stat(dir); err != nil || !stat.IsDir() {
		err = os.MkdirAll(dir, 0755)
		if err != nil {
			panic("Directory can't be created or error encountered")
		}
	}
	if stat, err := os.Stat(dir); err != nil || !stat.IsDir() {
		panic("Directory can't be created or error encountered")
	}
	return filepath.Join(dir, fileName)
}

// CleanCurrentRunDirectory deletes contents of a "current run" directory
func CleanCurrentRunDirectory(baseDir string) {
	dir := filepath.Join(baseDir, AptomiCurrentRunDir)
	err := DeleteDirectoryContents(dir)
	if err != nil {
		panic("Directory contents can't be deleted")
	}
}
<|endoftext|>


<commit_before>package tools

import (
	"fmt"
	"io"
	"os"
	"os/exec"
)

func FmtCode(path string, errOut io.Writer) {
	oscmd := exec.Command("goimports", "-format-only", "-w", path)
	oscmd.Stderr = errOut
	if err := oscmd.Run(); err != nil {
		fmt.Fprintln(os.Stderr, "run fmt tool: goimports: %w", err)
		return
	}
}
<commit_msg>pkg/tools: fmt errors<commit_after>package tools

import (
	"fmt"
	"io"
	"os"
	"os/exec"
)

func FmtCode(path string, errOut io.Writer) {
	oscmd := exec.Command("goimports", "-format-only", "-w", path)
	oscmd.Stderr = errOut
	if err := oscmd.Run(); err != nil {
		fmt.Fprintln(os.Stderr, fmt.Errorf("run fmt tool: goimports: %w", err))
		return
	}
}
<|endoftext|>


<commit_before>package main

import(
	"fmt"
	"io"
	"log"
	"net"
	"os/exec"
)

const(
	PBCOPY      = "pbcopy"
	LISTEN_ADDR = "127.0.0.1"
	LISTEN_PORT = "8377"
)

func main() {
	if _, err := exec.LookPath(PBCOPY); err != nil {
		log.Fatal(err.Error())
	}

	log.Print("Starting the server")
	address := fmt.Sprintf("%s:%s", LISTEN_ADDR, LISTEN_PORT)
	listener, err := net.Listen("tcp", address)
	if err != nil {
		log.Fatal(err.Error())
	}

	for {
		conn, err := listener.Accept()
		if err != nil {
			log.Print(err.Error())
			return
		}

		go handleConnection(conn)
	}
}

func handleConnection(conn net.Conn) {
	defer log.Print("Connection closed")
	defer conn.Close()

	cmd := exec.Command(PBCOPY)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		log.Print(err.Error())
		return
	}
	if err = cmd.Start(); err != nil {
		log.Print(err.Error())
		return
	}

	if copied, err := io.Copy(stdin, conn); err != nil {
		log.Print(err.Error())
	} else {
		log.Print("Echoed ", copied, " bytes")
	}
	stdin.Close()

	if err = cmd.Wait(); err != nil {
		log.Print(err.Error())
	}
}
<commit_msg>Add command-line flag parsing<commit_after>package main

import(
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"os/exec"
)

const(
	PBCOPY              = "pbcopy"
	DEFAULT_LISTEN_ADDR = "127.0.0.1"
	DEFAULT_LISTEN_PORT = 8377
)

func main() {
	address := flag.String("address", DEFAULT_LISTEN_ADDR, "address to bind to")
	port    := flag.Int("port", DEFAULT_LISTEN_PORT, "port to listen on")
	flag.Parse()

	if _, err := exec.LookPath(PBCOPY); err != nil {
		log.Fatal(err.Error())
	}

	log.Print("Starting the server")
	listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", *address, *port))
	if err != nil {
		log.Fatal(err.Error())
	}

	for {
		conn, err := listener.Accept()
		if err != nil {
			log.Print(err.Error())
			return
		}

		go handleConnection(conn)
	}
}

func handleConnection(conn net.Conn) {
	defer log.Print("Connection closed")
	defer conn.Close()

	cmd := exec.Command(PBCOPY)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		log.Print(err.Error())
		return
	}
	if err = cmd.Start(); err != nil {
		log.Print(err.Error())
		return
	}

	if copied, err := io.Copy(stdin, conn); err != nil {
		log.Print(err.Error())
	} else {
		log.Print("Echoed ", copied, " bytes")
	}
	stdin.Close()

	if err = cmd.Wait(); err != nil {
		log.Print(err.Error())
	}
}
<|endoftext|>