<commit_before>package cucumberexpressions

import (
	"fmt"
	"regexp"
	"strings"
)

type CucumberExpression struct {
	expression     string
	parameterTypes []*ParameterType
	treeRegexp     *TreeRegexp
}

func NewCucumberExpression(expression string, parameterTypeRegistry *ParameterTypeRegistry) (*CucumberExpression, error) {
	ESCAPE_REGEXP := regexp.MustCompile(`([\\^[$.|?*+])`)
	PARAMETER_REGEXP := regexp.MustCompile("{([^}]+)}")
	OPTIONAL_REGEXP := regexp.MustCompile(`(\\\\)?\([^)]+\)`)
	ALTERNATIVE_NON_WHITESPACE_TEXT_REGEXP := regexp.MustCompile(`([^\s^/]+)((\/[^\s^/]+)+)`)

	result := &CucumberExpression{expression: expression}
	parameterTypes := []*ParameterType{}
	r := "^"
	matchOffset := 0

	// Does not include (){} because they have special meaning
	fmt.Println(expression)
	expression = ESCAPE_REGEXP.ReplaceAllString(expression, "\\$1")
	fmt.Println(expression)

	// Create non-capturing, optional capture groups from parenthesis
	expression = OPTIONAL_REGEXP.ReplaceAllStringFunc(expression, func(match string) string {
		if strings.HasPrefix(match, "\\\\") {
			return fmt.Sprintf(`\(%s\)`, match[3:len(match)-1])
		}
		return fmt.Sprintf("(?:%s", match[1:])
	})

	expression = ALTERNATIVE_NON_WHITESPACE_TEXT_REGEXP.ReplaceAllStringFunc(expression, func(match string) string {
		return fmt.Sprintf("(?:%s)", strings.Replace(match, "/", "|", -1))
	})

	matches := PARAMETER_REGEXP.FindAllStringSubmatchIndex(expression[matchOffset:], -1)
	for _, indicies := range matches {
		typeName := expression[indicies[2]:indicies[3]]
		parameterType := parameterTypeRegistry.LookupByTypeName(typeName)
		if parameterType == nil {
			return nil, NewUndefinedParameterTypeError(typeName)
		}
		parameterTypes = append(parameterTypes, parameterType)
		text := expression[matchOffset:indicies[0]]
		captureRegexp := buildCaptureRegexp(parameterType.regexps)
		matchOffset = indicies[1]
		r += text
		r += captureRegexp
	}

	r += expression[matchOffset:] + "$"
	result.parameterTypes = parameterTypes
	result.treeRegexp = NewTreeRegexp(regexp.MustCompile(r))
	return result, nil
}

func (c *CucumberExpression) Match(text string) []*Argument {
	return BuildArguments(c.treeRegexp, text, c.parameterTypes)
}

func (c *CucumberExpression) Regexp() *regexp.Regexp {
	return c.treeRegexp.Regexp()
}

func (c *CucumberExpression) Source() string {
	return c.expression
}

func buildCaptureRegexp(regexps []*regexp.Regexp) string {
	if len(regexps) == 1 {
		return fmt.Sprintf("(%s)", regexps[0].String())
	}

	captureGroups := make([]string, len(regexps))
	for i, r := range regexps {
		captureGroups[i] = fmt.Sprintf("(?:%s)", r.String())
	}

	return fmt.Sprintf("(%s)", strings.Join(captureGroups, "|"))
}

// const Argument = require('./argument')
// const TreeRegexp = require('./tree_regexp')
// const { UndefinedParameterTypeError } = require('./errors')
//
// class CucumberExpression {
//   /**
//    * @param expression
//    * @param parameterTypeRegistry
//    */
//   constructor(expression, parameterTypeRegistry) {
//     // Does not include (){} characters because they have special meaning
//     const ESCAPE_REGEXP = /([\\^[$.|?*+])/g
//     const PARAMETER_REGEXP = /{([^}]+)}/g
//     const OPTIONAL_REGEXP = /(\\\\)?\(([^)]+)\)/g
//     const ALTERNATIVE_NON_WHITESPACE_TEXT_REGEXP = /([^\s^/]+)((\/[^\s^/]+)+)/g
//
//     this._expression = expression
//     this._parameterTypes = []
//     let regexp = '^'
//     let match
//     let matchOffset = 0
//
//     // Does not include (){} because they have special meaning
//
//     expression = expression.replace(ESCAPE_REGEXP, '\\$1')
//
//     // Create non-capturing, optional capture groups from parenthesis
//     expression = expression.replace(
//       OPTIONAL_REGEXP,
//       (match, p1, p2) => (p1 === '\\\\' ? `\\(${p2}\\)` : `(?:${p2})?`)
//     )
//
//     expression = expression.replace(
//       ALTERNATIVE_NON_WHITESPACE_TEXT_REGEXP,
//       (_, p1, p2) => `(?:${p1}${p2.replace(/\//g, '|')})`
//     )
//
//     while ((match = PARAMETER_REGEXP.exec(expression)) !== null) {
//       const typeName = match[1]
//
//       const parameterType = parameterTypeRegistry.lookupByTypeName(typeName)
//       if (!parameterType) throw new UndefinedParameterTypeError(typeName)
//       this._parameterTypes.push(parameterType)
//
//       const text = expression.slice(matchOffset, match.index)
//       const captureRegexp = buildCaptureRegexp(parameterType.regexps)
//       matchOffset = PARAMETER_REGEXP.lastIndex
//       regexp += text
//       regexp += captureRegexp
//     }
//     regexp += expression.slice(matchOffset)
//     regexp += '$'
//     this._treeRegexp = new TreeRegexp(regexp)
//   }
//
// }
//
// function buildCaptureRegexp(regexps) {
//   if (regexps.length === 1) {
//     return `(${regexps[0]})`
//   }
//
//   const captureGroups = regexps.map(group => {
//     return `(?:${group})`
//   })
//
//   return `(${captureGroups.join('|')})`
// }
//
// module.exports = CucumberExpression
<commit_msg>update<commit_after>package cucumberexpressions

import (
	"fmt"
	"regexp"
	"strings"
)

type CucumberExpression struct {
	expression     string
	parameterTypes []*ParameterType
	treeRegexp     *TreeRegexp
}

func NewCucumberExpression(expression string, parameterTypeRegistry *ParameterTypeRegistry) (*CucumberExpression, error) {
	ESCAPE_REGEXP := regexp.MustCompile(`([\\^[$.|?*+])`)
	PARAMETER_REGEXP := regexp.MustCompile("{([^}]+)}")
	OPTIONAL_REGEXP := regexp.MustCompile(`(\\\\\\\\)?\([^)]+\)`)
	ALTERNATIVE_NON_WHITESPACE_TEXT_REGEXP := regexp.MustCompile(`([^\s^/]+)((\/[^\s^/]+)+)`)

	result := &CucumberExpression{expression: expression}
	parameterTypes := []*ParameterType{}
	r := "^"
	matchOffset := 0

	// Does not include (){} because they have special meaning
	expression = ESCAPE_REGEXP.ReplaceAllString(expression, "\\$1")

	// Create non-capturing, optional capture groups from parenthesis
	expression = OPTIONAL_REGEXP.ReplaceAllStringFunc(expression, func(match string) string {
		if strings.HasPrefix(match, "\\\\\\\\") {
			return fmt.Sprintf(`\(%s\)`, match[5:len(match)-1])
		}
		return fmt.Sprintf("(?:%s", match[1:])
	})

	expression = ALTERNATIVE_NON_WHITESPACE_TEXT_REGEXP.ReplaceAllStringFunc(expression, func(match string) string {
		return fmt.Sprintf("(?:%s)", strings.Replace(match, "/", "|", -1))
	})

	matches := PARAMETER_REGEXP.FindAllStringSubmatchIndex(expression[matchOffset:], -1)
	for _, indicies := range matches {
		typeName := expression[indicies[2]:indicies[3]]
		parameterType := parameterTypeRegistry.LookupByTypeName(typeName)
		if parameterType == nil {
			return nil, NewUndefinedParameterTypeError(typeName)
		}
		parameterTypes = append(parameterTypes, parameterType)
		text := expression[matchOffset:indicies[0]]
		captureRegexp := buildCaptureRegexp(parameterType.regexps)
		matchOffset = indicies[1]
		r += text
		r += captureRegexp
	}

	r += expression[matchOffset:] + "$"
	result.parameterTypes = parameterTypes
	result.treeRegexp = NewTreeRegexp(regexp.MustCompile(r))
	return result, nil
}

func (c *CucumberExpression) Match(text string) []*Argument {
	return BuildArguments(c.treeRegexp, text, c.parameterTypes)
}

func (c *CucumberExpression) Regexp() *regexp.Regexp {
	return c.treeRegexp.Regexp()
}

func (c *CucumberExpression) Source() string {
	return c.expression
}

func buildCaptureRegexp(regexps []*regexp.Regexp) string {
	if len(regexps) == 1 {
		return fmt.Sprintf("(%s)", regexps[0].String())
	}

	captureGroups := make([]string, len(regexps))
	for i, r := range regexps {
		captureGroups[i] = fmt.Sprintf("(?:%s)", r.String())
	}

	return fmt.Sprintf("(%s)", strings.Join(captureGroups, "|"))
}

// const Argument = require('./argument')
// const TreeRegexp = require('./tree_regexp')
// const { UndefinedParameterTypeError } = require('./errors')
//
// class CucumberExpression {
//   /**
//    * @param expression
//    * @param parameterTypeRegistry
//    */
//   constructor(expression, parameterTypeRegistry) {
//     // Does not include (){} characters because they have special meaning
//     const ESCAPE_REGEXP = /([\\^[$.|?*+])/g
//     const PARAMETER_REGEXP = /{([^}]+)}/g
//     const OPTIONAL_REGEXP = /(\\\\)?\(([^)]+)\)/g
//     const ALTERNATIVE_NON_WHITESPACE_TEXT_REGEXP = /([^\s^/]+)((\/[^\s^/]+)+)/g
//
//     this._expression = expression
//     this._parameterTypes = []
//     let regexp = '^'
//     let match
//     let matchOffset = 0
//
//     // Does not include (){} because they have special meaning
//
//     expression = expression.replace(ESCAPE_REGEXP, '\\$1')
//
//     // Create non-capturing, optional capture groups from parenthesis
//     expression = expression.replace(
//       OPTIONAL_REGEXP,
//       (match, p1, p2) => (p1 === '\\\\' ? `\\(${p2}\\)` : `(?:${p2})?`)
//     )
//
//     expression = expression.replace(
//       ALTERNATIVE_NON_WHITESPACE_TEXT_REGEXP,
//       (_, p1, p2) => `(?:${p1}${p2.replace(/\//g, '|')})`
//     )
//
//     while ((match = PARAMETER_REGEXP.exec(expression)) !== null) {
//       const typeName = match[1]
//
//       const parameterType = parameterTypeRegistry.lookupByTypeName(typeName)
//       if (!parameterType) throw new UndefinedParameterTypeError(typeName)
//       this._parameterTypes.push(parameterType)
//
//       const text = expression.slice(matchOffset, match.index)
//       const captureRegexp = buildCaptureRegexp(parameterType.regexps)
//       matchOffset = PARAMETER_REGEXP.lastIndex
//       regexp += text
//       regexp += captureRegexp
//     }
//     regexp += expression.slice(matchOffset)
//     regexp += '$'
//     this._treeRegexp = new TreeRegexp(regexp)
//   }
//
// }
//
// function buildCaptureRegexp(regexps) {
//   if (regexps.length === 1) {
//     return `(${regexps[0]})`
//   }
//
//   const captureGroups = regexps.map(group => {
//     return `(?:${group})`
//   })
//
//   return `(${captureGroups.join('|')})`
// }
//
// module.exports = CucumberExpression
<|endoftext|>


<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package casclient provides remote-apis-sdks client with luci integration.
package casclient

import (
	"context"
	"fmt"
	"net"
	"runtime"
	"strings"

	"github.com/bazelbuild/remote-apis-sdks/go/pkg/cas"
	"github.com/bazelbuild/remote-apis-sdks/go/pkg/client"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/hardcoded/chromeinfra"
)

// AddrProd is the PROD CAS service address.
const AddrProd = "remotebuildexecution.googleapis.com:443"

// New returns luci auth configured Client for RBE-CAS.
func New(ctx context.Context, addr string, instance string, opts auth.Options, readOnly bool) (*cas.Client, error) {
	var dialParams client.DialParams
	useLocal, err := isLocalAddr(addr)
	if err != nil {
		return nil, errors.Annotate(err, "invalid addr").Err()
	}
	if useLocal {
		// Connect to local fake CAS server.
		// See also go.chromium.org/luci/tools/cmd/fakecas
		if instance != "" {
			return nil, errors.Reason("do not specify instance with local address").Err()
		}
		instance = "instance"
		dialParams = client.DialParams{
			Service:    addr,
			NoSecurity: true,
		}
	} else {
		creds, err := perRPCCreds(ctx, instance, opts, readOnly)
		if err != nil {
			return nil, err
		}

		dialParams = client.DialParams{
			Service:            addr,
			TransportCredsOnly: true,
			DialOpts:           []grpc.DialOption{grpc.WithPerRPCCredentials(creds)},
		}
	}

	conn, err := client.Dial(ctx, dialParams.Service, dialParams)
	if err != nil {
		return nil, errors.Annotate(err, "failed to dial RBE").Err()
	}

	cl, err := cas.NewClientWithConfig(ctx, conn, instance, DefaultConfig())
	if err != nil {
		return nil, errors.Annotate(err, "failed to create client").Err()
	}
	return cl, nil
}

// DefaultConfig returns default CAS client configuration.
func DefaultConfig() cas.ClientConfig {
	cfg := cas.DefaultClientConfig()
	cfg.CompressedBytestreamThreshold = 0 // compress always

	// Do not read file less than 10MiB twice.
	cfg.SmallFileThreshold = 10 * 1024 * 1024

	return cfg
}

func perRPCCreds(ctx context.Context, instance string, opts auth.Options, readOnly bool) (credentials.PerRPCCredentials, error) {
	project := strings.Split(instance, "/")[1]
	var role string
	if readOnly {
		role = "cas-read-only"
	} else {
		role = "cas-read-write"
	}

	// Construct auth.Options.
	opts.ActAsServiceAccount = fmt.Sprintf("%s@%s.iam.gserviceaccount.com", role, project)
	opts.ActViaLUCIRealm = fmt.Sprintf("@internal:%s/%s", project, role)
	opts.Scopes = []string{"https://www.googleapis.com/auth/cloud-platform"}

	if strings.HasSuffix(project, "-dev") || strings.HasSuffix(project, "-staging") {
		// use dev token server for dev/staging projects.
		opts.TokenServerHost = chromeinfra.TokenServerDevHost
	}

	creds, err := auth.NewAuthenticator(ctx, auth.SilentLogin, opts).PerRPCCredentials()
	if err != nil {
		return nil, errors.Annotate(err, "failed to get PerRPCCredentials").Err()
	}
	return creds, nil
}

// NewLegacy returns luci auth configured legacy Client for RBE.
// In general, NewClient is preferred.
// TODO(crbug.com/1225524): remove this.
func NewLegacy(ctx context.Context, addr string, instance string, opts auth.Options, readOnly bool) (*client.Client, error) {
	useLocal, err := isLocalAddr(addr)
	if err != nil {
		return nil, errors.Annotate(err, "invalid addr").Err()
	}
	if useLocal {
		// Connect to local fake CAS server.
		// See also go.chromium.org/luci/tools/cmd/fakecas
		if instance != "" {
			return nil, errors.Reason("do not specify instance with local address").Err()
		}
		dialParams := client.DialParams{
			Service:    addr,
			NoSecurity: true,
		}
		cl, err := client.NewClient(ctx, "instance", dialParams)
		if err != nil {
			return nil, errors.Annotate(err, "failed to create client").Err()
		}
		return cl, nil
	}

	creds, err := perRPCCreds(ctx, instance, opts, readOnly)
	if err != nil {
		return nil, err
	}
	dialParams := client.DialParams{
		Service:            "remotebuildexecution.googleapis.com:443",
		TransportCredsOnly: true,
	}

	cl, err := client.NewClient(ctx, instance, dialParams, Options(creds)...)
	if err != nil {
		logging.Errorf(ctx, "failed to create casclient: %+v", err)
		return nil, errors.Annotate(err, "failed to create client").Err()
	}
	return cl, nil
}

// Options returns CAS client options.
func Options(creds credentials.PerRPCCredentials) []client.Opt {
	casConcurrency := runtime.NumCPU() * 2
	if runtime.GOOS == "windows" {
		// This is for better file write performance on Windows (http://b/171672371#comment6).
		casConcurrency = runtime.NumCPU()
	}

	return []client.Opt{
		&client.PerRPCCreds{Creds: creds},
		client.CASConcurrency(casConcurrency),
		client.UtilizeLocality(true),
		&client.TreeSymlinkOpts{Preserved: true, FollowsTarget: false},
		// Set restricted permission for written files.
		client.DirMode(0700),
		client.ExecutableMode(0700),
		client.RegularMode(0600),
		client.CompressedBytestreamThreshold(0),

		// Do not set per RPC timeout.
		client.RPCTimeouts{},
	}
}

// ContextWithMetadata attaches RBE related metadata with tool name to the
// given context.
func ContextWithMetadata(ctx context.Context, toolName string) (context.Context, error) {
	ctx, err := client.ContextWithMetadata(ctx, &client.ContextMetadata{
		ToolName: toolName,
	})
	if err != nil {
		return nil, errors.Annotate(err, "failed to attach metadata").Err()
	}

	m, err := client.GetContextMetadata(ctx)
	if err != nil {
		return nil, errors.Annotate(err, "failed to extract metadata").Err()
	}

	logging.Infof(ctx, "context metadata: %#+v", *m)

	return ctx, nil
}

func isLocalAddr(addr string) (bool, error) {
	tcpaddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		return false, err
	}
	if tcpaddr.IP == nil {
		return true, nil
	}
	return tcpaddr.IP.IsLoopback(), nil
}
<commit_msg>[swarming] Ignore CAS instance when local address is specified<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package casclient provides remote-apis-sdks client with luci integration.
package casclient

import (
	"context"
	"fmt"
	"net"
	"runtime"
	"strings"

	"github.com/bazelbuild/remote-apis-sdks/go/pkg/cas"
	"github.com/bazelbuild/remote-apis-sdks/go/pkg/client"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/hardcoded/chromeinfra"
)

// AddrProd is the PROD CAS service address.
const AddrProd = "remotebuildexecution.googleapis.com:443"

// New returns luci auth configured Client for RBE-CAS.
func New(ctx context.Context, addr string, instance string, opts auth.Options, readOnly bool) (*cas.Client, error) {
	var dialParams client.DialParams
	useLocal, err := isLocalAddr(addr)
	if err != nil {
		return nil, errors.Annotate(err, "invalid addr").Err()
	}
	if useLocal {
		// Connect to local fake CAS server.
		// See also go.chromium.org/luci/tools/cmd/fakecas
		if instance != "" {
			return nil, errors.Reason("do not specify instance with local address").Err()
		}
		instance = "instance"
		dialParams = client.DialParams{
			Service:    addr,
			NoSecurity: true,
		}
	} else {
		creds, err := perRPCCreds(ctx, instance, opts, readOnly)
		if err != nil {
			return nil, err
		}

		dialParams = client.DialParams{
			Service:            addr,
			TransportCredsOnly: true,
			DialOpts:           []grpc.DialOption{grpc.WithPerRPCCredentials(creds)},
		}
	}

	conn, err := client.Dial(ctx, dialParams.Service, dialParams)
	if err != nil {
		return nil, errors.Annotate(err, "failed to dial RBE").Err()
	}

	cl, err := cas.NewClientWithConfig(ctx, conn, instance, DefaultConfig())
	if err != nil {
		return nil, errors.Annotate(err, "failed to create client").Err()
	}
	return cl, nil
}

// DefaultConfig returns default CAS client configuration.
func DefaultConfig() cas.ClientConfig {
	cfg := cas.DefaultClientConfig()
	cfg.CompressedBytestreamThreshold = 0 // compress always

	// Do not read file less than 10MiB twice.
	cfg.SmallFileThreshold = 10 * 1024 * 1024

	return cfg
}

func perRPCCreds(ctx context.Context, instance string, opts auth.Options, readOnly bool) (credentials.PerRPCCredentials, error) {
	project := strings.Split(instance, "/")[1]
	var role string
	if readOnly {
		role = "cas-read-only"
	} else {
		role = "cas-read-write"
	}

	// Construct auth.Options.
	opts.ActAsServiceAccount = fmt.Sprintf("%s@%s.iam.gserviceaccount.com", role, project)
	opts.ActViaLUCIRealm = fmt.Sprintf("@internal:%s/%s", project, role)
	opts.Scopes = []string{"https://www.googleapis.com/auth/cloud-platform"}

	if strings.HasSuffix(project, "-dev") || strings.HasSuffix(project, "-staging") {
		// use dev token server for dev/staging projects.
		opts.TokenServerHost = chromeinfra.TokenServerDevHost
	}

	creds, err := auth.NewAuthenticator(ctx, auth.SilentLogin, opts).PerRPCCredentials()
	if err != nil {
		return nil, errors.Annotate(err, "failed to get PerRPCCredentials").Err()
	}
	return creds, nil
}

// NewLegacy returns luci auth configured legacy Client for RBE.
// In general, NewClient is preferred.
// TODO(crbug.com/1225524): remove this.
func NewLegacy(ctx context.Context, addr string, instance string, opts auth.Options, readOnly bool) (*client.Client, error) {
	useLocal, err := isLocalAddr(addr)
	if err != nil {
		return nil, errors.Annotate(err, "invalid addr").Err()
	}
	if useLocal {
		// Connect to local fake CAS server.
		// See also go.chromium.org/luci/tools/cmd/fakecas
		if instance != "" {
			logging.Warningf(ctx, "instance `%s` is given, but will be ignored.", instance)
		}
		dialParams := client.DialParams{
			Service:    addr,
			NoSecurity: true,
		}
		cl, err := client.NewClient(ctx, "instance", dialParams)
		if err != nil {
			return nil, errors.Annotate(err, "failed to create client").Err()
		}
		return cl, nil
	}

	creds, err := perRPCCreds(ctx, instance, opts, readOnly)
	if err != nil {
		return nil, err
	}
	dialParams := client.DialParams{
		Service:            "remotebuildexecution.googleapis.com:443",
		TransportCredsOnly: true,
	}

	cl, err := client.NewClient(ctx, instance, dialParams, Options(creds)...)
	if err != nil {
		logging.Errorf(ctx, "failed to create casclient: %+v", err)
		return nil, errors.Annotate(err, "failed to create client").Err()
	}
	return cl, nil
}

// Options returns CAS client options.
func Options(creds credentials.PerRPCCredentials) []client.Opt {
	casConcurrency := runtime.NumCPU() * 2
	if runtime.GOOS == "windows" {
		// This is for better file write performance on Windows (http://b/171672371#comment6).
		casConcurrency = runtime.NumCPU()
	}

	return []client.Opt{
		&client.PerRPCCreds{Creds: creds},
		client.CASConcurrency(casConcurrency),
		client.UtilizeLocality(true),
		&client.TreeSymlinkOpts{Preserved: true, FollowsTarget: false},
		// Set restricted permission for written files.
		client.DirMode(0700),
		client.ExecutableMode(0700),
		client.RegularMode(0600),
		client.CompressedBytestreamThreshold(0),

		// Do not set per RPC timeout.
		client.RPCTimeouts{},
	}
}

// ContextWithMetadata attaches RBE related metadata with tool name to the
// given context.
func ContextWithMetadata(ctx context.Context, toolName string) (context.Context, error) {
	ctx, err := client.ContextWithMetadata(ctx, &client.ContextMetadata{
		ToolName: toolName,
	})
	if err != nil {
		return nil, errors.Annotate(err, "failed to attach metadata").Err()
	}

	m, err := client.GetContextMetadata(ctx)
	if err != nil {
		return nil, errors.Annotate(err, "failed to extract metadata").Err()
	}

	logging.Infof(ctx, "context metadata: %#+v", *m)

	return ctx, nil
}

func isLocalAddr(addr string) (bool, error) {
	tcpaddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		return false, err
	}
	if tcpaddr.IP == nil {
		return true, nil
	}
	return tcpaddr.IP.IsLoopback(), nil
}
<|endoftext|>


<commit_before>package test

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/adamdecaf/cert-manage/pkg/file"
)

var (
	debug = os.Getenv("DEBUG") != ""
)

type dockerfile struct {
	// Local fs path to the Dockerfile
	base string

	// Commands represents a series of commands to be ran in the image
	commands []*Cmd

	// -t flag with build/run
	tag string

	// any errors specific to the dockerfile from `docker` commands
	err error

	// only run build, tag and run steps once
	wg sync.WaitGroup

	// used for cert-manage init
	sync.Once
}

func Dockerfile(where string) *dockerfile {
	if !strings.HasSuffix(where, "Dockerfile") {
		where = filepath.Join(where, "/Dockerfile")
	}

	// Grab the env name (e.g. envs/$env_name/Dockerfile)
	dir := filepath.Dir(where)
	now := time.Now().Unix()
	tag := fmt.Sprintf("cert-manage:%s-%d", filepath.Base(dir), now)

	return &dockerfile{
		base: where,
		tag:  tag,
	}
}

func (d *dockerfile) Run(cmd string, args ...string) {
	d.commands = append(d.commands, Command(cmd, args...))
}

func (d *dockerfile) RunSplit(stmt string) {
	parts := strings.Split(stmt, " ")
	d.Run(parts[0], parts[1:]...)
}

func (d *dockerfile) ShouldFail(cmd string, args ...string) {
	d.Run("set +e")
	d.Run(cmd, args...)
	d.Run("set -e")
}

func (d *dockerfile) ExitCode(code, cmd string, args ...string) {
	d.Run("set +e")
	d.Run(cmd, args...)
	d.Run("code=$?")
	d.Run("set -e")
	d.Run("echo", "$code", "|", "grep", code)
}

func (d *dockerfile) CertManage(args ...string) {
	d.Do(func() {
		d.Run("chmod", "+x", "/bin/cert-manage")
	})
	d.Run("/bin/cert-manage", args...)
}

func (d *dockerfile) SuccessT(t *testing.T) {
	if !d.enabled() {
		t.Skip("docker isn't enabled")
	}
	if runtime.GOOS == "darwin" && inCI() {
		t.Fatal("travis-ci supports docker on OSX?? - https://docs.travis-ci.com/user/docker/")
	}

	d.prep()
	t.Helper()

	if d.err != nil {
		t.Fatal(d.err)
	}
}

func (d *dockerfile) build() {
	d.wg.Add(1)
	defer d.wg.Done()

	// Copy our original image's contents into the dst file
	dir, err := ioutil.TempDir("", d.tag)
	if err != nil {
		d.err = fmt.Errorf("tempfile create err=%v", err)
		return
	}

	// Copy cert-manage and whitelist to the temp directory and assume it's linux
	copyable := []string{
		"../bin/cert-manage-linux-amd64",
		"../testdata/Download.java",
		"../testdata/globalsign-whitelist.json",
	}
	for i := range copyable {
		name := filepath.Base(copyable[i])
		err = file.CopyFile(copyable[i], filepath.Join(dir, name))
		if err != nil {
			d.err = fmt.Errorf("error copying %s to tmp dir, err=%v", name, err)
			return
		}
	}

	dst, err := os.Create(filepath.Join(dir, "Dockerfile"))
	if err != nil {
		d.err = fmt.Errorf("tmp Dockerfile create err=%v", err)
		return
	}
	defer os.Remove(dst.Name())

	src, err := os.Open(d.base)
	if err != nil {
		d.err = fmt.Errorf("tmpfile open err=%v", err)
		return
	}
	if _, err := io.Copy(dst, src); err != nil {
		d.err = fmt.Errorf("src->dst copy err=%v", err)
		return
	}
	if err := src.Close(); err != nil {
		d.err = fmt.Errorf("src close err=%v", err)
		return
	}
	// Force all writes into our Dockerfile
	if err := dst.Sync(); err != nil {
		d.err = fmt.Errorf("dst fsync err=%v", err)
		return
	}

	// Add all commands to a script copied Dockerfile
	script, err := os.Create(filepath.Join(dir, "script.sh"))
	if err != nil {
		d.err = err
		return
	}
	defer os.Remove(script.Name())
	_, err = script.WriteString(`#!/bin/sh` + "\n")
	if err != nil {
		d.err = err
		return
	}
	for i := range d.commands {
		line := fmt.Sprintf("%s %s\n", d.commands[i].command, strings.Join(d.commands[i].args, " "))
		if _, err := script.WriteString(line); err != nil {
			d.err = fmt.Errorf("command=%q err=%v", line, err)
			return
		}
	}
	d.err = script.Sync()
	if d.err != nil {
		return
	}

	// Build docker image now
	out, err := exec.Command("docker", "build", "-t", d.tag, dir).CombinedOutput()
	if err != nil {
		d.err = fmt.Errorf("ERROR: err=%v\nOutput: %s", err, string(out))
	}
}

func (d *dockerfile) run() {
	d.wg.Add(1)
	defer d.wg.Done()

	// don't attempt anything if we've already failed
	if d.err != nil {
		return
	}

	out, err := exec.Command("docker", "run", "-t", d.tag).CombinedOutput()
	if err != nil {
		d.err = fmt.Errorf("ERROR: err=%v\nOutput: %s", err, string(out))
	}
	if debug {
		fmt.Println(string(out))
	}
}

func (d *dockerfile) prep() {
	d.build()
	d.run()
	d.wg.Wait()
}

func (d *dockerfile) enabled() bool {
	out, err := exec.Command("docker", "ps").CombinedOutput()
	if err != nil || bytes.Contains(out, []byte("Cannot connect to the Docker daemon")) {
		return false
	}
	return true
}
<commit_msg>test/docker: pass down DEBUG to containers if it's set<commit_after>package test

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/adamdecaf/cert-manage/pkg/file"
)

var (
	debug = os.Getenv("DEBUG") != ""
)

type dockerfile struct {
	// Local fs path to the Dockerfile
	base string

	// Commands represents a series of commands to be ran in the image
	commands []*Cmd

	// -t flag with build/run
	tag string

	// any errors specific to the dockerfile from `docker` commands
	err error

	// only run build, tag and run steps once
	wg sync.WaitGroup

	// used for cert-manage init
	sync.Once
}

func Dockerfile(where string) *dockerfile {
	if !strings.HasSuffix(where, "Dockerfile") {
		where = filepath.Join(where, "/Dockerfile")
	}

	// Grab the env name (e.g. envs/$env_name/Dockerfile)
	dir := filepath.Dir(where)
	now := time.Now().Unix()
	tag := fmt.Sprintf("cert-manage:%s-%d", filepath.Base(dir), now)

	return &dockerfile{
		base: where,
		tag:  tag,
	}
}

func (d *dockerfile) Run(cmd string, args ...string) {
	d.commands = append(d.commands, Command(cmd, args...))
}

func (d *dockerfile) RunSplit(stmt string) {
	parts := strings.Split(stmt, " ")
	d.Run(parts[0], parts[1:]...)
}

func (d *dockerfile) ShouldFail(cmd string, args ...string) {
	d.Run("set +e")
	d.Run(cmd, args...)
	d.Run("set -e")
}

func (d *dockerfile) ExitCode(code, cmd string, args ...string) {
	d.Run("set +e")
	d.Run(cmd, args...)
	d.Run("code=$?")
	d.Run("set -e")
	d.Run("echo", "$code", "|", "grep", code)
}

func (d *dockerfile) CertManage(args ...string) {
	d.Do(func() {
		d.Run("chmod", "+x", "/bin/cert-manage")
	})
	d.Run("/bin/cert-manage", args...)
}

func (d *dockerfile) SuccessT(t *testing.T) {
	if !d.enabled() {
		t.Skip("docker isn't enabled")
	}
	if runtime.GOOS == "darwin" && inCI() {
		t.Fatal("travis-ci supports docker on OSX?? - https://docs.travis-ci.com/user/docker/")
	}

	d.prep()
	t.Helper()

	if d.err != nil {
		t.Fatal(d.err)
	}
}

func (d *dockerfile) build() {
	d.wg.Add(1)
	defer d.wg.Done()

	// Copy our original image's contents into the dst file
	dir, err := ioutil.TempDir("", d.tag)
	if err != nil {
		d.err = fmt.Errorf("tempfile create err=%v", err)
		return
	}

	// Copy cert-manage and whitelist to the temp directory and assume it's linux
	copyable := []string{
		"../bin/cert-manage-linux-amd64",
		"../testdata/Download.java",
		"../testdata/globalsign-whitelist.json",
		"../testdata/localcert.pem",
	}
	for i := range copyable {
		name := filepath.Base(copyable[i])
		err = file.CopyFile(copyable[i], filepath.Join(dir, name))
		if err != nil {
			d.err = fmt.Errorf("error copying %s to tmp dir, err=%v", name, err)
			return
		}
	}

	dst, err := os.Create(filepath.Join(dir, "Dockerfile"))
	if err != nil {
		d.err = fmt.Errorf("tmp Dockerfile create err=%v", err)
		return
	}
	defer os.Remove(dst.Name())

	src, err := os.Open(d.base)
	if err != nil {
		d.err = fmt.Errorf("tmpfile open err=%v", err)
		return
	}
	if _, err := io.Copy(dst, src); err != nil {
		d.err = fmt.Errorf("src->dst copy err=%v", err)
		return
	}
	if err := src.Close(); err != nil {
		d.err = fmt.Errorf("src close err=%v", err)
		return
	}
	// Force all writes into our Dockerfile
	if err := dst.Sync(); err != nil {
		d.err = fmt.Errorf("dst fsync err=%v", err)
		return
	}

	// Add all commands to a script copied Dockerfile
	script, err := os.Create(filepath.Join(dir, "script.sh"))
	if err != nil {
		d.err = err
		return
	}
	defer os.Remove(script.Name())
	_, err = script.WriteString(`#!/bin/sh` + "\n")
	if err != nil {
		d.err = err
		return
	}
	for i := range d.commands {
		line := fmt.Sprintf("%s %s\n", d.commands[i].command, strings.Join(d.commands[i].args, " "))
		if _, err := script.WriteString(line); err != nil {
			d.err = fmt.Errorf("command=%q err=%v", line, err)
			return
		}
	}
	d.err = script.Sync()
	if d.err != nil {
		return
	}

	// Build docker image now
	out, err := exec.Command("docker", "build", "-t", d.tag, dir).CombinedOutput()
	if err != nil {
		d.err = fmt.Errorf("ERROR: err=%v\nOutput: %s", err, string(out))
	}
}

func (d *dockerfile) run() {
	d.wg.Add(1)
	defer d.wg.Done()

	// don't attempt anything if we've already failed
	if d.err != nil {
		return
	}

	// build `docker run` flags
	args := []string{"run"}
	if debug {
		args = append(args, "-e", "DEBUG=true")
	}
	args = append(args, "-t", d.tag)

	out, err := exec.Command("docker", args...).CombinedOutput()
	if err != nil {
		d.err = fmt.Errorf("ERROR: err=%v\nOutput: %s", err, string(out))
	}
	if debug {
		fmt.Println(string(out))
	}
}

func (d *dockerfile) prep() {
	d.build()
	d.run()
	d.wg.Wait()
}

func (d *dockerfile) enabled() bool {
	out, err := exec.Command("docker", "ps").CombinedOutput()
	if err != nil || bytes.Contains(out, []byte("Cannot connect to the Docker daemon")) {
		return false
	}
	return true
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package app

import (
	"reflect"
	"testing"
)

func TestValidateAuthFlow(t *testing.T) {
	type FlagResult struct {
		Name  string
		Flow  string
		Error error
	}
	tests := []FlagResult{
		{Name: "validate gcr auth flow", Flow: gcrAuthFlow, Error: nil},
		{Name: "validate docker-cfg auth flow option", Flow: dockerConfigAuthFlow, Error: nil},
		{Name: "validate docker-cfg-url auth flow option", Flow: dockerConfigURLAuthFlow, Error: nil},
		{Name: "bad auth flow option", Flow: "bad-flow", Error: &AuthFlowFlagError{flagValue: "bad-flow"}},
		{Name: "empty auth flow option", Flow: "", Error: &AuthFlowFlagError{flagValue: ""}},
		{Name: "case-sensitive auth flow", Flow: "Gcrauthflow", Error: &AuthFlowFlagError{flagValue: "Gcrauthflow"}},
	}
	for _, tc := range tests {
		t.Run(tc.Name, func(t *testing.T) {
			err := validateFlags(&CredentialOptions{AuthFlow: tc.Flow})
			if err != nil && tc.Error == nil {
				t.Fatalf("with flow %q unexpected error %q", tc.Flow, err)
			}
			if err == nil && tc.Error != nil {
				t.Fatalf("with flow %q did not get expected error %q", tc.Flow, err)
			}
			if err != nil && tc.Error != nil {
				if reflect.TypeOf(err) != reflect.TypeOf(tc.Error) {
					t.Fatalf("with flow %q got unexpected error type %q (expected %q)", tc.Flow, reflect.TypeOf(err), reflect.TypeOf(tc.Error))
				}
			}
		})
	}
}

func TestProviderFromFlow(t *testing.T) {
	type ProviderResult struct {
		Name  string
		Flow  string
		Type  string
		Error error
	}
	tests := []ProviderResult{
		{Name: "gcr auth provider selection", Flow: gcrAuthFlow, Type: "ContainerRegistryProvider", Error: nil},
		{Name: "docker-cfg auth provider selection", Flow: dockerConfigAuthFlow, Type: "DockerConfigKeyProvider", Error: nil},
		{Name: "docker-cfg-url auth provider selection", Flow: dockerConfigURLAuthFlow, Type: "DockerConfigURLKeyProvider", Error: nil},
		{Name: "non-existent auth provider request", Flow: "bad-flow", Type: "", Error: &AuthFlowTypeError{requestedFlow: "bad-flow"}},
		{Name: "empty auth provider request", Flow: "", Type: "", Error: &AuthFlowTypeError{requestedFlow: ""}},
	}
	for _, tc := range tests {
		t.Run(tc.Name, func(t *testing.T) {
			provider, err := providerFromFlow(tc.Flow)
			if err != nil && tc.Error == nil {
				t.Fatalf("with flow %q unexpected error %q", tc.Flow, err)
			}
			if err == nil && tc.Error != nil {
				t.Fatalf("with flow %q did not get expected error %q", tc.Flow, err)
			}
			if err != nil && tc.Error != nil {
				if reflect.TypeOf(err) != reflect.TypeOf(tc.Error) {
					t.Fatalf("with flow %q got unexpected error type %q (expected %q)", tc.Flow, reflect.TypeOf(err), reflect.TypeOf(tc.Error))
				}
			}
			if tc.Type == "" && provider != nil {
				t.Fatalf("with flow %q got unexpectedly non-nil provider %q", tc.Flow, provider)
			}
			if provider != nil {
				providerType := reflect.TypeOf(provider).String()
				if providerType != "*gcpcredential."+tc.Type {
					t.Errorf("with flow %q unexpected provider type %q", tc.Flow, providerType)
				}
			}
		})
	}
}
<commit_msg>Refactor test validation in getcredentials_test.go.<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package app

import (
	"reflect"
	"testing"
)

func TestValidateAuthFlow(t *testing.T) {
	type FlagResult struct {
		Name  string
		Flow  string
		Error error
	}
	tests := []FlagResult{
		{Name: "validate gcr auth flow", Flow: gcrAuthFlow, Error: nil},
		{Name: "validate docker-cfg auth flow option", Flow: dockerConfigAuthFlow, Error: nil},
		{Name: "validate docker-cfg-url auth flow option", Flow: dockerConfigURLAuthFlow, Error: nil},
		{Name: "bad auth flow option", Flow: "bad-flow", Error: &AuthFlowFlagError{flagValue: "bad-flow"}},
		{Name: "empty auth flow option", Flow: "", Error: &AuthFlowFlagError{flagValue: ""}},
		{Name: "case-sensitive auth flow", Flow: "Gcrauthflow", Error: &AuthFlowFlagError{flagValue: "Gcrauthflow"}},
	}
	for _, tc := range tests {
		t.Run(tc.Name, func(t *testing.T) {
			err := validateFlags(&CredentialOptions{AuthFlow: tc.Flow})
			if tc.Error != nil {
				if err == nil {
					t.Fatalf("with flow %q did not get expected error %q", tc.Flow, err)
				}
				if reflect.TypeOf(err) != reflect.TypeOf(tc.Error) {
					t.Fatalf("with flow %q got unexpected error type %q (expected %q)", tc.Flow, reflect.TypeOf(err), reflect.TypeOf(tc.Error))
				}
				return
			}
			if err != nil {
				t.Fatalf("with flow %q unexpected error %q", tc.Flow, err)
			}
		})
	}
}

func TestProviderFromFlow(t *testing.T) {
	type ProviderResult struct {
		Name  string
		Flow  string
		Type  string
		Error error
	}
	tests := []ProviderResult{
		{Name: "gcr auth provider selection", Flow: gcrAuthFlow, Type: "ContainerRegistryProvider", Error: nil},
		{Name: "docker-cfg auth provider selection", Flow: dockerConfigAuthFlow, Type: "DockerConfigKeyProvider", Error: nil},
		{Name: "docker-cfg-url auth provider selection", Flow: dockerConfigURLAuthFlow, Type: "DockerConfigURLKeyProvider", Error: nil},
		{Name: "non-existent auth provider request", Flow: "bad-flow", Type: "", Error: &AuthFlowTypeError{requestedFlow: "bad-flow"}},
		{Name: "empty auth provider request", Flow: "", Type: "", Error: &AuthFlowTypeError{requestedFlow: ""}},
	}
	for _, tc := range tests {
		t.Run(tc.Name, func(t *testing.T) {
			provider, err := providerFromFlow(tc.Flow)
			if tc.Error != nil {
				if err == nil {
					t.Fatalf("with flow %q did not get expected error %q", tc.Flow, err)
				}
				if reflect.TypeOf(err) != reflect.TypeOf(tc.Error) {
					t.Fatalf("with flow %q got unexpected error type %q (expected %q)", tc.Flow, reflect.TypeOf(err), reflect.TypeOf(tc.Error))
				}
				return
			}
			if err != nil {
				t.Fatalf("with flow %q unexpected error %q", tc.Flow, err)
			}
			providerType := reflect.TypeOf(provider).String()
			if providerType != "*gcpcredential."+tc.Type {
				t.Errorf("with flow %q unexpected provider type %q", tc.Flow, providerType)
			}
		})
	}
}
<|endoftext|>


<commit_before>/* a-b-tester is a utility program that makes it easy to test and see if a
/* change to the core library is helping us build a better model. Normal usage
/* is to provide it a relativedifficulties.csv file and then it will output
/* r2, but you can also compare multiple configs and have it report the best
/* one. To do that, create different branches with each configuration set.
/* Then run a-b-tester with -b and a space delimited string of branch names to
/* try. a-b-tester will run each in turn, save out analysis and solves files
/* for each, and then report which one has the best r2.*/

package main

import (
	"flag"
	"fmt"
	"github.com/gosuri/uitable"
	"log"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"
)

const pathToDokugenAnalysis = "../../"
const pathFromDokugenAnalysis = "internal/a-b-tester/"

const pathToWekaTrainer = "../weka-trainer/"
const pathFromWekaTrainer = "../a-b-tester/"

const rowSeparator = "****************"

//TODO: amek this resilient to not being run in the package's directory

type appOptions struct {
	relativeDifficultiesFile string
	solvesFile               string
	analysisFile             string
	branches                 string
	branchesList             []string
	help                     bool
	flagSet                  *flag.FlagSet
}

func (a *appOptions) defineFlags() {
	if a.flagSet == nil {
		return
	}
	a.flagSet.StringVar(&a.branches, "b", "", "Git branch to checkout. Can also be a space delimited list of multiple branches to checkout.")
	a.flagSet.StringVar(&a.relativeDifficultiesFile, "r", "relativedifficulties_SAMPLED.csv", "The file to use as relative difficulties input")
	a.flagSet.StringVar(&a.solvesFile, "s", "solves.csv", "The file to output solves to")
	a.flagSet.StringVar(&a.analysisFile, "a", "analysis.txt", "The file to output analysis to")
	a.flagSet.BoolVar(&a.help, "h", false, "If provided, will print help and exit.")
}

func (a *appOptions) fixUp() {
	a.branchesList = strings.Split(a.branches, " ")
	a.solvesFile = strings.Replace(a.solvesFile, ".csv", "", -1)
	a.analysisFile = strings.Replace(a.analysisFile, ".txt", "", -1)
}

func (a *appOptions) parse(args []string) {
	a.flagSet.Parse(args)
	a.fixUp()
}

func newAppOptions(flagSet *flag.FlagSet) *appOptions {
	a := &appOptions{
		flagSet: flagSet,
	}
	a.defineFlags()
	return a
}

func main() {
	a := newAppOptions(flag.CommandLine)
	a.parse(os.Args[1:])

	results := make(map[string]float64)

	startingBranch := gitCurrentBranch()

	for _, branch := range a.branchesList {

		if branch == "" {
			log.Println("Staying on the current branch.")
		} else {
			log.Println("Switching to branch", branch)
		}

		//a.analysisFile and a.solvesFile have had their extension removed, if they had one.
		effectiveSolvesFile := a.solvesFile + ".csv"
		effectiveAnalysisFile := a.analysisFile + ".txt"

		if branch != "" {

			effectiveSolvesFile = a.solvesFile + "_" + strings.ToUpper(branch) + ".csv"
			effectiveAnalysisFile = a.analysisFile + "_" + strings.ToUpper(branch) + ".txt"
		}

		if !checkoutGitBranch(branch) {
			log.Println("Couldn't switch to branch", branch, " (perhaps you have uncommitted changes?). Quitting.")
			return
		}

		runSolves(a.relativeDifficultiesFile, effectiveSolvesFile)

		branchKey := branch

		if branchKey == "" {
			branchKey = "<default>"
		}

		results[branchKey] = runWeka(effectiveSolvesFile, effectiveAnalysisFile)
	}

	if len(results) > 1 {
		//We only need to go to the trouble of painting the table if more than
		//one branch was run
		printR2Table(results)
	}

	if gitCurrentBranch() != startingBranch {
		checkoutGitBranch(startingBranch)
	}
}

func printR2Table(results map[string]float64) {
	bestR2 := 0.0
	bestR2Branch := ""

	for key, val := range results {
		if val > bestR2 {
			bestR2 = val
			bestR2Branch = key
		}
	}

	fmt.Println(rowSeparator)
	fmt.Println("Results:")
	fmt.Println(rowSeparator)

	table := uitable.New()

	table.AddRow("Best?", "Branch", "R2")

	for key, val := range results {
		isBest := " "
		if key == bestR2Branch {
			isBest = "*"
		}
		table.AddRow(isBest, key, val)
	}

	fmt.Println(table.String())
	fmt.Println(rowSeparator)
}

func runSolves(difficultiesFile, solvesOutputFile string) {

	os.Chdir(pathToDokugenAnalysis)

	defer func() {
		os.Chdir(pathFromDokugenAnalysis)
	}()

	//Build the dokugen-analysis executable to make sure we get the freshest version of the sudoku pacakge.
	cmd := exec.Command("go", "build")
	err := cmd.Run()

	if err != nil {
		log.Println(err)
		return
	}

	outFile, err := os.Create(path.Join(pathFromDokugenAnalysis, solvesOutputFile))

	if err != nil {
		log.Println(err)
		return
	}

	analysisCmd := exec.Command("./dokugen-analysis", "-a", "-v", "-w", "-t", "-h", "-no-cache", path.Join(pathFromDokugenAnalysis, difficultiesFile))
	analysisCmd.Stdout = outFile
	analysisCmd.Stderr = os.Stderr
	err = analysisCmd.Run()

	if err != nil {
		log.Println(err)
	}
}

func runWeka(solvesFile string, analysisFile string) float64 {

	os.Chdir(pathToWekaTrainer)

	defer func() {
		os.Chdir(pathFromWekaTrainer)
	}()

	//Build the weka-trainer executable to make sure we get the freshest version of the sudoku pacakge.
	cmd := exec.Command("go", "build")
	err := cmd.Run()

	if err != nil {
		log.Println(err)
		return 0.0
	}

	trainCmd := exec.Command("./weka-trainer", "-i", path.Join(pathFromWekaTrainer, solvesFile), "-o", path.Join(pathFromWekaTrainer, analysisFile))
	trainCmd.Stderr = os.Stderr
	output, err := trainCmd.Output()

	if err != nil {
		log.Println(err)
		return 0.0
	}

	fmt.Printf("%s", string(output))

	return extractR2(string(output))

}

//extractR2 extracts R2 out of the string formatted like "R2 = <float>"
func extractR2(input string) float64 {

	input = strings.TrimPrefix(input, "R2 = ")
	input = strings.TrimSpace(input)

	result, _ := strconv.ParseFloat(input, 64)

	return result

}

//gitCurrentBranch returns the current branch that the current repo is in.
func gitCurrentBranch() string {
	branchCmd := exec.Command("git", "branch")

	output, err := branchCmd.Output()

	if err != nil {
		log.Println(err)
		return ""
	}

	for _, line := range strings.Split(string(output), "\n") {
		if strings.Contains(line, "*") {
			//Found it!
			line = strings.Replace(line, "*", "", -1)
			line = strings.TrimSpace(line)
			return line
		}
	}

	return ""
}

func checkoutGitBranch(branch string) bool {

	if branch == "" {
		return true
	}

	checkoutCmd := exec.Command("git", "checkout", branch)
	checkoutCmd.Run()

	if gitCurrentBranch() != branch {
		return false
	}

	return true

}
<commit_msg>Implement -s for stash mode: making it easy to do a-b-testing with changes that are not checked in but uncommitted (or one level deep in stash). Fixes #241.<commit_after>/* a-b-tester is a utility program that makes it easy to test and see if a
/* change to the core library is helping us build a better model. Normal usage
/* is to provide it a relativedifficulties.csv file and then it will output
/* r2, but you can also compare multiple configs and have it report the best
/* one. To do that, create different branches with each configuration set.
/* Then run a-b-tester with -b and a space delimited string of branch names to
/* try. a-b-tester will run each in turn, save out analysis and solves files
/* for each, and then report which one has the best r2.*/

package main

import (
	"errors"
	"flag"
	"fmt"
	"github.com/gosuri/uitable"
	"log"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"
)

const pathToDokugenAnalysis = "../../"
const pathFromDokugenAnalysis = "internal/a-b-tester/"

const pathToWekaTrainer = "../weka-trainer/"
const pathFromWekaTrainer = "../a-b-tester/"

const rowSeparator = "****************"

const uncommittedChangesBranchName = "STASHED"
const committedChangesBranchName = "COMMITTED"

//TODO: amek this resilient to not being run in the package's directory

type appOptions struct {
	relativeDifficultiesFile       string
	solvesFile                     string
	analysisFile                   string
	stashMode                      bool
	startingWithUncommittedChanges bool
	branches                       string
	branchesList                   []string
	help                           bool
	flagSet                        *flag.FlagSet
}

func (a *appOptions) defineFlags() {
	if a.flagSet == nil {
		return
	}
	a.flagSet.BoolVar(&a.stashMode, "s", false, "If in stash mode, will do the a-b test between uncommitted and committed changes, automatically figuring out which state we're currently in. Cannot be combined with -b")
	a.flagSet.StringVar(&a.branches, "b", "", "Git branch to checkout. Can also be a space delimited list of multiple branches to checkout.")
	a.flagSet.StringVar(&a.relativeDifficultiesFile, "r", "relativedifficulties_SAMPLED.csv", "The file to use as relative difficulties input")
	a.flagSet.StringVar(&a.solvesFile, "o", "solves.csv", "The file to output solves to")
	a.flagSet.StringVar(&a.analysisFile, "a", "analysis.txt", "The file to output analysis to")
	a.flagSet.BoolVar(&a.help, "h", false, "If provided, will print help and exit.")
}

func (a *appOptions) fixUp() error {
	if a.branches != "" && a.stashMode {
		return errors.New("-b and -s cannot both be passed")
	}
	if a.stashMode {
		a.startingWithUncommittedChanges = gitUncommittedChanges()
		if a.startingWithUncommittedChanges {
			a.branchesList = []string{
				uncommittedChangesBranchName,
				committedChangesBranchName,
			}
		} else {
			a.branchesList = []string{
				committedChangesBranchName,
				uncommittedChangesBranchName,
			}
		}
	} else {
		a.branchesList = strings.Split(a.branches, " ")
	}
	a.solvesFile = strings.Replace(a.solvesFile, ".csv", "", -1)
	a.analysisFile = strings.Replace(a.analysisFile, ".txt", "", -1)
	return nil
}

func (a *appOptions) parse(args []string) error {
	a.flagSet.Parse(args)
	return a.fixUp()
}

func newAppOptions(flagSet *flag.FlagSet) *appOptions {
	a := &appOptions{
		flagSet: flagSet,
	}
	a.defineFlags()
	return a
}

func main() {
	a := newAppOptions(flag.CommandLine)
	if err := a.parse(os.Args[1:]); err != nil {
		log.Println("Invalid options provided:", err.Error())
		return
	}

	results := make(map[string]float64)

	startingBranch := gitCurrentBranch()

	branchSwitchMessage := "Switching to branch"

	if a.stashMode {
		branchSwitchMessage = "Calculating on"
	}

	for i, branch := range a.branchesList {

		if branch == "" {
			log.Println("Staying on the current branch.")
		} else {
			log.Println(branchSwitchMessage, branch)
		}

		//a.analysisFile and a.solvesFile have had their extension removed, if they had one.
		effectiveSolvesFile := a.solvesFile + ".csv"
		effectiveAnalysisFile := a.analysisFile + ".txt"

		if branch != "" {

			effectiveSolvesFile = a.solvesFile + "_" + strings.ToUpper(branch) + ".csv"
			effectiveAnalysisFile = a.analysisFile + "_" + strings.ToUpper(branch) + ".txt"
		}

		//Get the repo in the right state for this run.
		if a.stashMode {
			// if i == 0
			switch i {
			case 0:
				//do nothing, we already ahve the right changes to start with
			case 1:
				//If we have uncommitted changes right now, stash them. Otherwise, stash pop.
				if !gitStash(a.startingWithUncommittedChanges) {
					log.Println("We couldn't stash/stash-pop.")
					return
				}
			default:
				//This should never happen
				//Note: panicing here will mean we don't do any clean up.
				panic("Got more than 2 'branches' in stash mode")
			}
		} else {
			if !checkoutGitBranch(branch) {
				log.Println("Couldn't switch to branch", branch, " (perhaps you have uncommitted changes?). Quitting.")
				return
			}
		}

		runSolves(a.relativeDifficultiesFile, effectiveSolvesFile)

		branchKey := branch

		if branchKey == "" {
			branchKey = "<default>"
		}

		results[branchKey] = runWeka(effectiveSolvesFile, effectiveAnalysisFile)
	}

	if len(results) > 1 {
		//We only need to go to the trouble of painting the table if more than
		//one branch was run
		printR2Table(results)
	}

	//Put the repo back in the state it was when we found it.
	if a.stashMode {
		//Reverse the gitStash operation to put it back

		if a.startingWithUncommittedChanges {
			log.Println("Unstashing changes to put repo back in starting state")
		} else {
			log.Println("Stashing changes to put repo back in starting state")
		}

		if !gitStash(!a.startingWithUncommittedChanges) {
			log.Println("We couldn't unstash/unpop to put the repo back in the same state.")
		}
	} else {
		//If we aren't in the branch we started in, switch back to that branch
		if gitCurrentBranch() != startingBranch {
			log.Println("Checking out", startingBranch, "to put repo back in the starting state.")
			checkoutGitBranch(startingBranch)
		}
	}

}

func printR2Table(results map[string]float64) {
	bestR2 := 0.0
	bestR2Branch := ""

	for key, val := range results {
		if val > bestR2 {
			bestR2 = val
			bestR2Branch = key
		}
	}

	fmt.Println(rowSeparator)
	fmt.Println("Results:")
	fmt.Println(rowSeparator)

	table := uitable.New()

	table.AddRow("Best?", "Branch", "R2")

	for key, val := range results {
		isBest := " "
		if key == bestR2Branch {
			isBest = "*"
		}
		table.AddRow(isBest, key, val)
	}

	fmt.Println(table.String())
	fmt.Println(rowSeparator)
}

func runSolves(difficultiesFile, solvesOutputFile string) {

	os.Chdir(pathToDokugenAnalysis)

	defer func() {
		os.Chdir(pathFromDokugenAnalysis)
	}()

	//Build the dokugen-analysis executable to make sure we get the freshest version of the sudoku pacakge.
	cmd := exec.Command("go", "build")
	err := cmd.Run()

	if err != nil {
		log.Println(err)
		return
	}

	outFile, err := os.Create(path.Join(pathFromDokugenAnalysis, solvesOutputFile))

	if err != nil {
		log.Println(err)
		return
	}

	analysisCmd := exec.Command("./dokugen-analysis", "-a", "-v", "-w", "-t", "-h", "-no-cache", path.Join(pathFromDokugenAnalysis, difficultiesFile))
	analysisCmd.Stdout = outFile
	analysisCmd.Stderr = os.Stderr
	err = analysisCmd.Run()

	if err != nil {
		log.Println(err)
	}
}

func runWeka(solvesFile string, analysisFile string) float64 {

	os.Chdir(pathToWekaTrainer)

	defer func() {
		os.Chdir(pathFromWekaTrainer)
	}()

	//Build the weka-trainer executable to make sure we get the freshest version of the sudoku pacakge.
	cmd := exec.Command("go", "build")
	err := cmd.Run()

	if err != nil {
		log.Println(err)
		return 0.0
	}

	trainCmd := exec.Command("./weka-trainer", "-i", path.Join(pathFromWekaTrainer, solvesFile), "-o", path.Join(pathFromWekaTrainer, analysisFile))
	trainCmd.Stderr = os.Stderr
	output, err := trainCmd.Output()

	if err != nil {
		log.Println(err)
		return 0.0
	}

	fmt.Printf("%s", string(output))

	return extractR2(string(output))

}

//extractR2 extracts R2 out of the string formatted like "R2 = <float>"
func extractR2(input string) float64 {

	input = strings.TrimPrefix(input, "R2 = ")
	input = strings.TrimSpace(input)

	result, _ := strconv.ParseFloat(input, 64)

	return result

}

//gitStash will use git stash if true, git stash pop if false.
func gitStash(stashChanges bool) bool {
	var stashCmd *exec.Cmd

	if stashChanges {
		stashCmd = exec.Command("git", "stash")
		if !gitUncommittedChanges() {
			//That's weird, there aren't any changes to stash
			log.Println("Can't stash: no uncommitted changes!")
			return false
		}
	} else {
		stashCmd = exec.Command("git", "stash", "pop")
		if gitUncommittedChanges() {
			//That's weird, there are uncommitted changes that this would overwrite.
			log.Println("Can't stash pop: uncommitted changes that would be overwritten")
			return false
		}
	}

	err := stashCmd.Run()

	if err != nil {
		log.Println(err)
		return false
	}

	//Verify it worked
	if stashChanges {
		//Stashing apaprently didn't work
		if gitUncommittedChanges() {
			log.Println("Stashing didn't work; there are still uncommitted changes")
			return false
		}
	} else {
		//Weird, stash popping didn't do anything.
		if !gitUncommittedChanges() {
			log.Println("Stash popping didn't work; there are no uncommitted changes that resulted.	")
			return false
		}
	}

	return true
}

//Returns true if there are currently uncommitted changes
func gitUncommittedChanges() bool {

	statusCmd := exec.Command("git", "status", "-s")

	output, err := statusCmd.Output()

	if err != nil {
		log.Println(err)
		return false
	}

	//In git status -s(hort), each line starts with two characters. ?? is hte
	//only prefix that we should ignore, since it means untracked files.

	for _, line := range strings.Split(string(output), "\n") {
		if strings.TrimSpace(line) == "" {
			continue
		}
		if !strings.HasPrefix(line, "??") {
			//Found a non-committed change
			return true
		}
	}

	return false

}

//gitCurrentBranch returns the current branch that the current repo is in.
func gitCurrentBranch() string {
	branchCmd := exec.Command("git", "branch")

	output, err := branchCmd.Output()

	if err != nil {
		log.Println(err)
		return ""
	}

	for _, line := range strings.Split(string(output), "\n") {
		if strings.Contains(line, "*") {
			//Found it!
			line = strings.Replace(line, "*", "", -1)
			line = strings.TrimSpace(line)
			return line
		}
	}

	return ""
}

func checkoutGitBranch(branch string) bool {

	if branch == "" {
		return true
	}

	checkoutCmd := exec.Command("git", "checkout", branch)
	checkoutCmd.Run()

	if gitCurrentBranch() != branch {
		return false
	}

	return true

}
<|endoftext|>


<commit_before>package root

import (
	"github.com/Aptomi/aptomi/cmd/aptomictl/dependency"
	"github.com/Aptomi/aptomi/cmd/aptomictl/gen"
	"github.com/Aptomi/aptomi/cmd/aptomictl/login"
	"github.com/Aptomi/aptomi/cmd/aptomictl/policy"
	"github.com/Aptomi/aptomi/cmd/aptomictl/revision"
	"github.com/Aptomi/aptomi/cmd/aptomictl/state"
	"github.com/Aptomi/aptomi/cmd/aptomictl/version"
	"github.com/Aptomi/aptomi/cmd/common"
	"github.com/Aptomi/aptomi/pkg/config"
	log "github.com/Sirupsen/logrus"
	"github.com/mitchellh/go-homedir"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"path"
	"time"
)

const (
	// EnvPrefix is the prefix for all environment variables used by aptomictl
	EnvPrefix = "APTOMICTL"
)

var (
	// Config is the global instance of client config
	Config = &config.Client{}

	// ConfigFile is the path to config file used to read config
	ConfigFile = new(string)

	// Command is the main (root) cobra command for aptomictl
	Command = &cobra.Command{
		Use:   "aptomictl",
		Short: "aptomictl controls Aptomi",
		Long:  "aptomictl controls Aptomi",

		PersistentPreRun: preRun,

		Run: func(cmd *cobra.Command, args []string) {
			// fall back on default help if no args/flags are passed
			cmd.HelpFunc()(cmd, args)
		},
	}
)

func init() {
	viper.SetEnvPrefix(EnvPrefix)

	common.AddDefaultFlags(Command, EnvPrefix)

	common.AddStringFlag(Command, "output", "output", "o", "text", EnvPrefix+"_OUTPUT", "Output format. One of: text (default), json, yaml")

	common.AddDurationFlag(Command, "http.timeout", "timeout", "", 15*time.Second, EnvPrefix+"_TIMEOUT", "HTTP Timeout")

	// Add sub commands
	Command.AddCommand(
		login.NewCommand(Config, ConfigFile),
		dependency.NewCommand(Config),
		policy.NewCommand(Config),
		revision.NewCommand(Config),
		state.NewCommand(Config),
		gen.NewCommand(Config),
		version.NewCommand(Config),
	)
}

func preRun(command *cobra.Command, args []string) {
	if command.Parent() != nil {
		err := common.ReadConfig(viper.GetViper(), Config, defaultConfigDir())
		if err != nil {
			log.Fatalf("error while loading config: %s", err)
		}

		usedConfigFile := viper.ConfigFileUsed()
		*ConfigFile = usedConfigFile

		log.Infof("Using config file: %s", usedConfigFile)
	}
}

func defaultConfigDir() string {
	home, err := homedir.Dir()
	if err != nil {
		log.Fatalf("can't find home dir: %s", err)
	}

	return path.Join(home, ".aptomi")
}
<commit_msg>default client 15s timeout is often not enough to receive a reply from the server (e.g. receiving endpoints)<commit_after>package root

import (
	"github.com/Aptomi/aptomi/cmd/aptomictl/dependency"
	"github.com/Aptomi/aptomi/cmd/aptomictl/gen"
	"github.com/Aptomi/aptomi/cmd/aptomictl/login"
	"github.com/Aptomi/aptomi/cmd/aptomictl/policy"
	"github.com/Aptomi/aptomi/cmd/aptomictl/revision"
	"github.com/Aptomi/aptomi/cmd/aptomictl/state"
	"github.com/Aptomi/aptomi/cmd/aptomictl/version"
	"github.com/Aptomi/aptomi/cmd/common"
	"github.com/Aptomi/aptomi/pkg/config"
	log "github.com/Sirupsen/logrus"
	"github.com/mitchellh/go-homedir"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"path"
	"time"
)

const (
	// EnvPrefix is the prefix for all environment variables used by aptomictl
	EnvPrefix = "APTOMICTL"
)

var (
	// Config is the global instance of client config
	Config = &config.Client{}

	// ConfigFile is the path to config file used to read config
	ConfigFile = new(string)

	// Command is the main (root) cobra command for aptomictl
	Command = &cobra.Command{
		Use:   "aptomictl",
		Short: "aptomictl controls Aptomi",
		Long:  "aptomictl controls Aptomi",

		PersistentPreRun: preRun,

		Run: func(cmd *cobra.Command, args []string) {
			// fall back on default help if no args/flags are passed
			cmd.HelpFunc()(cmd, args)
		},
	}
)

func init() {
	viper.SetEnvPrefix(EnvPrefix)

	common.AddDefaultFlags(Command, EnvPrefix)

	common.AddStringFlag(Command, "output", "output", "o", "text", EnvPrefix+"_OUTPUT", "Output format. One of: text (default), json, yaml")

	common.AddDurationFlag(Command, "http.timeout", "timeout", "", 60*time.Second, EnvPrefix+"_TIMEOUT", "Specifies time limit for receiving a reply from the server")

	// Add sub commands
	Command.AddCommand(
		login.NewCommand(Config, ConfigFile),
		dependency.NewCommand(Config),
		policy.NewCommand(Config),
		revision.NewCommand(Config),
		state.NewCommand(Config),
		gen.NewCommand(Config),
		version.NewCommand(Config),
	)
}

func preRun(command *cobra.Command, args []string) {
	if command.Parent() != nil {
		err := common.ReadConfig(viper.GetViper(), Config, defaultConfigDir())
		if err != nil {
			log.Fatalf("error while loading config: %s", err)
		}

		usedConfigFile := viper.ConfigFileUsed()
		*ConfigFile = usedConfigFile

		log.Infof("Using config file: %s", usedConfigFile)
	}
}

func defaultConfigDir() string {
	home, err := homedir.Dir()
	if err != nil {
		log.Fatalf("can't find home dir: %s", err)
	}

	return path.Join(home, ".aptomi")
}
<|endoftext|>


<commit_before>// Command deployctl implements single-command operator's interface to manage
// deployments running under deploy-registry and deploy-agent
package main

import (
	"crypto/sha256"
	"flag"
	"fmt"
	"io"
	"net"
	"os"
	"strings"
	"time"

	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/agent"

	"github.com/artyom/autoflags"
	"github.com/artyom/deploy-tools/internal/shared"
	"github.com/pkg/errors"
	"github.com/pkg/sftp"
)

func main() {
	args := &runArgs{
		Addr: os.Getenv("DEPLOYCTL_ADDR"),
		Fp:   os.Getenv("DEPLOYCTL_FINGERPRINT"),
	}
	fs := flag.NewFlagSet("deployctl", flag.ExitOnError)
	fs.Usage = usageFunc(fs.PrintDefaults)
	autoflags.DefineFlagSet(fs, args)
	fs.Parse(os.Args[1:])
	if err := args.Validate(); err != nil {
		fmt.Fprintf(os.Stderr, "%v\n\n", err)
		fs.Usage()
		os.Exit(2)
	}
	if len(fs.Args()) == 0 {
		fs.Usage()
		os.Exit(2)
	}
	if err := dispatch(args.Addr, args.Fp, fs.Args()); err != nil {
		if err == errFlagParseError {
			os.Exit(2)
		}
		if _, ok := err.(*ssh.ExitError); !ok { // don't write "Process exited with status 1"
			fmt.Fprintln(os.Stderr, err)
		}
		os.Exit(1)
	}
}

func dispatch(addr, fingerprint string, rawArgs []string) error {
	if len(rawArgs) == 0 {
		return errors.New("nothing to do")
	}
	cmd, args := rawArgs[0], rawArgs[1:]
	switch cmd {
	case "components", "configurations":
		return proxyCommand(addr, fingerprint, rawArgs)
	case "addver":
		val := &shared.ArgsAddVersionByFile{}
		if err := parseArgs(cmd, val, os.Stderr, args); err != nil {
			return err
		}
		return uploadAndUpdate(addr, fingerprint, val)
	}
	val, err := validatorForCommand(cmd)
	if err != nil {
		return err
	}
	if err := parseArgs(cmd, val, os.Stderr, args); err != nil {
		return err
	}
	return proxyCommand(addr, fingerprint, rawArgs)
}

func uploadAndUpdate(addr, fingerprint string, args *shared.ArgsAddVersionByFile) error {
	src, err := os.Open(args.File)
	if err != nil {
		return err
	}
	defer src.Close()
	client, cancel, err := dialSSH(addr, fingerprint)
	if err != nil {
		return err
	}
	defer cancel()
	sftpconn, err := sftp.NewClient(client)
	if err != nil {
		return err
	}
	defer sftpconn.Close()
	dst, err := sftpconn.Create("upload")
	if err != nil {
		return err
	}
	defer dst.Close()
	// TODO: check whether file is valid tar.gz archive by unpacking
	// & discarding it as we upload
	h := sha256.New()
	tr := io.TeeReader(src, h)
	if _, err := io.Copy(dst, tr); err != nil {
		return errors.WithMessage(err, "upload failure")
	}
	if err := dst.Close(); err != nil {
		return err
	}
	session, err := client.NewSession()
	if err != nil {
		return err
	}
	defer session.Close()
	session.Stdout = os.Stdout
	session.Stderr = os.Stderr
	return session.Run(fmt.Sprintf("addver -name=%q -version=%q -hash=%x",
		args.Name, args.Version, h.Sum(nil)))
}

func dialSSH(addr, fingerprint string) (client *ssh.Client, closeFunc func(), err error) {
	agentConn, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK"))
	if err != nil {
		return nil, nil, errors.WithMessage(err, "cannot connect to ssh-agent, check if SSH_AUTH_SOCK is set")
	}
	defer func() {
		if err != nil {
			agentConn.Close()
		}
	}()
	sshAgent := agent.NewClient(agentConn)
	var signers []ssh.Signer
	signers, err = sshAgent.Signers()
	if err != nil {
		return nil, nil, err
	}
	config := &ssh.ClientConfig{
		User:    os.Getenv("USER"),
		Auth:    []ssh.AuthMethod{ssh.PublicKeys(signers...)},
		Timeout: 30 * time.Second,
		HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error {
			if hostFp := ssh.FingerprintSHA256(key); hostFp != fingerprint {
				return errors.Errorf("host key fingerprint mismatch: %v", hostFp)
			}
			return nil
		},
	}
	client, err = ssh.Dial("tcp", addr, config)
	if err != nil {
		return nil, nil, err
	}
	closeFunc = func() { client.Close(); agentConn.Close() }
	return client, closeFunc, nil
}

func proxyCommand(addr, fingerprint string, args []string) error {
	client, cancel, err := dialSSH(addr, fingerprint)
	if err != nil {
		return err
	}
	defer cancel()
	session, err := client.NewSession()
	if err != nil {
		return err
	}
	defer session.Close()
	session.Stdout = os.Stdout
	session.Stderr = os.Stderr
	return session.Run(strings.Join(args, " "))
}

type validator interface {
	Validate() error
}

func validatorForCommand(name string) (validator, error) {
	v, ok := map[string]validator{
		"delver":     &shared.ArgsDelVersion{},
		"delcomp":    &shared.ArgsDelComponent{},
		"addconf":    &shared.ArgsAddConfiguration{},
		"delconf":    &shared.ArgsDelConfiguration{},
		"changeconf": &shared.ArgsUpdateConfiguration{},
		"showconf":   &shared.ArgsShowConfiguration{},
		"showcomp":   &shared.ArgsShowComponent{},
	}[name]
	if !ok {
		return nil, errors.Errorf("unknown command: %q", name)
	}
	return v, nil
}

// errFlagParseError is a sentinel error value used to determine whether error
// originates from flagset that already reported error to stderr so its
// reporting can be omitted
var errFlagParseError = errors.New("flag parse error")

// parseArgs defines new flag set with flags from argStruct that writes its
// errors to w, then calls flag set Parse method on provided raw arguments and
// calls Validate() method on provided argStruct. If parseArgs returns
// errFlagParseError, it means that flag set already reported error to w.
func parseArgs(command string, argStruct validator, w io.Writer, raw []string) error {
	fs := flag.NewFlagSet(command, flag.ContinueOnError)
	fs.SetOutput(w)
	autoflags.DefineFlagSet(fs, argStruct)
	if err := fs.Parse(raw); err != nil {
		return errFlagParseError
	}
	return argStruct.Validate()
}

type runArgs struct {
	Addr string `flag:"addr,$DEPLOYCTL_ADDR, registry host address (host:port)"`
	Fp   string `flag:"fp,$DEPLOYCTL_FINGERPRINT, sha256 host key fingerprint (sha256:...)"`
}

func (a *runArgs) Validate() error {
	if a.Addr == "" || a.Fp == "" {
		return errors.New("both addr and fp should be set")
	}
	return nil
}

func usageFunc(printDefaults func()) func() {
	return func() {
		fmt.Fprintln(os.Stderr, "Usage: deployctl [flags] subcommand [subcommand flags]")
		printDefaults()
		fmt.Fprintln(os.Stderr, "\nSubcommands:\n")
		fmt.Fprintln(os.Stderr, strings.TrimSpace(shared.CommandsListing))
	}
}

var knownCommands = []string{"addver", "addconf",
	"changeconf", "showconf",
	"components", "configurations",
	"showcomp",
	"delver",
	"delcomp",
	"delconf",
}
<commit_msg>deployctl: replace map lookup with switch statement<commit_after>// Command deployctl implements single-command operator's interface to manage
// deployments running under deploy-registry and deploy-agent
package main

import (
	"crypto/sha256"
	"flag"
	"fmt"
	"io"
	"net"
	"os"
	"strings"
	"time"

	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/agent"

	"github.com/artyom/autoflags"
	"github.com/artyom/deploy-tools/internal/shared"
	"github.com/pkg/errors"
	"github.com/pkg/sftp"
)

func main() {
	args := &runArgs{
		Addr: os.Getenv("DEPLOYCTL_ADDR"),
		Fp:   os.Getenv("DEPLOYCTL_FINGERPRINT"),
	}
	fs := flag.NewFlagSet("deployctl", flag.ExitOnError)
	fs.Usage = usageFunc(fs.PrintDefaults)
	autoflags.DefineFlagSet(fs, args)
	fs.Parse(os.Args[1:])
	if err := args.Validate(); err != nil {
		fmt.Fprintf(os.Stderr, "%v\n\n", err)
		fs.Usage()
		os.Exit(2)
	}
	if len(fs.Args()) == 0 {
		fs.Usage()
		os.Exit(2)
	}
	if err := dispatch(args.Addr, args.Fp, fs.Args()); err != nil {
		if err == errFlagParseError {
			os.Exit(2)
		}
		if _, ok := err.(*ssh.ExitError); !ok { // don't write "Process exited with status 1"
			fmt.Fprintln(os.Stderr, err)
		}
		os.Exit(1)
	}
}

func dispatch(addr, fingerprint string, rawArgs []string) error {
	if len(rawArgs) == 0 {
		return errors.New("nothing to do")
	}
	cmd, args := rawArgs[0], rawArgs[1:]
	switch cmd {
	case "components", "configurations":
		return proxyCommand(addr, fingerprint, rawArgs)
	case "addver":
		val := &shared.ArgsAddVersionByFile{}
		if err := parseArgs(cmd, val, os.Stderr, args); err != nil {
			return err
		}
		return uploadAndUpdate(addr, fingerprint, val)
	}
	val, err := validatorForCommand(cmd)
	if err != nil {
		return err
	}
	if err := parseArgs(cmd, val, os.Stderr, args); err != nil {
		return err
	}
	return proxyCommand(addr, fingerprint, rawArgs)
}

func uploadAndUpdate(addr, fingerprint string, args *shared.ArgsAddVersionByFile) error {
	src, err := os.Open(args.File)
	if err != nil {
		return err
	}
	defer src.Close()
	client, cancel, err := dialSSH(addr, fingerprint)
	if err != nil {
		return err
	}
	defer cancel()
	sftpconn, err := sftp.NewClient(client)
	if err != nil {
		return err
	}
	defer sftpconn.Close()
	dst, err := sftpconn.Create("upload")
	if err != nil {
		return err
	}
	defer dst.Close()
	// TODO: check whether file is valid tar.gz archive by unpacking
	// & discarding it as we upload
	h := sha256.New()
	tr := io.TeeReader(src, h)
	if _, err := io.Copy(dst, tr); err != nil {
		return errors.WithMessage(err, "upload failure")
	}
	if err := dst.Close(); err != nil {
		return err
	}
	session, err := client.NewSession()
	if err != nil {
		return err
	}
	defer session.Close()
	session.Stdout = os.Stdout
	session.Stderr = os.Stderr
	return session.Run(fmt.Sprintf("addver -name=%q -version=%q -hash=%x",
		args.Name, args.Version, h.Sum(nil)))
}

func dialSSH(addr, fingerprint string) (client *ssh.Client, closeFunc func(), err error) {
	agentConn, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK"))
	if err != nil {
		return nil, nil, errors.WithMessage(err, "cannot connect to ssh-agent, check if SSH_AUTH_SOCK is set")
	}
	defer func() {
		if err != nil {
			agentConn.Close()
		}
	}()
	sshAgent := agent.NewClient(agentConn)
	var signers []ssh.Signer
	signers, err = sshAgent.Signers()
	if err != nil {
		return nil, nil, err
	}
	config := &ssh.ClientConfig{
		User:    os.Getenv("USER"),
		Auth:    []ssh.AuthMethod{ssh.PublicKeys(signers...)},
		Timeout: 30 * time.Second,
		HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error {
			if hostFp := ssh.FingerprintSHA256(key); hostFp != fingerprint {
				return errors.Errorf("host key fingerprint mismatch: %v", hostFp)
			}
			return nil
		},
	}
	client, err = ssh.Dial("tcp", addr, config)
	if err != nil {
		return nil, nil, err
	}
	closeFunc = func() { client.Close(); agentConn.Close() }
	return client, closeFunc, nil
}

func proxyCommand(addr, fingerprint string, args []string) error {
	client, cancel, err := dialSSH(addr, fingerprint)
	if err != nil {
		return err
	}
	defer cancel()
	session, err := client.NewSession()
	if err != nil {
		return err
	}
	defer session.Close()
	session.Stdout = os.Stdout
	session.Stderr = os.Stderr
	return session.Run(strings.Join(args, " "))
}

type validator interface {
	Validate() error
}

func validatorForCommand(name string) (validator, error) {
	switch name {
	case "delver":
		return &shared.ArgsDelVersion{}, nil
	case "delcomp":
		return &shared.ArgsDelComponent{}, nil
	case "addconf":
		return &shared.ArgsAddConfiguration{}, nil
	case "delconf":
		return &shared.ArgsDelConfiguration{}, nil
	case "changeconf":
		return &shared.ArgsUpdateConfiguration{}, nil
	case "showconf":
		return &shared.ArgsShowConfiguration{}, nil
	case "showcomp":
		return &shared.ArgsShowComponent{}, nil
	}
	return nil, errors.Errorf("unknown command: %q", name)
}

// errFlagParseError is a sentinel error value used to determine whether error
// originates from flagset that already reported error to stderr so its
// reporting can be omitted
var errFlagParseError = errors.New("flag parse error")

// parseArgs defines new flag set with flags from argStruct that writes its
// errors to w, then calls flag set Parse method on provided raw arguments and
// calls Validate() method on provided argStruct. If parseArgs returns
// errFlagParseError, it means that flag set already reported error to w.
func parseArgs(command string, argStruct validator, w io.Writer, raw []string) error {
	fs := flag.NewFlagSet(command, flag.ContinueOnError)
	fs.SetOutput(w)
	autoflags.DefineFlagSet(fs, argStruct)
	if err := fs.Parse(raw); err != nil {
		return errFlagParseError
	}
	return argStruct.Validate()
}

type runArgs struct {
	Addr string `flag:"addr,$DEPLOYCTL_ADDR, registry host address (host:port)"`
	Fp   string `flag:"fp,$DEPLOYCTL_FINGERPRINT, sha256 host key fingerprint (sha256:...)"`
}

func (a *runArgs) Validate() error {
	if a.Addr == "" || a.Fp == "" {
		return errors.New("both addr and fp should be set")
	}
	return nil
}

func usageFunc(printDefaults func()) func() {
	return func() {
		fmt.Fprintln(os.Stderr, "Usage: deployctl [flags] subcommand [subcommand flags]")
		printDefaults()
		fmt.Fprintln(os.Stderr, "\nSubcommands:\n")
		fmt.Fprintln(os.Stderr, strings.TrimSpace(shared.CommandsListing))
	}
}

var knownCommands = []string{"addver", "addconf",
	"changeconf", "showconf",
	"components", "configurations",
	"showcomp",
	"delver",
	"delcomp",
	"delconf",
}
<|endoftext|>


<commit_before>// +build !fasthttp

package main

import (
	"encoding/json"
	"io"
	"net/http"
	"strings"
	"time"

	"github.com/funkygao/gafka/cmd/kateway/manager"
	"github.com/funkygao/gafka/cmd/kateway/meta"
	"github.com/funkygao/gafka/cmd/kateway/store"
	"github.com/funkygao/gafka/mpool"
	log "github.com/funkygao/log4go"
	"github.com/julienschmidt/httprouter"
)

// /topics/:topic/:ver?key=mykey&async=1&delay=100
func (this *Gateway) pubHandler(w http.ResponseWriter, r *http.Request,
	params httprouter.Params) {
	t1 := time.Now()

	if options.Ratelimit && !this.leakyBuckets.Pour(r.RemoteAddr, 1) {
		this.writeQuotaExceeded(w)
		return
	}

	appid := r.Header.Get(HttpHeaderAppid)
	topic := params.ByName(UrlParamTopic) // params[0].Value
	ver := params.ByName(UrlParamVersion) // params[1].Value
	if err := manager.Default.AuthPub(appid, r.Header.Get(HttpHeaderPubkey), topic); err != nil {
		log.Warn("pub[%s] %s(%s) {topic:%s, ver:%s} %s",
			appid, r.RemoteAddr, getHttpRemoteIp(r), topic, ver, err)

		this.writeAuthFailure(w, err)
		return
	}

	if r.Header.Get(HttpHeaderConnection) == "close" {
		log.Warn("pub[%s] %s(%s) {topic:%s, ver:%s} not keep-alive",
			appid, r.RemoteAddr, getHttpRemoteIp(r), topic, ver)
	}

	// get the raw POST message
	msgLen := int(r.ContentLength)
	switch {
	case msgLen == -1:
		log.Warn("pub[%s] %s(%s) {topic:%s, ver:%s} invalid content length: %d",
			appid, r.RemoteAddr, getHttpRemoteIp(r), topic, ver, msgLen)
		this.writeInvalidContentLength(w)
		return

	case int64(msgLen) > options.MaxPubSize:
		log.Warn("pub[%s] %s(%s) {topic:%s, ver:%s} too big content length: %d",
			appid, r.RemoteAddr, getHttpRemoteIp(r), topic, ver, msgLen)
		this.writeErrorResponse(w, ErrTooBigPubMessage.Error(), http.StatusBadRequest)
		return

	case msgLen < options.MinPubSize:
		log.Warn("pub[%s] %s(%s) {topic:%s, ver:%s} too small content length: %d",
			appid, r.RemoteAddr, getHttpRemoteIp(r), topic, ver, msgLen)
		this.writeErrorResponse(w, ErrTooSmallPubMessage.Error(), http.StatusBadRequest)
		return
	}

	lbr := io.LimitReader(r.Body, options.MaxPubSize+1)
	msg := mpool.NewMessage(msgLen)
	msg.Body = msg.Body[0:msgLen]
	if _, err := io.ReadAtLeast(lbr, msg.Body, msgLen); err != nil {
		msg.Free()

		log.Error("pub[%s] %s(%s) {topic:%s, ver:%s} %s",
			appid, r.RemoteAddr, getHttpRemoteIp(r), topic, ver, err)
		this.writeErrorResponse(w, ErrTooBigPubMessage.Error(), http.StatusBadRequest)
		return
	}

	if options.Debug {
		log.Debug("pub[%s] %s(%s) {topic:%s, ver:%s} %s",
			appid, r.RemoteAddr, getHttpRemoteIp(r), topic, ver, string(msg.Body))
	}

	if !options.DisableMetrics {
		this.pubMetrics.PubQps.Mark(1)
		this.pubMetrics.PubMsgSize.Update(int64(len(msg.Body)))
	}

	query := r.URL.Query() // reuse the query will save 100ns

	pubMethod := store.DefaultPubStore.SyncPub
	if query.Get(UrlQueryAsync) == "1" {
		pubMethod = store.DefaultPubStore.AsyncPub
	}

	cluster, found := manager.Default.LookupCluster(appid)
	if !found {
		log.Warn("pub[%s] %s(%s) {topic:%s, ver:%s} cluster not found",
			appid, r.RemoteAddr, getHttpRemoteIp(r), topic, ver)

		http.Error(w, "invalid appid", http.StatusBadRequest)
		return
	}

	err := pubMethod(cluster, appid+"."+topic+"."+ver,
		[]byte(query.Get(UrlQueryKey)), msg.Body)
	if err != nil {
		msg.Free() // defer is costly

		if !options.DisableMetrics {
			this.pubMetrics.PubFail(appid, topic, ver)
		}

		log.Error("pub[%s] %s(%s) {topic:%s, ver:%s} %s",
			appid, r.RemoteAddr, getHttpRemoteIp(r), topic, ver, err)
		this.writeErrorResponse(w, err.Error(), http.StatusInternalServerError)
		return
	}

	msg.Free()
	this.writeKatewayHeader(w)
	if _, err = w.Write(ResponseOk); err != nil {
		log.Error("%s: %v", r.RemoteAddr, err)
		this.pubMetrics.ClientError.Inc(1)
	}

	if !options.DisableMetrics {
		this.pubMetrics.PubOk(appid, topic, ver)
		this.pubMetrics.PubLatency.Update(time.Since(t1).Nanoseconds() / 1e6) // in ms
	}

}

// /raw/topics/:topic/:ver
func (this *Gateway) pubRawHandler(w http.ResponseWriter, r *http.Request,
	params httprouter.Params) {
	var (
		topic string
		ver   string
		appid string
	)

	ver = params.ByName(UrlParamVersion)
	topic = params.ByName(UrlParamTopic)
	appid = r.Header.Get(HttpHeaderAppid)

	if err := manager.Default.AuthSub(appid, r.Header.Get(HttpHeaderPubkey), topic); err != nil {
		log.Error("app[%s] %s %+v: %s", appid, r.RemoteAddr, params, err)

		this.writeAuthFailure(w, err)
		return
	}

	cluster, found := manager.Default.LookupCluster(appid)
	if !found {
		log.Error("cluster not found for app: %s", appid)

		http.Error(w, "invalid appid", http.StatusBadRequest)
		return
	}

	this.writeKatewayHeader(w)
	var out = map[string]string{
		"store":       "kafka",
		"broker.list": strings.Join(meta.Default.BrokerList(cluster), ","),
		"topic":       meta.KafkaTopic(appid, topic, ver),
	}

	b, _ := json.Marshal(out)
	w.Header().Set(ContentTypeText, ContentTypeJson)
	w.Write(b)
}

// /ws/topics/:topic/:ver
func (this *Gateway) pubWsHandler(w http.ResponseWriter, r *http.Request,
	params httprouter.Params) {
	ws, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Error("%s: %v", r.RemoteAddr, err)
		return
	}

	defer ws.Close()
}

func (this *Gateway) pubCheckHandler(w http.ResponseWriter, r *http.Request,
	params httprouter.Params) {
	this.writeKatewayHeader(w)
	w.Write(ResponseOk)
}
<commit_msg>Pub none keep-alive conn will not be warned<commit_after>// +build !fasthttp

package main

import (
	"encoding/json"
	"io"
	"net/http"
	"strings"
	"time"

	"github.com/funkygao/gafka/cmd/kateway/manager"
	"github.com/funkygao/gafka/cmd/kateway/meta"
	"github.com/funkygao/gafka/cmd/kateway/store"
	"github.com/funkygao/gafka/mpool"
	log "github.com/funkygao/log4go"
	"github.com/julienschmidt/httprouter"
)

// /topics/:topic/:ver?key=mykey&async=1&delay=100
func (this *Gateway) pubHandler(w http.ResponseWriter, r *http.Request,
	params httprouter.Params) {
	t1 := time.Now()

	if options.Ratelimit && !this.leakyBuckets.Pour(r.RemoteAddr, 1) {
		this.writeQuotaExceeded(w)
		return
	}

	appid := r.Header.Get(HttpHeaderAppid)
	topic := params.ByName(UrlParamTopic) // params[0].Value
	ver := params.ByName(UrlParamVersion) // params[1].Value
	if err := manager.Default.AuthPub(appid, r.Header.Get(HttpHeaderPubkey), topic); err != nil {
		log.Warn("pub[%s] %s(%s) {topic:%s, ver:%s} %s",
			appid, r.RemoteAddr, getHttpRemoteIp(r), topic, ver, err)

		this.writeAuthFailure(w, err)
		return
	}

	if false && r.Header.Get(HttpHeaderConnection) == "close" {
		log.Debug("pub[%s] %s(%s) {topic:%s, ver:%s} better keep-alive",
			appid, r.RemoteAddr, getHttpRemoteIp(r), topic, ver)
	}

	// get the raw POST message
	msgLen := int(r.ContentLength)
	switch {
	case msgLen == -1:
		log.Warn("pub[%s] %s(%s) {topic:%s, ver:%s} invalid content length: %d",
			appid, r.RemoteAddr, getHttpRemoteIp(r), topic, ver, msgLen)
		this.writeInvalidContentLength(w)
		return

	case int64(msgLen) > options.MaxPubSize:
		log.Warn("pub[%s] %s(%s) {topic:%s, ver:%s} too big content length: %d",
			appid, r.RemoteAddr, getHttpRemoteIp(r), topic, ver, msgLen)
		this.writeErrorResponse(w, ErrTooBigPubMessage.Error(), http.StatusBadRequest)
		return

	case msgLen < options.MinPubSize:
		log.Warn("pub[%s] %s(%s) {topic:%s, ver:%s} too small content length: %d",
			appid, r.RemoteAddr, getHttpRemoteIp(r), topic, ver, msgLen)
		this.writeErrorResponse(w, ErrTooSmallPubMessage.Error(), http.StatusBadRequest)
		return
	}

	lbr := io.LimitReader(r.Body, options.MaxPubSize+1)
	msg := mpool.NewMessage(msgLen)
	msg.Body = msg.Body[0:msgLen]
	if _, err := io.ReadAtLeast(lbr, msg.Body, msgLen); err != nil {
		msg.Free()

		log.Error("pub[%s] %s(%s) {topic:%s, ver:%s} %s",
			appid, r.RemoteAddr, getHttpRemoteIp(r), topic, ver, err)
		this.writeErrorResponse(w, ErrTooBigPubMessage.Error(), http.StatusBadRequest)
		return
	}

	if options.Debug {
		log.Debug("pub[%s] %s(%s) {topic:%s, ver:%s} %s",
			appid, r.RemoteAddr, getHttpRemoteIp(r), topic, ver, string(msg.Body))
	}

	if !options.DisableMetrics {
		this.pubMetrics.PubQps.Mark(1)
		this.pubMetrics.PubMsgSize.Update(int64(len(msg.Body)))
	}

	query := r.URL.Query() // reuse the query will save 100ns

	pubMethod := store.DefaultPubStore.SyncPub
	if query.Get(UrlQueryAsync) == "1" {
		pubMethod = store.DefaultPubStore.AsyncPub
	}

	cluster, found := manager.Default.LookupCluster(appid)
	if !found {
		log.Warn("pub[%s] %s(%s) {topic:%s, ver:%s} cluster not found",
			appid, r.RemoteAddr, getHttpRemoteIp(r), topic, ver)

		http.Error(w, "invalid appid", http.StatusBadRequest)
		return
	}

	err := pubMethod(cluster, appid+"."+topic+"."+ver,
		[]byte(query.Get(UrlQueryKey)), msg.Body)
	if err != nil {
		msg.Free() // defer is costly

		if !options.DisableMetrics {
			this.pubMetrics.PubFail(appid, topic, ver)
		}

		log.Error("pub[%s] %s(%s) {topic:%s, ver:%s} %s",
			appid, r.RemoteAddr, getHttpRemoteIp(r), topic, ver, err)
		this.writeErrorResponse(w, err.Error(), http.StatusInternalServerError)
		return
	}

	msg.Free()
	this.writeKatewayHeader(w)
	if _, err = w.Write(ResponseOk); err != nil {
		log.Error("%s: %v", r.RemoteAddr, err)
		this.pubMetrics.ClientError.Inc(1)
	}

	if !options.DisableMetrics {
		this.pubMetrics.PubOk(appid, topic, ver)
		this.pubMetrics.PubLatency.Update(time.Since(t1).Nanoseconds() / 1e6) // in ms
	}

}

// /raw/topics/:topic/:ver
func (this *Gateway) pubRawHandler(w http.ResponseWriter, r *http.Request,
	params httprouter.Params) {
	var (
		topic string
		ver   string
		appid string
	)

	ver = params.ByName(UrlParamVersion)
	topic = params.ByName(UrlParamTopic)
	appid = r.Header.Get(HttpHeaderAppid)

	if err := manager.Default.AuthSub(appid, r.Header.Get(HttpHeaderPubkey), topic); err != nil {
		log.Error("app[%s] %s %+v: %s", appid, r.RemoteAddr, params, err)

		this.writeAuthFailure(w, err)
		return
	}

	cluster, found := manager.Default.LookupCluster(appid)
	if !found {
		log.Error("cluster not found for app: %s", appid)

		http.Error(w, "invalid appid", http.StatusBadRequest)
		return
	}

	this.writeKatewayHeader(w)
	var out = map[string]string{
		"store":       "kafka",
		"broker.list": strings.Join(meta.Default.BrokerList(cluster), ","),
		"topic":       meta.KafkaTopic(appid, topic, ver),
	}

	b, _ := json.Marshal(out)
	w.Header().Set(ContentTypeText, ContentTypeJson)
	w.Write(b)
}

// /ws/topics/:topic/:ver
func (this *Gateway) pubWsHandler(w http.ResponseWriter, r *http.Request,
	params httprouter.Params) {
	ws, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Error("%s: %v", r.RemoteAddr, err)
		return
	}

	defer ws.Close()
}

func (this *Gateway) pubCheckHandler(w http.ResponseWriter, r *http.Request,
	params httprouter.Params) {
	this.writeKatewayHeader(w)
	w.Write(ResponseOk)
}
<|endoftext|>


<commit_before>package mapgen
//
// import (
// 	"buildblast/coords"
// )
//
// type Map struct {
// 	generator ChunkSource
// 	chunks map[coords.Chunk]mapgen.Chunk
// 	lock sync.Mutex
// }
//
// func (m *Map) Block(wc coords.World) Block {
// 	m.lock.Lock()
// 	defer m.lock.Unlock()
//
// 	cc := wc.Chunk()
// 	oc := wc.Offset()
// 	chunk := m.chunks[cc]
// 	if chunk == nil {
// 		return BLOCK_NIL
// 	}
// 	return chunk.Block(oc)
// }
//
// func (m *Map) RequestChunk(cc coords.Chunk) Chunk {
// 	m.lock.Lock()
// 	defer m.lock.Unlock()
//
// 	chunk := m.chunks[cc]
//
// }
<commit_msg>Remove map.go (not sure how it got there...)<commit_after><|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/mitchellh/go-homedir"

	"github.com/puma/puma-dev/dev"
)

var (
	fDebug    = flag.Bool("debug", false, "enable debug output")
	fDomains  = flag.String("d", "dev", "domains to handle, separate with :")
	fHTTPPort = flag.Int("http-port", 9280, "port to listen on http for")
	fTLSPort  = flag.Int("https-port", 9283, "port to listen on https for")
	fSysBind  = flag.Bool("sysbind", false, "bind to ports 80 and 443")
	fDir      = flag.String("dir", "~/.puma-dev", "directory to watch for apps")
	fTimeout  = flag.Duration("timeout", 15*60*time.Second, "how long to let an app idle for")
)

func main() {
	flag.Parse()

	allCheck()

	domains := strings.Split(*fDomains, ":")

	if *fSysBind {
		*fHTTPPort = 80
		*fTLSPort = 443
	}

	dir, err := homedir.Expand(*fDir)
	if err != nil {
		log.Fatalf("Unable to expand dir: %s", err)
	}

	err = os.MkdirAll(dir, 0755)
	if err != nil {
		log.Fatalf("Unable to create dir '%s': %s", dir, err)
	}

	var events dev.Events

	var pool dev.AppPool
	pool.Dir = dir
	pool.IdleTime = *fTimeout
	pool.Events = &events

	purge := make(chan os.Signal, 1)

	signal.Notify(purge, syscall.SIGUSR1)

	go func() {
		for {
			<-purge
			pool.Purge()
		}
	}()

	stop := make(chan os.Signal, 1)

	signal.Notify(stop, os.Interrupt, syscall.SIGQUIT, syscall.SIGTERM)

	go func() {
		<-stop
		fmt.Printf("! Shutdown requested\n")
		pool.Purge()
		os.Exit(0)
	}()

	err = dev.SetupOurCert()
	if err != nil {
		log.Fatalf("Unable to setup TLS cert: %s", err)
	}

	fmt.Printf("* Directory for apps: %s\n", dir)
	fmt.Printf("* Domains: %s\n", strings.Join(domains, ", "))
	fmt.Printf("* HTTP Server port: %d\n", *fHTTPPort)
	fmt.Printf("* HTTPS Server port: %d\n", *fTLSPort)

	var http dev.HTTPServer

	http.Address = fmt.Sprintf("127.0.0.1:%d", *fHTTPPort)
	http.TLSAddress = fmt.Sprintf("127.0.0.1:%d", *fTLSPort)
	http.Pool = &pool
	http.Debug = *fDebug
	http.Events = &events

	http.Setup()

	fmt.Printf("! Puma dev listening on http and https\n")

	go http.ServeTLS()

	err = http.Serve()
	if err != nil {
		log.Fatalf("Error listening: %s", err)
	}
}
<commit_msg>Listen on 0.0.0.0. Fixes #65<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/mitchellh/go-homedir"

	"github.com/puma/puma-dev/dev"
)

var (
	fDebug    = flag.Bool("debug", false, "enable debug output")
	fDomains  = flag.String("d", "dev", "domains to handle, separate with :")
	fHTTPPort = flag.Int("http-port", 9280, "port to listen on http for")
	fTLSPort  = flag.Int("https-port", 9283, "port to listen on https for")
	fSysBind  = flag.Bool("sysbind", false, "bind to ports 80 and 443")
	fDir      = flag.String("dir", "~/.puma-dev", "directory to watch for apps")
	fTimeout  = flag.Duration("timeout", 15*60*time.Second, "how long to let an app idle for")
)

func main() {
	flag.Parse()

	allCheck()

	domains := strings.Split(*fDomains, ":")

	if *fSysBind {
		*fHTTPPort = 80
		*fTLSPort = 443
	}

	dir, err := homedir.Expand(*fDir)
	if err != nil {
		log.Fatalf("Unable to expand dir: %s", err)
	}

	err = os.MkdirAll(dir, 0755)
	if err != nil {
		log.Fatalf("Unable to create dir '%s': %s", dir, err)
	}

	var events dev.Events

	var pool dev.AppPool
	pool.Dir = dir
	pool.IdleTime = *fTimeout
	pool.Events = &events

	purge := make(chan os.Signal, 1)

	signal.Notify(purge, syscall.SIGUSR1)

	go func() {
		for {
			<-purge
			pool.Purge()
		}
	}()

	stop := make(chan os.Signal, 1)

	signal.Notify(stop, os.Interrupt, syscall.SIGQUIT, syscall.SIGTERM)

	go func() {
		<-stop
		fmt.Printf("! Shutdown requested\n")
		pool.Purge()
		os.Exit(0)
	}()

	err = dev.SetupOurCert()
	if err != nil {
		log.Fatalf("Unable to setup TLS cert: %s", err)
	}

	fmt.Printf("* Directory for apps: %s\n", dir)
	fmt.Printf("* Domains: %s\n", strings.Join(domains, ", "))
	fmt.Printf("* HTTP Server port: %d\n", *fHTTPPort)
	fmt.Printf("* HTTPS Server port: %d\n", *fTLSPort)

	var http dev.HTTPServer

	http.Address = fmt.Sprintf(":%d", *fHTTPPort)
	http.TLSAddress = fmt.Sprintf(":%d", *fTLSPort)
	http.Pool = &pool
	http.Debug = *fDebug
	http.Events = &events

	http.Setup()

	fmt.Printf("! Puma dev listening on http and https\n")

	go http.ServeTLS()

	err = http.Serve()
	if err != nil {
		log.Fatalf("Error listening: %s", err)
	}
}
<|endoftext|>