<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/hashicorp/consul/api"
	"github.com/nu7hatch/gouuid"

	"code.cloudfoundry.org/auctioneer"
	"code.cloudfoundry.org/auctioneer/auctionmetricemitterdelegate"
	"code.cloudfoundry.org/auctioneer/auctionrunnerdelegate"
	"code.cloudfoundry.org/auctioneer/handlers"
	"code.cloudfoundry.org/bbs"
	"code.cloudfoundry.org/consuladapter"
	"code.cloudfoundry.org/locket"
	"code.cloudfoundry.org/rep"
	"github.com/cloudfoundry-incubator/cf-debug-server"
	cf_lager "github.com/cloudfoundry-incubator/cf-lager"
	"github.com/cloudfoundry-incubator/cf_http"
	"github.com/pivotal-golang/lager"
	"github.com/pivotal-golang/localip"

	"code.cloudfoundry.org/auction/auctionrunner"
	"code.cloudfoundry.org/auction/auctiontypes"
	"github.com/cloudfoundry/dropsonde"
	"github.com/cloudfoundry/gunk/workpool"
	"github.com/pivotal-golang/clock"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"
)

var communicationTimeout = flag.Duration(
	"communicationTimeout",
	10*time.Second,
	"Timeout applied to all HTTP requests.",
)

var cellStateTimeout = flag.Duration(
	"cellStateTimeout",
	1*time.Second,
	"Timeout applied to HTTP requests to the Cell State endpoint.",
)

var consulCluster = flag.String(
	"consulCluster",
	"",
	"comma-separated list of consul server addresses (ip:port)",
)

var dropsondePort = flag.Int(
	"dropsondePort",
	3457,
	"port the local metron agent is listening on",
)

var lockTTL = flag.Duration(
	"lockTTL",
	locket.LockTTL,
	"TTL for service lock",
)

var lockRetryInterval = flag.Duration(
	"lockRetryInterval",
	locket.RetryInterval,
	"interval to wait before retrying a failed lock acquisition",
)

var listenAddr = flag.String(
	"listenAddr",
	"0.0.0.0:9016",
	"host:port to serve auction and LRP stop requests on",
)

var bbsAddress = flag.String(
	"bbsAddress",
	"",
	"Address to the BBS Server",
)

var bbsCACert = flag.String(
	"bbsCACert",
	"",
	"path to certificate authority cert used for mutually authenticated TLS BBS communication",
)

var bbsClientCert = flag.String(
	"bbsClientCert",
	"",
	"path to client cert used for mutually authenticated TLS BBS communication",
)

var bbsClientKey = flag.String(
	"bbsClientKey",
	"",
	"path to client key used for mutually authenticated TLS BBS communication",
)

var bbsClientSessionCacheSize = flag.Int(
	"bbsClientSessionCacheSize",
	0,
	"Capacity of the ClientSessionCache option on the TLS configuration. If zero, golang's default will be used",
)

var bbsMaxIdleConnsPerHost = flag.Int(
	"bbsMaxIdleConnsPerHost",
	0,
	"Controls the maximum number of idle (keep-alive) connctions per host. If zero, golang's default will be used",
)

var auctionRunnerWorkers = flag.Int(
	"auctionRunnerWorkers",
	1000,
	"Max concurrency for cell operations in the auction runner",
)

var startingContainerWeight = flag.Float64(
	"startingContainerWeight",
	0.25,
	"Factor to bias against cells with starting containers (0.0 - 1.0)",
)

const (
	auctionRunnerTimeout = 10 * time.Second
	dropsondeOrigin      = "auctioneer"
	serverProtocol       = "http"
)

func main() {
	cf_debug_server.AddFlags(flag.CommandLine)
	cf_lager.AddFlags(flag.CommandLine)
	flag.Parse()

	cf_http.Initialize(*communicationTimeout)

	logger, reconfigurableSink := cf_lager.New("auctioneer")
	initializeDropsonde(logger)

	if err := validateBBSAddress(); err != nil {
		logger.Fatal("invalid-bbs-address", err)
	}

	consulClient, err := consuladapter.NewClientFromUrl(*consulCluster)
	if err != nil {
		logger.Fatal("new-client-failed", err)
	}

	port, err := strconv.Atoi(strings.Split(*listenAddr, ":")[1])
	if err != nil {
		logger.Fatal("invalid-port", err)
	}

	clock := clock.NewClock()
	auctioneerServiceClient := auctioneer.NewServiceClient(consulClient, clock)

	auctionRunner := initializeAuctionRunner(logger, *cellStateTimeout,
		initializeBBSClient(logger), *startingContainerWeight)
	auctionServer := initializeAuctionServer(logger, auctionRunner)
	lockMaintainer := initializeLockMaintainer(logger, auctioneerServiceClient, port)
	registrationRunner := initializeRegistrationRunner(logger, consulClient, clock, port)

	members := grouper.Members{
		{"lock-maintainer", lockMaintainer},
		{"auction-runner", auctionRunner},
		{"auction-server", auctionServer},
		{"registration-runner", registrationRunner},
	}

	if dbgAddr := cf_debug_server.DebugAddress(flag.CommandLine); dbgAddr != "" {
		members = append(grouper.Members{
			{"debug-server", cf_debug_server.Runner(dbgAddr, reconfigurableSink)},
		}, members...)
	}

	group := grouper.NewOrdered(os.Interrupt, members)

	monitor := ifrit.Invoke(sigmon.New(group))

	logger.Info("started")

	err = <-monitor.Wait()
	if err != nil {
		logger.Error("exited-with-failure", err)
		os.Exit(1)
	}

	logger.Info("exited")
}

func initializeAuctionRunner(logger lager.Logger, cellStateTimeout time.Duration, bbsClient bbs.InternalClient, startingContainerWeight float64) auctiontypes.AuctionRunner {
	httpClient := cf_http.NewClient()
	stateClient := cf_http.NewCustomTimeoutClient(cellStateTimeout)
	repClientFactory := rep.NewClientFactory(httpClient, stateClient)

	delegate := auctionrunnerdelegate.New(repClientFactory, bbsClient, logger)
	metricEmitter := auctionmetricemitterdelegate.New()
	workPool, err := workpool.NewWorkPool(*auctionRunnerWorkers)
	if err != nil {
		logger.Fatal("failed-to-construct-auction-runner-workpool", err, lager.Data{"num-workers": *auctionRunnerWorkers}) // should never happen
	}

	return auctionrunner.New(
		logger,
		delegate,
		metricEmitter,
		clock.NewClock(),
		workPool,
		startingContainerWeight,
	)
}

func initializeDropsonde(logger lager.Logger) {
	dropsondeDestination := fmt.Sprint("localhost:", *dropsondePort)
	err := dropsonde.Initialize(dropsondeDestination, dropsondeOrigin)
	if err != nil {
		logger.Error("failed to initialize dropsonde: %v", err)
	}
}

func initializeAuctionServer(logger lager.Logger, runner auctiontypes.AuctionRunner) ifrit.Runner {
	return http_server.New(*listenAddr, handlers.New(runner, logger))
}

func initializeRegistrationRunner(logger lager.Logger, consulClient consuladapter.Client, clock clock.Clock, port int) ifrit.Runner {
	registration := &api.AgentServiceRegistration{
		Name: "auctioneer",
		Port: port,
		Check: &api.AgentServiceCheck{
			TTL: "3s",
		},
	}
	return locket.NewRegistrationRunner(logger, registration, consulClient, locket.RetryInterval, clock)
}

func initializeLockMaintainer(logger lager.Logger, serviceClient auctioneer.ServiceClient, port int) ifrit.Runner {
	uuid, err := uuid.NewV4()
	if err != nil {
		logger.Fatal("Couldn't generate uuid", err)
	}

	localIP, err := localip.LocalIP()
	if err != nil {
		logger.Fatal("Couldn't determine local IP", err)
	}

	address := fmt.Sprintf("%s://%s:%d", serverProtocol, localIP, port)
	auctioneerPresence := auctioneer.NewPresence(uuid.String(), address)

	lockMaintainer, err := serviceClient.NewAuctioneerLockRunner(logger, auctioneerPresence, *lockRetryInterval, *lockTTL)
	if err != nil {
		logger.Fatal("Couldn't create lock maintainer", err)
	}

	return lockMaintainer
}

func validateBBSAddress() error {
	if *bbsAddress == "" {
		return errors.New("bbsAddress is required")
	}
	return nil
}

func initializeBBSClient(logger lager.Logger) bbs.InternalClient {
	bbsURL, err := url.Parse(*bbsAddress)
	if err != nil {
		logger.Fatal("Invalid BBS URL", err)
	}

	if bbsURL.Scheme != "https" {
		return bbs.NewClient(*bbsAddress)
	}

	bbsClient, err := bbs.NewSecureClient(*bbsAddress, *bbsCACert, *bbsClientCert, *bbsClientKey, *bbsClientSessionCacheSize, *bbsMaxIdleConnsPerHost)
	if err != nil {
		logger.Fatal("Failed to configure secure BBS client", err)
	}
	return bbsClient
}
<commit_msg>Update and rename cf-debug-server -> debugserver<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/hashicorp/consul/api"
	"github.com/nu7hatch/gouuid"

	"code.cloudfoundry.org/auctioneer"
	"code.cloudfoundry.org/auctioneer/auctionmetricemitterdelegate"
	"code.cloudfoundry.org/auctioneer/auctionrunnerdelegate"
	"code.cloudfoundry.org/auctioneer/handlers"
	"code.cloudfoundry.org/bbs"
	"code.cloudfoundry.org/consuladapter"
	"code.cloudfoundry.org/debugserver"
	"code.cloudfoundry.org/locket"
	"code.cloudfoundry.org/rep"
	cf_lager "github.com/cloudfoundry-incubator/cf-lager"
	"github.com/cloudfoundry-incubator/cf_http"
	"github.com/pivotal-golang/lager"
	"github.com/pivotal-golang/localip"

	"code.cloudfoundry.org/auction/auctionrunner"
	"code.cloudfoundry.org/auction/auctiontypes"
	"github.com/cloudfoundry/dropsonde"
	"github.com/cloudfoundry/gunk/workpool"
	"github.com/pivotal-golang/clock"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"
)

var communicationTimeout = flag.Duration(
	"communicationTimeout",
	10*time.Second,
	"Timeout applied to all HTTP requests.",
)

var cellStateTimeout = flag.Duration(
	"cellStateTimeout",
	1*time.Second,
	"Timeout applied to HTTP requests to the Cell State endpoint.",
)

var consulCluster = flag.String(
	"consulCluster",
	"",
	"comma-separated list of consul server addresses (ip:port)",
)

var dropsondePort = flag.Int(
	"dropsondePort",
	3457,
	"port the local metron agent is listening on",
)

var lockTTL = flag.Duration(
	"lockTTL",
	locket.LockTTL,
	"TTL for service lock",
)

var lockRetryInterval = flag.Duration(
	"lockRetryInterval",
	locket.RetryInterval,
	"interval to wait before retrying a failed lock acquisition",
)

var listenAddr = flag.String(
	"listenAddr",
	"0.0.0.0:9016",
	"host:port to serve auction and LRP stop requests on",
)

var bbsAddress = flag.String(
	"bbsAddress",
	"",
	"Address to the BBS Server",
)

var bbsCACert = flag.String(
	"bbsCACert",
	"",
	"path to certificate authority cert used for mutually authenticated TLS BBS communication",
)

var bbsClientCert = flag.String(
	"bbsClientCert",
	"",
	"path to client cert used for mutually authenticated TLS BBS communication",
)

var bbsClientKey = flag.String(
	"bbsClientKey",
	"",
	"path to client key used for mutually authenticated TLS BBS communication",
)

var bbsClientSessionCacheSize = flag.Int(
	"bbsClientSessionCacheSize",
	0,
	"Capacity of the ClientSessionCache option on the TLS configuration. If zero, golang's default will be used",
)

var bbsMaxIdleConnsPerHost = flag.Int(
	"bbsMaxIdleConnsPerHost",
	0,
	"Controls the maximum number of idle (keep-alive) connctions per host. If zero, golang's default will be used",
)

var auctionRunnerWorkers = flag.Int(
	"auctionRunnerWorkers",
	1000,
	"Max concurrency for cell operations in the auction runner",
)

var startingContainerWeight = flag.Float64(
	"startingContainerWeight",
	0.25,
	"Factor to bias against cells with starting containers (0.0 - 1.0)",
)

const (
	auctionRunnerTimeout = 10 * time.Second
	dropsondeOrigin      = "auctioneer"
	serverProtocol       = "http"
)

func main() {
	debugserver.AddFlags(flag.CommandLine)
	cf_lager.AddFlags(flag.CommandLine)
	flag.Parse()

	cf_http.Initialize(*communicationTimeout)

	logger, reconfigurableSink := cf_lager.New("auctioneer")
	initializeDropsonde(logger)

	if err := validateBBSAddress(); err != nil {
		logger.Fatal("invalid-bbs-address", err)
	}

	consulClient, err := consuladapter.NewClientFromUrl(*consulCluster)
	if err != nil {
		logger.Fatal("new-client-failed", err)
	}

	port, err := strconv.Atoi(strings.Split(*listenAddr, ":")[1])
	if err != nil {
		logger.Fatal("invalid-port", err)
	}

	clock := clock.NewClock()
	auctioneerServiceClient := auctioneer.NewServiceClient(consulClient, clock)

	auctionRunner := initializeAuctionRunner(logger, *cellStateTimeout,
		initializeBBSClient(logger), *startingContainerWeight)
	auctionServer := initializeAuctionServer(logger, auctionRunner)
	lockMaintainer := initializeLockMaintainer(logger, auctioneerServiceClient, port)
	registrationRunner := initializeRegistrationRunner(logger, consulClient, clock, port)

	members := grouper.Members{
		{"lock-maintainer", lockMaintainer},
		{"auction-runner", auctionRunner},
		{"auction-server", auctionServer},
		{"registration-runner", registrationRunner},
	}

	if dbgAddr := debugserver.DebugAddress(flag.CommandLine); dbgAddr != "" {
		members = append(grouper.Members{
			{"debug-server", debugserver.Runner(dbgAddr, reconfigurableSink)},
		}, members...)
	}

	group := grouper.NewOrdered(os.Interrupt, members)

	monitor := ifrit.Invoke(sigmon.New(group))

	logger.Info("started")

	err = <-monitor.Wait()
	if err != nil {
		logger.Error("exited-with-failure", err)
		os.Exit(1)
	}

	logger.Info("exited")
}

func initializeAuctionRunner(logger lager.Logger, cellStateTimeout time.Duration, bbsClient bbs.InternalClient, startingContainerWeight float64) auctiontypes.AuctionRunner {
	httpClient := cf_http.NewClient()
	stateClient := cf_http.NewCustomTimeoutClient(cellStateTimeout)
	repClientFactory := rep.NewClientFactory(httpClient, stateClient)

	delegate := auctionrunnerdelegate.New(repClientFactory, bbsClient, logger)
	metricEmitter := auctionmetricemitterdelegate.New()
	workPool, err := workpool.NewWorkPool(*auctionRunnerWorkers)
	if err != nil {
		logger.Fatal("failed-to-construct-auction-runner-workpool", err, lager.Data{"num-workers": *auctionRunnerWorkers}) // should never happen
	}

	return auctionrunner.New(
		logger,
		delegate,
		metricEmitter,
		clock.NewClock(),
		workPool,
		startingContainerWeight,
	)
}

func initializeDropsonde(logger lager.Logger) {
	dropsondeDestination := fmt.Sprint("localhost:", *dropsondePort)
	err := dropsonde.Initialize(dropsondeDestination, dropsondeOrigin)
	if err != nil {
		logger.Error("failed to initialize dropsonde: %v", err)
	}
}

func initializeAuctionServer(logger lager.Logger, runner auctiontypes.AuctionRunner) ifrit.Runner {
	return http_server.New(*listenAddr, handlers.New(runner, logger))
}

func initializeRegistrationRunner(logger lager.Logger, consulClient consuladapter.Client, clock clock.Clock, port int) ifrit.Runner {
	registration := &api.AgentServiceRegistration{
		Name: "auctioneer",
		Port: port,
		Check: &api.AgentServiceCheck{
			TTL: "3s",
		},
	}
	return locket.NewRegistrationRunner(logger, registration, consulClient, locket.RetryInterval, clock)
}

func initializeLockMaintainer(logger lager.Logger, serviceClient auctioneer.ServiceClient, port int) ifrit.Runner {
	uuid, err := uuid.NewV4()
	if err != nil {
		logger.Fatal("Couldn't generate uuid", err)
	}

	localIP, err := localip.LocalIP()
	if err != nil {
		logger.Fatal("Couldn't determine local IP", err)
	}

	address := fmt.Sprintf("%s://%s:%d", serverProtocol, localIP, port)
	auctioneerPresence := auctioneer.NewPresence(uuid.String(), address)

	lockMaintainer, err := serviceClient.NewAuctioneerLockRunner(logger, auctioneerPresence, *lockRetryInterval, *lockTTL)
	if err != nil {
		logger.Fatal("Couldn't create lock maintainer", err)
	}

	return lockMaintainer
}

func validateBBSAddress() error {
	if *bbsAddress == "" {
		return errors.New("bbsAddress is required")
	}
	return nil
}

func initializeBBSClient(logger lager.Logger) bbs.InternalClient {
	bbsURL, err := url.Parse(*bbsAddress)
	if err != nil {
		logger.Fatal("Invalid BBS URL", err)
	}

	if bbsURL.Scheme != "https" {
		return bbs.NewClient(*bbsAddress)
	}

	bbsClient, err := bbs.NewSecureClient(*bbsAddress, *bbsCACert, *bbsClientCert, *bbsClientKey, *bbsClientSessionCacheSize, *bbsMaxIdleConnsPerHost)
	if err != nil {
		logger.Fatal("Failed to configure secure BBS client", err)
	}
	return bbsClient
}
<|endoftext|>


<commit_before>package catalog

import (
	"fmt"
	"strconv"
	"strings"
)

func CommentString(
	intNames, floatNames []string, order, sizes []int,
) string {

	tokens := []string{"# Column contents:"}
	for i := range intNames {
		tokens = append(tokens, fmt.Sprintf("%s", intNames[i]))
	}
	for i := range floatNames {
		tokens = append(tokens, fmt.Sprintf("%s", floatNames[i]))
	}

	orderedTokens := []string{tokens[0]}
	orderedSizes := []int{}
	for _, idx := range order {
		if idx >= len(intNames)+len(floatNames) {
			panic("Column ordering out of range.")
		}

		orderedTokens = append(orderedTokens, tokens[idx+1])
		orderedSizes = append(orderedSizes, sizes[idx])

	}

	n := 0
	for i := 1; i < len(orderedTokens); i++ {
		if orderedSizes[i-1] == 1 {
			orderedTokens[i] = fmt.Sprintf("%s(%d)", orderedTokens[i], n)
		} else {
			orderedTokens[i] = fmt.Sprintf("%s(%d-%d)", orderedTokens[i],
				n, n+orderedSizes[i-1]-1)
		}
		n += orderedSizes[i-1]
	}

	return strings.Join(orderedTokens, " ")
}

func FormatCols(intCols [][]int, floatCols [][]float64, order []int) []string {
	if (len(intCols) == 0 && len(floatCols) == 0) ||
		(len(intCols) > 0 && len(intCols[0]) == 0) ||
		(len(floatCols) > 0 && len(floatCols[0]) == 0) {
		return []string{}
	}

	formattedIntCols := make([][]string, len(intCols))
	formattedFloatCols := make([][]string, len(floatCols))

	height := -1
	for i := range intCols {
		formattedIntCols[i] = formatIntCol(intCols[i])
		if height == -1 {
			height = len(intCols[i])
		} else if height != len(intCols[i]) {
			panic("Columns of unequal height.")
		}
	}

	for i := range floatCols {
		formattedFloatCols[i] = formatFloatCol(floatCols[i])
		if height == -1 {
			height = len(floatCols[i])
		} else if height != len(floatCols[i]) {
			panic("Columns of unequal height.")
		}
	}

	orderedCols := [][]string{}
	for _, idx := range order {
		if idx >= len(intCols)+len(floatCols) {
			panic("Column ordering out of range.")
		}

		if idx < len(intCols) {
			orderedCols = append(orderedCols, formattedIntCols[idx])
		} else {
			idx -= len(intCols)
			orderedCols = append(orderedCols, formattedFloatCols[idx])
		}
	}

	lines := []string{}
	tokens := make([]string, len(intCols)+len(floatCols))
	for i := 0; i < height; i++ {
		for j := range orderedCols {
			tokens[j] = orderedCols[j][i]
		}
		line := strings.Join(tokens, " ")
		lines = append(lines, line)
	}

	return lines
}

func formatIntCol(col []int) []string {
	width := len(fmt.Sprintf("%d", col[0]))
	for i := 1; i < len(col); i++ {
		n := len(fmt.Sprintf("%d", col[i]))
		if n > width {
			width = n
		}
	}

	out := []string{}
	for i := range col {
		out = append(out, fmt.Sprintf("%*d", width, col[i]))
	}

	return out
}

func formatFloatCol(col []float64) []string {
	width := len(fmt.Sprintf("%.6g", col[0]))
	for i := 1; i < len(col); i++ {
		n := len(fmt.Sprintf("%.6g", col[i]))
		if n > width {
			width = n
		}
	}

	out := []string{}
	for i := range col {
		out = append(out, fmt.Sprintf("%*.6g", width, col[i]))
	}

	return out
}

func Uncomment(lines []string) (out []string, lineNums []int) {
	for i := range lines {
		idx := strings.Index(lines[i], "#")
		if idx >= 0 {
			lines[i] = lines[i][:idx]
		}
	}

	out = []string{}
	lineNums = []int{}
	for i := range lines {
		trimmed := strings.Trim(lines[i], " \t")
		if len(trimmed) > 0 {
			out = append(out, trimmed)
			lineNums = append(lineNums, i+1)
		}
	}
	return out, lineNums
}

func ParseCols(
	lines []string, intIdxs, floatIdxs []int,
) ([][]int, [][]float64, error) {
	if len(intIdxs) == 0 && len(floatIdxs) == 0 {
		return nil, nil, nil
	}

	fLines, lineNums := Uncomment(lines)
	minWidth := -1
	for _, x := range intIdxs {
		if x > minWidth {
			minWidth = x
		}
	}
	for _, x := range floatIdxs {
		if x > minWidth {
			minWidth = x
		}
	}
	minWidth++

	intCols := make([][]int, len(intIdxs))
	floatCols := make([][]float64, len(floatIdxs))

	for i := range fLines {
		toks := tokenize(fLines[i])

		if len(toks) < minWidth {
			return nil, nil, fmt.Errorf(
				"Line %d has %d columns, but I need %d columns.",
				lineNums[i], len(toks), minWidth,
			)
		} else {
			for colIdx, j := range intIdxs {
				n, err := strconv.Atoi(toks[j])
				if err != nil {
					return nil, nil, fmt.Errorf("Cannot parse column %d of "+
						"line %d, '%s', to an int.", j, lineNums[i], toks[j])
				}
				intCols[colIdx] = append(intCols[j], n)
			}

			for colIdx, j := range floatIdxs {
				x, err := strconv.ParseFloat(toks[j], 64)
				if err != nil {
					return nil, nil, fmt.Errorf("Cannot parse column %d of "+
						"line %d, '%s', to a float.", j, lineNums[i], toks[j])
				}
				floatCols[colIdx] = append(floatCols[colIdx], x)
			}
		}
	}

	return intCols, floatCols, nil
}

func tokenize(line string) []string {
	toks := strings.Split(line, " ")
	fToks := []string{}
	for i := range toks {
		if len(toks[i]) > 0 {
			fToks = append(fToks, toks[i])
		}
	}
	return fToks
}
<commit_msg>Fixed bug in parsing of int columns.<commit_after>package catalog

import (
	"fmt"
	"strconv"
	"strings"
)

func CommentString(
	intNames, floatNames []string, order, sizes []int,
) string {

	tokens := []string{"# Column contents:"}
	for i := range intNames {
		tokens = append(tokens, fmt.Sprintf("%s", intNames[i]))
	}
	for i := range floatNames {
		tokens = append(tokens, fmt.Sprintf("%s", floatNames[i]))
	}

	orderedTokens := []string{tokens[0]}
	orderedSizes := []int{}
	for _, idx := range order {
		if idx >= len(intNames)+len(floatNames) {
			panic("Column ordering out of range.")
		}

		orderedTokens = append(orderedTokens, tokens[idx+1])
		orderedSizes = append(orderedSizes, sizes[idx])

	}

	n := 0
	for i := 1; i < len(orderedTokens); i++ {
		if orderedSizes[i-1] == 1 {
			orderedTokens[i] = fmt.Sprintf("%s(%d)", orderedTokens[i], n)
		} else {
			orderedTokens[i] = fmt.Sprintf("%s(%d-%d)", orderedTokens[i],
				n, n+orderedSizes[i-1]-1)
		}
		n += orderedSizes[i-1]
	}

	return strings.Join(orderedTokens, " ")
}

func FormatCols(intCols [][]int, floatCols [][]float64, order []int) []string {
	if (len(intCols) == 0 && len(floatCols) == 0) ||
		(len(intCols) > 0 && len(intCols[0]) == 0) ||
		(len(floatCols) > 0 && len(floatCols[0]) == 0) {
		return []string{}
	}

	formattedIntCols := make([][]string, len(intCols))
	formattedFloatCols := make([][]string, len(floatCols))

	height := -1
	for i := range intCols {
		formattedIntCols[i] = formatIntCol(intCols[i])
		if height == -1 {
			height = len(intCols[i])
		} else if height != len(intCols[i]) {
			panic("Columns of unequal height.")
		}
	}

	for i := range floatCols {
		formattedFloatCols[i] = formatFloatCol(floatCols[i])
		if height == -1 {
			height = len(floatCols[i])
		} else if height != len(floatCols[i]) {
			panic("Columns of unequal height.")
		}
	}

	orderedCols := [][]string{}
	for _, idx := range order {
		if idx >= len(intCols)+len(floatCols) {
			panic("Column ordering out of range.")
		}

		if idx < len(intCols) {
			orderedCols = append(orderedCols, formattedIntCols[idx])
		} else {
			idx -= len(intCols)
			orderedCols = append(orderedCols, formattedFloatCols[idx])
		}
	}

	lines := []string{}
	tokens := make([]string, len(intCols)+len(floatCols))
	for i := 0; i < height; i++ {
		for j := range orderedCols {
			tokens[j] = orderedCols[j][i]
		}
		line := strings.Join(tokens, " ")
		lines = append(lines, line)
	}

	return lines
}

func formatIntCol(col []int) []string {
	width := len(fmt.Sprintf("%d", col[0]))
	for i := 1; i < len(col); i++ {
		n := len(fmt.Sprintf("%d", col[i]))
		if n > width {
			width = n
		}
	}

	out := []string{}
	for i := range col {
		out = append(out, fmt.Sprintf("%*d", width, col[i]))
	}

	return out
}

func formatFloatCol(col []float64) []string {
	width := len(fmt.Sprintf("%.6g", col[0]))
	for i := 1; i < len(col); i++ {
		n := len(fmt.Sprintf("%.6g", col[i]))
		if n > width {
			width = n
		}
	}

	out := []string{}
	for i := range col {
		out = append(out, fmt.Sprintf("%*.6g", width, col[i]))
	}

	return out
}

func Uncomment(lines []string) (out []string, lineNums []int) {
	for i := range lines {
		idx := strings.Index(lines[i], "#")
		if idx >= 0 {
			lines[i] = lines[i][:idx]
		}
	}

	out = []string{}
	lineNums = []int{}
	for i := range lines {
		trimmed := strings.Trim(lines[i], " \t")
		if len(trimmed) > 0 {
			out = append(out, trimmed)
			lineNums = append(lineNums, i+1)
		}
	}
	return out, lineNums
}

func ParseCols(
	lines []string, intIdxs, floatIdxs []int,
) ([][]int, [][]float64, error) {
	if len(intIdxs) == 0 && len(floatIdxs) == 0 {
		return nil, nil, nil
	}

	fLines, lineNums := Uncomment(lines)
	minWidth := -1
	for _, x := range intIdxs {
		if x > minWidth {
			minWidth = x
		}
	}
	for _, x := range floatIdxs {
		if x > minWidth {
			minWidth = x
		}
	}
	minWidth++

	intCols := make([][]int, len(intIdxs))
	floatCols := make([][]float64, len(floatIdxs))

	for i := range fLines {
		toks := tokenize(fLines[i])

		if len(toks) < minWidth {
			return nil, nil, fmt.Errorf(
				"Line %d has %d columns, but I need %d columns.",
				lineNums[i], len(toks), minWidth,
			)
		} else {
			for colIdx, j := range intIdxs {
				n, err := strconv.Atoi(toks[j])
				if err != nil {
					return nil, nil, fmt.Errorf("Cannot parse column %d of "+
						"line %d, '%s', to an int.", j, lineNums[i], toks[j])
				}
				intCols[colIdx] = append(intCols[colIdx], n)
			}

			for colIdx, j := range floatIdxs {
				x, err := strconv.ParseFloat(toks[j], 64)
				if err != nil {
					return nil, nil, fmt.Errorf("Cannot parse column %d of "+
						"line %d, '%s', to a float.", j, lineNums[i], toks[j])
				}
				floatCols[colIdx] = append(floatCols[colIdx], x)
			}
		}
	}

	return intCols, floatCols, nil
}

func tokenize(line string) []string {
	toks := strings.Split(line, " ")
	fToks := []string{}
	for i := range toks {
		if len(toks[i]) > 0 {
			fToks = append(fToks, toks[i])
		}
	}
	return fToks
}
<|endoftext|>


<commit_before>package cliedit

import (
	"fmt"
	"os"

	"github.com/elves/elvish/cli"
	"github.com/elves/elvish/diag"
	"github.com/elves/elvish/eval"
	"github.com/elves/elvish/eval/vals"
	"github.com/elves/elvish/eval/vars"
	"github.com/xiaq/persistent/hashmap"
)

func initAPI(app *cli.App, ev *eval.Evaler, ns eval.Ns) {
	initMaxHeight(app, ns)
	initBeforeReadline(app, ev, ns)
	initAfterReadline(app, ev, ns)
	initInsert(app, ev, ns)

	initMiscBuiltins(app, ns)
	initBufferBuiltins(app, ns)
}

func initMaxHeight(app *cli.App, ns eval.Ns) {
	maxHeight := -1
	maxHeightVar := vars.FromPtr(&maxHeight)
	app.Config.MaxHeight = func() int { return maxHeightVar.Get().(int) }
	ns.Add("max-height", maxHeightVar)
}

func initBeforeReadline(app *cli.App, ev *eval.Evaler, ns eval.Ns) {
	hook := vals.EmptyList
	hookVar := vars.FromPtr(&hook)
	ns["before-readline"] = hookVar
	app.Config.BeforeReadline = func() {
		i := -1
		hook := hookVar.Get().(vals.List)
		for it := hook.Iterator(); it.HasElem(); it.Next() {
			i++
			name := fmt.Sprintf("$before-readline[%d]", i)
			fn, ok := it.Elem().(eval.Callable)
			if !ok {
				// TODO(xiaq): This is not testable as it depends on stderr.
				// Make it testable.
				diag.Complainf("%s not function", name)
				continue
			}
			// TODO(xiaq): This should use stdPorts, but stdPorts is currently
			// unexported from eval.
			ports := []*eval.Port{
				{File: os.Stdin}, {File: os.Stdout}, {File: os.Stderr}}
			fm := eval.NewTopFrame(ev, eval.NewInternalSource(name), ports)
			fm.Call(fn, eval.NoArgs, eval.NoOpts)
		}
	}
}

func initAfterReadline(app *cli.App, ev *eval.Evaler, ns eval.Ns) {
	hook := vals.EmptyList
	hookVar := vars.FromPtr(&hook)
	ns["after-readline"] = hookVar
	app.Config.AfterReadline = func(code string) {
		i := -1
		hook := hookVar.Get().(vals.List)
		for it := hook.Iterator(); it.HasElem(); it.Next() {
			i++
			name := fmt.Sprintf("$after-readline[%d]", i)
			fn, ok := it.Elem().(eval.Callable)
			if !ok {
				// TODO(xiaq): This is not testable as it depends on stderr.
				// Make it testable.
				diag.Complainf("%s not function", name)
				continue
			}
			// TODO(xiaq): This should use stdPorts, but stdPorts is currently
			// unexported from eval.
			ports := []*eval.Port{
				{File: os.Stdin}, {File: os.Stdout}, {File: os.Stderr}}
			fm := eval.NewTopFrame(ev, eval.NewInternalSource(name), ports)
			fm.Call(fn, []interface{}{code}, eval.NoOpts)
		}
	}
}

func initInsert(app *cli.App, ev *eval.Evaler, ns eval.Ns) {
	abbr := vals.EmptyMap
	abbrVar := vars.FromPtr(&abbr)
	app.CodeArea.Abbreviations = makeMapIterator(abbrVar)

	// TODO(xiaq): Synchronize properly.
	binding := emptyBindingMap
	bindingVar := vars.FromPtr(&binding)
	app.CodeArea.OverlayHandler = newMapBinding(app, ev, &binding)

	quotePaste := false
	quotePasteVar := vars.FromPtr(&quotePaste)
	app.CodeArea.QuotePaste = func() bool { return quotePasteVar.Get().(bool) }

	ns.AddNs("insert", eval.Ns{
		"abbr":        abbrVar,
		"binding":     bindingVar,
		"quote-paste": quotePasteVar,
	})
}

func makeMapIterator(mv vars.Var) func(func(a, b string)) {
	return func(f func(a, b string)) {
		for it := mv.Get().(hashmap.Map).Iterator(); it.HasElem(); it.Next() {
			k, v := it.Elem()
			ks, kok := k.(string)
			vs, vok := v.(string)
			if !kok || !vok {
				continue
			}
			f(ks, vs)
		}
	}
}
<commit_msg>cliedit: Use PtrVar.GetRaw.<commit_after>package cliedit

import (
	"fmt"
	"os"

	"github.com/elves/elvish/cli"
	"github.com/elves/elvish/diag"
	"github.com/elves/elvish/eval"
	"github.com/elves/elvish/eval/vals"
	"github.com/elves/elvish/eval/vars"
	"github.com/xiaq/persistent/hashmap"
)

func initAPI(app *cli.App, ev *eval.Evaler, ns eval.Ns) {
	initMaxHeight(app, ns)
	initBeforeReadline(app, ev, ns)
	initAfterReadline(app, ev, ns)
	initInsert(app, ev, ns)

	initMiscBuiltins(app, ns)
	initBufferBuiltins(app, ns)
}

func initMaxHeight(app *cli.App, ns eval.Ns) {
	maxHeight := -1
	maxHeightVar := vars.FromPtr(&maxHeight)
	app.Config.MaxHeight = func() int { return maxHeightVar.GetRaw().(int) }
	ns.Add("max-height", maxHeightVar)
}

func initBeforeReadline(app *cli.App, ev *eval.Evaler, ns eval.Ns) {
	hook := vals.EmptyList
	hookVar := vars.FromPtr(&hook)
	ns["before-readline"] = hookVar
	app.Config.BeforeReadline = func() {
		i := -1
		hook := hookVar.GetRaw().(vals.List)
		for it := hook.Iterator(); it.HasElem(); it.Next() {
			i++
			name := fmt.Sprintf("$before-readline[%d]", i)
			fn, ok := it.Elem().(eval.Callable)
			if !ok {
				// TODO(xiaq): This is not testable as it depends on stderr.
				// Make it testable.
				diag.Complainf("%s not function", name)
				continue
			}
			// TODO(xiaq): This should use stdPorts, but stdPorts is currently
			// unexported from eval.
			ports := []*eval.Port{
				{File: os.Stdin}, {File: os.Stdout}, {File: os.Stderr}}
			fm := eval.NewTopFrame(ev, eval.NewInternalSource(name), ports)
			fm.Call(fn, eval.NoArgs, eval.NoOpts)
		}
	}
}

func initAfterReadline(app *cli.App, ev *eval.Evaler, ns eval.Ns) {
	hook := vals.EmptyList
	hookVar := vars.FromPtr(&hook)
	ns["after-readline"] = hookVar
	app.Config.AfterReadline = func(code string) {
		i := -1
		hook := hookVar.GetRaw().(vals.List)
		for it := hook.Iterator(); it.HasElem(); it.Next() {
			i++
			name := fmt.Sprintf("$after-readline[%d]", i)
			fn, ok := it.Elem().(eval.Callable)
			if !ok {
				// TODO(xiaq): This is not testable as it depends on stderr.
				// Make it testable.
				diag.Complainf("%s not function", name)
				continue
			}
			// TODO(xiaq): This should use stdPorts, but stdPorts is currently
			// unexported from eval.
			ports := []*eval.Port{
				{File: os.Stdin}, {File: os.Stdout}, {File: os.Stderr}}
			fm := eval.NewTopFrame(ev, eval.NewInternalSource(name), ports)
			fm.Call(fn, []interface{}{code}, eval.NoOpts)
		}
	}
}

func initInsert(app *cli.App, ev *eval.Evaler, ns eval.Ns) {
	abbr := vals.EmptyMap
	abbrVar := vars.FromPtr(&abbr)
	app.CodeArea.Abbreviations = makeMapIterator(abbrVar)

	// TODO(xiaq): Synchronize properly.
	binding := emptyBindingMap
	bindingVar := vars.FromPtr(&binding)
	app.CodeArea.OverlayHandler = newMapBinding(app, ev, &binding)

	quotePaste := false
	quotePasteVar := vars.FromPtr(&quotePaste)
	app.CodeArea.QuotePaste = func() bool { return quotePasteVar.GetRaw().(bool) }

	ns.AddNs("insert", eval.Ns{
		"abbr":        abbrVar,
		"binding":     bindingVar,
		"quote-paste": quotePasteVar,
	})
}

func makeMapIterator(mv vars.PtrVar) func(func(a, b string)) {
	return func(f func(a, b string)) {
		for it := mv.GetRaw().(hashmap.Map).Iterator(); it.HasElem(); it.Next() {
			k, v := it.Elem()
			ks, kok := k.(string)
			vs, vok := v.(string)
			if !kok || !vok {
				continue
			}
			f(ks, vs)
		}
	}
}
<|endoftext|>


<commit_before>/*
   - support for RelayState
   - does not seem to work: incl. Capitalization content-security-policy: referrer no-referrer;
   - redo no-referer - current version does not work !!!
   - MDQ lookup by location also for hub md
   - Trusted proxy
   - wayf:wayf i hub_ops metadata
        - AttributeNameFormat for Krib -> WAYF SPS - ie if none -> repeat wayf error both formats but error
        - schacHomeOrganization
        - schacHomeOrganizationType
   - collect schema errors
   - illegal attributes from IdP - ignore or provoke error
*/

package gohybrid

import (
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"github.com/gorilla/securecookie"
	"github.com/wayf-dk/go-libxml2/types"
	"github.com/wayf-dk/gosaml"
	"github.com/wayf-dk/goxml"
	"html/template"
	"log"
	"net/http"
	"net/url"
	"regexp"
	"sync"
)

type (
	formdata struct {
		Acs          string
		Samlresponse string
		RelayState   string
		Ard template.JS
	}

	idpsppair struct {
		idp string
		sp  string
	}

	Conf struct {
		DiscoveryService        string
		Domain                  string
		HubEntityID             string
		EptidSalt               string
		HubRequestedAttributes  *goxml.Xp
		Internal, External, Hub gosaml.Md
		SecureCookieHashKey     string
		PostFormTemplate        string
		AttributeReleaseTemplate        string
		Basic2uri               map[string]string
		StdTiming               gosaml.IdAndTiming
		ElementsToSign          []string
		AttributeHandler        func(*goxml.Xp, *goxml.Xp, *goxml.Xp, *goxml.Xp) (error, map[string][]string)
	}
)

const (
	idpCertQuery = `./md:IDPSSODescriptor/md:KeyDescriptor[@use="signing" or not(@use)]/ds:KeyInfo/ds:X509Data/ds:X509Certificate`
)

var (
	_     = log.Printf // For debugging; delete when done.
	_     = fmt.Printf
	remap = map[string]idpsppair{
		"https://nemlogin.wayf.dk": idpsppair{"https://saml.nemlog-in.dk", "https://nemlogin.wayf.dk"},
	}

	contextmutex sync.RWMutex
	context      = make(map[*http.Request]map[string]string)
	bify         = regexp.MustCompile("^(https?://)(.*)$")
	debify       = regexp.MustCompile("^(https?://)(?:(?:birk|krib)\\.wayf.dk/(?:birk|krib)\\.php/)(.+)$")

	postForm,attributeReleaseForm  *template.Template
	hashKey   []byte
	seccookie *securecookie.SecureCookie
	config    = Conf{}
)

func Config(configuration Conf) {
	config = configuration
	hashKey, _ := hex.DecodeString(config.SecureCookieHashKey)
	seccookie = securecookie.New(hashKey, nil)
	postForm = template.Must(template.New("post").Parse(config.PostFormTemplate))
	attributeReleaseForm = template.Must(template.New("post").Parse(config.AttributeReleaseTemplate))
}

func SsoService(w http.ResponseWriter, r *http.Request) (err error) {
	defer r.Body.Close()
	// handle non ok urls gracefully
	// var err error
	// check issuer and acs in md
	// receiveRequest -> request, issuer md, receiver md
	//     check for IDPList 1st in md, then in request then in query
	//     sanitize idp from query or request
	request, spmd, _, relayState, err := gosaml.ReceiveSAMLRequest(r, config.Internal, config.Hub)
	if err != nil {
		return
	}
	entityID := spmd.Query1(nil, "@entityID")
	idp := spmd.Query1(nil, "//IDPList/ProviderID") // Need to find a place for IDPList
	if idp == "" {
		idp = request.Query1(nil, "IDPList/ProviderID")
	}
	if idp == "" {
		idp = r.URL.Query().Get("idpentityid")
	}
	if idp == "" {
		data := url.Values{}
		data.Set("return", "https://"+r.Host+r.RequestURI)
		data.Set("returnIDParam", "idpentityid")
		data.Set("entityID", entityID)
		http.Redirect(w, r, config.DiscoveryService+data.Encode(), http.StatusFound)
	} else {
		var idpmd *goxml.Xp
		/**/
		// check overlap btw ad-hoc feds for the idp and the sp
		kribID := bify.ReplaceAllString(entityID, "${1}krib.wayf.dk/krib.php/$2")
		if kribID == entityID {
			kribID = "urn:oid:1.3.6.1.4.1.39153:42:" + entityID
		}

		request.QueryDashP(nil, "/saml:Issuer", kribID, nil)
		acs := request.Query1(nil, "@AssertionConsumerServiceURL")
		acsurl := bify.ReplaceAllString(acs, "${1}krib.wayf.dk/krib.php/$2")
		request.QueryDashP(nil, "@AssertionConsumerServiceURL", acsurl, nil)
		/**/
		idpmd, err = config.External.MDQ(idp)
		if err != nil {
			return
		}
		const ssoquery = "./md:IDPSSODescriptor/md:SingleSignOnService[@Binding='urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect']/@Location"
		ssoservice := idpmd.Query1(nil, ssoquery)
		if ssoservice == "" {

		}
		request.QueryDashP(nil, "@Destination", ssoservice, nil)
		u, _ := gosaml.SAMLRequest2Url(request, relayState, "", "", "")
		log.Println(request.Doc.Dump(true))
		http.Redirect(w, r, u.String(), http.StatusFound)
	}
	return
}

func BirkService(w http.ResponseWriter, r *http.Request) (err error) {
	// use incoming request for crafting the new one
	// remember to add the Scoping element to inform the IdP of requesterID - if stated in metadata for the IdP
	// check ad-hoc feds overlab
	defer r.Body.Close()
	// get the sp as well to check for allowed acs
	request, _, mdbirkidp, relayState, err := gosaml.ReceiveSAMLRequest(r, config.External, config.External)
	if err != nil {
		return
	}
	// Save the issuer and destination in a cookie for when the response comes back

	cookievalue, err := seccookie.Encode("BIRK", gosaml.Deflate(request.Doc.Dump(true)))
	http.SetCookie(w, &http.Cookie{Name: "BIRK", Value: cookievalue, Domain: config.Domain, Path: "/", Secure: true, HttpOnly: true})

	idp := debify.ReplaceAllString(mdbirkidp.Query1(nil, "@entityID"), "$1$2")

	var mdhub, mdidp *goxml.Xp
	// are we remapping - for now only use case is https://nemlogin.wayf.dk -> https://saml.nemlog-in.dk
	if rm, ok := remap[idp]; ok {
		mdidp, err = config.Internal.MDQ(rm.idp)
		mdhub, err = config.Internal.MDQ(rm.sp)
	} else {
		mdidp, err = config.Internal.MDQ(idp)
		mdhub, err = config.Hub.MDQ(config.HubEntityID)
	}
	// use a std request - we take care of NameID etc in acsService below
	newrequest := gosaml.NewAuthnRequest(config.StdTiming.Refresh(), mdhub, mdidp)
	u, _ := gosaml.SAMLRequest2Url(newrequest, relayState, "", "", "") // not signed so blank key, pw and algo
	http.Redirect(w, r, u.String(), http.StatusFound)
	return
}

func AcsService(w http.ResponseWriter, r *http.Request) (err error) {
	defer r.Body.Close()
	birk, err := r.Cookie("BIRK")
	if err != nil {
		return err
	}

	value := []byte{}
	if err = seccookie.Decode("BIRK", birk.Value, &value); err != nil {
		return
	}

	// we checked the request when we received in birkService - we can use it without fear ie. we just parse it
	log.Println("cookie", string(gosaml.Inflate(value)))
	request := goxml.NewXp(string(gosaml.Inflate(value)))

	http.SetCookie(w, &http.Cookie{Name: "BIRK", Value: "", Domain: config.Domain, Path: "/", Secure: true, HttpOnly: true, MaxAge: -1})
	sp_md, err := config.External.MDQ(request.Query1(nil, "/samlp:AuthnRequest/saml:Issuer"))
	if err != nil {
		return
	}

	response, idp_md, _, relayState, err := gosaml.ReceiveSAMLResponse(r, config.Internal, config.Hub)
	if err != nil {
		return
	}

	err, ard := config.AttributeHandler(idp_md, config.HubRequestedAttributes, sp_md, response)
	if err != nil {
		return
	}

	birkmd, err := config.External.MDQ(request.Query1(nil, "/samlp:AuthnRequest/@Destination"))
	if err != nil {
		return
	}
	nameid := response.Query(nil, "./saml:Assertion/saml:Subject/saml:NameID")[0]
	// respect nameID in req, give persistent id + all computed attributes + nameformat conversion
	nameidformat := sp_md.Query1(nil, "./md:SPSSODescriptor/md:NameIDFormat")
	if nameidformat == gosaml.Persistent {
		response.QueryDashP(nameid, "@Format", gosaml.Persistent, nil)
		eptid := response.Query1(nil, `./saml:Assertion/saml:AttributeStatement/saml:Attribute[@FriendlyName="eduPersonTargetedID"]/saml:AttributeValue`)
		response.QueryDashP(nameid, ".", eptid, nil)
	} else if nameidformat == gosaml.Transient {
		response.QueryDashP(nameid, ".", gosaml.Id(), nil)
	}

	newresponse := gosaml.NewResponse(config.StdTiming.Refresh(), birkmd, sp_md, request, response)

	for _, q := range config.ElementsToSign {
		err = gosaml.SignResponse(newresponse, q, birkmd)
		if err != nil {
			return
		}
	}

	// Prepare data for the attributerelease form

	// when consent as a service is ready - we will post to that
	acs := newresponse.Query1(nil, "@Destination")

    ardjson, err := json.Marshal(ard)
	data := formdata{Acs: acs, Samlresponse: base64.StdEncoding.EncodeToString([]byte(newresponse.Doc.Dump(false))), RelayState: relayState, Ard: template.JS(ardjson)}
	attributeReleaseForm.Execute(w, data)
	return
}

func KribService(w http.ResponseWriter, r *http.Request) (err error) {
	// check ad-hoc feds overlap
	defer r.Body.Close()

	response, _, _, relayState, err := gosaml.ReceiveSAMLResponse(r, config.External, config.External)
	if err != nil {
		return
	}
	destination := debify.ReplaceAllString(response.Query1(nil, "@Destination"), "$1$2")
	response.QueryDashP(nil, "@Destination", destination, nil)
	response.QueryDashP(nil, "./saml:Assertion/saml:Subject/saml:SubjectConfirmation/saml:SubjectConfirmationData/@Recipient", destination, nil)
	issuer := config.HubEntityID
	response.QueryDashP(nil, "./saml:Issuer", issuer, nil)
	response.QueryDashP(nil, "./saml:Assertion/saml:Issuer", issuer, nil)
	// Krib always receives attributes with nameformat=urn. Before sending to the real SP we need to look into
	// the metadata for SP to determine the actual nameformat - as WAYF supports both for internal SPs.
	mdsp, err := config.Internal.MDQ(destination)
	if err != nil {
		return
	}
	requestedattributes := mdsp.Query(nil, "./md:SPSSODescriptor/md:AttributeConsumingService/md:RequestedAttribute")
	attributestatement := response.Query(nil, "./saml:Assertion/saml:AttributeStatement")[0]
	for _, attr := range requestedattributes {
		nameFormat, _ := attr.(types.Element).GetAttribute("NameFormat")
		if nameFormat.NodeValue() == gosaml.Basic {
			basicname, _ := attr.(types.Element).GetAttribute("Name")
			uriname := config.Basic2uri[basicname.NodeValue()]
			responseattribute := response.Query(attributestatement, "saml:Attribute[@Name='"+uriname+"']")
			if len(responseattribute) > 0 {
				responseattribute[0].(types.Element).SetAttribute("Name", basicname.NodeValue())
				responseattribute[0].(types.Element).SetAttribute("NameFormat", gosaml.Basic)
			}
		}
	}

	mdhub, err := config.Hub.MDQ(config.HubEntityID)
	if err != nil {
		return
	}

	for _, q := range config.ElementsToSign {
		err = gosaml.SignResponse(response, q, mdhub)
		if err != nil {
			return
		}
	}

	data := formdata{Acs: destination, Samlresponse: base64.StdEncoding.EncodeToString([]byte(response.Doc.Dump(false))), RelayState: relayState}
	postForm.Execute(w, data)
	return
}
<commit_msg>added support for attributerelaseform<commit_after>/*
   - support for RelayState
   - does not seem to work: incl. Capitalization content-security-policy: referrer no-referrer;
   - redo no-referer - current version does not work !!!
   - MDQ lookup by location also for hub md
   - Trusted proxy
   - wayf:wayf i hub_ops metadata
        - AttributeNameFormat for Krib -> WAYF SPS - ie if none -> repeat wayf error both formats but error
        - schacHomeOrganization
        - schacHomeOrganizationType
   - collect schema errors
   - illegal attributes from IdP - ignore or provoke error
*/

package gohybrid

import (
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"github.com/gorilla/securecookie"
	"github.com/wayf-dk/go-libxml2/types"
	"github.com/wayf-dk/gosaml"
	"github.com/wayf-dk/goxml"
	"html/template"
	"log"
	"net/http"
	"net/url"
	"regexp"
	"sync"
)

type (
	formdata struct {
		Acs          string
		Samlresponse string
		RelayState   string
		Ard          template.JS
	}

	idpsppair struct {
		idp string
		sp  string
	}

	AttributeReleaseData struct {
		Values         map[string][]string
		IdPDisplayName map[string]string
		IdPLogo        string
		SPDisplayName  map[string]string
		SPDescription  map[string]string
		SPLogo         string
		SPEntityID     string
		Key            string
		Hash           string
	}

	Conf struct {
		DiscoveryService         string
		Domain                   string
		HubEntityID              string
		EptidSalt                string
		HubRequestedAttributes   *goxml.Xp
		Internal, External, Hub  gosaml.Md
		SecureCookieHashKey      string
		PostFormTemplate         string
		AttributeReleaseTemplate string
		Basic2uri                map[string]string
		StdTiming                gosaml.IdAndTiming
		ElementsToSign           []string
		AttributeHandler         func(*goxml.Xp, *goxml.Xp, *goxml.Xp, *goxml.Xp) (error, AttributeReleaseData)
	}
)

const (
	idpCertQuery = `./md:IDPSSODescriptor/md:KeyDescriptor[@use="signing" or not(@use)]/ds:KeyInfo/ds:X509Data/ds:X509Certificate`
)

var (
	_     = log.Printf // For debugging; delete when done.
	_     = fmt.Printf
	remap = map[string]idpsppair{
		"https://nemlogin.wayf.dk": idpsppair{"https://saml.nemlog-in.dk", "https://nemlogin.wayf.dk"},
	}

	contextmutex sync.RWMutex
	context      = make(map[*http.Request]map[string]string)
	bify         = regexp.MustCompile("^(https?://)(.*)$")
	debify       = regexp.MustCompile("^(https?://)(?:(?:birk|krib)\\.wayf.dk/(?:birk|krib)\\.php/)(.+)$")

	postForm, attributeReleaseForm *template.Template
	hashKey                        []byte
	seccookie                      *securecookie.SecureCookie
	config                         = Conf{}
)

func Config(configuration Conf) {
	config = configuration
	hashKey, _ := hex.DecodeString(config.SecureCookieHashKey)
	seccookie = securecookie.New(hashKey, nil)
	postForm = template.Must(template.New("post").Parse(config.PostFormTemplate))
	attributeReleaseForm = template.Must(template.New("post").Parse(config.AttributeReleaseTemplate))
}

func SsoService(w http.ResponseWriter, r *http.Request) (err error) {
	defer r.Body.Close()
	// handle non ok urls gracefully
	// var err error
	// check issuer and acs in md
	// receiveRequest -> request, issuer md, receiver md
	//     check for IDPList 1st in md, then in request then in query
	//     sanitize idp from query or request
	request, spmd, _, relayState, err := gosaml.ReceiveSAMLRequest(r, config.Internal, config.Hub)
	if err != nil {
		return
	}
	entityID := spmd.Query1(nil, "@entityID")
	idp := spmd.Query1(nil, "//IDPList/ProviderID") // Need to find a place for IDPList
	if idp == "" {
		idp = request.Query1(nil, "IDPList/ProviderID")
	}
	if idp == "" {
		idp = r.URL.Query().Get("idpentityid")
	}
	if idp == "" {
		data := url.Values{}
		data.Set("return", "https://"+r.Host+r.RequestURI)
		data.Set("returnIDParam", "idpentityid")
		data.Set("entityID", entityID)
		http.Redirect(w, r, config.DiscoveryService+data.Encode(), http.StatusFound)
	} else {
		var idpmd *goxml.Xp
		/**/
		// check overlap btw ad-hoc feds for the idp and the sp
		kribID := bify.ReplaceAllString(entityID, "${1}krib.wayf.dk/krib.php/$2")
		if kribID == entityID {
			kribID = "urn:oid:1.3.6.1.4.1.39153:42:" + entityID
		}

		request.QueryDashP(nil, "/saml:Issuer", kribID, nil)
		acs := request.Query1(nil, "@AssertionConsumerServiceURL")
		acsurl := bify.ReplaceAllString(acs, "${1}krib.wayf.dk/krib.php/$2")
		request.QueryDashP(nil, "@AssertionConsumerServiceURL", acsurl, nil)
		/**/
		idpmd, err = config.External.MDQ(idp)
		if err != nil {
			return
		}
		const ssoquery = "./md:IDPSSODescriptor/md:SingleSignOnService[@Binding='urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect']/@Location"
		ssoservice := idpmd.Query1(nil, ssoquery)
		if ssoservice == "" {

		}
		request.QueryDashP(nil, "@Destination", ssoservice, nil)
		u, _ := gosaml.SAMLRequest2Url(request, relayState, "", "", "")
		http.Redirect(w, r, u.String(), http.StatusFound)
	}
	return
}

func BirkService(w http.ResponseWriter, r *http.Request) (err error) {
	// use incoming request for crafting the new one
	// remember to add the Scoping element to inform the IdP of requesterID - if stated in metadata for the IdP
	// check ad-hoc feds overlab
	defer r.Body.Close()
	// get the sp as well to check for allowed acs
	request, _, mdbirkidp, relayState, err := gosaml.ReceiveSAMLRequest(r, config.External, config.External)
	if err != nil {
		return
	}
	// Save the issuer and destination in a cookie for when the response comes back

	cookievalue, err := seccookie.Encode("BIRK", gosaml.Deflate(request.Doc.Dump(true)))
	http.SetCookie(w, &http.Cookie{Name: "BIRK", Value: cookievalue, Domain: config.Domain, Path: "/", Secure: true, HttpOnly: true})

	idp := debify.ReplaceAllString(mdbirkidp.Query1(nil, "@entityID"), "$1$2")

	var mdhub, mdidp *goxml.Xp
	// are we remapping - for now only use case is https://nemlogin.wayf.dk -> https://saml.nemlog-in.dk
	if rm, ok := remap[idp]; ok {
		mdidp, err = config.Internal.MDQ(rm.idp)
		if err != nil {
			return
		}
		mdhub, err = config.Internal.MDQ(rm.sp)
		if err != nil {
			return
		}
	} else {
		mdidp, err = config.Internal.MDQ(idp)
		if err != nil {
			return
		}
		mdhub, err = config.Hub.MDQ(config.HubEntityID)
		if err != nil {
			return
		}
	}
	// use a std request - we take care of NameID etc in acsService below
	newrequest := gosaml.NewAuthnRequest(config.StdTiming.Refresh(), mdhub, mdidp)
	u, _ := gosaml.SAMLRequest2Url(newrequest, relayState, "", "", "") // not signed so blank key, pw and algo
	http.Redirect(w, r, u.String(), http.StatusFound)
	return
}

func AcsService(w http.ResponseWriter, r *http.Request) (err error) {
	defer r.Body.Close()
	birk, err := r.Cookie("BIRK")
	if err != nil {
		return err
	}

	value := []byte{}
	if err = seccookie.Decode("BIRK", birk.Value, &value); err != nil {
		return
	}

	// we checked the request when we received in birkService - we can use it without fear ie. we just parse it
	request := goxml.NewXp(string(gosaml.Inflate(value)))

	http.SetCookie(w, &http.Cookie{Name: "BIRK", Value: "", Domain: config.Domain, Path: "/", Secure: true, HttpOnly: true, MaxAge: -1})
	sp_md, err := config.External.MDQ(request.Query1(nil, "/samlp:AuthnRequest/saml:Issuer"))
	if err != nil {
		return
	}

	response, idp_md, _, relayState, err := gosaml.ReceiveSAMLResponse(r, config.Internal, config.Hub)
	if err != nil {
		return
	}

	err, ard := config.AttributeHandler(idp_md, config.HubRequestedAttributes, sp_md, response)
	if err != nil {
		return
	}

	birkmd, err := config.External.MDQ(request.Query1(nil, "/samlp:AuthnRequest/@Destination"))
	if err != nil {
		return
	}
	nameid := response.Query(nil, "./saml:Assertion/saml:Subject/saml:NameID")[0]
	// respect nameID in req, give persistent id + all computed attributes + nameformat conversion
	nameidformat := sp_md.Query1(nil, "./md:SPSSODescriptor/md:NameIDFormat")
	if nameidformat == gosaml.Persistent {
		response.QueryDashP(nameid, "@Format", gosaml.Persistent, nil)
		eptid := response.Query1(nil, `./saml:Assertion/saml:AttributeStatement/saml:Attribute[@FriendlyName="eduPersonTargetedID"]/saml:AttributeValue`)
		response.QueryDashP(nameid, ".", eptid, nil)
	} else if nameidformat == gosaml.Transient {
		response.QueryDashP(nameid, ".", gosaml.Id(), nil)
	}

	newresponse := gosaml.NewResponse(config.StdTiming.Refresh(), birkmd, sp_md, request, response)

	for _, q := range config.ElementsToSign {
		err = gosaml.SignResponse(newresponse, q, birkmd)
		if err != nil {
			return
		}
	}

	// Prepare data for the attributerelease form

	// when consent as a service is ready - we will post to that
	acs := newresponse.Query1(nil, "@Destination")

	ardjson, err := json.Marshal(ard)
	data := formdata{Acs: acs, Samlresponse: base64.StdEncoding.EncodeToString([]byte(newresponse.Doc.Dump(false))), RelayState: relayState, Ard: template.JS(ardjson)}
	attributeReleaseForm.Execute(w, data)
	return
}

func KribService(w http.ResponseWriter, r *http.Request) (err error) {
	// check ad-hoc feds overlap
	defer r.Body.Close()

	response, _, _, relayState, err := gosaml.ReceiveSAMLResponse(r, config.External, config.External)
	if err != nil {
		return
	}
	destination := debify.ReplaceAllString(response.Query1(nil, "@Destination"), "$1$2")
	response.QueryDashP(nil, "@Destination", destination, nil)
	response.QueryDashP(nil, "./saml:Assertion/saml:Subject/saml:SubjectConfirmation/saml:SubjectConfirmationData/@Recipient", destination, nil)
	issuer := config.HubEntityID
	response.QueryDashP(nil, "./saml:Issuer", issuer, nil)
	response.QueryDashP(nil, "./saml:Assertion/saml:Issuer", issuer, nil)
	// Krib always receives attributes with nameformat=urn. Before sending to the real SP we need to look into
	// the metadata for SP to determine the actual nameformat - as WAYF supports both for internal SPs.
	mdsp, err := config.Internal.MDQ(destination)
	if err != nil {
		return
	}
	requestedattributes := mdsp.Query(nil, "./md:SPSSODescriptor/md:AttributeConsumingService/md:RequestedAttribute")
	attributestatement := response.Query(nil, "./saml:Assertion/saml:AttributeStatement")[0]
	for _, attr := range requestedattributes {
		nameFormat, _ := attr.(types.Element).GetAttribute("NameFormat")
		if nameFormat.NodeValue() == gosaml.Basic {
			basicname, _ := attr.(types.Element).GetAttribute("Name")
			uriname := config.Basic2uri[basicname.NodeValue()]
			responseattribute := response.Query(attributestatement, "saml:Attribute[@Name='"+uriname+"']")
			if len(responseattribute) > 0 {
				responseattribute[0].(types.Element).SetAttribute("Name", basicname.NodeValue())
				responseattribute[0].(types.Element).SetAttribute("NameFormat", gosaml.Basic)
			}
		}
	}

	mdhub, err := config.Hub.MDQ(config.HubEntityID)
	if err != nil {
		return
	}

	for _, q := range config.ElementsToSign {
		err = gosaml.SignResponse(response, q, mdhub)
		if err != nil {
			return
		}
	}

	data := formdata{Acs: destination, Samlresponse: base64.StdEncoding.EncodeToString([]byte(response.Doc.Dump(false))), RelayState: relayState}
	postForm.Execute(w, data)
	return
}
<|endoftext|>


<commit_before>package sockaddr

import (
	"fmt"
	"net"
)

// IfAddr is a union of a SockAddr and a net.Interface.
type IfAddr struct {
	SockAddr
	net.Interface
}

// Attr returns the named attribute as a string
func (ifAddr IfAddr) Attr(attrName AttrName) string {
	sa := ifAddr.SockAddr
	switch sockType := sa.Type(); {
	case sockType&TypeIP != 0:
		ip := *ToIPAddr(sa)
		attrVal := IPAddrAttr(ip, attrName)
		if attrVal != "" {
			return attrVal
		}

		if sa.Type() == TypeIPv4 {
			ipv4 := *ToIPv4Addr(sa)
			attrVal := IPAddrAttr(ipv4, attrName)
			if attrVal != "" {
				return attrVal
			}
		}

		if sa.Type() == TypeIPv6 {
			ipv6 := *ToIPv6Addr(sa)
			attrVal := IPAddrAttr(ipv6, attrName)
			if attrVal != "" {
				return attrVal
			}
		}

		// Random attribute names that are Interface specific
		switch attrName {
		case "name":
			return ifAddr.Interface.Name
		case "flags":
			return ifAddr.Interface.Flags.String()
		}
	case sockType == TypeUnix:
		us := *ToUnixSock(sa)
		attrVal := UnixSockAttr(us, attrName)
		if attrVal != "" {
			return attrVal
		}
	}

	// Non type-specific attributes
	switch attrName {
	case "string":
		return sa.String()
	case "type":
		return sa.Type().String()
	}

	return fmt.Sprintf("<unsupported attribute name %q>", attrName)
}
<commit_msg>Fix two issues when taking type of an IPAddr.<commit_after>package sockaddr

import (
	"fmt"
	"net"
)

// IfAddr is a union of a SockAddr and a net.Interface.
type IfAddr struct {
	SockAddr
	net.Interface
}

// Attr returns the named attribute as a string
func (ifAddr IfAddr) Attr(attrName AttrName) string {
	sa := ifAddr.SockAddr
	switch sockType := sa.Type(); {
	case sockType&TypeIP != 0:
		ip := *ToIPAddr(sa)
		attrVal := IPAddrAttr(ip, attrName)
		if attrVal != "" {
			return attrVal
		}

		if sa.Type() == TypeIPv4 {
			ipv4 := *ToIPv4Addr(sa)
			attrVal := IPv4AddrAttr(ipv4, attrName)
			if attrVal != "" {
				return attrVal
			}
		}

		if sa.Type() == TypeIPv6 {
			ipv6 := *ToIPv6Addr(sa)
			attrVal := IPv6AddrAttr(ipv6, attrName)
			if attrVal != "" {
				return attrVal
			}
		}

		// Random attribute names that are Interface specific
		switch attrName {
		case "name":
			return ifAddr.Interface.Name
		case "flags":
			return ifAddr.Interface.Flags.String()
		}
	case sockType == TypeUnix:
		us := *ToUnixSock(sa)
		attrVal := UnixSockAttr(us, attrName)
		if attrVal != "" {
			return attrVal
		}
	}

	// Non type-specific attributes
	switch attrName {
	case "string":
		return sa.String()
	case "type":
		return sa.Type().String()
	}

	return fmt.Sprintf("<unsupported attribute name %q>", attrName)
}
<|endoftext|>


<commit_before>package lxd

import (
	"crypto/tls"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"

	"github.com/lxc/lxd/shared"
)

func tlsHTTPClient(client *http.Client, tlsClientCert string, tlsClientKey string, tlsCA string, tlsServerCert string, insecureSkipVerify bool, proxy func(req *http.Request) (*url.URL, error)) (*http.Client, error) {
	// Get the TLS configuration
	tlsConfig, err := shared.GetTLSConfigMem(tlsClientCert, tlsClientKey, tlsCA, tlsServerCert, insecureSkipVerify)
	if err != nil {
		return nil, err
	}

	// Support disabling of strict ciphers
	if shared.IsTrue(os.Getenv("LXD_INSECURE_TLS")) {
		tlsConfig.CipherSuites = nil
	}

	// Define the http transport
	transport := &http.Transport{
		TLSClientConfig:   tlsConfig,
		Dial:              shared.RFC3493Dialer,
		Proxy:             shared.ProxyFromEnvironment,
		DisableKeepAlives: true,
	}

	// Allow overriding the proxy
	if proxy != nil {
		transport.Proxy = proxy
	}

	// Special TLS handling
	//lint:ignore SA1019 DialContext doesn't exist in Go 1.13
	transport.DialTLS = func(network string, addr string) (net.Conn, error) {
		tlsDial := func(network string, addr string, config *tls.Config, resetName bool) (net.Conn, error) {
			// TCP connection
			//lint:ignore SA1019 DialContext doesn't exist in Go 1.13
			conn, err := transport.Dial(network, addr)
			if err != nil {
				return nil, err
			}

			// Setup TLS
			if resetName {
				hostName, _, err := net.SplitHostPort(addr)
				if err != nil {
					hostName = addr
				}

				config = config.Clone()
				config.ServerName = hostName
			}
			tlsConn := tls.Client(conn, config)

			// Validate the connection
			err = tlsConn.Handshake()
			if err != nil {
				conn.Close()
				return nil, err
			}

			if !config.InsecureSkipVerify {
				err := tlsConn.VerifyHostname(config.ServerName)
				if err != nil {
					conn.Close()
					return nil, err
				}
			}

			return tlsConn, nil
		}

		conn, err := tlsDial(network, addr, transport.TLSClientConfig, false)
		if err != nil {
			// We may have gotten redirected to a non-LXD machine
			return tlsDial(network, addr, transport.TLSClientConfig, true)
		}

		return conn, nil
	}

	// Define the http client
	if client == nil {
		client = &http.Client{}
	}
	client.Transport = transport

	// Setup redirect policy
	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
		// Replicate the headers
		req.Header = via[len(via)-1].Header

		return nil
	}

	return client, nil
}

func unixHTTPClient(client *http.Client, path string) (*http.Client, error) {
	// Setup a Unix socket dialer
	unixDial := func(network, addr string) (net.Conn, error) {
		raddr, err := net.ResolveUnixAddr("unix", path)
		if err != nil {
			return nil, err
		}

		return net.DialUnix("unix", nil, raddr)
	}

	// Define the http transport
	transport := &http.Transport{
		Dial:              unixDial,
		DisableKeepAlives: true,
	}

	// Define the http client
	if client == nil {
		client = &http.Client{}
	}
	client.Transport = transport

	// Setup redirect policy
	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
		// Replicate the headers
		req.Header = via[len(via)-1].Header

		return nil
	}

	return client, nil
}

// remoteOperationResult used for storing the error that occurred for a particular remote URL.
type remoteOperationResult struct {
	URL   string
	Error error
}

func remoteOperationError(msg string, errors []remoteOperationResult) error {
	// Check if empty
	if len(errors) == 0 {
		return nil
	}

	// Check if all identical
	var err error
	for _, entry := range errors {
		if err != nil && entry.Error.Error() != err.Error() {
			errorStrs := make([]string, 0, len(errors))
			for _, error := range errors {
				errorStrs = append(errorStrs, fmt.Sprintf("%s: %v", error.URL, error.Error))
			}

			return fmt.Errorf("%s:\n - %s", msg, strings.Join(errorStrs, "\n - "))
		}

		err = entry.Error
	}

	// Check if successful
	if err != nil {
		return fmt.Errorf("%s: %s", msg, err)
	}

	return nil
}

// Set the value of a query parameter in the given URI.
func setQueryParam(uri, param, value string) (string, error) {
	fields, err := url.Parse(uri)
	if err != nil {
		return "", err
	}

	values := fields.Query()
	values.Set(param, url.QueryEscape(value))

	fields.RawQuery = values.Encode()

	return fields.String(), nil
}
<commit_msg>client/util: Adds urlsToResourceNames function<commit_after>package lxd

import (
	"crypto/tls"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"

	"github.com/lxc/lxd/shared"
)

func tlsHTTPClient(client *http.Client, tlsClientCert string, tlsClientKey string, tlsCA string, tlsServerCert string, insecureSkipVerify bool, proxy func(req *http.Request) (*url.URL, error)) (*http.Client, error) {
	// Get the TLS configuration
	tlsConfig, err := shared.GetTLSConfigMem(tlsClientCert, tlsClientKey, tlsCA, tlsServerCert, insecureSkipVerify)
	if err != nil {
		return nil, err
	}

	// Support disabling of strict ciphers
	if shared.IsTrue(os.Getenv("LXD_INSECURE_TLS")) {
		tlsConfig.CipherSuites = nil
	}

	// Define the http transport
	transport := &http.Transport{
		TLSClientConfig:   tlsConfig,
		Dial:              shared.RFC3493Dialer,
		Proxy:             shared.ProxyFromEnvironment,
		DisableKeepAlives: true,
	}

	// Allow overriding the proxy
	if proxy != nil {
		transport.Proxy = proxy
	}

	// Special TLS handling
	//lint:ignore SA1019 DialContext doesn't exist in Go 1.13
	transport.DialTLS = func(network string, addr string) (net.Conn, error) {
		tlsDial := func(network string, addr string, config *tls.Config, resetName bool) (net.Conn, error) {
			// TCP connection
			//lint:ignore SA1019 DialContext doesn't exist in Go 1.13
			conn, err := transport.Dial(network, addr)
			if err != nil {
				return nil, err
			}

			// Setup TLS
			if resetName {
				hostName, _, err := net.SplitHostPort(addr)
				if err != nil {
					hostName = addr
				}

				config = config.Clone()
				config.ServerName = hostName
			}
			tlsConn := tls.Client(conn, config)

			// Validate the connection
			err = tlsConn.Handshake()
			if err != nil {
				conn.Close()
				return nil, err
			}

			if !config.InsecureSkipVerify {
				err := tlsConn.VerifyHostname(config.ServerName)
				if err != nil {
					conn.Close()
					return nil, err
				}
			}

			return tlsConn, nil
		}

		conn, err := tlsDial(network, addr, transport.TLSClientConfig, false)
		if err != nil {
			// We may have gotten redirected to a non-LXD machine
			return tlsDial(network, addr, transport.TLSClientConfig, true)
		}

		return conn, nil
	}

	// Define the http client
	if client == nil {
		client = &http.Client{}
	}
	client.Transport = transport

	// Setup redirect policy
	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
		// Replicate the headers
		req.Header = via[len(via)-1].Header

		return nil
	}

	return client, nil
}

func unixHTTPClient(client *http.Client, path string) (*http.Client, error) {
	// Setup a Unix socket dialer
	unixDial := func(network, addr string) (net.Conn, error) {
		raddr, err := net.ResolveUnixAddr("unix", path)
		if err != nil {
			return nil, err
		}

		return net.DialUnix("unix", nil, raddr)
	}

	// Define the http transport
	transport := &http.Transport{
		Dial:              unixDial,
		DisableKeepAlives: true,
	}

	// Define the http client
	if client == nil {
		client = &http.Client{}
	}
	client.Transport = transport

	// Setup redirect policy
	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
		// Replicate the headers
		req.Header = via[len(via)-1].Header

		return nil
	}

	return client, nil
}

// remoteOperationResult used for storing the error that occurred for a particular remote URL.
type remoteOperationResult struct {
	URL   string
	Error error
}

func remoteOperationError(msg string, errors []remoteOperationResult) error {
	// Check if empty
	if len(errors) == 0 {
		return nil
	}

	// Check if all identical
	var err error
	for _, entry := range errors {
		if err != nil && entry.Error.Error() != err.Error() {
			errorStrs := make([]string, 0, len(errors))
			for _, error := range errors {
				errorStrs = append(errorStrs, fmt.Sprintf("%s: %v", error.URL, error.Error))
			}

			return fmt.Errorf("%s:\n - %s", msg, strings.Join(errorStrs, "\n - "))
		}

		err = entry.Error
	}

	// Check if successful
	if err != nil {
		return fmt.Errorf("%s: %s", msg, err)
	}

	return nil
}

// Set the value of a query parameter in the given URI.
func setQueryParam(uri, param, value string) (string, error) {
	fields, err := url.Parse(uri)
	if err != nil {
		return "", err
	}

	values := fields.Query()
	values.Set(param, url.QueryEscape(value))

	fields.RawQuery = values.Encode()

	return fields.String(), nil
}

// urlsToResourceNames returns a list of resource names extracted from one or more URLs of the same resource type.
// The resource type path prefix to match is provided by the matchPathPrefix argument.
func urlsToResourceNames(matchPathPrefix string, urls ...string) ([]string, error) {
	var resourceNames []string

	for _, urlRaw := range urls {
		u, err := url.Parse(urlRaw)
		if err != nil {
			return nil, fmt.Errorf("Failed parsing URL %q: %w", urlRaw, err)
		}

		fields := strings.Split(u.Path, fmt.Sprintf("%s/", matchPathPrefix))
		if len(fields) != 2 {
			return nil, fmt.Errorf("Unexpected URL path %q", u)
		}

		resourceNames = append(resourceNames, fields[len(fields)-1])
	}

	return resourceNames, nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"

	log "github.com/sirupsen/logrus"
	"github.com/urfave/cli"
	"github.com/usefathom/fathom/pkg/models"
)

var registerCmd = cli.Command{
	Name:    "register",
	Aliases: []string{"r"},
	Usage:   "register a new admin user",
	Action:  register,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "email, e",
			Usage: "user email",
		},
		cli.StringFlag{
			Name:  "password, p",
			Usage: "user password",
		},
	},
}

func register(c *cli.Context) error {
	email := c.String("email")
	if email == "" {
		return errors.New("Invalid arguments: missing email")
	}

	password := c.String("password")
	if password == "" {
		return errors.New("Invalid arguments: missing password")
	}

	user := models.NewUser(email, password)
	err := app.database.SaveUser(&user)

	if err != nil {
		return fmt.Errorf("Error creating user: %s", err)
	}

	log.Infof("Created user %s", user.Email)
	return nil
}
<commit_msg>remove alias for register command<commit_after>package main

import (
	"errors"
	"fmt"

	log "github.com/sirupsen/logrus"
	"github.com/urfave/cli"
	"github.com/usefathom/fathom/pkg/models"
)

var registerCmd = cli.Command{
	Name:   "register",
	Usage:  "register a new admin user",
	Action: register,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "email, e",
			Usage: "user email",
		},
		cli.StringFlag{
			Name:  "password, p",
			Usage: "user password",
		},
	},
}

func register(c *cli.Context) error {
	email := c.String("email")
	if email == "" {
		return errors.New("Invalid arguments: missing email")
	}

	password := c.String("password")
	if password == "" {
		return errors.New("Invalid arguments: missing password")
	}

	user := models.NewUser(email, password)
	err := app.database.SaveUser(&user)

	if err != nil {
		return fmt.Errorf("Error creating user: %s", err)
	}

	log.Infof("Created user %s", user.Email)
	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package main

import (
	"fmt"
	"log"
	"os"
	"regexp"
	"sort"
	"strings"

	"github.com/mvdan/adb"

	"github.com/mvdan/fdroidcl"
)

var cmdSearch = &Command{
	UsageLine: "search [<regexp...>]",
	Short:     "Search available apps",
}

var (
	quiet     = cmdSearch.Flag.Bool("q", false, "Print package names only")
	installed = cmdSearch.Flag.Bool("i", false, "Filter installed apps")
	updates   = cmdSearch.Flag.Bool("u", false, "Filter apps with updates")
	category  = cmdSearch.Flag.String("c", "", "Filter apps by category")
	sortBy    = cmdSearch.Flag.String("o", "", "Sort order (added, updated)")
)

func init() {
	cmdSearch.Run = runSearch
}

func runSearch(args []string) {
	if *installed && *updates {
		fmt.Fprintf(os.Stderr, "-i is redundant if -u is specified\n")
		cmdSearch.Flag.Usage()
	}
	sfunc, err := sortFunc(*sortBy)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		cmdSearch.Flag.Usage()
	}
	var device *adb.Device
	if *installed || *updates {
		device = mustOneDevice()
	}
	apps := filterAppsSearch(mustLoadIndexes(), args)
	instPkgs := mustInstalled(device)
	if *installed {
		apps = filterAppsInstalled(apps, instPkgs)
	}
	if *updates {
		apps = filterAppsUpdates(apps, instPkgs)
	}
	if *category != "" {
		apps = filterAppsCategory(apps, *category)
		if apps == nil {
			fmt.Fprintf(os.Stderr, "No such category: %s\n", *category)
			cmdSearch.Flag.Usage()
		}
	}
	if sfunc != nil {
		apps = sortApps(apps, sfunc)
	}
	if *quiet {
		for _, app := range apps {
			fmt.Println(app.ID)
		}
	} else {
		printApps(apps, instPkgs)
	}
}

func filterAppsSearch(apps []fdroidcl.App, terms []string) []fdroidcl.App {
	regexes := make([]*regexp.Regexp, len(terms))
	for i, term := range terms {
		regexes[i] = regexp.MustCompile(term)
	}
	var result []fdroidcl.App
	for _, app := range apps {
		fields := []string{
			strings.ToLower(app.ID),
			strings.ToLower(app.Name),
			strings.ToLower(app.Summary),
			strings.ToLower(app.Desc),
		}
		if !appMatches(fields, regexes) {
			continue
		}
		result = append(result, app)
	}
	return result
}

func appMatches(fields []string, regexes []*regexp.Regexp) bool {
fieldLoop:
	for _, field := range fields {
		for _, regex := range regexes {
			if !regex.MatchString(field) {
				continue fieldLoop
			}
		}
		return true
	}
	return false
}

func printApps(apps []fdroidcl.App, inst map[string]adb.Package) {
	maxIDLen := 0
	for _, app := range apps {
		if len(app.ID) > maxIDLen {
			maxIDLen = len(app.ID)
		}
	}
	for _, app := range apps {
		var pkg *adb.Package
		p, e := inst[app.ID]
		if e {
			pkg = &p
		}
		printApp(app, maxIDLen, pkg)
	}
}

func descVersion(app fdroidcl.App, inst *adb.Package) string {
	cur := app.CurApk()
	if cur == nil {
		return "(no version available)"
	}
	if inst == nil {
		return fmt.Sprintf("%s (%d)", cur.VName, cur.VCode)
	}
	if inst.VCode < cur.VCode {
		return fmt.Sprintf("%s (%d) -> %s (%d)", inst.VName, inst.VCode,
			cur.VName, cur.VCode)
	}
	if !*installed {
		return fmt.Sprintf("%s (%d) [installed]", cur.VName, cur.VCode)
	}
	return fmt.Sprintf("%s (%d)", cur.VName, cur.VCode)
}

func printApp(app fdroidcl.App, IDLen int, inst *adb.Package) {
	fmt.Printf("%s%s %s - %s\n", app.ID, strings.Repeat(" ", IDLen-len(app.ID)),
		app.Name, descVersion(app, inst))
	fmt.Printf("    %s\n", app.Summary)
}

func mustInstalled(device *adb.Device) map[string]adb.Package {
	if device == nil {
		return nil
	}
	inst, err := device.Installed()
	if err != nil {
		log.Fatalf("Could not get installed packages: %v", err)
	}
	return inst
}

func filterAppsInstalled(apps []fdroidcl.App, inst map[string]adb.Package) []fdroidcl.App {
	var result []fdroidcl.App
	for _, app := range apps {
		if _, e := inst[app.ID]; !e {
			continue
		}
		result = append(result, app)
	}
	return result
}

func filterAppsUpdates(apps []fdroidcl.App, inst map[string]adb.Package) []fdroidcl.App {
	var result []fdroidcl.App
	for _, app := range apps {
		p, e := inst[app.ID]
		if !e {
			continue
		}
		cur := app.CurApk()
		if cur == nil {
			continue
		}
		if p.VCode >= cur.VCode {
			continue
		}
		result = append(result, app)
	}
	return result
}

func contains(l []string, s string) bool {
	for _, s1 := range l {
		if s1 == s {
			return true
		}
	}
	return false
}

func filterAppsCategory(apps []fdroidcl.App, categ string) []fdroidcl.App {
	var result []fdroidcl.App
	for _, app := range apps {
		if !contains(app.Categs, categ) {
			continue
		}
		result = append(result, app)
	}
	return result
}

func cmpAdded(a, b *fdroidcl.App) bool {
	return a.Added.Before(b.Added.Time)
}

func cmpUpdated(a, b *fdroidcl.App) bool {
	return a.Updated.Before(b.Updated.Time)
}

func sortFunc(sortBy string) (func(a, b *fdroidcl.App) bool, error) {
	switch sortBy {
	case "added":
		return cmpAdded, nil
	case "updated":
		return cmpUpdated, nil
	case "":
		return nil, nil
	}
	return nil, fmt.Errorf("Unknown sort order: %s", sortBy)
}

type appList struct {
	l []fdroidcl.App
	f func(a, b *fdroidcl.App) bool
}

func (al appList) Len() int           { return len(al.l) }
func (al appList) Swap(i, j int)      { al.l[i], al.l[j] = al.l[j], al.l[i] }
func (al appList) Less(i, j int) bool { return al.f(&al.l[i], &al.l[j]) }

func sortApps(apps []fdroidcl.App, f func(a, b *fdroidcl.App) bool) []fdroidcl.App {
	sort.Sort(appList{l: apps, f: f})
	return apps
}
<commit_msg>search: consider suggestions<commit_after>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package main

import (
	"fmt"
	"log"
	"os"
	"regexp"
	"sort"
	"strings"

	"github.com/mvdan/adb"

	"github.com/mvdan/fdroidcl"
)

var cmdSearch = &Command{
	UsageLine: "search [<regexp...>]",
	Short:     "Search available apps",
}

var (
	quiet     = cmdSearch.Flag.Bool("q", false, "Print package names only")
	installed = cmdSearch.Flag.Bool("i", false, "Filter installed apps")
	updates   = cmdSearch.Flag.Bool("u", false, "Filter apps with updates")
	category  = cmdSearch.Flag.String("c", "", "Filter apps by category")
	sortBy    = cmdSearch.Flag.String("o", "", "Sort order (added, updated)")
)

func init() {
	cmdSearch.Run = runSearch
}

func runSearch(args []string) {
	if *installed && *updates {
		fmt.Fprintf(os.Stderr, "-i is redundant if -u is specified\n")
		cmdSearch.Flag.Usage()
	}
	sfunc, err := sortFunc(*sortBy)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		cmdSearch.Flag.Usage()
	}
	var device *adb.Device
	if *installed || *updates {
		device = mustOneDevice()
	}
	apps := filterAppsSearch(mustLoadIndexes(), args)
	if *installed {
		apps = filterAppsInstalled(apps, device)
	}
	if *updates {
		apps = filterAppsUpdates(apps, device)
	}
	if *category != "" {
		apps = filterAppsCategory(apps, *category)
		if apps == nil {
			fmt.Fprintf(os.Stderr, "No such category: %s\n", *category)
			cmdSearch.Flag.Usage()
		}
	}
	if sfunc != nil {
		apps = sortApps(apps, sfunc)
	}
	if *quiet {
		for _, app := range apps {
			fmt.Println(app.ID)
		}
	} else {
		printApps(apps, device)
	}
}

func filterAppsSearch(apps []fdroidcl.App, terms []string) []fdroidcl.App {
	regexes := make([]*regexp.Regexp, len(terms))
	for i, term := range terms {
		regexes[i] = regexp.MustCompile(term)
	}
	var result []fdroidcl.App
	for _, app := range apps {
		fields := []string{
			strings.ToLower(app.ID),
			strings.ToLower(app.Name),
			strings.ToLower(app.Summary),
			strings.ToLower(app.Desc),
		}
		if !appMatches(fields, regexes) {
			continue
		}
		result = append(result, app)
	}
	return result
}

func appMatches(fields []string, regexes []*regexp.Regexp) bool {
fieldLoop:
	for _, field := range fields {
		for _, regex := range regexes {
			if !regex.MatchString(field) {
				continue fieldLoop
			}
		}
		return true
	}
	return false
}

func printApps(apps []fdroidcl.App, device *adb.Device) {
	maxIDLen := 0
	for _, app := range apps {
		if len(app.ID) > maxIDLen {
			maxIDLen = len(app.ID)
		}
	}
	inst := mustInstalled(device)
	for _, app := range apps {
		var pkg *adb.Package
		p, e := inst[app.ID]
		if e {
			pkg = &p
		}
		printApp(app, maxIDLen, pkg, device)
	}
}

func descVersion(app fdroidcl.App, inst *adb.Package, device *adb.Device) string {
	// With "-u" or "-i" option there must be a connected device
	if *updates || *installed {
		suggested := app.SuggestedApk(device)
		if suggested != nil && inst.VCode < suggested.VCode {
			return fmt.Sprintf("%s (%d) -> %s (%d)", inst.VName, inst.VCode,
				suggested.VName, suggested.VCode)
		}
		return fmt.Sprintf("%s (%d)", inst.VName, inst.VCode)
	}
	// Without "-u" or "-i" we only have repositories indices
	return fmt.Sprintf("%s (%d)", app.CVName, app.CVCode)
}

func printApp(app fdroidcl.App, IDLen int, inst *adb.Package, device *adb.Device) {
	fmt.Printf("%s%s %s - %s\n", app.ID, strings.Repeat(" ", IDLen-len(app.ID)),
		app.Name, descVersion(app, inst, device))
	fmt.Printf("    %s\n", app.Summary)
}

func mustInstalled(device *adb.Device) map[string]adb.Package {
	if device == nil {
		return nil
	}
	inst, err := device.Installed()
	if err != nil {
		log.Fatalf("Could not get installed packages: %v", err)
	}
	return inst
}

func filterAppsInstalled(apps []fdroidcl.App, device *adb.Device) []fdroidcl.App {
	var result []fdroidcl.App
	inst := mustInstalled(device)
	for _, app := range apps {
		if _, e := inst[app.ID]; !e {
			continue
		}
		result = append(result, app)
	}
	return result
}

func filterAppsUpdates(apps []fdroidcl.App, device *adb.Device) []fdroidcl.App {
	var result []fdroidcl.App
	inst := mustInstalled(device)
	for _, app := range apps {
		p, e := inst[app.ID]
		if !e {
			continue
		}
		suggested := app.SuggestedApk(device)
		if suggested == nil {
			continue
		}
		if p.VCode >= suggested.VCode {
			continue
		}
		result = append(result, app)
	}
	return result
}

func contains(l []string, s string) bool {
	for _, s1 := range l {
		if s1 == s {
			return true
		}
	}
	return false
}

func filterAppsCategory(apps []fdroidcl.App, categ string) []fdroidcl.App {
	var result []fdroidcl.App
	for _, app := range apps {
		if !contains(app.Categs, categ) {
			continue
		}
		result = append(result, app)
	}
	return result
}

func cmpAdded(a, b *fdroidcl.App) bool {
	return a.Added.Before(b.Added.Time)
}

func cmpUpdated(a, b *fdroidcl.App) bool {
	return a.Updated.Before(b.Updated.Time)
}

func sortFunc(sortBy string) (func(a, b *fdroidcl.App) bool, error) {
	switch sortBy {
	case "added":
		return cmpAdded, nil
	case "updated":
		return cmpUpdated, nil
	case "":
		return nil, nil
	}
	return nil, fmt.Errorf("Unknown sort order: %s", sortBy)
}

type appList struct {
	l []fdroidcl.App
	f func(a, b *fdroidcl.App) bool
}

func (al appList) Len() int           { return len(al.l) }
func (al appList) Swap(i, j int)      { al.l[i], al.l[j] = al.l[j], al.l[i] }
func (al appList) Less(i, j int) bool { return al.f(&al.l[i], &al.l[j]) }

func sortApps(apps []fdroidcl.App, f func(a, b *fdroidcl.App) bool) []fdroidcl.App {
	sort.Sort(appList{l: apps, f: f})
	return apps
}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/goauth2/oauth"
	"code.google.com/p/google-api-go-client/storage/v1"
	"fmt"
	"github.com/TheHippo/gcssync"
	"github.com/codegangsta/cli"
	"github.com/dustin/go-humanize"
	"os"
	"path/filepath"
)

const (
	_                = iota
	errorAuthInfo    = iota
	errorProjectInfo = iota
	errorClientInit  = iota
	errorListFiles   = iota
	errorUploadFiles = iota
	errorSyncFiles   = iota
)

const (
	scope       = storage.DevstorageFull_controlScope
	authURL     = "https://accounts.google.com/o/oauth2/auth"
	tokenURL    = "https://accounts.google.com/o/oauth2/token"
	entityName  = "allUsers"
	redirectURL = "urn:ietf:wg:oauth:2.0:oob"
)

func main() {
	app := cli.NewApp()
	app.Name = "gcssync"
	app.Usage = "Sync files with Google Cloud Storage"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "cachefile",
			Value:  "cache.json",
			Usage:  "Cache file for caching auth tokens",
			EnvVar: "AUTH_CACHE_FILE",
		},
		cli.StringFlag{
			Name:   "bucketname, b",
			Value:  "",
			Usage:  "Name of bucket",
			EnvVar: "BUCKET_NAME",
		},
		cli.StringFlag{
			Name:   "projectid, p",
			Value:  "",
			Usage:  "Google project",
			EnvVar: "PROJECT_ID",
		},
		cli.StringFlag{
			Name:   "clientid, c",
			Value:  "",
			Usage:  "Auth client id",
			EnvVar: "AUTH_CLIENT_ID",
		},
		cli.StringFlag{
			Name:   "clientsecret, s",
			Value:  "",
			Usage:  "Client secrect",
			EnvVar: "AUTH_CLIENT_SECRET",
		},
		cli.StringFlag{
			Name:   "code",
			Value:  "",
			Usage:  "Authorization Code",
			EnvVar: "AUTH_CODE",
		},
	}
	app.Commands = []cli.Command{
		{
			Name:      "list",
			ShortName: "l",
			Usage:     "List remote files",
			Action:    listFiles,
		},
		{
			Name:      "upload",
			ShortName: "u",
			Usage:     "Upload a single file",
			Action:    uploadFile,
		},
		{
			Name:      "sync",
			ShortName: "s",
			Usage:     "Syncs a folder to a Google Cloudstorage bucket",
			Action:    syncFolder,
		},
	}
	app.Run(os.Args)
}

func generateOAuthConfig(c *cli.Context) (*oauth.Config, error) {
	clientId := c.GlobalString("clientid")
	if clientId == "" {
		return &oauth.Config{}, fmt.Errorf("Could not find Client ID")
	}
	clientSecret := c.GlobalString("clientsecret")
	if clientSecret == "" {
		return &oauth.Config{}, fmt.Errorf("Could not find Client Secret")
	}

	return &oauth.Config{
		ClientId:     clientId,
		ClientSecret: clientSecret,
		Scope:        scope,
		AuthURL:      authURL,
		TokenURL:     tokenURL,
		TokenCache:   oauth.CacheFile(c.GlobalString("cachefile")),
		RedirectURL:  redirectURL,
	}, nil
}

func generateServiceConfig(c *cli.Context) (*gcssync.ServiceConfig, error) {
	projectID := c.GlobalString("projectid")
	if projectID == "" {
		return &gcssync.ServiceConfig{}, fmt.Errorf("Could not find project id")
	}
	bucketName := c.GlobalString("bucketname")
	if bucketName == "" {
		return &gcssync.ServiceConfig{}, fmt.Errorf("Cloud not find bucket name")
	}
	return &gcssync.ServiceConfig{
		ProjectID:  projectID,
		BucketName: bucketName,
	}, nil
}

func getClient(c *cli.Context) *gcssync.Client {
	oauthConfig, err := generateOAuthConfig(c)
	if err != nil {
		fmt.Println("Missing auth informations", err.Error())
		os.Exit(errorAuthInfo)
	}
	serviceConfig, err := generateServiceConfig(c)
	if err != nil {
		fmt.Println("Missing project config", err.Error())
		os.Exit(errorProjectInfo)
	}

	client, err := gcssync.NewClient(oauthConfig, c.GlobalString("code"), serviceConfig)
	if err != nil {
		fmt.Println("Error initilizing client: ", err.Error())
		os.Exit(errorClientInit)
	}

	return client
}

func listFiles(c *cli.Context) {
	client := getClient(c)
	files, err := client.ListFiles()
	if err != nil {
		fmt.Println(err)
		os.Exit(errorListFiles)
		return
	}
	for _, object := range files {
		fmt.Printf("%s %s\n", object.Name, humanize.Bytes(object.Size))
	}
	fmt.Printf("Objects in %s - %d\n", client.GetBucketname(), len(files))
}

func uploadFile(c *cli.Context) {
	client := getClient(c)
	if len(c.Args()) != 2 {
		fmt.Println("Need local and remote name!")
		os.Exit(errorUploadFiles)
	}

	success, object, err := client.UploadFile(c.Args().Get(0), c.Args().Get(1))
	if !success {
		fmt.Println(err.Error())
		os.Exit(errorUploadFiles)
		return
	}

	fmt.Printf("Uploaded file to %s\n", client.GetBucketname())
	fmt.Printf("%s %s\n", object.Name, humanize.Bytes(object.Size))

}

func syncFolder(c *cli.Context) {
	client := getClient(c)
	var local, remote string
	switch len(c.Args()) {
	case 0:
		local = ""
		remote = ""
	case 1:
		local = c.Args().Get(0)
		remote = ""
	case 2:
		local = c.Args().Get(0)
		remote = c.Args().Get(1)
	default:
		fmt.Println("To many arguments")
		os.Exit(errorSyncFiles)
	}
	local, err := filepath.Abs(local)
	if err != nil {
		fmt.Println("Could not get absolute path")
		os.Exit(errorSyncFiles)
	}
	client.SyncFolder(local, remote)
}
<commit_msg>fix(version): added versionnumber for first release<commit_after>package main

import (
	"code.google.com/p/goauth2/oauth"
	"code.google.com/p/google-api-go-client/storage/v1"
	"fmt"
	"github.com/TheHippo/gcssync"
	"github.com/codegangsta/cli"
	"github.com/dustin/go-humanize"
	"os"
	"path/filepath"
)

const (
	_                = iota
	errorAuthInfo    = iota
	errorProjectInfo = iota
	errorClientInit  = iota
	errorListFiles   = iota
	errorUploadFiles = iota
	errorSyncFiles   = iota
)

const (
	version = "0.1.0"
)

const (
	scope       = storage.DevstorageFull_controlScope
	authURL     = "https://accounts.google.com/o/oauth2/auth"
	tokenURL    = "https://accounts.google.com/o/oauth2/token"
	entityName  = "allUsers"
	redirectURL = "urn:ietf:wg:oauth:2.0:oob"
)

func main() {
	app := cli.NewApp()
	app.Name = "gcssync"
	app.Usage = "Sync files with Google Cloud Storage"
	app.Version = version
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "cachefile",
			Value:  "cache.json",
			Usage:  "Cache file for caching auth tokens",
			EnvVar: "AUTH_CACHE_FILE",
		},
		cli.StringFlag{
			Name:   "bucketname, b",
			Value:  "",
			Usage:  "Name of bucket",
			EnvVar: "BUCKET_NAME",
		},
		cli.StringFlag{
			Name:   "projectid, p",
			Value:  "",
			Usage:  "Google project",
			EnvVar: "PROJECT_ID",
		},
		cli.StringFlag{
			Name:   "clientid, c",
			Value:  "",
			Usage:  "Auth client id",
			EnvVar: "AUTH_CLIENT_ID",
		},
		cli.StringFlag{
			Name:   "clientsecret, s",
			Value:  "",
			Usage:  "Client secrect",
			EnvVar: "AUTH_CLIENT_SECRET",
		},
		cli.StringFlag{
			Name:   "code",
			Value:  "",
			Usage:  "Authorization Code",
			EnvVar: "AUTH_CODE",
		},
	}
	app.Commands = []cli.Command{
		{
			Name:      "list",
			ShortName: "l",
			Usage:     "List remote files",
			Action:    listFiles,
		},
		{
			Name:      "upload",
			ShortName: "u",
			Usage:     "Upload a single file",
			Action:    uploadFile,
		},
		{
			Name:      "sync",
			ShortName: "s",
			Usage:     "Syncs a folder to a Google Cloudstorage bucket",
			Action:    syncFolder,
		},
	}
	app.Run(os.Args)
}

func generateOAuthConfig(c *cli.Context) (*oauth.Config, error) {
	clientId := c.GlobalString("clientid")
	if clientId == "" {
		return &oauth.Config{}, fmt.Errorf("Could not find Client ID")
	}
	clientSecret := c.GlobalString("clientsecret")
	if clientSecret == "" {
		return &oauth.Config{}, fmt.Errorf("Could not find Client Secret")
	}

	return &oauth.Config{
		ClientId:     clientId,
		ClientSecret: clientSecret,
		Scope:        scope,
		AuthURL:      authURL,
		TokenURL:     tokenURL,
		TokenCache:   oauth.CacheFile(c.GlobalString("cachefile")),
		RedirectURL:  redirectURL,
	}, nil
}

func generateServiceConfig(c *cli.Context) (*gcssync.ServiceConfig, error) {
	projectID := c.GlobalString("projectid")
	if projectID == "" {
		return &gcssync.ServiceConfig{}, fmt.Errorf("Could not find project id")
	}
	bucketName := c.GlobalString("bucketname")
	if bucketName == "" {
		return &gcssync.ServiceConfig{}, fmt.Errorf("Cloud not find bucket name")
	}
	return &gcssync.ServiceConfig{
		ProjectID:  projectID,
		BucketName: bucketName,
	}, nil
}

func getClient(c *cli.Context) *gcssync.Client {
	oauthConfig, err := generateOAuthConfig(c)
	if err != nil {
		fmt.Println("Missing auth informations", err.Error())
		os.Exit(errorAuthInfo)
	}
	serviceConfig, err := generateServiceConfig(c)
	if err != nil {
		fmt.Println("Missing project config", err.Error())
		os.Exit(errorProjectInfo)
	}

	client, err := gcssync.NewClient(oauthConfig, c.GlobalString("code"), serviceConfig)
	if err != nil {
		fmt.Println("Error initilizing client: ", err.Error())
		os.Exit(errorClientInit)
	}

	return client
}

func listFiles(c *cli.Context) {
	client := getClient(c)
	files, err := client.ListFiles()
	if err != nil {
		fmt.Println(err)
		os.Exit(errorListFiles)
		return
	}
	for _, object := range files {
		fmt.Printf("%s %s\n", object.Name, humanize.Bytes(object.Size))
	}
	fmt.Printf("Objects in %s - %d\n", client.GetBucketname(), len(files))
}

func uploadFile(c *cli.Context) {
	client := getClient(c)
	if len(c.Args()) != 2 {
		fmt.Println("Need local and remote name!")
		os.Exit(errorUploadFiles)
	}

	success, object, err := client.UploadFile(c.Args().Get(0), c.Args().Get(1))
	if !success {
		fmt.Println(err.Error())
		os.Exit(errorUploadFiles)
		return
	}

	fmt.Printf("Uploaded file to %s\n", client.GetBucketname())
	fmt.Printf("%s %s\n", object.Name, humanize.Bytes(object.Size))

}

func syncFolder(c *cli.Context) {
	client := getClient(c)
	var local, remote string
	switch len(c.Args()) {
	case 0:
		local = ""
		remote = ""
	case 1:
		local = c.Args().Get(0)
		remote = ""
	case 2:
		local = c.Args().Get(0)
		remote = c.Args().Get(1)
	default:
		fmt.Println("To many arguments")
		os.Exit(errorSyncFiles)
	}
	local, err := filepath.Abs(local)
	if err != nil {
		fmt.Println("Could not get absolute path")
		os.Exit(errorSyncFiles)
	}
	client.SyncFolder(local, remote)
}
<|endoftext|>


<commit_before>// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Govulncheck reports known vulnerabilities that affect Go code. It uses static
analysis of source code or a binary's symbol table to narrow down reports to
only those that potentially affect the application. For more information about
the API behind govulncheck, see https://go.dev/security/vuln/vulncheck.

By default, govulncheck uses the Go vulnerability database at
https://vuln.go.dev. Set the GOVULNDB environment variable to specify a
different database.  The database must follow the specification at
https://go.dev/security/vuln/database.

Govulncheck requires Go version 1.18 or higher to run.

# Usage

To analyze source code, run govulncheck from the module directory, using the
same package path syntax that the go command uses:

	$ cd my-module
	$ govulncheck ./...

If no vulnerabilities are found, govulncheck will display a short message. If
there are vulnerabilities, each is displayed briefly, with a summary of a call
stack.

The call stack summary shows in brief how the package calls a vulnerable
function. For example, it might say

	main.go:[line]:[column]: mypackage.main calls golang.org/x/text/language.Parse

For a more detailed call path that resembles Go panic stack traces, use the -v
flag.

To control which files are processed, use the -tags flag to provide a
comma-separated list of build tags, and the -tests flag to indicate that test
files should be included.

To run govulncheck on a compiled binary, pass it the path to the binary file:

	$ govulncheck $HOME/go/bin/my-go-program

Govulncheck uses the binary's symbol information to find mentions of vulnerable
functions. Its output and exit codes are as described above, except that
without source it cannot produce call stacks.

# Other Modes

A few flags control govulncheck's output. Regardless of output, govulncheck
exits successfully if there are no vulnerabilities, and exits unsuccessfully if
there are.

The -v flag outputs more information about call stacks when run on source. It
has no effect when run on a binary.

The -json flag outputs a JSON object with vulnerability information. The output
corresponds to the type golang.org/x/vuln/vulncheck.Result.

# Weaknesses

Govulncheck is built on top of golang.org/x/vuln/vulncheck library and thus
shares its limitations described at
https://go.dev/security/vulncheck#limitations.
*/
package main
<commit_msg>cmd/govulncheck: update docs<commit_after>// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Govulncheck reports known vulnerabilities that affect Go code. It uses static
analysis of source code or a binary's symbol table to narrow down reports to
only those that could affect the application.

By default, govulncheck makes requests to the Go vulnerability database at
https://vuln.go.dev. Requests to the vulnerability database contain only module
paths, not code or other properties of your program. See
https://vuln.go.dev/privacy.html for more. Set the GOVULNDB environment
variable to specify a different database, which must implement the
specification at https://go.dev/security/vuln/database.

Govulncheck looks for vulnerabilities in Go programs using a specific build
configuration. For analyzing source code, that configuration is the operating
system, architecture, and Go version specified by GOOS, GOARCH, and the “go”
command found on the PATH. For binaries, the build configuration is the one
used to build the binary. Note that different build configurations may have
different known vulnerabilities. For example, a dependency with a
Windows-specific vulnerability will not be reported for a Linux build.

Govulncheck must be built with Go version 1.18 or later.

# Usage

To analyze source code, run govulncheck from the module directory, using the
same package path syntax that the go command uses:

	$ cd my-module
	$ govulncheck ./...

If no vulnerabilities are found, govulncheck will display a short message. If
there are vulnerabilities, each is displayed briefly, with a summary of a call
stack.

The call stack summary shows in brief how the package calls a vulnerable
function. For example, it might say

	main.go:[line]:[column]: mypackage.main calls golang.org/x/text/language.Parse

For a more detailed call path that resembles Go panic stack traces, use the -v flag.

To control which files are processed, use the -tags flag to provide a
comma-separated list of build tags, and the -test flag to indicate that test
files should be included.

To run govulncheck on a compiled binary, pass it the path to the binary file:

	$ govulncheck $HOME/go/bin/my-go-program

Govulncheck uses the binary's symbol information to find mentions of vulnerable
functions. Its output omits call stacks, which require source code analysis.

Govulncheck exits successfully (exit code 0) if there are no vulnerabilities,
and exits unsuccessfully if there are.

# Flags

A few flags control govulncheck's behavior.

The -v flag causes govulncheck to output more information about call stacks
when run on source. It has no effect when run on a binary.

The -json flag causes govulncheck to print its output as a JSON object
corresponding to the type [golang.org/x/vuln/vulncheck.Result].

The -tags flag accepts a comma-separated list of build tags to control which
files should be included in loaded packages for source analysis.

The -test flag causes govulncheck to include test files in the source analysis.

# Limitations

Govulncheck uses [golang.org/x/vuln/vulncheck], which has these limitations:

  - Govulncheck analyzes function pointer and interface calls conservatively,
    which may result in false positives or inaccurate call stacks in some cases.
  - Calls to functions made using package reflect are not visible to static
    analysis. Vulnerable code reachable only through those calls will not be
    reported.
  - Because Go binaries do not contain detailed call information, vulncheck
    cannot show the call graphs for detected vulnerabilities. It may also
    report false positives for code that is in the binary but unreachable.
  - Govulncheck does not report vulnerabilities in
    vendored packages for binaries.
  - There is no support for silencing vulnerability findings.
  - Govulncheck only reads binaries compiled with Go 1.18 and later.

# Feedback

Govulncheck is an experimental tool under active development. To share
feedback, see https://go.dev/security/vuln#feedback.
*/
package main
<|endoftext|>