<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"strings"
	"sync"
	"time"

	docker "github.com/fsouza/go-dockerclient"
	"github.com/litl/galaxy/config"
	"github.com/litl/galaxy/log"
	"github.com/litl/galaxy/registry"
	"github.com/litl/galaxy/runtime"
	"github.com/litl/galaxy/utils"
)

var (
	stopCutoff      int64
	apps            []string
	redisHost       string
	env             string
	pool            string
	loop            bool
	shuttleHost     string
	statsdHost      string
	debug           bool
	runOnce         bool
	version         bool
	buildVersion    string
	serviceConfigs  []*config.ServiceConfig
	serviceRegistry *registry.ServiceRegistry
	configStore     *config.ConfigStore
	serviceRuntime  *runtime.ServiceRuntime
	workerChans     map[string]chan string
	wg              sync.WaitGroup
)

func initOrDie() {

	serviceRegistry = registry.NewServiceRegistry(
		"",
		registry.DefaultTTL,
		"",
	)
	serviceRegistry.Connect(redisHost)

	configStore = config.NewConfigStore(
		"",
		registry.DefaultTTL,
		"",
	)
	configStore.Connect(redisHost)

	serviceRuntime = runtime.NewServiceRuntime(serviceRegistry, shuttleHost, statsdHost)

	apps, err := configStore.ListAssignments(env, pool)
	if err != nil {
		log.Fatalf("ERROR: Could not retrieve service configs for /%s/%s: %s\n", env, pool, err)
	}

	workerChans = make(map[string]chan string)
	for _, app := range apps {
		serviceConfig, err := configStore.Get(app, env)
		if err != nil {
			log.Fatalf("ERROR: Could not retrieve service config for /%s/%s: %s\n", env, pool, err)
		}

		workerChans[serviceConfig.Name] = make(chan string)
	}
}

func pullImageAsync(serviceConfig config.ServiceConfig, errChan chan error) {
	// err logged via pullImage
	_, err := pullImage(&serviceConfig)
	if err != nil {
		errChan <- err
		return
	}
	errChan <- nil
}

func pullImage(serviceConfig *config.ServiceConfig) (*docker.Image, error) {

	image, err := serviceRuntime.InspectImage(serviceConfig.Version())
	if image != nil && image.ID == serviceConfig.VersionID() || serviceConfig.VersionID() == "" {
		return image, nil
	}

	log.Printf("Pulling %s version %s\n", serviceConfig.Name, serviceConfig.Version())
	image, err = serviceRuntime.PullImage(serviceConfig.Version(),
		serviceConfig.VersionID(), true)
	if image == nil || err != nil {
		log.Errorf("ERROR: Could not pull image %s: %s",
			serviceConfig.Version(), err)
		return nil, err
	}

	if image.ID != serviceConfig.VersionID() && len(serviceConfig.VersionID()) > 12 {
		log.Errorf("ERROR: Pulled image for %s does not match expected ID. Expected: %s: Got: %s",
			serviceConfig.Version(),
			image.ID[0:12], serviceConfig.VersionID()[0:12])
		return nil, errors.New(fmt.Sprintf("failed to pull image ID %s", serviceConfig.VersionID()[0:12]))
	}

	log.Printf("Pulled %s\n", serviceConfig.Version())
	return image, nil
}

func startService(serviceConfig *config.ServiceConfig, logStatus bool) {
	started, container, err := serviceRuntime.StartIfNotRunning(env, serviceConfig)
	if err != nil {
		log.Errorf("ERROR: Could not start container for %s: %s", serviceConfig.Version(), err)
		return
	}

	if started {
		log.Printf("Started %s version %s as %s\n", serviceConfig.Name, serviceConfig.Version(), container.ID[0:12])
	}

	if logStatus && !debug {
		log.Printf("%s version %s running as %s\n", serviceConfig.Name, serviceConfig.Version(), container.ID[0:12])
	}

	log.Debugf("%s version %s running as %s\n", serviceConfig.Name, serviceConfig.Version(), container.ID[0:12])

	err = serviceRuntime.StopAllButLatestService(serviceConfig.Name, stopCutoff)
	if err != nil {
		log.Errorf("ERROR: Could not stop containers: %s", err)
	}
}

func appAssigned(app string) (bool, error) {
	assignments, err := configStore.ListAssignments(env, pool)
	if err != nil {
		return false, err
	}

	if !utils.StringInSlice(app, assignments) {
		return false, nil
	}
	return true, nil
}

func restartContainers(app string, cmdChan chan string) {
	defer wg.Done()
	logOnce := true

	ticker := time.NewTicker(10 * time.Second)

	for {

		select {

		case cmd := <-cmdChan:

			assigned, err := appAssigned(app)
			if err != nil {
				log.Errorf("ERROR: Error retrieving assignments for %s: %s", app, err)
				if !loop {
					return
				}
				continue
			}

			if !assigned {
				continue
			}

			serviceConfig, err := configStore.Get(app, env)
			if err != nil {
				log.Errorf("ERROR: Error retrieving service config for %s: %s", app, err)
				if !loop {
					return
				}
				continue
			}

			if serviceConfig.Version() == "" {
				if !loop {
					return
				}
				continue
			}

			if cmd == "deploy" {
				_, err = pullImage(serviceConfig)
				if err != nil {
					if !loop {
						return
					}
					continue
				}
				startService(serviceConfig, logOnce)
			}

			if cmd == "restart" {
				err := serviceRuntime.Stop(serviceConfig)
				if err != nil {
					log.Errorf("ERROR: Could not stop %s: %s",
						serviceConfig.Version(), err)
					if !loop {
						return
					}

					startService(serviceConfig, logOnce)
					continue
				}
			}

			logOnce = false
		case <-ticker.C:

			serviceConfig, err := configStore.Get(app, env)
			if err != nil {
				log.Errorf("ERROR: Error retrieving service config for %s: %s", app, err)
				continue
			}

			assigned, err := appAssigned(app)
			if err != nil {
				log.Errorf("ERROR: Error retrieving service config for %s: %s", app, err)
				if !loop {
					return
				}

				continue
			}

			if serviceConfig == nil || !assigned {
				log.Errorf("%s no longer exists.  Stopping worker.", app)
				serviceRuntime.StopAllMatching(app)
				delete(workerChans, app)
				return
			}

			if serviceConfig.Version() == "" {
				continue
			}

			_, err = pullImage(serviceConfig)
			if err != nil {
				if !loop {
					return
				}
				log.Errorf("ERROR: Could not pull images: %s", err)
				continue
			}

			started, container, err := serviceRuntime.StartIfNotRunning(env, serviceConfig)
			if err != nil {
				log.Errorf("ERROR: Could not start containers: %s", err)
				continue
			}

			if started {
				log.Printf("Started %s version %s as %s\n", serviceConfig.Name, serviceConfig.Version(), container.ID[0:12])
			}

			log.Debugf("%s version %s running as %s\n", serviceConfig.Name, serviceConfig.Version(), container.ID[0:12])

			err = serviceRuntime.StopAllButCurrentVersion(serviceConfig)
			if err != nil {
				log.Errorf("ERROR: Could not stop containers: %s", err)
			}
		}

		if !loop {
			return
		}

	}
}

func monitorService(changedConfigs chan *config.ConfigChange) {

	for {

		var changedConfig *config.ConfigChange
		select {

		case changedConfig = <-changedConfigs:

			if changedConfig.Error != nil {
				log.Errorf("ERROR: Error watching changes: %s", changedConfig.Error)
				continue
			}

			if changedConfig.ServiceConfig == nil {
				continue
			}

			assigned, err := appAssigned(changedConfig.ServiceConfig.Name)
			if err != nil {
				log.Errorf("ERROR: Error retrieving service config for %s: %s", changedConfig.ServiceConfig.Name, err)
				if !loop {
					return
				}
				continue
			}

			if !assigned {
				continue
			}

			ch, ok := workerChans[changedConfig.ServiceConfig.Name]
			if !ok {
				name := changedConfig.ServiceConfig.Name
				ch := make(chan string)
				workerChans[name] = ch
				wg.Add(1)
				go restartContainers(name, ch)
				ch <- "deploy"

				log.Printf("Started new worker for %s\n", name)
				continue
			}

			if changedConfig.Restart {
				log.Printf("Restarting %s", changedConfig.ServiceConfig.Name)
				ch <- "restart"
			} else {
				ch <- "deploy"
			}
		}
	}

}

func main() {
	flag.Int64Var(&stopCutoff, "cutoff", 10, "Seconds to wait before stopping old containers")
	flag.StringVar(&redisHost, "redis", utils.GetEnv("GALAXY_REDIS_HOST", utils.DefaultRedisHost), "redis host")
	flag.StringVar(&env, "env", utils.GetEnv("GALAXY_ENV", ""), "Environment namespace")
	flag.StringVar(&pool, "pool", utils.GetEnv("GALAXY_POOL", ""), "Pool namespace")
	flag.StringVar(&shuttleHost, "shuttleAddr", "", "IP where containers can reach shuttle proxy. Defaults to docker0 IP.")
	flag.StringVar(&statsdHost, "statsdAddr", utils.GetEnv("GALAXY_STATSD_HOST", ""), "IP where containers can reach a statsd service. Defaults to docker0 IP:8125.")
	flag.BoolVar(&debug, "debug", false, "verbose logging")
	flag.BoolVar(&version, "v", false, "display version info")

	flag.Usage = func() {
		println("Usage: commander [options] <command> [<args>]\n")
		println("Available commands are:")
		println("   agent           Runs commander agent")
		println("   start           Starts one or more apps")
		println("   stop            Stops one or more apps")
		println("\nOptions:\n")
		flag.PrintDefaults()

	}

	flag.Parse()

	if version {
		fmt.Println(buildVersion)
		return
	}

	if strings.TrimSpace(env) == "" {
		fmt.Println("Need an env")
		flag.PrintDefaults()
		os.Exit(1)
	}

	if strings.TrimSpace(pool) == "" {
		fmt.Println("Need a pool")
		flag.PrintDefaults()
		os.Exit(1)
	}

	if debug {
		log.DefaultLogger.Level = log.DEBUG
	}

	if flag.NArg() < 1 {
		fmt.Println("Need a command")
		flag.PrintDefaults()
		os.Exit(1)
	}

	initOrDie()

	switch flag.Args()[0] {
	case "agent":
		loop = true
		agentFs := flag.NewFlagSet("agent", flag.ExitOnError)
		agentFs.Usage = func() {
			println("Usage: commander agent [options]\n")
			println("    Runs commander continuously\n\n")
			println("Options:\n\n")
			agentFs.PrintDefaults()
		}
		agentFs.Parse(flag.Args()[1:])
	case "start":

		startFs := flag.NewFlagSet("start", flag.ExitOnError)
		startFs.Usage = func() {
			println("Usage: commander start [options] [<app>]*\n")
			println("    Starts one or more apps. If no apps are specified, starts all apps.\n\n")
			println("Options:\n\n")
			startFs.PrintDefaults()
		}
		startFs.Parse(flag.Args()[1:])

		apps = startFs.Args()

		break
	case "stop":
		stopFs := flag.NewFlagSet("stop", flag.ExitOnError)
		stopFs.Usage = func() {
			println("Usage: commander stop [options] [<app>]*\n")
			println("    Stops one or more apps. If no apps are specified, stops all apps.\n\n")
			println("Options:\n\n")
			stopFs.PrintDefaults()
		}
		stopFs.Parse(flag.Args()[1:])

		apps = stopFs.Args()

		for _, app := range apps {
			err := serviceRuntime.StopAllMatching(app)
			if err != nil {
				log.Fatalf("ERROR: Unable able to stop all containers: %s", err)
			}
		}
		if len(apps) > 0 {
			return
		}

		err := serviceRuntime.StopAll(env)
		if err != nil {
			log.Fatalf("ERROR: Unable able to stop all containers: %s", err)
		}
		return
	}

	log.Printf("Starting commander %s", buildVersion)
	log.Printf("Using env = %s, pool = %s",
		env, pool)

	for app, ch := range workerChans {
		if len(apps) == 0 || utils.StringInSlice(app, apps) {
			wg.Add(1)
			go restartContainers(app, ch)
			ch <- "deploy"
		}
	}

	if loop {

		cancelChan := make(chan struct{})
		// do we need to cancel ever?

		restartChan := configStore.Watch(env, cancelChan)
		monitorService(restartChan)
	}

	wg.Wait()
}
<commit_msg>Print Usage() when no args passed<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"strings"
	"sync"
	"time"

	docker "github.com/fsouza/go-dockerclient"
	"github.com/litl/galaxy/config"
	"github.com/litl/galaxy/log"
	"github.com/litl/galaxy/registry"
	"github.com/litl/galaxy/runtime"
	"github.com/litl/galaxy/utils"
)

var (
	stopCutoff      int64
	apps            []string
	redisHost       string
	env             string
	pool            string
	loop            bool
	shuttleHost     string
	statsdHost      string
	debug           bool
	runOnce         bool
	version         bool
	buildVersion    string
	serviceConfigs  []*config.ServiceConfig
	serviceRegistry *registry.ServiceRegistry
	configStore     *config.ConfigStore
	serviceRuntime  *runtime.ServiceRuntime
	workerChans     map[string]chan string
	wg              sync.WaitGroup
)

func initOrDie() {

	serviceRegistry = registry.NewServiceRegistry(
		"",
		registry.DefaultTTL,
		"",
	)
	serviceRegistry.Connect(redisHost)

	configStore = config.NewConfigStore(
		"",
		registry.DefaultTTL,
		"",
	)
	configStore.Connect(redisHost)

	serviceRuntime = runtime.NewServiceRuntime(serviceRegistry, shuttleHost, statsdHost)

	apps, err := configStore.ListAssignments(env, pool)
	if err != nil {
		log.Fatalf("ERROR: Could not retrieve service configs for /%s/%s: %s\n", env, pool, err)
	}

	workerChans = make(map[string]chan string)
	for _, app := range apps {
		serviceConfig, err := configStore.Get(app, env)
		if err != nil {
			log.Fatalf("ERROR: Could not retrieve service config for /%s/%s: %s\n", env, pool, err)
		}

		workerChans[serviceConfig.Name] = make(chan string)
	}
}

func pullImageAsync(serviceConfig config.ServiceConfig, errChan chan error) {
	// err logged via pullImage
	_, err := pullImage(&serviceConfig)
	if err != nil {
		errChan <- err
		return
	}
	errChan <- nil
}

func pullImage(serviceConfig *config.ServiceConfig) (*docker.Image, error) {

	image, err := serviceRuntime.InspectImage(serviceConfig.Version())
	if image != nil && image.ID == serviceConfig.VersionID() || serviceConfig.VersionID() == "" {
		return image, nil
	}

	log.Printf("Pulling %s version %s\n", serviceConfig.Name, serviceConfig.Version())
	image, err = serviceRuntime.PullImage(serviceConfig.Version(),
		serviceConfig.VersionID(), true)
	if image == nil || err != nil {
		log.Errorf("ERROR: Could not pull image %s: %s",
			serviceConfig.Version(), err)
		return nil, err
	}

	if image.ID != serviceConfig.VersionID() && len(serviceConfig.VersionID()) > 12 {
		log.Errorf("ERROR: Pulled image for %s does not match expected ID. Expected: %s: Got: %s",
			serviceConfig.Version(),
			image.ID[0:12], serviceConfig.VersionID()[0:12])
		return nil, errors.New(fmt.Sprintf("failed to pull image ID %s", serviceConfig.VersionID()[0:12]))
	}

	log.Printf("Pulled %s\n", serviceConfig.Version())
	return image, nil
}

func startService(serviceConfig *config.ServiceConfig, logStatus bool) {
	started, container, err := serviceRuntime.StartIfNotRunning(env, serviceConfig)
	if err != nil {
		log.Errorf("ERROR: Could not start container for %s: %s", serviceConfig.Version(), err)
		return
	}

	if started {
		log.Printf("Started %s version %s as %s\n", serviceConfig.Name, serviceConfig.Version(), container.ID[0:12])
	}

	if logStatus && !debug {
		log.Printf("%s version %s running as %s\n", serviceConfig.Name, serviceConfig.Version(), container.ID[0:12])
	}

	log.Debugf("%s version %s running as %s\n", serviceConfig.Name, serviceConfig.Version(), container.ID[0:12])

	err = serviceRuntime.StopAllButLatestService(serviceConfig.Name, stopCutoff)
	if err != nil {
		log.Errorf("ERROR: Could not stop containers: %s", err)
	}
}

func appAssigned(app string) (bool, error) {
	assignments, err := configStore.ListAssignments(env, pool)
	if err != nil {
		return false, err
	}

	if !utils.StringInSlice(app, assignments) {
		return false, nil
	}
	return true, nil
}

func restartContainers(app string, cmdChan chan string) {
	defer wg.Done()
	logOnce := true

	ticker := time.NewTicker(10 * time.Second)

	for {

		select {

		case cmd := <-cmdChan:

			assigned, err := appAssigned(app)
			if err != nil {
				log.Errorf("ERROR: Error retrieving assignments for %s: %s", app, err)
				if !loop {
					return
				}
				continue
			}

			if !assigned {
				continue
			}

			serviceConfig, err := configStore.Get(app, env)
			if err != nil {
				log.Errorf("ERROR: Error retrieving service config for %s: %s", app, err)
				if !loop {
					return
				}
				continue
			}

			if serviceConfig.Version() == "" {
				if !loop {
					return
				}
				continue
			}

			if cmd == "deploy" {
				_, err = pullImage(serviceConfig)
				if err != nil {
					if !loop {
						return
					}
					continue
				}
				startService(serviceConfig, logOnce)
			}

			if cmd == "restart" {
				err := serviceRuntime.Stop(serviceConfig)
				if err != nil {
					log.Errorf("ERROR: Could not stop %s: %s",
						serviceConfig.Version(), err)
					if !loop {
						return
					}

					startService(serviceConfig, logOnce)
					continue
				}
			}

			logOnce = false
		case <-ticker.C:

			serviceConfig, err := configStore.Get(app, env)
			if err != nil {
				log.Errorf("ERROR: Error retrieving service config for %s: %s", app, err)
				continue
			}

			assigned, err := appAssigned(app)
			if err != nil {
				log.Errorf("ERROR: Error retrieving service config for %s: %s", app, err)
				if !loop {
					return
				}

				continue
			}

			if serviceConfig == nil || !assigned {
				log.Errorf("%s no longer exists.  Stopping worker.", app)
				serviceRuntime.StopAllMatching(app)
				delete(workerChans, app)
				return
			}

			if serviceConfig.Version() == "" {
				continue
			}

			_, err = pullImage(serviceConfig)
			if err != nil {
				if !loop {
					return
				}
				log.Errorf("ERROR: Could not pull images: %s", err)
				continue
			}

			started, container, err := serviceRuntime.StartIfNotRunning(env, serviceConfig)
			if err != nil {
				log.Errorf("ERROR: Could not start containers: %s", err)
				continue
			}

			if started {
				log.Printf("Started %s version %s as %s\n", serviceConfig.Name, serviceConfig.Version(), container.ID[0:12])
			}

			log.Debugf("%s version %s running as %s\n", serviceConfig.Name, serviceConfig.Version(), container.ID[0:12])

			err = serviceRuntime.StopAllButCurrentVersion(serviceConfig)
			if err != nil {
				log.Errorf("ERROR: Could not stop containers: %s", err)
			}
		}

		if !loop {
			return
		}

	}
}

func monitorService(changedConfigs chan *config.ConfigChange) {

	for {

		var changedConfig *config.ConfigChange
		select {

		case changedConfig = <-changedConfigs:

			if changedConfig.Error != nil {
				log.Errorf("ERROR: Error watching changes: %s", changedConfig.Error)
				continue
			}

			if changedConfig.ServiceConfig == nil {
				continue
			}

			assigned, err := appAssigned(changedConfig.ServiceConfig.Name)
			if err != nil {
				log.Errorf("ERROR: Error retrieving service config for %s: %s", changedConfig.ServiceConfig.Name, err)
				if !loop {
					return
				}
				continue
			}

			if !assigned {
				continue
			}

			ch, ok := workerChans[changedConfig.ServiceConfig.Name]
			if !ok {
				name := changedConfig.ServiceConfig.Name
				ch := make(chan string)
				workerChans[name] = ch
				wg.Add(1)
				go restartContainers(name, ch)
				ch <- "deploy"

				log.Printf("Started new worker for %s\n", name)
				continue
			}

			if changedConfig.Restart {
				log.Printf("Restarting %s", changedConfig.ServiceConfig.Name)
				ch <- "restart"
			} else {
				ch <- "deploy"
			}
		}
	}

}

func main() {
	flag.Int64Var(&stopCutoff, "cutoff", 10, "Seconds to wait before stopping old containers")
	flag.StringVar(&redisHost, "redis", utils.GetEnv("GALAXY_REDIS_HOST", utils.DefaultRedisHost), "redis host")
	flag.StringVar(&env, "env", utils.GetEnv("GALAXY_ENV", ""), "Environment namespace")
	flag.StringVar(&pool, "pool", utils.GetEnv("GALAXY_POOL", ""), "Pool namespace")
	flag.StringVar(&shuttleHost, "shuttleAddr", "", "IP where containers can reach shuttle proxy. Defaults to docker0 IP.")
	flag.StringVar(&statsdHost, "statsdAddr", utils.GetEnv("GALAXY_STATSD_HOST", ""), "IP where containers can reach a statsd service. Defaults to docker0 IP:8125.")
	flag.BoolVar(&debug, "debug", false, "verbose logging")
	flag.BoolVar(&version, "v", false, "display version info")

	flag.Usage = func() {
		println("Usage: commander [options] <command> [<args>]\n")
		println("Available commands are:")
		println("   agent           Runs commander agent")
		println("   start           Starts one or more apps")
		println("   stop            Stops one or more apps")
		println("\nOptions:\n")
		flag.PrintDefaults()

	}

	flag.Parse()

	if version {
		fmt.Println(buildVersion)
		return
	}

	if strings.TrimSpace(env) == "" {
		fmt.Println("Need an env")
		flag.PrintDefaults()
		os.Exit(1)
	}

	if strings.TrimSpace(pool) == "" {
		fmt.Println("Need a pool")
		flag.PrintDefaults()
		os.Exit(1)
	}

	if debug {
		log.DefaultLogger.Level = log.DEBUG
	}

	if flag.NArg() < 1 {
		fmt.Println("Need a command")
		flag.Usage()
		os.Exit(1)
	}

	initOrDie()

	switch flag.Args()[0] {
	case "agent":
		loop = true
		agentFs := flag.NewFlagSet("agent", flag.ExitOnError)
		agentFs.Usage = func() {
			println("Usage: commander agent [options]\n")
			println("    Runs commander continuously\n\n")
			println("Options:\n\n")
			agentFs.PrintDefaults()
		}
		agentFs.Parse(flag.Args()[1:])
	case "start":

		startFs := flag.NewFlagSet("start", flag.ExitOnError)
		startFs.Usage = func() {
			println("Usage: commander start [options] [<app>]*\n")
			println("    Starts one or more apps. If no apps are specified, starts all apps.\n\n")
			println("Options:\n\n")
			startFs.PrintDefaults()
		}
		startFs.Parse(flag.Args()[1:])

		apps = startFs.Args()

		break
	case "stop":
		stopFs := flag.NewFlagSet("stop", flag.ExitOnError)
		stopFs.Usage = func() {
			println("Usage: commander stop [options] [<app>]*\n")
			println("    Stops one or more apps. If no apps are specified, stops all apps.\n\n")
			println("Options:\n\n")
			stopFs.PrintDefaults()
		}
		stopFs.Parse(flag.Args()[1:])

		apps = stopFs.Args()

		for _, app := range apps {
			err := serviceRuntime.StopAllMatching(app)
			if err != nil {
				log.Fatalf("ERROR: Unable able to stop all containers: %s", err)
			}
		}
		if len(apps) > 0 {
			return
		}

		err := serviceRuntime.StopAll(env)
		if err != nil {
			log.Fatalf("ERROR: Unable able to stop all containers: %s", err)
		}
		return
	}

	log.Printf("Starting commander %s", buildVersion)
	log.Printf("Using env = %s, pool = %s",
		env, pool)

	for app, ch := range workerChans {
		if len(apps) == 0 || utils.StringInSlice(app, apps) {
			wg.Add(1)
			go restartContainers(app, ch)
			ch <- "deploy"
		}
	}

	if loop {

		cancelChan := make(chan struct{})
		// do we need to cancel ever?

		restartChan := configStore.Watch(env, cancelChan)
		monitorService(restartChan)
	}

	wg.Wait()
}
<|endoftext|>


<commit_before>package memqueue

import (
	"context"
	"errors"
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/vmihailenco/taskq/v2"
	"github.com/vmihailenco/taskq/v2/internal"
)

type Queue struct {
	opt *taskq.QueueOptions

	sync    bool
	noDelay bool

	wg       sync.WaitGroup
	consumer *taskq.Consumer

	_closed int32
}

var _ taskq.Queue = (*Queue)(nil)

func NewQueue(opt *taskq.QueueOptions) *Queue {
	opt.Init()

	q := &Queue{
		opt: opt,
	}
	q.consumer = taskq.NewConsumer(q)
	if err := q.consumer.Start(context.Background()); err != nil {
		panic(err)
	}

	return q
}

func (q *Queue) Name() string {
	return q.opt.Name
}

func (q *Queue) String() string {
	return fmt.Sprintf("queue=%q", q.Name())
}

func (q *Queue) Options() *taskq.QueueOptions {
	return q.opt
}

func (q *Queue) Consumer() *taskq.Consumer {
	return q.consumer
}

func (q *Queue) SetSync(sync bool) {
	q.sync = sync
}

func (q *Queue) SetNoDelay(noDelay bool) {
	q.noDelay = noDelay
}

// Close is like CloseTimeout with 30 seconds timeout.
func (q *Queue) Close() error {
	return q.CloseTimeout(30 * time.Second)
}

// CloseTimeout closes the queue waiting for pending messages to be processed.
func (q *Queue) CloseTimeout(timeout time.Duration) error {
	if !atomic.CompareAndSwapInt32(&q._closed, 0, 1) {
		return fmt.Errorf("taskq: %s is already closed", q)
	}

	done := make(chan struct{}, 1)
	go func() {
		q.wg.Wait()
		done <- struct{}{}
	}()

	select {
	case <-done:
	case <-time.After(timeout):
		return fmt.Errorf("taskq: %s: messages are not processed after %s", q.consumer, timeout)
	}

	_ = q.consumer.StopTimeout(timeout)

	return nil
}

func (q *Queue) Len() (int, error) {
	return q.consumer.Len(), nil
}

// Add adds message to the queue.
func (q *Queue) Add(msg *taskq.Message) error {
	if q.closed() {
		return fmt.Errorf("taskq: %s is closed", q)
	}
	if msg.TaskName == "" {
		return internal.ErrTaskNameRequired
	}
	if q.isDuplicate(msg) {
		return taskq.ErrDuplicate
	}
	q.wg.Add(1)
	return q.enqueueMessage(msg)
}

func (q *Queue) enqueueMessage(msg *taskq.Message) error {
	if (q.noDelay || q.sync) && msg.Delay > 0 {
		msg.Delay = 0
	}
	msg.ReservedCount++

	if q.sync {
		return q.consumer.Process(msg)
	}
	return q.consumer.Add(msg)
}

func (q *Queue) ReserveN(n int, waitTimeout time.Duration) ([]taskq.Message, error) {
	return nil, internal.ErrNotSupported
}

func (q *Queue) Release(msg *taskq.Message) error {
	return q.enqueueMessage(msg)
}

func (q *Queue) Delete(msg *taskq.Message) error {
	q.wg.Done()
	return nil
}

func (q *Queue) DeleteBatch(msgs []*taskq.Message) error {
	if len(msgs) == 0 {
		return errors.New("taskq: no messages to delete")
	}
	for _, msg := range msgs {
		if err := q.Delete(msg); err != nil {
			return err
		}
	}
	return nil
}

func (q *Queue) Purge() error {
	return q.consumer.Purge()
}

func (q *Queue) closed() bool {
	return atomic.LoadInt32(&q._closed) == 1
}

func (q *Queue) isDuplicate(msg *taskq.Message) bool {
	if msg.Name == "" {
		return false
	}
	return q.opt.Storage.Exists("taskq:" + q.opt.Name + ":" + msg.Name)
}
<commit_msg>memqueue: extract WaitTimeout<commit_after>package memqueue

import (
	"context"
	"errors"
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/vmihailenco/taskq/v2"
	"github.com/vmihailenco/taskq/v2/internal"
)

type Queue struct {
	opt *taskq.QueueOptions

	sync    bool
	noDelay bool

	wg       sync.WaitGroup
	consumer *taskq.Consumer

	_closed int32
}

var _ taskq.Queue = (*Queue)(nil)

func NewQueue(opt *taskq.QueueOptions) *Queue {
	opt.Init()

	q := &Queue{
		opt: opt,
	}
	q.consumer = taskq.NewConsumer(q)
	if err := q.consumer.Start(context.Background()); err != nil {
		panic(err)
	}

	return q
}

func (q *Queue) Name() string {
	return q.opt.Name
}

func (q *Queue) String() string {
	return fmt.Sprintf("queue=%q", q.Name())
}

func (q *Queue) Options() *taskq.QueueOptions {
	return q.opt
}

func (q *Queue) Consumer() *taskq.Consumer {
	return q.consumer
}

func (q *Queue) SetSync(sync bool) {
	q.sync = sync
}

func (q *Queue) SetNoDelay(noDelay bool) {
	q.noDelay = noDelay
}

// Close is like CloseTimeout with 30 seconds timeout.
func (q *Queue) Close() error {
	return q.CloseTimeout(30 * time.Second)
}

// CloseTimeout closes the queue waiting for pending messages to be processed.
func (q *Queue) CloseTimeout(timeout time.Duration) error {
	if !atomic.CompareAndSwapInt32(&q._closed, 0, 1) {
		return fmt.Errorf("taskq: %s is already closed", q)
	}
	err := q.WaitTimeout(timeout)
	_ = q.consumer.StopTimeout(timeout)
	return err
}

func (q *Queue) WaitTimeout(timeout time.Duration) error {
	done := make(chan struct{}, 1)
	go func() {
		q.wg.Wait()
		done <- struct{}{}
	}()

	select {
	case <-done:
	case <-time.After(timeout):
		return fmt.Errorf("taskq: %s: messages are not processed after %s", q.consumer, timeout)
	}

	return nil
}

func (q *Queue) Len() (int, error) {
	return q.consumer.Len(), nil
}

// Add adds message to the queue.
func (q *Queue) Add(msg *taskq.Message) error {
	if q.closed() {
		return fmt.Errorf("taskq: %s is closed", q)
	}
	if msg.TaskName == "" {
		return internal.ErrTaskNameRequired
	}
	if q.isDuplicate(msg) {
		return taskq.ErrDuplicate
	}
	q.wg.Add(1)
	return q.enqueueMessage(msg)
}

func (q *Queue) enqueueMessage(msg *taskq.Message) error {
	if (q.noDelay || q.sync) && msg.Delay > 0 {
		msg.Delay = 0
	}
	msg.ReservedCount++

	if q.sync {
		return q.consumer.Process(msg)
	}
	return q.consumer.Add(msg)
}

func (q *Queue) ReserveN(n int, waitTimeout time.Duration) ([]taskq.Message, error) {
	return nil, internal.ErrNotSupported
}

func (q *Queue) Release(msg *taskq.Message) error {
	return q.enqueueMessage(msg)
}

func (q *Queue) Delete(msg *taskq.Message) error {
	q.wg.Done()
	return nil
}

func (q *Queue) DeleteBatch(msgs []*taskq.Message) error {
	if len(msgs) == 0 {
		return errors.New("taskq: no messages to delete")
	}
	for _, msg := range msgs {
		if err := q.Delete(msg); err != nil {
			return err
		}
	}
	return nil
}

func (q *Queue) Purge() error {
	return q.consumer.Purge()
}

func (q *Queue) closed() bool {
	return atomic.LoadInt32(&q._closed) == 1
}

func (q *Queue) isDuplicate(msg *taskq.Message) bool {
	if msg.Name == "" {
		return false
	}
	return q.opt.Storage.Exists("taskq:" + q.opt.Name + ":" + msg.Name)
}
<|endoftext|>


<commit_before>/*******************************************************************************
The MIT License (MIT)

Copyright (c) 2015 Hajime Nakagami

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*******************************************************************************/

package toybroker

import (
	"errors"
	"reflect"
	"testing"
)

func TestTopics(t *testing.T) {
	var err error
	topics := NewMemoryTopics()
	topics.Add("foo/bar", "client1")
	topics.Add("foo/bar", "client2")
	topics.Add("foo/baz", "client1")

	if !reflect.DeepEqual(topics.TopicList(), []string{"foo/bar", "foo/baz"}) {
		err = errors.New("topic.TopicList()")
	}

	if !reflect.DeepEqual(topics.List("foo/bar"), []string{"client1", "client2"}) {
		err = errors.New("topics.List(\"foo/bar\")")
	}

	if !reflect.DeepEqual(topics.List("foo/baz"), []string{"client1"}) {
		err = errors.New("topics.List(\"foo/baz\")")
	}

	topics.Remove("foo/bar", "client1")
	if !reflect.DeepEqual(topics.List("foo/bar"), []string{"client2"}) {
		err = errors.New("Remove(\"boo/bar\", \"client1\")")
	}

	if err != nil {
		t.Error(err.Error())
	}
}
<commit_msg>fix test<commit_after>/*******************************************************************************
The MIT License (MIT)

Copyright (c) 2015 Hajime Nakagami

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*******************************************************************************/

package toybroker

import (
	"errors"
	"reflect"
	"sort"
	"testing"
)

func TestTopics(t *testing.T) {
	var err error
	topics := NewMemoryTopics()
	topics.Add("foo/bar", "client1")
	topics.Add("foo/bar", "client2")
	topics.Add("foo/baz", "client1")

	if !reflect.DeepEqual(topics.TopicList(), []string{"foo/bar", "foo/baz"}) {
		err = errors.New("topic.TopicList()")
	}

	r := topics.List("foo/bar")
	sort.Strings(r)
	if !reflect.DeepEqual(r, []string{"client1", "client2"}) {
		err = errors.New("topics.List(\"foo/bar\")")
	}

	if !reflect.DeepEqual(topics.List("foo/baz"), []string{"client1"}) {
		err = errors.New("topics.List(\"foo/baz\")")
	}

	topics.Remove("foo/bar", "client1")
	if !reflect.DeepEqual(topics.List("foo/bar"), []string{"client2"}) {
		err = errors.New("Remove(\"boo/bar\", \"client1\")")
	}

	if err != nil {
		t.Error(err.Error())
	}
}
<|endoftext|>


<commit_before>package realms

import (
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	log "github.com/Sirupsen/logrus"
	"github.com/google/go-github/github"
	"github.com/matrix-org/go-neb/database"
	"github.com/matrix-org/go-neb/services/github/client"
	"github.com/matrix-org/go-neb/types"
	"io/ioutil"
	"net/http"
	"net/url"
)

// GithubRealm can handle OAuth processes with github.com
type GithubRealm struct {
	id           string
	redirectURL  string
	ClientSecret string
	ClientID     string
	StarterLink  string
}

// GithubSession represents an authenticated github session
type GithubSession struct {
	// The client-supplied URL to redirect them to after the auth process is complete.
	ClientsRedirectURL string
	// AccessToken is the github access token for the user
	AccessToken string
	// Scopes are the set of *ALLOWED* scopes (which may not be the same as the requested scopes)
	Scopes  string
	id      string
	userID  string
	realmID string
}

// Authenticated returns true if the user has completed the auth process
func (s *GithubSession) Authenticated() bool {
	return s.AccessToken != ""
}

// Info returns a list of possible repositories that this session can integrate with.
func (s *GithubSession) Info() interface{} {
	logger := log.WithFields(log.Fields{
		"user_id":  s.userID,
		"realm_id": s.realmID,
	})
	cli := client.New(s.AccessToken)
	// query for a list of possible projects
	rs, _, err := cli.Repositories.List("", &github.RepositoryListOptions{
		Type: "all",
	})
	if err != nil {
		logger.WithError(err).Print("Failed to query github projects on github.com")
		return nil
	}

	var repos []client.TrimmedRepository

	for _, r := range rs {
		repos = append(repos, client.TrimRepository(r))
	}

	return struct {
		Repos []client.TrimmedRepository
	}{repos}
}

// UserID returns the user_id who authorised with Github
func (s *GithubSession) UserID() string {
	return s.userID
}

// RealmID returns the realm ID of the realm which performed the authentication
func (s *GithubSession) RealmID() string {
	return s.realmID
}

// ID returns the session ID
func (s *GithubSession) ID() string {
	return s.id
}

// ID returns the realm ID
func (r *GithubRealm) ID() string {
	return r.id
}

// Type is github
func (r *GithubRealm) Type() string {
	return "github"
}

// Init does nothing.
func (r *GithubRealm) Init() error {
	return nil
}

// Register does nothing.
func (r *GithubRealm) Register() error {
	return nil
}

// RequestAuthSession generates an OAuth2 URL for this user to auth with github via.
func (r *GithubRealm) RequestAuthSession(userID string, req json.RawMessage) interface{} {
	state, err := randomString(10)
	if err != nil {
		log.WithError(err).Print("Failed to generate state param")
		return nil
	}

	u, _ := url.Parse("https://github.com/login/oauth/authorize")
	q := u.Query()
	q.Set("client_id", r.ClientID)
	q.Set("client_secret", r.ClientSecret)
	q.Set("state", state)
	q.Set("redirect_uri", r.redirectURL)
	u.RawQuery = q.Encode()
	session := &GithubSession{
		id:      state, // key off the state for redirects
		userID:  userID,
		realmID: r.ID(),
	}

	// check if they supplied a redirect URL
	var reqBody struct {
		RedirectURL string
	}
	if err = json.Unmarshal(req, &reqBody); err != nil {
		log.WithError(err).Print("Failed to decode request body")
		return nil
	}
	session.ClientsRedirectURL = reqBody.RedirectURL

	_, err = database.GetServiceDB().StoreAuthSession(session)
	if err != nil {
		log.WithError(err).Print("Failed to store new auth session")
		return nil
	}

	return &struct {
		URL string
	}{u.String()}
}

// OnReceiveRedirect processes OAuth redirect requests from Github
func (r *GithubRealm) OnReceiveRedirect(w http.ResponseWriter, req *http.Request) {
	// parse out params from the request
	code := req.URL.Query().Get("code")
	state := req.URL.Query().Get("state")
	logger := log.WithFields(log.Fields{
		"state": state,
	})
	logger.WithField("code", code).Print("GithubRealm: OnReceiveRedirect")
	if code == "" || state == "" {
		failWith(logger, w, 400, "code and state are required", nil)
		return
	}
	// load the session (we keyed off the state param)
	session, err := database.GetServiceDB().LoadAuthSessionByID(r.ID(), state)
	if err != nil {
		// most likely cause
		failWith(logger, w, 400, "Provided ?state= param is not recognised.", err)
		return
	}
	ghSession, ok := session.(*GithubSession)
	if !ok {
		failWith(logger, w, 500, "Unexpected session found.", nil)
		return
	}
	logger.WithField("user_id", ghSession.UserID()).Print("Mapped redirect to user")

	if ghSession.AccessToken != "" && ghSession.Scopes != "" {
		failWith(logger, w, 400, "You have already authenticated with Github", nil)
		return
	}

	// exchange code for access_token
	res, err := http.PostForm("https://github.com/login/oauth/access_token",
		url.Values{"client_id": {r.ClientID}, "client_secret": {r.ClientSecret}, "code": {code}})
	if err != nil {
		failWith(logger, w, 502, "Failed to exchange code for token", err)
		return
	}
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		failWith(logger, w, 502, "Failed to read token response", err)
		return
	}
	vals, err := url.ParseQuery(string(body))
	if err != nil {
		failWith(logger, w, 502, "Failed to parse token response", err)
		return
	}

	// update database and return
	ghSession.AccessToken = vals.Get("access_token")
	ghSession.Scopes = vals.Get("scope")
	logger.WithField("scope", ghSession.Scopes).Print("Scopes granted.")
	_, err = database.GetServiceDB().StoreAuthSession(ghSession)
	if err != nil {
		failWith(logger, w, 500, "Failed to persist session", err)
		return
	}
	if ghSession.ClientsRedirectURL != "" {
		w.WriteHeader(302)
		w.Header().Set("Location", ghSession.ClientsRedirectURL)
		// technically don't need a body but *shrug*
		w.Write([]byte(ghSession.ClientsRedirectURL))
	} else {
		w.WriteHeader(200)
		w.Write([]byte("You have successfully linked your Github account to " + ghSession.UserID()))
	}
}

// AuthSession returns a GithubSession for this user
func (r *GithubRealm) AuthSession(id, userID, realmID string) types.AuthSession {
	return &GithubSession{
		id:      id,
		userID:  userID,
		realmID: realmID,
	}
}

func failWith(logger *log.Entry, w http.ResponseWriter, code int, msg string, err error) {
	logger.WithError(err).Print(msg)
	w.WriteHeader(code)
	w.Write([]byte(msg))
}

// Generate a cryptographically secure pseudorandom string with the given number of bytes (length).
// Returns a hex string of the bytes.
func randomString(length int) (string, error) {
	b := make([]byte, length)
	_, err := rand.Read(b)
	if err != nil {
		return "", err
	}
	return hex.EncodeToString(b), nil
}

func init() {
	types.RegisterAuthRealm(func(realmID, redirectURL string) types.AuthRealm {
		return &GithubRealm{id: realmID, redirectURL: redirectURL}
	})
}
<commit_msg>Actually specify scopes...<commit_after>package realms

import (
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	log "github.com/Sirupsen/logrus"
	"github.com/google/go-github/github"
	"github.com/matrix-org/go-neb/database"
	"github.com/matrix-org/go-neb/services/github/client"
	"github.com/matrix-org/go-neb/types"
	"io/ioutil"
	"net/http"
	"net/url"
)

// GithubRealm can handle OAuth processes with github.com
type GithubRealm struct {
	id           string
	redirectURL  string
	ClientSecret string
	ClientID     string
	StarterLink  string
}

// GithubSession represents an authenticated github session
type GithubSession struct {
	// The client-supplied URL to redirect them to after the auth process is complete.
	ClientsRedirectURL string
	// AccessToken is the github access token for the user
	AccessToken string
	// Scopes are the set of *ALLOWED* scopes (which may not be the same as the requested scopes)
	Scopes  string
	id      string
	userID  string
	realmID string
}

// Authenticated returns true if the user has completed the auth process
func (s *GithubSession) Authenticated() bool {
	return s.AccessToken != ""
}

// Info returns a list of possible repositories that this session can integrate with.
func (s *GithubSession) Info() interface{} {
	logger := log.WithFields(log.Fields{
		"user_id":  s.userID,
		"realm_id": s.realmID,
	})
	cli := client.New(s.AccessToken)
	// query for a list of possible projects
	rs, _, err := cli.Repositories.List("", &github.RepositoryListOptions{
		Type: "all",
	})
	if err != nil {
		logger.WithError(err).Print("Failed to query github projects on github.com")
		return nil
	}

	var repos []client.TrimmedRepository

	for _, r := range rs {
		repos = append(repos, client.TrimRepository(r))
	}

	return struct {
		Repos []client.TrimmedRepository
	}{repos}
}

// UserID returns the user_id who authorised with Github
func (s *GithubSession) UserID() string {
	return s.userID
}

// RealmID returns the realm ID of the realm which performed the authentication
func (s *GithubSession) RealmID() string {
	return s.realmID
}

// ID returns the session ID
func (s *GithubSession) ID() string {
	return s.id
}

// ID returns the realm ID
func (r *GithubRealm) ID() string {
	return r.id
}

// Type is github
func (r *GithubRealm) Type() string {
	return "github"
}

// Init does nothing.
func (r *GithubRealm) Init() error {
	return nil
}

// Register does nothing.
func (r *GithubRealm) Register() error {
	return nil
}

// RequestAuthSession generates an OAuth2 URL for this user to auth with github via.
func (r *GithubRealm) RequestAuthSession(userID string, req json.RawMessage) interface{} {
	state, err := randomString(10)
	if err != nil {
		log.WithError(err).Print("Failed to generate state param")
		return nil
	}

	u, _ := url.Parse("https://github.com/login/oauth/authorize")
	q := u.Query()
	q.Set("client_id", r.ClientID)
	q.Set("client_secret", r.ClientSecret)
	q.Set("state", state)
	q.Set("redirect_uri", r.redirectURL)
	q.Set("scope", "admin:repo_hook,admin:org_hook,repo")
	u.RawQuery = q.Encode()
	session := &GithubSession{
		id:      state, // key off the state for redirects
		userID:  userID,
		realmID: r.ID(),
	}

	// check if they supplied a redirect URL
	var reqBody struct {
		RedirectURL string
	}
	if err = json.Unmarshal(req, &reqBody); err != nil {
		log.WithError(err).Print("Failed to decode request body")
		return nil
	}
	session.ClientsRedirectURL = reqBody.RedirectURL

	_, err = database.GetServiceDB().StoreAuthSession(session)
	if err != nil {
		log.WithError(err).Print("Failed to store new auth session")
		return nil
	}

	return &struct {
		URL string
	}{u.String()}
}

// OnReceiveRedirect processes OAuth redirect requests from Github
func (r *GithubRealm) OnReceiveRedirect(w http.ResponseWriter, req *http.Request) {
	// parse out params from the request
	code := req.URL.Query().Get("code")
	state := req.URL.Query().Get("state")
	logger := log.WithFields(log.Fields{
		"state": state,
	})
	logger.WithField("code", code).Print("GithubRealm: OnReceiveRedirect")
	if code == "" || state == "" {
		failWith(logger, w, 400, "code and state are required", nil)
		return
	}
	// load the session (we keyed off the state param)
	session, err := database.GetServiceDB().LoadAuthSessionByID(r.ID(), state)
	if err != nil {
		// most likely cause
		failWith(logger, w, 400, "Provided ?state= param is not recognised.", err)
		return
	}
	ghSession, ok := session.(*GithubSession)
	if !ok {
		failWith(logger, w, 500, "Unexpected session found.", nil)
		return
	}
	logger.WithField("user_id", ghSession.UserID()).Print("Mapped redirect to user")

	if ghSession.AccessToken != "" && ghSession.Scopes != "" {
		failWith(logger, w, 400, "You have already authenticated with Github", nil)
		return
	}

	// exchange code for access_token
	res, err := http.PostForm("https://github.com/login/oauth/access_token",
		url.Values{"client_id": {r.ClientID}, "client_secret": {r.ClientSecret}, "code": {code}})
	if err != nil {
		failWith(logger, w, 502, "Failed to exchange code for token", err)
		return
	}
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		failWith(logger, w, 502, "Failed to read token response", err)
		return
	}
	vals, err := url.ParseQuery(string(body))
	if err != nil {
		failWith(logger, w, 502, "Failed to parse token response", err)
		return
	}

	// update database and return
	ghSession.AccessToken = vals.Get("access_token")
	ghSession.Scopes = vals.Get("scope")
	logger.WithField("scope", ghSession.Scopes).Print("Scopes granted.")
	_, err = database.GetServiceDB().StoreAuthSession(ghSession)
	if err != nil {
		failWith(logger, w, 500, "Failed to persist session", err)
		return
	}
	if ghSession.ClientsRedirectURL != "" {
		w.WriteHeader(302)
		w.Header().Set("Location", ghSession.ClientsRedirectURL)
		// technically don't need a body but *shrug*
		w.Write([]byte(ghSession.ClientsRedirectURL))
	} else {
		w.WriteHeader(200)
		w.Write([]byte("You have successfully linked your Github account to " + ghSession.UserID()))
	}
}

// AuthSession returns a GithubSession for this user
func (r *GithubRealm) AuthSession(id, userID, realmID string) types.AuthSession {
	return &GithubSession{
		id:      id,
		userID:  userID,
		realmID: realmID,
	}
}

func failWith(logger *log.Entry, w http.ResponseWriter, code int, msg string, err error) {
	logger.WithError(err).Print(msg)
	w.WriteHeader(code)
	w.Write([]byte(msg))
}

// Generate a cryptographically secure pseudorandom string with the given number of bytes (length).
// Returns a hex string of the bytes.
func randomString(length int) (string, error) {
	b := make([]byte, length)
	_, err := rand.Read(b)
	if err != nil {
		return "", err
	}
	return hex.EncodeToString(b), nil
}

func init() {
	types.RegisterAuthRealm(func(realmID, redirectURL string) types.AuthRealm {
		return &GithubRealm{id: realmID, redirectURL: redirectURL}
	})
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package metrics

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"go.opencensus.io/stats"
	"go.uber.org/zap"
	"knative.dev/pkg/metrics/metricskey"
)

const (
	DomainEnv = "METRICS_DOMAIN"
)

// metricsBackend specifies the backend to use for metrics
type metricsBackend string

const (
	// The following keys are used to configure metrics reporting.
	// See https://github.com/knative/serving/blob/master/config/config-observability.yaml
	// for details.
	AllowStackdriverCustomMetricsKey    = "metrics.allow-stackdriver-custom-metrics"
	BackendDestinationKey               = "metrics.backend-destination"
	ReportingPeriodKey                  = "metrics.reporting-period-seconds"
	StackdriverCustomMetricSubDomainKey = "metrics.stackdriver-custom-metrics-subdomain"
	// Stackdriver client configuration keys
	StackdriverProjectIDKey   = "metrics.stackdriver-project-id"
	StackdriverGCPLocationKey = "metrics.stackdriver-gcp-location"
	StackdriverClusterNameKey = "metrics.stackdriver-cluster-name"
	StackdriverUseSecretKey   = "metrics.stackdriver-use-secret"

	// Stackdriver is used for Stackdriver backend
	Stackdriver metricsBackend = "stackdriver"
	// Prometheus is used for Prometheus backend
	Prometheus metricsBackend = "prometheus"

	defaultBackendEnvName = "DEFAULT_METRICS_BACKEND"

	defaultPrometheusPort = 9090
	maxPrometheusPort     = 65535
	minPrometheusPort     = 1024
)

type metricsConfig struct {
	// The metrics domain. e.g. "serving.knative.dev" or "build.knative.dev".
	domain string
	// The component that emits the metrics. e.g. "activator", "autoscaler".
	component string
	// The metrics backend destination.
	backendDestination metricsBackend
	// reportingPeriod specifies the interval between reporting aggregated views.
	// If duration is less than or equal to zero, it enables the default behavior.
	reportingPeriod time.Duration

	// recorder provides a hook for performing custom transformations before
	// writing the metrics to the stats.RecordWithOptions interface.
	recorder func(context.Context, stats.Measurement, ...stats.Options) error

	// ---- Prometheus specific below ----
	// prometheusPort is the port where metrics are exposed in Prometheus
	// format. It defaults to 9090.
	prometheusPort int

	// ---- Stackdriver specific below ----
	// True if backendDestination equals to "stackdriver". Store this in a variable
	// to reduce string comparison operations.
	isStackdriverBackend bool
	// stackdriverMetricTypePrefix is the metric domain joins component, e.g.
	// "knative.dev/serving/activator". Store this in a variable to reduce string
	// join operations.
	stackdriverMetricTypePrefix string
	// stackdriverCustomMetricTypePrefix is "custom.googleapis.com" joined with the subdomain and component.
	// E.g., "custom.googleapis.com/<subdomain>/<component>".
	// Store this in a variable to reduce string join operations.
	stackdriverCustomMetricTypePrefix string
	// stackdriverClientConfig is the metadata to configure the metrics exporter's Stackdriver client.
	stackdriverClientConfig StackdriverClientConfig
}

// StackdriverClientConfig encapsulates the metadata required to configure a Stackdriver client.
type StackdriverClientConfig struct {
	// ProjectID is the stackdriver project ID to which data is uploaded.
	// This is not necessarily the GCP project ID where the Kubernetes cluster is hosted.
	// Required when the Kubernetes cluster is not hosted on GCE.
	ProjectID string
	// GCPLocation is the GCP region or zone to which data is uploaded.
	// This is not necessarily the GCP location where the Kubernetes cluster is hosted.
	// Required when the Kubernetes cluster is not hosted on GCE.
	GCPLocation string
	// ClusterName is the cluster name with which the data will be associated in Stackdriver.
	// Required when the Kubernetes cluster is not hosted on GCE.
	ClusterName string
	// UseSecret is whether the credentials stored in a Kubernetes Secret should be used to
	// authenticate with Stackdriver. The Secret name and namespace can be specified by calling
	// metrics.SetStackdriverSecretLocation.
	// If UseSecret is false, Google Application Default Credentials
	// will be used (https://cloud.google.com/docs/authentication/production).
	UseSecret bool
}

// NewStackdriverClientConfigFromMap creates a stackdriverClientConfig from the given map
func NewStackdriverClientConfigFromMap(config map[string]string) *StackdriverClientConfig {
	return &StackdriverClientConfig{
		ProjectID:   config[StackdriverProjectIDKey],
		GCPLocation: config[StackdriverGCPLocationKey],
		ClusterName: config[StackdriverClusterNameKey],
		UseSecret:   strings.EqualFold(config[StackdriverUseSecretKey], "true"),
	}
}

// Record applies the `ros` Options to `ms` and then records the resulting
// measurements in the metricsConfig's designated backend.
func (mc *metricsConfig) Record(ctx context.Context, ms stats.Measurement, ros ...stats.Options) error {
	if mc == nil || mc.recorder == nil {
		return stats.RecordWithOptions(ctx, append(ros, stats.WithMeasurements(ms))...)
	}
	return mc.recorder(ctx, ms, ros...)
}

func createMetricsConfig(ops ExporterOptions, logger *zap.SugaredLogger) (*metricsConfig, error) {
	var mc metricsConfig

	if ops.Domain == "" {
		return nil, errors.New("metrics domain cannot be empty")
	}
	mc.domain = ops.Domain

	if ops.Component == "" {
		return nil, errors.New("metrics component name cannot be empty")
	}
	mc.component = ops.Component

	if ops.ConfigMap == nil {
		return nil, errors.New("metrics config map cannot be empty")
	}
	m := ops.ConfigMap
	// Read backend setting from environment variable first
	backend := os.Getenv(defaultBackendEnvName)
	if backend == "" {
		// Use Prometheus if DEFAULT_METRICS_BACKEND does not exist or is empty
		backend = string(Prometheus)
	}
	// Override backend if it is setting in config map.
	if backendFromConfig, ok := m[BackendDestinationKey]; ok {
		backend = backendFromConfig
	}
	lb := metricsBackend(strings.ToLower(backend))
	switch lb {
	case Stackdriver, Prometheus:
		mc.backendDestination = lb
	default:
		return nil, fmt.Errorf("unsupported metrics backend value %q", backend)
	}

	if mc.backendDestination == Prometheus {
		pp := ops.PrometheusPort
		if pp == 0 {
			pp = defaultPrometheusPort
		}
		if pp < minPrometheusPort || pp > maxPrometheusPort {
			return nil, fmt.Errorf("invalid port %v, should between %v and %v", pp, minPrometheusPort, maxPrometheusPort)
		}
		mc.prometheusPort = pp
	}

	// If stackdriverClientConfig is not provided for stackdriver backend destination, OpenCensus will try to
	// use the application default credentials. If that is not available, Opencensus would fail to create the
	// metrics exporter.
	if mc.backendDestination == Stackdriver {
		scc := NewStackdriverClientConfigFromMap(m)
		mc.stackdriverClientConfig = *scc
		mc.isStackdriverBackend = true
		var allowCustomMetrics bool
		var err error
		mc.stackdriverMetricTypePrefix = path.Join(mc.domain, mc.component)

		customMetricsSubDomain := m[StackdriverCustomMetricSubDomainKey]
		if customMetricsSubDomain == "" {
			customMetricsSubDomain = defaultCustomMetricSubDomain
		}
		mc.stackdriverCustomMetricTypePrefix = path.Join(customMetricTypePrefix, customMetricsSubDomain, mc.component)
		if ascmStr := m[AllowStackdriverCustomMetricsKey]; ascmStr != "" {
			allowCustomMetrics, err = strconv.ParseBool(ascmStr)
			if err != nil {
				return nil, fmt.Errorf("invalid %s value %q", AllowStackdriverCustomMetricsKey, ascmStr)
			}
		}

		if !allowCustomMetrics {
			servingOrEventing := metricskey.KnativeRevisionMetrics.Union(
				metricskey.KnativeTriggerMetrics)
			mc.recorder = func(ctx context.Context, ms stats.Measurement, ros ...stats.Options) error {
				metricType := path.Join(mc.stackdriverMetricTypePrefix, ms.Measure().Name())

				if servingOrEventing.Has(metricType) {
					return stats.RecordWithOptions(ctx, append(ros, stats.WithMeasurements(ms))...)
				}
				// Otherwise, skip (because it won't be accepted)
				return nil
			}
		}
	}

	// If reporting period is specified, use the value from the configuration.
	// If not, set a default value based on the selected backend.
	// Each exporter makes different promises about what the lowest supported
	// reporting period is. For Stackdriver, this value is 1 minute.
	// For Prometheus, we will use a lower value since the exporter doesn't
	// push anything but just responds to pull requests, and shorter durations
	// do not really hurt the performance and we rely on the scraping configuration.
	if repStr, ok := m[ReportingPeriodKey]; ok && repStr != "" {
		repInt, err := strconv.Atoi(repStr)
		if err != nil {
			return nil, fmt.Errorf("invalid %s value %q", ReportingPeriodKey, repStr)
		}
		mc.reportingPeriod = time.Duration(repInt) * time.Second
	} else if mc.backendDestination == Stackdriver {
		mc.reportingPeriod = 60 * time.Second
	} else if mc.backendDestination == Prometheus {
		mc.reportingPeriod = 5 * time.Second
	}

	return &mc, nil
}

// Domain holds the metrics domain to use for surfacing metrics.
func Domain() string {
	if domain := os.Getenv(DomainEnv); domain != "" {
		return domain
	}

	panic(fmt.Sprintf(`The environment variable %q is not set

If this is a process running on Kubernetes, then it should be specifying
this via:

  env:
  - name: %s
    value: knative.dev/some-repository

If this is a Go unit test consuming metric.Domain() then it should add the
following import:

import (
	_ "knative.dev/pkg/metrics/testing"
)`, DomainEnv, DomainEnv))
}

// JsonToMetricsOptions converts a json string of a
// ExporterOptions. Returns a non-nil ExporterOptions always.
func JsonToMetricsOptions(jsonOpts string) (*ExporterOptions, error) {
	var opts ExporterOptions
	if jsonOpts == "" {
		return nil, errors.New("json options string is empty")
	}

	if err := json.Unmarshal([]byte(jsonOpts), &opts); err != nil {
		return nil, err
	}

	return &opts, nil
}

// MetricsOptionsToJson converts a ExporterOptions to a json string.
func MetricsOptionsToJson(opts *ExporterOptions) (string, error) {
	if opts == nil {
		return "", nil
	}

	jsonOpts, err := json.Marshal(opts)
	if err != nil {
		return "", err
	}

	return string(jsonOpts), nil
}
<commit_msg>adding knative_broker metrics (#944)<commit_after>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package metrics

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"go.opencensus.io/stats"
	"go.uber.org/zap"
	"knative.dev/pkg/metrics/metricskey"
)

const (
	DomainEnv = "METRICS_DOMAIN"
)

// metricsBackend specifies the backend to use for metrics
type metricsBackend string

const (
	// The following keys are used to configure metrics reporting.
	// See https://github.com/knative/serving/blob/master/config/config-observability.yaml
	// for details.
	AllowStackdriverCustomMetricsKey    = "metrics.allow-stackdriver-custom-metrics"
	BackendDestinationKey               = "metrics.backend-destination"
	ReportingPeriodKey                  = "metrics.reporting-period-seconds"
	StackdriverCustomMetricSubDomainKey = "metrics.stackdriver-custom-metrics-subdomain"
	// Stackdriver client configuration keys
	StackdriverProjectIDKey   = "metrics.stackdriver-project-id"
	StackdriverGCPLocationKey = "metrics.stackdriver-gcp-location"
	StackdriverClusterNameKey = "metrics.stackdriver-cluster-name"
	StackdriverUseSecretKey   = "metrics.stackdriver-use-secret"

	// Stackdriver is used for Stackdriver backend
	Stackdriver metricsBackend = "stackdriver"
	// Prometheus is used for Prometheus backend
	Prometheus metricsBackend = "prometheus"

	defaultBackendEnvName = "DEFAULT_METRICS_BACKEND"

	defaultPrometheusPort = 9090
	maxPrometheusPort     = 65535
	minPrometheusPort     = 1024
)

type metricsConfig struct {
	// The metrics domain. e.g. "serving.knative.dev" or "build.knative.dev".
	domain string
	// The component that emits the metrics. e.g. "activator", "autoscaler".
	component string
	// The metrics backend destination.
	backendDestination metricsBackend
	// reportingPeriod specifies the interval between reporting aggregated views.
	// If duration is less than or equal to zero, it enables the default behavior.
	reportingPeriod time.Duration

	// recorder provides a hook for performing custom transformations before
	// writing the metrics to the stats.RecordWithOptions interface.
	recorder func(context.Context, stats.Measurement, ...stats.Options) error

	// ---- Prometheus specific below ----
	// prometheusPort is the port where metrics are exposed in Prometheus
	// format. It defaults to 9090.
	prometheusPort int

	// ---- Stackdriver specific below ----
	// True if backendDestination equals to "stackdriver". Store this in a variable
	// to reduce string comparison operations.
	isStackdriverBackend bool
	// stackdriverMetricTypePrefix is the metric domain joins component, e.g.
	// "knative.dev/serving/activator". Store this in a variable to reduce string
	// join operations.
	stackdriverMetricTypePrefix string
	// stackdriverCustomMetricTypePrefix is "custom.googleapis.com" joined with the subdomain and component.
	// E.g., "custom.googleapis.com/<subdomain>/<component>".
	// Store this in a variable to reduce string join operations.
	stackdriverCustomMetricTypePrefix string
	// stackdriverClientConfig is the metadata to configure the metrics exporter's Stackdriver client.
	stackdriverClientConfig StackdriverClientConfig
}

// StackdriverClientConfig encapsulates the metadata required to configure a Stackdriver client.
type StackdriverClientConfig struct {
	// ProjectID is the stackdriver project ID to which data is uploaded.
	// This is not necessarily the GCP project ID where the Kubernetes cluster is hosted.
	// Required when the Kubernetes cluster is not hosted on GCE.
	ProjectID string
	// GCPLocation is the GCP region or zone to which data is uploaded.
	// This is not necessarily the GCP location where the Kubernetes cluster is hosted.
	// Required when the Kubernetes cluster is not hosted on GCE.
	GCPLocation string
	// ClusterName is the cluster name with which the data will be associated in Stackdriver.
	// Required when the Kubernetes cluster is not hosted on GCE.
	ClusterName string
	// UseSecret is whether the credentials stored in a Kubernetes Secret should be used to
	// authenticate with Stackdriver. The Secret name and namespace can be specified by calling
	// metrics.SetStackdriverSecretLocation.
	// If UseSecret is false, Google Application Default Credentials
	// will be used (https://cloud.google.com/docs/authentication/production).
	UseSecret bool
}

// NewStackdriverClientConfigFromMap creates a stackdriverClientConfig from the given map
func NewStackdriverClientConfigFromMap(config map[string]string) *StackdriverClientConfig {
	return &StackdriverClientConfig{
		ProjectID:   config[StackdriverProjectIDKey],
		GCPLocation: config[StackdriverGCPLocationKey],
		ClusterName: config[StackdriverClusterNameKey],
		UseSecret:   strings.EqualFold(config[StackdriverUseSecretKey], "true"),
	}
}

// Record applies the `ros` Options to `ms` and then records the resulting
// measurements in the metricsConfig's designated backend.
func (mc *metricsConfig) Record(ctx context.Context, ms stats.Measurement, ros ...stats.Options) error {
	if mc == nil || mc.recorder == nil {
		return stats.RecordWithOptions(ctx, append(ros, stats.WithMeasurements(ms))...)
	}
	return mc.recorder(ctx, ms, ros...)
}

func createMetricsConfig(ops ExporterOptions, logger *zap.SugaredLogger) (*metricsConfig, error) {
	var mc metricsConfig

	if ops.Domain == "" {
		return nil, errors.New("metrics domain cannot be empty")
	}
	mc.domain = ops.Domain

	if ops.Component == "" {
		return nil, errors.New("metrics component name cannot be empty")
	}
	mc.component = ops.Component

	if ops.ConfigMap == nil {
		return nil, errors.New("metrics config map cannot be empty")
	}
	m := ops.ConfigMap
	// Read backend setting from environment variable first
	backend := os.Getenv(defaultBackendEnvName)
	if backend == "" {
		// Use Prometheus if DEFAULT_METRICS_BACKEND does not exist or is empty
		backend = string(Prometheus)
	}
	// Override backend if it is setting in config map.
	if backendFromConfig, ok := m[BackendDestinationKey]; ok {
		backend = backendFromConfig
	}
	lb := metricsBackend(strings.ToLower(backend))
	switch lb {
	case Stackdriver, Prometheus:
		mc.backendDestination = lb
	default:
		return nil, fmt.Errorf("unsupported metrics backend value %q", backend)
	}

	if mc.backendDestination == Prometheus {
		pp := ops.PrometheusPort
		if pp == 0 {
			pp = defaultPrometheusPort
		}
		if pp < minPrometheusPort || pp > maxPrometheusPort {
			return nil, fmt.Errorf("invalid port %v, should between %v and %v", pp, minPrometheusPort, maxPrometheusPort)
		}
		mc.prometheusPort = pp
	}

	// If stackdriverClientConfig is not provided for stackdriver backend destination, OpenCensus will try to
	// use the application default credentials. If that is not available, Opencensus would fail to create the
	// metrics exporter.
	if mc.backendDestination == Stackdriver {
		scc := NewStackdriverClientConfigFromMap(m)
		mc.stackdriverClientConfig = *scc
		mc.isStackdriverBackend = true
		var allowCustomMetrics bool
		var err error
		mc.stackdriverMetricTypePrefix = path.Join(mc.domain, mc.component)

		customMetricsSubDomain := m[StackdriverCustomMetricSubDomainKey]
		if customMetricsSubDomain == "" {
			customMetricsSubDomain = defaultCustomMetricSubDomain
		}
		mc.stackdriverCustomMetricTypePrefix = path.Join(customMetricTypePrefix, customMetricsSubDomain, mc.component)
		if ascmStr := m[AllowStackdriverCustomMetricsKey]; ascmStr != "" {
			allowCustomMetrics, err = strconv.ParseBool(ascmStr)
			if err != nil {
				return nil, fmt.Errorf("invalid %s value %q", AllowStackdriverCustomMetricsKey, ascmStr)
			}
		}

		if !allowCustomMetrics {
			servingOrEventing := metricskey.KnativeRevisionMetrics.Union(
				metricskey.KnativeTriggerMetrics).Union(metricskey.KnativeBrokerMetrics)
			mc.recorder = func(ctx context.Context, ms stats.Measurement, ros ...stats.Options) error {
				metricType := path.Join(mc.stackdriverMetricTypePrefix, ms.Measure().Name())

				if servingOrEventing.Has(metricType) {
					return stats.RecordWithOptions(ctx, append(ros, stats.WithMeasurements(ms))...)
				}
				// Otherwise, skip (because it won't be accepted)
				return nil
			}
		}
	}

	// If reporting period is specified, use the value from the configuration.
	// If not, set a default value based on the selected backend.
	// Each exporter makes different promises about what the lowest supported
	// reporting period is. For Stackdriver, this value is 1 minute.
	// For Prometheus, we will use a lower value since the exporter doesn't
	// push anything but just responds to pull requests, and shorter durations
	// do not really hurt the performance and we rely on the scraping configuration.
	if repStr, ok := m[ReportingPeriodKey]; ok && repStr != "" {
		repInt, err := strconv.Atoi(repStr)
		if err != nil {
			return nil, fmt.Errorf("invalid %s value %q", ReportingPeriodKey, repStr)
		}
		mc.reportingPeriod = time.Duration(repInt) * time.Second
	} else if mc.backendDestination == Stackdriver {
		mc.reportingPeriod = 60 * time.Second
	} else if mc.backendDestination == Prometheus {
		mc.reportingPeriod = 5 * time.Second
	}

	return &mc, nil
}

// Domain holds the metrics domain to use for surfacing metrics.
func Domain() string {
	if domain := os.Getenv(DomainEnv); domain != "" {
		return domain
	}

	panic(fmt.Sprintf(`The environment variable %q is not set

If this is a process running on Kubernetes, then it should be specifying
this via:

  env:
  - name: %s
    value: knative.dev/some-repository

If this is a Go unit test consuming metric.Domain() then it should add the
following import:

import (
	_ "knative.dev/pkg/metrics/testing"
)`, DomainEnv, DomainEnv))
}

// JsonToMetricsOptions converts a json string of a
// ExporterOptions. Returns a non-nil ExporterOptions always.
func JsonToMetricsOptions(jsonOpts string) (*ExporterOptions, error) {
	var opts ExporterOptions
	if jsonOpts == "" {
		return nil, errors.New("json options string is empty")
	}

	if err := json.Unmarshal([]byte(jsonOpts), &opts); err != nil {
		return nil, err
	}

	return &opts, nil
}

// MetricsOptionsToJson converts a ExporterOptions to a json string.
func MetricsOptionsToJson(opts *ExporterOptions) (string, error) {
	if opts == nil {
		return "", nil
	}

	jsonOpts, err := json.Marshal(opts)
	if err != nil {
		return "", err
	}

	return string(jsonOpts), nil
}
<|endoftext|>


<commit_before>package test

import (
	"fmt"
	"os"
	"path/filepath"
	"testing"

	argoexec "github.com/argoproj/pkg/exec"
	"github.com/stretchr/testify/assert"
)

// TestBuildManifests makes sure we are consistent in naming, and all kustomization.yamls are buildable
func TestBuildManifests(t *testing.T) {
	out, err := argoexec.RunCommand("kustomize", "version")
	assert.NoError(t, err)
	if !assert.Contains(t, out, "KustomizeVersion:2", "kustomize should be version 2") {
		t.FailNow()
	}

	err = filepath.Walk("../manifests", func(path string, f os.FileInfo, err error) error {
		switch filepath.Base(path) {
		case "kustomization.yaml":
			// noop
		case "Kustomization", "kustomization.yml":
			// These are valid, but we want to be consistent with filenames
			return fmt.Errorf("Please name file 'kustomization.yaml' instead of '%s'", filepath.Base(path))
		case "Kustomize", "kustomize.yaml", "kustomize.yml":
			// These are not even valid kustomization filenames but sometimes get mistaken for them
			return fmt.Errorf("'%s' is not a valid kustomize name", filepath.Base(path))
		default:
			return nil
		}
		dirName := filepath.Dir(path)
		_, err = argoexec.RunCommand("kustomize", "build", dirName)
		return err
	})
	assert.NoError(t, err)
}
<commit_msg>Make test more tolerant (#1177)<commit_after>package test

import (
	"fmt"
	"os"
	"path/filepath"
	. "strings"
	"testing"

	argoexec "github.com/argoproj/pkg/exec"
	"github.com/stretchr/testify/assert"
)

// TestBuildManifests makes sure we are consistent in naming, and all kustomization.yamls are buildable
func TestBuildManifests(t *testing.T) {

	out, err := argoexec.RunCommand("kustomize", "version")
	assert.NoError(t, err)
	assert.True(t, Contains(out, "KustomizeVersion:2") || Contains(out, "KustomizeVersion:v2"), "kustomize should be version 2")

	err = filepath.Walk("../manifests", func(path string, f os.FileInfo, err error) error {
		switch filepath.Base(path) {
		case "kustomization.yaml":
			// noop
		case "Kustomization", "kustomization.yml":
			// These are valid, but we want to be consistent with filenames
			return fmt.Errorf("Please name file 'kustomization.yaml' instead of '%s'", filepath.Base(path))
		case "Kustomize", "kustomize.yaml", "kustomize.yml":
			// These are not even valid kustomization filenames but sometimes get mistaken for them
			return fmt.Errorf("'%s' is not a valid kustomize name", filepath.Base(path))
		default:
			return nil
		}
		dirName := filepath.Dir(path)
		_, err = argoexec.RunCommand("kustomize", "build", dirName)
		return err
	})
	assert.NoError(t, err)
}
<|endoftext|>


<commit_before>package lexer

import "fmt"

type lexFn func(l *Lexer) (Token, lexFn)

// Lexer represents an HTML tokenizer.
type Lexer struct {
	input        string
	position     int // current position in the input (current char)
	readPosition int // current reading position in the input (after current char)

	line             int
	lineFound        bool // delay line increment after newline char
	linePosition     int
	lineReadPosition int

	ch  byte // current char
	lex lexFn
}

// New instanciates a new Lexer.
func New(input string) *Lexer {
	l := &Lexer{input: input, lex: lexNode}
	l.readChar()
	return l
}

func (l *Lexer) readChar() {
	if l.readPosition >= len(l.input) {
		l.ch = 0
	} else {
		l.ch = l.input[l.readPosition]
	}
	if l.lineFound {
		l.line++
		l.linePosition = 0
		l.lineReadPosition = 0
	}
	l.position = l.readPosition
	l.readPosition++
	l.linePosition = l.lineReadPosition
	l.lineReadPosition++
	l.lineFound = l.ch == '\n'
}

// NextToken returns the next token read by the lexer.
func (l *Lexer) NextToken() Token {
	tok, fn := l.lex(l)
	l.lex = fn
	return tok
}

// lexNode tokenizes an HTML node.
func lexNode(l *Lexer) (tok Token, fn lexFn) {
	l.skipWhitespace()

	tok = newToken(l)
	fn = lexNode
	switch l.ch {
	case '!':
		tok.Type = TokenBang
		tok.Literal = string(l.ch)
	case '=':
		tok.Type = TokenEqual
		tok.Literal = string(l.ch)
	case '/':
		tok.Type = TokenSlash
		tok.Literal = string(l.ch)
	case '\'', '"':
		tok.Type = TokenString
		tok.Literal = l.readString()
	case '<':
		tok.Type = TokenLBracket
		tok.Literal = string(l.ch)
	case '>':
		tok.Type = TokenRBracket
		tok.Literal = string(l.ch)
		fn = lexText
	case '0':
		tok.Literal, tok.Type = "", TokenEOF
	default:
		if isLetter(l.ch) {
			tok.Type = TokenIdent
			tok.Literal = l.readIdentifier()
			return
		}
		tok.Type = TokenError
		tok.Literal = fmt.Sprintf("illegal character %q", l.ch)
	}
	l.readChar()
	return
}

// lexText tokenizes the text between two HTML nodes.
func lexText(l *Lexer) (tok Token, fn lexFn) {
	if l.ch == 0 {
		tok.Type = TokenEOF
		fn = lexText
		return
	}
	tok = newToken(l)
	for {
		if l.ch == '<' || l.ch == 0 {
			tok.Type = TokenText
			tok.Literal = l.input[tok.Position:l.position]
			fn = lexNode
			break
		}
		//TODO: detect HTML character
		l.readChar()
	}
	// if the text node is empty we call the next lexer
	if l.position-tok.Position == 0 {
		tok, fn = fn(l)
	}
	return
}

func newToken(l *Lexer) Token {
	return Token{
		Position:     l.position,
		Line:         l.line,
		LinePosition: l.linePosition,
	}
}

func (l *Lexer) readIdentifier() string {
	position := l.position
	for isLetter(l.ch) {
		l.readChar()
	}
	return l.input[position:l.position]
}

func isLetter(ch byte) bool {
	return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' || ch == '-'
}

func isWhitespace(ch byte) bool {
	return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r'
}

func (l *Lexer) skipWhitespace() {
	for isWhitespace(l.ch) {
		l.readChar()
	}
}

func (l *Lexer) readString() string {
	position := l.position + 1
	for {
		l.readChar()
		if l.ch == '"' || l.ch == '\'' || l.ch == 0 {
			break
		}
	}
	return l.input[position:l.position]
}
<commit_msg>Fix text lexing<commit_after>package lexer

import (
	"fmt"
	"strings"
)

type lexFn func(l *Lexer) (Token, lexFn)

// Lexer represents an HTML tokenizer.
type Lexer struct {
	input        string
	position     int // current position in the input (current char)
	readPosition int // current reading position in the input (after current char)

	line             int
	lineFound        bool // delay line increment after newline char
	linePosition     int
	lineReadPosition int

	ch  byte // current char
	lex lexFn
}

// New instanciates a new Lexer.
func New(input string) *Lexer {
	l := &Lexer{input: input, lex: lexNode}
	l.readChar()
	return l
}

func (l *Lexer) readChar() {
	if l.readPosition >= len(l.input) {
		l.ch = 0
	} else {
		l.ch = l.input[l.readPosition]
	}
	if l.lineFound {
		l.line++
		l.linePosition = 0
		l.lineReadPosition = 0
	}
	l.position = l.readPosition
	l.readPosition++
	l.linePosition = l.lineReadPosition
	l.lineReadPosition++
	l.lineFound = l.ch == '\n'
}

// NextToken returns the next token read by the lexer.
func (l *Lexer) NextToken() Token {
	tok, fn := l.lex(l)
	l.lex = fn
	return tok
}

// lexNode tokenizes an HTML node.
func lexNode(l *Lexer) (tok Token, fn lexFn) {
	l.skipWhitespace()

	tok = newToken(l)
	fn = lexNode
	switch l.ch {
	case '!':
		tok.Type = TokenBang
		tok.Literal = string(l.ch)
	case '=':
		tok.Type = TokenEqual
		tok.Literal = string(l.ch)
	case '/':
		tok.Type = TokenSlash
		tok.Literal = string(l.ch)
	case '\'', '"':
		tok.Type = TokenString
		tok.Literal = l.readString()
	case '<':
		tok.Type = TokenLBracket
		tok.Literal = string(l.ch)
	case '>':
		tok.Type = TokenRBracket
		tok.Literal = string(l.ch)
		fn = lexText
	case '0':
		tok.Literal, tok.Type = "", TokenEOF
	default:
		if isLetter(l.ch) {
			tok.Type = TokenIdent
			tok.Literal = l.readIdentifier()
			return
		}
		tok.Type = TokenError
		tok.Literal = fmt.Sprintf("illegal character %q", l.ch)
	}
	l.readChar()
	return
}

// lexText tokenizes the text between two HTML nodes.
func lexText(l *Lexer) (tok Token, fn lexFn) {
	if l.ch == 0 {
		tok.Type = TokenEOF
		fn = lexText
		return
	}
	tok = newToken(l)
	for {
		if l.ch == '<' || l.ch == 0 {
			text := l.input[tok.Position:l.position]
			trimmedText := strings.TrimSpace(text)
			if trimmedText == "" {
				return lexNode(l)
			}

			tok.Type = TokenText
			tok.Literal = trimmedText
			fn = lexNode
			break
		}
		//TODO: detect HTML character
		l.readChar()
	}
	// if the text node is empty we call the next lexer
	if l.position-tok.Position == 0 {
		tok, fn = fn(l)
	}
	return
}

func newToken(l *Lexer) Token {
	return Token{
		Position:     l.position,
		Line:         l.line,
		LinePosition: l.linePosition,
	}
}

func (l *Lexer) readIdentifier() string {
	position := l.position
	for isLetter(l.ch) {
		l.readChar()
	}
	return l.input[position:l.position]
}

func isLetter(ch byte) bool {
	return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' || ch == '-'
}

func isWhitespace(ch byte) bool {
	return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r'
}

func (l *Lexer) skipWhitespace() {
	for isWhitespace(l.ch) {
		l.readChar()
	}
}

func (l *Lexer) readString() string {
	position := l.position + 1
	for {
		l.readChar()
		if l.ch == '"' || l.ch == '\'' || l.ch == 0 {
			break
		}
	}
	return l.input[position:l.position]
}
<|endoftext|>


<commit_before>// This file is mostly copied from Go's path/match_test.go

package doublestar

import (
  "testing"
  "path/filepath"
)

type MatchTest struct {
  pattern, s string
  match      bool
  err        error
  testGlob   bool
}

var matchTests = []MatchTest{
  {"abc", "abc", true, nil, true},
  {"*", "abc", true, nil, true},
  {"*c", "abc", true, nil, true},
  {"a*", "a", true, nil, true},
  {"a*", "abc", true, nil, true},
  {"a*", filepath.Join("ab", "c"), false, nil, true},
  {filepath.Join("a*", "b"), filepath.Join("abc", "b"), true, nil, true},
  {filepath.Join("a*", "b"), filepath.Join("a", "c", "b"), false, nil, true},
  {filepath.Join("a*b*c*d*e*", "f"), filepath.Join("axbxcxdxe", "f"), true, nil, true},
  {filepath.Join("a*b*c*d*e*", "f"), filepath.Join("axbxcxdxexxx", "f"), true, nil, true},
  {filepath.Join("a*b*c*d*e*", "f"), filepath.Join("axbxcxdxe", "xxx", "f"), false, nil, true},
  {filepath.Join("a*b*c*d*e*", "f"), filepath.Join("axbxcxdxexxx", "fff"), false, nil, true},
  {"a*b?c*x", "abxbbxdbxebxczzx", true, nil, true},
  {"a*b?c*x", "abxbbxdbxebxczzy", false, nil, true},
  {"ab[c]", "abc", true, nil, true},
  {"ab[b-d]", "abc", true, nil, true},
  {"ab[e-g]", "abc", false, nil, true},
  {"ab[^c]", "abc", false, nil, true},
  {"ab[^b-d]", "abc", false, nil, true},
  {"ab[^e-g]", "abc", true, nil, true},
  {"a\\*b", "ab", false, nil, true},
  {"a?b", "a☺b", true, nil, true},
  {"a[^a]b", "a☺b", true, nil, true},
  {"a???b", "a☺b", false, nil, true},
  {"a[^a][^a][^a]b", "a☺b", false, nil, true},
  {"[a-ζ]*", "α", true, nil, true},
  {"*[a-ζ]", "A", false, nil, true},
  {"a?b", filepath.Join("a", "b"), false, nil, true},
  {"a*b", filepath.Join("a", "b"), false, nil, true},
  {"[\\]a]", "]", true, nil, true},
  {"[\\-]", "-", true, nil, true},
  {"[x\\-]", "x", true, nil, true},
  {"[x\\-]", "-", true, nil, true},
  {"[x\\-]", "z", false, nil, true},
  {"[\\-x]", "x", true, nil, true},
  {"[\\-x]", "-", true, nil, true},
  {"[\\-x]", "a", false, nil, true},
  {"[]a]", "]", false, ErrBadPattern, true},
  {"[-]", "-", false, ErrBadPattern, true},
  {"[x-]", "x", false, ErrBadPattern, true},
  {"[x-]", "-", false, ErrBadPattern, true},
  {"[x-]", "z", false, ErrBadPattern, true},
  {"[-x]", "x", false, ErrBadPattern, true},
  {"[-x]", "-", false, ErrBadPattern, true},
  {"[-x]", "a", false, ErrBadPattern, true},
  {"\\", "a", false, ErrBadPattern, true},
  {"[a-b-c]", "a", false, ErrBadPattern, true},
  {"[", "a", false, ErrBadPattern, true},
  {"[^", "a", false, ErrBadPattern, true},
  {"[^bc", "a", false, ErrBadPattern, true},
  {"a[", "a", false, nil, false},
  {"a[", "ab", false, ErrBadPattern, true},
  {"*x", "xxx", true, nil, true},
  {filepath.Join("a", "**"), "a", false, nil, true},
  {filepath.Join("a", "**"), filepath.Join("a", "b"), true, nil, true},
  {filepath.Join("a", "**"), filepath.Join("a", "b", "c"), true, nil, true},
  {filepath.Join("**", "c"), "c", true, nil, true},
  {filepath.Join("**", "c"), filepath.Join("b", "c"), true, nil, true},
  {filepath.Join("**", "c"), filepath.Join("a", "b", "c"), true, nil, true},
  {filepath.Join("**", "c"), filepath.Join("a", "b"), false, nil, true},
  {filepath.Join("**", "c"), "abcd", false, nil, true},
  {filepath.Join("**", "c"), filepath.Join("a", "abc"), false, nil, true},
  {filepath.Join("a", "**", "b"), filepath.Join("a", "b"), true, nil, true},
  {filepath.Join("a", "**", "c"), filepath.Join("a", "b", "c"), true, nil, true},
  {filepath.Join("a", "**", "d"), filepath.Join("a", "b", "c", "d"), true, nil, true},
  {filepath.Join("a", "\\**"), filepath.Join("a", "b", "c"), false, nil, true},
  {"ab{c,d}", "abc", true, nil, true},
  {"ab{c,d,*}", "abcde", true, nil, true},
  {"ab{c,d}[", "abcd", false, ErrBadPattern, true},
  {"abc**", "abc", true, nil, true},
  {"**abc", "abc", true, nil, true},
  {"broken-symlink", "broken-symlink", true, nil, true},
  {filepath.Join("working-symlink", "c", "*"), filepath.Join("working-symlink", "c", "d"), true, nil, true},
  {filepath.Join("working-sym*", "*"), filepath.Join("working-symlink", "c"), true, nil, true},
  {filepath.Join("b", "**", "f"), filepath.Join("b", "symlink-dir", "f"), true, nil, true},
}

func TestMatch(t *testing.T) {
  for idx, tt := range matchTests {
    testMatchWith(t, idx, tt)
  }
}

func testMatchWith(t *testing.T, idx int, tt MatchTest) {
  defer func() {
    if r := recover(); r != nil {
      t.Errorf("#%v. Match(%#q, %#q) panicked: %#v", idx, tt.pattern, tt.s, r)
    }
  }()

  ok, err := Match(tt.pattern, tt.s)
  if ok != tt.match || err != tt.err {
    t.Errorf("#%v. Match(%#q, %#q) = %v, %v want %v, %v", idx, tt.pattern, tt.s, ok, err, tt.match, tt.err)
  }
}

func TestGlob(t *testing.T) {
  for idx, tt := range matchTests {
    if tt.testGlob {
      testGlobWith(t, idx, tt)
    }
  }
}

func testGlobWith(t *testing.T, idx int, tt MatchTest) {
  defer func() {
    if r := recover(); r != nil {
      t.Errorf("#%v. Glob(%#q) panicked: %#v", idx, tt.pattern, r)
    }
  }()

  matches, err := Glob(filepath.Join("test", tt.pattern))
  if inSlice(filepath.Join("test", tt.s), matches) != tt.match {
    if tt.match {
      t.Errorf("#%v. Glob(%#q) = %#v - doesn't contain %v, but should", idx, tt.pattern, matches, tt.s)
    } else {
      t.Errorf("#%v. Glob(%#q) = %#v - contains %v, but shouldn't", idx, tt.pattern, matches, tt.s)
    }
  }
  if err != tt.err {
    t.Errorf("#%v. Glob(%#q) has error %v, but should be %v", idx, tt.pattern, err, tt.err)
  }
}

func inSlice(s string, a []string) bool {
  for _, i := range a {
    if i == s { return true }
  }
  return false
}

<commit_msg>updated tests for windows and absolute paths<commit_after>// This file is mostly copied from Go's path/match_test.go

package doublestar

import (
  "os"
  "runtime"
  "testing"
  "path"
  "path/filepath"
)

type MatchTest struct {
  pattern, testPath []string   // a pattern and path to test the pattern on
  shouldMatch       bool       // true if the pattern should match the path
  expectedErr       error      // an expected error
  testOnDisk        bool       // true: test pattern against files in "test" directory
}

// Tests which contain escapes and symlinks will not work on Windows
var onWindows = runtime.GOOS == "windows"

var matchTests = []MatchTest{
  {[]string{"abc"},                       []string{"abc"},                       true, nil, true},
  {[]string{"*"},                         []string{"abc"},                       true, nil, true},
  {[]string{"*c"},                        []string{"abc"},                       true, nil, true},
  {[]string{"a*"},                        []string{"a"},                         true, nil, true},
  {[]string{"a*"},                        []string{"abc"},                       true, nil, true},
  {[]string{"a*"},                        []string{"ab", "c"},                   false, nil, true},
  {[]string{"a*", "b"},                   []string{"abc", "b"},                  true, nil, true},
  {[]string{"a*", "b"},                   []string{"a", "c", "b"},               false, nil, true},
  {[]string{"a*b*c*d*e*", "f"},           []string{"axbxcxdxe", "f"},            true, nil, true},
  {[]string{"a*b*c*d*e*", "f"},           []string{"axbxcxdxexxx", "f"},         true, nil, true},
  {[]string{"a*b*c*d*e*", "f"},           []string{"axbxcxdxe", "xxx", "f"},     false, nil, true},
  {[]string{"a*b*c*d*e*", "f"},           []string{"axbxcxdxexxx", "fff"},       false, nil, true},
  {[]string{"a*b?c*x"},                   []string{"abxbbxdbxebxczzx"},          true, nil, true},
  {[]string{"a*b?c*x"},                   []string{"abxbbxdbxebxczzy"},          false, nil, true},
  {[]string{"ab[c]"},                     []string{"abc"},                       true, nil, true},
  {[]string{"ab[b-d]"},                   []string{"abc"},                       true, nil, true},
  {[]string{"ab[e-g]"},                   []string{"abc"},                       false, nil, true},
  {[]string{"ab[^c]"},                    []string{"abc"},                       false, nil, true},
  {[]string{"ab[^b-d]"},                  []string{"abc"},                       false, nil, true},
  {[]string{"ab[^e-g]"},                  []string{"abc"},                       true, nil, true},
  {[]string{"a\\*b"},                     []string{"ab"},                        false, nil, true},
  {[]string{"a?b"},                       []string{"a☺b"},                       true, nil, true},
  {[]string{"a[^a]b"},                    []string{"a☺b"},                       true, nil, true},
  {[]string{"a???b"},                     []string{"a☺b"},                       false, nil, true},
  {[]string{"a[^a][^a][^a]b"},            []string{"a☺b"},                       false, nil, true},
  {[]string{"[a-ζ]*"},                    []string{"α"},                         true, nil, true},
  {[]string{"*[a-ζ]"},                    []string{"A"},                         false, nil, true},
  {[]string{"a?b"},                       []string{"a", "b"},                    false, nil, true},
  {[]string{"a*b"},                       []string{"a", "b"},                    false, nil, true},
  {[]string{"[\\]a]"},                    []string{"]"},                         true, nil, !onWindows},
  {[]string{"[\\-]"},                     []string{"-"},                         true, nil, !onWindows},
  {[]string{"[x\\-]"},                    []string{"x"},                         true, nil, !onWindows},
  {[]string{"[x\\-]"},                    []string{"-"},                         true, nil, !onWindows},
  {[]string{"[x\\-]"},                    []string{"z"},                         false, nil, !onWindows},
  {[]string{"[\\-x]"},                    []string{"x"},                         true, nil, !onWindows},
  {[]string{"[\\-x]"},                    []string{"-"},                         true, nil, !onWindows},
  {[]string{"[\\-x]"},                    []string{"a"},                         false, nil, !onWindows},
  {[]string{"[]a]"},                      []string{"]"},                         false, ErrBadPattern, true},
  {[]string{"[-]"},                       []string{"-"},                         false, ErrBadPattern, true},
  {[]string{"[x-]"},                      []string{"x"},                         false, ErrBadPattern, true},
  {[]string{"[x-]"},                      []string{"-"},                         false, ErrBadPattern, true},
  {[]string{"[x-]"},                      []string{"z"},                         false, ErrBadPattern, true},
  {[]string{"[-x]"},                      []string{"x"},                         false, ErrBadPattern, true},
  {[]string{"[-x]"},                      []string{"-"},                         false, ErrBadPattern, true},
  {[]string{"[-x]"},                      []string{"a"},                         false, ErrBadPattern, true},
  {[]string{"\\"},                        []string{"a"},                         false, ErrBadPattern, !onWindows},
  {[]string{"[a-b-c]"},                   []string{"a"},                         false, ErrBadPattern, true},
  {[]string{"["},                         []string{"a"},                         false, ErrBadPattern, true},
  {[]string{"[^"},                        []string{"a"},                         false, ErrBadPattern, true},
  {[]string{"[^bc"},                      []string{"a"},                         false, ErrBadPattern, true},
  {[]string{"a["},                        []string{"a"},                         false, nil, false},
  {[]string{"a["},                        []string{"ab"},                        false, ErrBadPattern, true},
  {[]string{"*x"},                        []string{"xxx"},                       true, nil, true},
  {[]string{"a", "**"},                   []string{"a"},                         false, nil, true},
  {[]string{"a", "**"},                   []string{"a", "b"},                    true, nil, true},
  {[]string{"a", "**"},                   []string{"a", "b", "c"},               true, nil, true},
  {[]string{"**", "c"},                   []string{"c"},                         true, nil, true},
  {[]string{"**", "c"},                   []string{"b", "c"},                    true, nil, true},
  {[]string{"**", "c"},                   []string{"a", "b", "c"},               true, nil, true},
  {[]string{"**", "c"},                   []string{"a", "b"},                    false, nil, true},
  {[]string{"**", "c"},                   []string{"abcd"},                      false, nil, true},
  {[]string{"**", "c"},                   []string{"a", "abc"},                  false, nil, true},
  {[]string{"a", "**", "b"},              []string{"a", "b"},                    true, nil, true},
  {[]string{"a", "**", "c"},              []string{"a", "b", "c"},               true, nil, true},
  {[]string{"a", "**", "d"},              []string{"a", "b", "c", "d"},          true, nil, true},
  {[]string{"a", "\\**"},                 []string{"a", "b", "c"},               false, nil, !onWindows},
  {[]string{"ab{c,d}"},                   []string{"abc"},                       true, nil, true},
  {[]string{"ab{c,d,*}"},                 []string{"abcde"},                     true, nil, true},
  {[]string{"ab{c,d}["},                  []string{"abcd"},                      false, ErrBadPattern, true},
  {[]string{"abc**"},                     []string{"abc"},                       true, nil, true},
  {[]string{"**abc"},                     []string{"abc"},                       true, nil, true},
  {[]string{"broken-symlink"},            []string{"broken-symlink"},            true, nil, !onWindows},
  {[]string{"working-symlink", "c", "*"}, []string{"working-symlink", "c", "d"}, true, nil, !onWindows},
  {[]string{"working-sym*", "*"},         []string{"working-symlink", "c"},      true, nil, !onWindows},
  {[]string{"b", "**", "f"},              []string{"b", "symlink-dir", "f"},     true, nil, !onWindows},
}

func TestMatch(t *testing.T) {
  for idx, tt := range matchTests {
    // Since Match() always uses "/" as the separator, we
    // don't need to worry about the tt.testOnDisk flag
    testMatchWith(t, idx, tt)
  }
}

func testMatchWith(t *testing.T, idx int, tt MatchTest) {
  defer func() {
    if r := recover(); r != nil {
      t.Errorf("#%v. Match(%#q, %#q) panicked: %#v", idx, tt.pattern, tt.testPath, r)
    }
  }()

  // Match() always uses "/" as the separator
  ok, err := Match(path.Join(tt.pattern...), path.Join(tt.testPath...))
  if ok != tt.shouldMatch || err != tt.expectedErr {
    t.Errorf("#%v. Match(%#q, %#q) = %v, %v want %v, %v", idx, tt.pattern, tt.testPath, ok, err, tt.shouldMatch, tt.expectedErr)
  }
}

func TestPathMatch(t *testing.T) {
  for idx, tt := range matchTests {
    // Even though we aren't actually matching paths on disk, we are using
    // PathMatch() which will use the system's separator. As a result, any
    // patterns that might cause problems on-disk need to also be avoided
    // here in this test.
    if tt.testOnDisk {
      testPathMatchWith(t, idx, tt)
    }
  }
}

func testPathMatchWith(t *testing.T, idx int, tt MatchTest) {
  defer func() {
    if r := recover(); r != nil {
      t.Errorf("#%v. Match(%#q, %#q) panicked: %#v", idx, tt.pattern, tt.testPath, r)
    }
  }()

  ok, err := PathMatch(filepath.Join(tt.pattern...), filepath.Join(tt.testPath...))
  if ok != tt.shouldMatch || err != tt.expectedErr {
    t.Errorf("#%v. Match(%#q, %#q) = %v, %v want %v, %v", idx, tt.pattern, tt.testPath, ok, err, tt.shouldMatch, tt.expectedErr)
  }
}

func TestGlob(t *testing.T) {
  abspath, err := os.Getwd()
  if err != nil {
    t.Errorf("Error getting current working directory: %v", err)
    return
  }

  for idx, tt := range matchTests {
    if tt.testOnDisk {
      // test both relative paths and absolute paths
      testGlobWith(t, idx, tt, "")
      testGlobWith(t, idx, tt, abspath)
    }
  }
}

func testGlobWith(t *testing.T, idx int, tt MatchTest, basepath string) {
  defer func() {
    if r := recover(); r != nil {
      t.Errorf("#%v. Glob(%#q) panicked: %#v", idx, tt.pattern, r)
    }
  }()

  pattern := filepath.Join(basepath, "test", filepath.Join(tt.pattern...))
  testPath := filepath.Join(basepath, "test", filepath.Join(tt.testPath...))
  matches, err := Glob(pattern)
  if inSlice(testPath, matches) != tt.shouldMatch {
    if tt.shouldMatch {
      t.Errorf("#%v. Glob(%#q) = %#v - doesn't contain %v, but should", idx, tt.pattern, matches, tt.testPath)
    } else {
      t.Errorf("#%v. Glob(%#q) = %#v - contains %v, but shouldn't", idx, tt.pattern, matches, tt.testPath)
    }
  }
  if err != tt.expectedErr {
    t.Errorf("#%v. Glob(%#q) has error %v, but should be %v", idx, tt.pattern, err, tt.expectedErr)
  }
}

func inSlice(s string, a []string) bool {
  for _, i := range a {
    if i == s { return true }
  }
  return false
}

<|endoftext|>