<commit_before>package main

import (
	"context"
	"net/http"
	"os"
	"time"

	log "github.com/sirupsen/logrus"

	"github.com/kawamuray/prometheus-kafka-consumer-group-exporter/kafka"
	kafkaprom "github.com/kawamuray/prometheus-kafka-consumer-group-exporter/prometheus"
	"github.com/kawamuray/prometheus-kafka-consumer-group-exporter/sync"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/urfave/cli"
)

const consumerGroupCommandName = "kafka-consumer-groups.sh"
const version = "0.0.5"

func main() {
	app := cli.NewApp()
	app.Name = "kafka_consumer_group_exporter"
	app.Version = version
	app.Usage = "[OPTIONS] BOOTSTRAP_SERVER#1,BOOTSTRAP_SERVER#2,..."
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "consumer-group-command-path",
			Usage: "Path to `kafka-consumer-groups.sh`.",
			Value: consumerGroupCommandName,
		},
		cli.StringFlag{
			Name:  "listen",
			Usage: "Interface and port to listen on.",
			Value: ":7979",
		},
		cli.DurationFlag{
			Name:  "kafka-command-timeout",
			Usage: "The maximum time the Kafka command is allowed to take before we kill it. We've seen it block forever in production at times (most likely during rebalances).",
			Value: 5 * time.Minute,
		},
		cli.IntFlag{
			Name:  "max-concurrent-group-queries",
			Usage: "The maximum number of consumer groups that are queried concurrently.",
			// Given that Kafka defaults maximum heap size to 256M for the
			// `kafka-consumer-groups.sh` script, the upper heap allocation
			// could be Value*256 MB.
			Value: 4,
		},
	}

	app.Action = func(c *cli.Context) {
		if c.NArg() == 0 {
			log.Fatal("Bootstrap server(s) missing.")
		}
		bootstrapServers := c.Args().Get(0)

		consumerGroupCommandPath := c.String("consumer-group-command-path")
		if data, err := os.Stat(consumerGroupCommandPath); os.IsNotExist(err) {
			log.Fatal("`consumer-group-command-path` does not exist. File: ", consumerGroupCommandPath)
		} else if err != nil {
			log.Fatal("Unable to to stat() `consumer-group-command-path`. Error: ", err)
		} else if perm := data.Mode().Perm(); perm&0111 == 0 {
			log.Fatal("`consumer-group-command-path` does not have executable bit set. File: ", consumerGroupCommandPath)
		}

		kafkaClient := kafka.ConsumerGroupsCommandClient{
			Parser:                   kafka.DefaultDescribeGroupParser(),
			BootstrapServers:         bootstrapServers,
			ConsumerGroupCommandPath: consumerGroupCommandPath,
		}
		fanInClient := sync.FanInConsumerGroupInfoClient{
			Delegate: &kafkaClient,
		}
		collector := kafkaprom.NewPartitionInfoCollector(
			context.Background(),
			&fanInClient,
			c.Duration("kafka-command-timeout"),
			c.Int("max-concurrent-group-queries"),
		)
		prometheus.DefaultRegisterer.MustRegister(collector)

		log.Fatal(http.ListenAndServe(c.String("listen"), promhttp.Handler()))
	}

	err := app.Run(os.Args)
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Release 0.0.6<commit_after>package main

import (
	"context"
	"net/http"
	"os"
	"time"

	log "github.com/sirupsen/logrus"

	"github.com/kawamuray/prometheus-kafka-consumer-group-exporter/kafka"
	kafkaprom "github.com/kawamuray/prometheus-kafka-consumer-group-exporter/prometheus"
	"github.com/kawamuray/prometheus-kafka-consumer-group-exporter/sync"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/urfave/cli"
)

const consumerGroupCommandName = "kafka-consumer-groups.sh"
const version = "0.0.6"

func main() {
	app := cli.NewApp()
	app.Name = "kafka_consumer_group_exporter"
	app.Version = version
	app.Usage = "[OPTIONS] BOOTSTRAP_SERVER#1,BOOTSTRAP_SERVER#2,..."
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "consumer-group-command-path",
			Usage: "Path to `kafka-consumer-groups.sh`.",
			Value: consumerGroupCommandName,
		},
		cli.StringFlag{
			Name:  "listen",
			Usage: "Interface and port to listen on.",
			Value: ":7979",
		},
		cli.DurationFlag{
			Name:  "kafka-command-timeout",
			Usage: "The maximum time the Kafka command is allowed to take before we kill it. We've seen it block forever in production at times (most likely during rebalances).",
			Value: 5 * time.Minute,
		},
		cli.IntFlag{
			Name:  "max-concurrent-group-queries",
			Usage: "The maximum number of consumer groups that are queried concurrently.",
			// Given that Kafka defaults maximum heap size to 256M for the
			// `kafka-consumer-groups.sh` script, the upper heap allocation
			// could be Value*256 MB.
			Value: 4,
		},
	}

	app.Action = func(c *cli.Context) {
		if c.NArg() == 0 {
			log.Fatal("Bootstrap server(s) missing.")
		}
		bootstrapServers := c.Args().Get(0)

		consumerGroupCommandPath := c.String("consumer-group-command-path")
		if data, err := os.Stat(consumerGroupCommandPath); os.IsNotExist(err) {
			log.Fatal("`consumer-group-command-path` does not exist. File: ", consumerGroupCommandPath)
		} else if err != nil {
			log.Fatal("Unable to to stat() `consumer-group-command-path`. Error: ", err)
		} else if perm := data.Mode().Perm(); perm&0111 == 0 {
			log.Fatal("`consumer-group-command-path` does not have executable bit set. File: ", consumerGroupCommandPath)
		}

		kafkaClient := kafka.ConsumerGroupsCommandClient{
			Parser:                   kafka.DefaultDescribeGroupParser(),
			BootstrapServers:         bootstrapServers,
			ConsumerGroupCommandPath: consumerGroupCommandPath,
		}
		fanInClient := sync.FanInConsumerGroupInfoClient{
			Delegate: &kafkaClient,
		}
		collector := kafkaprom.NewPartitionInfoCollector(
			context.Background(),
			&fanInClient,
			c.Duration("kafka-command-timeout"),
			c.Int("max-concurrent-group-queries"),
		)
		prometheus.DefaultRegisterer.MustRegister(collector)

		log.Fatal(http.ListenAndServe(c.String("listen"), promhttp.Handler()))
	}

	err := app.Run(os.Args)
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
	"reflect"
	"testing"

	"k8s.io/ingress/core/pkg/ingress"
	"k8s.io/ingress/core/pkg/ingress/annotations/auth"
	"k8s.io/ingress/core/pkg/ingress/annotations/authreq"
	"k8s.io/ingress/core/pkg/ingress/annotations/authtls"
	"k8s.io/ingress/core/pkg/ingress/annotations/ipwhitelist"
	"k8s.io/ingress/core/pkg/ingress/annotations/proxy"
	"k8s.io/ingress/core/pkg/ingress/annotations/ratelimit"
	"k8s.io/ingress/core/pkg/ingress/annotations/redirect"
	"k8s.io/ingress/core/pkg/ingress/annotations/rewrite"
)

type fakeError struct{}

func (fe *fakeError) Error() string {
	return "fakeError"
}

func TestMergeLocationAnnotations(t *testing.T) {
	// initial parameters
	loc := ingress.Location{}
	annotations := map[string]interface{}{
		"Path":               "/checkpath",
		"IsDefBackend":       true,
		"Backend":            "foo_backend",
		"BasicDigestAuth":    auth.BasicDigest{},
		DeniedKeyName:        &fakeError{},
		"EnableCORS":         true,
		"ExternalAuth":       authreq.External{},
		"RateLimit":          ratelimit.RateLimit{},
		"Redirect":           redirect.Redirect{},
		"Rewrite":            rewrite.Redirect{},
		"Whitelist":          ipwhitelist.SourceRange{},
		"Proxy":              proxy.Configuration{},
		"CertificateAuth":    authtls.AuthSSLConfig{},
		"UsePortInRedirects": true,
	}

	// create test table
	type fooMergeLocationAnnotationsStruct struct {
		fName string
		er    interface{}
	}
	fooTests := []fooMergeLocationAnnotationsStruct{}
	for name, value := range annotations {
		fva := fooMergeLocationAnnotationsStruct{name, value}
		fooTests = append(fooTests, fva)
	}

	// execute test
	mergeLocationAnnotations(&loc, annotations)

	// check result
	for _, foo := range fooTests {
		fv := reflect.ValueOf(loc).FieldByName(foo.fName).Interface()
		if !reflect.DeepEqual(fv, foo.er) {
			t.Errorf("Returned %v but expected %v for the field %s", fv, foo.er, foo.fName)
		}
	}
	if _, ok := annotations[DeniedKeyName]; ok {
		t.Errorf("%s should be removed after mergeLocationAnnotations", DeniedKeyName)
	}
}
<commit_msg>Fix tests<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
	"reflect"
	"testing"

	"k8s.io/ingress/core/pkg/ingress"
	"k8s.io/ingress/core/pkg/ingress/annotations/auth"
	"k8s.io/ingress/core/pkg/ingress/annotations/authreq"
	"k8s.io/ingress/core/pkg/ingress/annotations/ipwhitelist"
	"k8s.io/ingress/core/pkg/ingress/annotations/proxy"
	"k8s.io/ingress/core/pkg/ingress/annotations/ratelimit"
	"k8s.io/ingress/core/pkg/ingress/annotations/redirect"
	"k8s.io/ingress/core/pkg/ingress/annotations/rewrite"
)

type fakeError struct{}

func (fe *fakeError) Error() string {
	return "fakeError"
}

func TestMergeLocationAnnotations(t *testing.T) {
	// initial parameters
	loc := ingress.Location{}
	annotations := map[string]interface{}{
		"Path":               "/checkpath",
		"IsDefBackend":       true,
		"Backend":            "foo_backend",
		"BasicDigestAuth":    auth.BasicDigest{},
		DeniedKeyName:        &fakeError{},
		"EnableCORS":         true,
		"ExternalAuth":       authreq.External{},
		"RateLimit":          ratelimit.RateLimit{},
		"Redirect":           redirect.Redirect{},
		"Rewrite":            rewrite.Redirect{},
		"Whitelist":          ipwhitelist.SourceRange{},
		"Proxy":              proxy.Configuration{},
		"UsePortInRedirects": true,
	}

	// create test table
	type fooMergeLocationAnnotationsStruct struct {
		fName string
		er    interface{}
	}
	fooTests := []fooMergeLocationAnnotationsStruct{}
	for name, value := range annotations {
		fva := fooMergeLocationAnnotationsStruct{name, value}
		fooTests = append(fooTests, fva)
	}

	// execute test
	mergeLocationAnnotations(&loc, annotations)

	// check result
	for _, foo := range fooTests {
		fv := reflect.ValueOf(loc).FieldByName(foo.fName).Interface()
		if !reflect.DeepEqual(fv, foo.er) {
			t.Errorf("Returned %v but expected %v for the field %s", fv, foo.er, foo.fName)
		}
	}
	if _, ok := annotations[DeniedKeyName]; ok {
		t.Errorf("%s should be removed after mergeLocationAnnotations", DeniedKeyName)
	}
}
<|endoftext|>


<commit_before>// +build windows

package logger

import (
	"fmt"
	"os"

	"golang.org/x/sys/windows"
)

// Windows 10 Build 16257 added support for ANSI color output if we enable them

func init() {
	var mode uint32
	stdout := windows.Handle(os.Stdout.Fd())

	if err := windows.GetConsoleMode(stdout, &mode); err != nil {
		fmt.Printf("Error getting console mode: %v\n", err)
		return
	}

	// See https://docs.microsoft.com/en-us/windows/console/getconsolemode
	mode = mode | windows.ENABLE_VIRTUAL_TERMINAL_PROCESSING

	if err := windows.SetConsoleMode(stdout, mode); err == nil {
		windowsColors = true
	} else {
		fmt.Printf("Error setting console mode: %v\n", err)
	}
}
<commit_msg>Suppress GetConsoleMode errors<commit_after>// +build windows

package logger

import (
	"fmt"
	"os"

	"golang.org/x/sys/windows"
)

// Windows 10 Build 16257 added support for ANSI color output if we enable them

func init() {
	var mode uint32
	stdout := windows.Handle(os.Stdout.Fd())

	if err := windows.GetConsoleMode(stdout, &mode); err != nil {
		return
	}

	// See https://docs.microsoft.com/en-us/windows/console/getconsolemode
	mode = mode | windows.ENABLE_VIRTUAL_TERMINAL_PROCESSING

	if err := windows.SetConsoleMode(stdout, mode); err == nil {
		windowsColors = true
	} else {
		fmt.Printf("Error setting console mode: %v\n", err)
	}
}
<|endoftext|>


<commit_before>// Package libdocker provides docker related library functions.
package libdocker

import (
	dockerclient "github.com/fsouza/go-dockerclient"
	"os"
	"path"
	"sync"
)

type CompleteDockerContainer struct {
	NiceImageName string `json:"NiceImageName,omitempty" yaml:"NiceImageName,omitempty"`
	Command       string `json:"Command,omitempty" yaml:"Command,omitempty"`
	Status        string `json:"Status,omitempty" yaml:"Status,omitempty"`
	dockerclient.Container
}

type CompleteDockerImage struct {
	RepoTags    []string `json:"RepoTags,omitempty" yaml:"RepoTags,omitempty"`
	VirtualSize int64    `json:"VirtualSize,omitempty" yaml:"VirtualSize,omitempty"`
	ParentID    string   `json:"ParentId,omitempty" yaml:"ParentId,omitempty"`
	dockerclient.Image
}

// DockerClient returns dockerclient.Client which handles Docker connection.
func DockerClient(endpoint string) (*dockerclient.Client, error) {
	if endpoint == "" {
		endpoint = os.Getenv("DOCKER_HOST")
		if endpoint == "" {
			endpoint = "unix:///var/run/docker.sock"
		}
	}

	dockerCertPath := os.Getenv("DOCKER_CERT_PATH")
	if dockerCertPath != "" {
		cert := path.Join(dockerCertPath, "cert.pem")
		key := path.Join(dockerCertPath, "key.pem")
		ca := path.Join(dockerCertPath, "ca.pem")

		return dockerclient.NewTLSClient(endpoint, cert, key, ca)
	} else {
		return dockerclient.NewClient(endpoint)
	}
}

// AllContainers is a convenience function to fetch a slice of all containers data.
func AllContainers(endpoint string) ([]dockerclient.APIContainers, error) {
	client, err := DockerClient(endpoint)
	if err != nil {
		return nil, err
	}

	return client.ListContainers(dockerclient.ListContainersOptions{})
}

// AllInspectedContainers is a convenience function to fetch a slice of all inspected containers data.
func AllInspectedContainers(endpoint string) ([]*CompleteDockerContainer, error) {
	client, err := DockerClient(endpoint)
	if err != nil {
		return nil, err
	}

	shortDescContainers, err := client.ListContainers(dockerclient.ListContainersOptions{})
	if err != nil {
		return nil, err
	}

	containersChan := make(chan *CompleteDockerContainer)
	var wg sync.WaitGroup

	for _, shortDescContainer := range shortDescContainers {
		container := &CompleteDockerContainer{}
		container.NiceImageName = shortDescContainer.Image
		container.Command = shortDescContainer.Command
		container.Status = shortDescContainer.Status

		wg.Add(1)

		go func(container *CompleteDockerContainer) {
			defer wg.Done()

			fullDescContainer, err := client.InspectContainer(shortDescContainer.ID)
			if err == nil && fullDescContainer != nil {
				container.ID = fullDescContainer.ID
				container.Created = fullDescContainer.Created
				container.Path = fullDescContainer.Path
				container.Args = fullDescContainer.Args
				container.Config = fullDescContainer.Config
				container.State = fullDescContainer.State
				container.Image = fullDescContainer.Image
				container.NetworkSettings = fullDescContainer.NetworkSettings
				container.SysInitPath = fullDescContainer.SysInitPath
				container.ResolvConfPath = fullDescContainer.ResolvConfPath
				container.HostnamePath = fullDescContainer.HostnamePath
				container.HostsPath = fullDescContainer.HostsPath
				container.Name = fullDescContainer.Name
				container.Driver = fullDescContainer.Driver
				container.Volumes = fullDescContainer.Volumes
				container.VolumesRW = fullDescContainer.VolumesRW
				container.HostConfig = fullDescContainer.HostConfig

				containersChan <- container
			}
		}(container)
	}

	containers := make([]*CompleteDockerContainer, 0)

	go func() {
		for container := range containersChan {
			containers = append(containers, container)
		}
	}()

	wg.Wait()

	return containers, nil
}

// AllImages is a convenience function to fetch a slice of all images data.
func AllImages(endpoint string) ([]dockerclient.APIImages, error) {
	client, err := DockerClient(endpoint)
	if err != nil {
		return nil, err
	}

	return client.ListImages(dockerclient.ListImagesOptions{})
}

// AllInspectedImages is a convenience function to fetch a slice of all inspected images data.
func AllInspectedImages(endpoint string) ([]*CompleteDockerImage, error) {
	client, err := DockerClient(endpoint)
	if err != nil {
		return nil, err
	}

	shortDescImages, err := client.ListImages(dockerclient.ListImagesOptions{})
	if err != nil {
		return nil, err
	}

	imagesChan := make(chan *CompleteDockerImage)
	var wg sync.WaitGroup

	for _, shortDescImage := range shortDescImages {
		img := &CompleteDockerImage{}
		img.ID = shortDescImage.ID
		img.RepoTags = shortDescImage.RepoTags
		img.VirtualSize = shortDescImage.VirtualSize
		img.ParentID = shortDescImage.ParentID

		wg.Add(1)

		go func(img *CompleteDockerImage) {
			defer wg.Done()

			fullDescImage, err := client.InspectImage(img.ID)
			if err == nil && fullDescImage != nil {
				img.Parent = fullDescImage.Parent
				img.Comment = fullDescImage.Comment
				img.Created = fullDescImage.Created
				img.Container = fullDescImage.Container
				img.ContainerConfig = fullDescImage.ContainerConfig
				img.DockerVersion = fullDescImage.DockerVersion
				img.Author = fullDescImage.Author
				img.Config = fullDescImage.Config
				img.Architecture = fullDescImage.Architecture
				img.Size = fullDescImage.Size

				imagesChan <- img
			}
		}(img)
	}

	images := make([]*CompleteDockerImage, 0)

	go func() {
		for image := range imagesChan {
			images = append(images, image)
		}
	}()

	wg.Wait()

	return images, nil
}
<commit_msg>Pool docker connection so we are not leaking memory.<commit_after>// Package libdocker provides docker related library functions.
package libdocker

import (
	dockerclient "github.com/fsouza/go-dockerclient"
	"os"
	"path"
	"sync"
)

var connections map[string]*dockerclient.Client

type CompleteDockerContainer struct {
	NiceImageName string `json:"NiceImageName,omitempty" yaml:"NiceImageName,omitempty"`
	Command       string `json:"Command,omitempty" yaml:"Command,omitempty"`
	Status        string `json:"Status,omitempty" yaml:"Status,omitempty"`
	dockerclient.Container
}

type CompleteDockerImage struct {
	RepoTags    []string `json:"RepoTags,omitempty" yaml:"RepoTags,omitempty"`
	VirtualSize int64    `json:"VirtualSize,omitempty" yaml:"VirtualSize,omitempty"`
	ParentID    string   `json:"ParentId,omitempty" yaml:"ParentId,omitempty"`
	dockerclient.Image
}

// DockerClient returns dockerclient.Client which handles Docker connection.
func DockerClient(endpoint string) (*dockerclient.Client, error) {
	var conn *dockerclient.Client
	var err error

	if endpoint == "" {
		endpoint = os.Getenv("DOCKER_HOST")
		if endpoint == "" {
			endpoint = "unix:///var/run/docker.sock"
		}
	}

	if connections == nil {
		connections = make(map[string]*dockerclient.Client)
	}

	// Do not create connection if one already exist.
	if existingConnection, ok := connections[endpoint]; ok && existingConnection != nil {
		return existingConnection, nil
	}

	dockerCertPath := os.Getenv("DOCKER_CERT_PATH")
	if dockerCertPath != "" {
		cert := path.Join(dockerCertPath, "cert.pem")
		key := path.Join(dockerCertPath, "key.pem")
		ca := path.Join(dockerCertPath, "ca.pem")

		conn, err = dockerclient.NewTLSClient(endpoint, cert, key, ca)
	} else {
		conn, err = dockerclient.NewClient(endpoint)
	}

	if err == nil && conn != nil {
		connections[endpoint] = conn
	}

	return conn, err
}

// AllContainers is a convenience function to fetch a slice of all containers data.
func AllContainers(endpoint string) ([]dockerclient.APIContainers, error) {
	client, err := DockerClient(endpoint)
	if err != nil {
		return nil, err
	}

	return client.ListContainers(dockerclient.ListContainersOptions{})
}

// AllInspectedContainers is a convenience function to fetch a slice of all inspected containers data.
func AllInspectedContainers(endpoint string) ([]*CompleteDockerContainer, error) {
	client, err := DockerClient(endpoint)
	if err != nil {
		return nil, err
	}

	shortDescContainers, err := client.ListContainers(dockerclient.ListContainersOptions{})
	if err != nil {
		return nil, err
	}

	containersChan := make(chan *CompleteDockerContainer)
	var wg sync.WaitGroup

	for _, shortDescContainer := range shortDescContainers {
		container := &CompleteDockerContainer{}
		container.NiceImageName = shortDescContainer.Image
		container.Command = shortDescContainer.Command
		container.Status = shortDescContainer.Status

		wg.Add(1)

		go func(container *CompleteDockerContainer) {
			defer wg.Done()

			fullDescContainer, err := client.InspectContainer(shortDescContainer.ID)
			if err == nil && fullDescContainer != nil {
				container.ID = fullDescContainer.ID
				container.Created = fullDescContainer.Created
				container.Path = fullDescContainer.Path
				container.Args = fullDescContainer.Args
				container.Config = fullDescContainer.Config
				container.State = fullDescContainer.State
				container.Image = fullDescContainer.Image
				container.NetworkSettings = fullDescContainer.NetworkSettings
				container.SysInitPath = fullDescContainer.SysInitPath
				container.ResolvConfPath = fullDescContainer.ResolvConfPath
				container.HostnamePath = fullDescContainer.HostnamePath
				container.HostsPath = fullDescContainer.HostsPath
				container.Name = fullDescContainer.Name
				container.Driver = fullDescContainer.Driver
				container.Volumes = fullDescContainer.Volumes
				container.VolumesRW = fullDescContainer.VolumesRW
				container.HostConfig = fullDescContainer.HostConfig

				containersChan <- container
			}
		}(container)
	}

	containers := make([]*CompleteDockerContainer, 0)

	go func() {
		for container := range containersChan {
			containers = append(containers, container)
		}
	}()

	wg.Wait()

	return containers, nil
}

// AllImages is a convenience function to fetch a slice of all images data.
func AllImages(endpoint string) ([]dockerclient.APIImages, error) {
	client, err := DockerClient(endpoint)
	if err != nil {
		return nil, err
	}

	return client.ListImages(dockerclient.ListImagesOptions{})
}

// AllInspectedImages is a convenience function to fetch a slice of all inspected images data.
func AllInspectedImages(endpoint string) ([]*CompleteDockerImage, error) {
	client, err := DockerClient(endpoint)
	if err != nil {
		return nil, err
	}

	shortDescImages, err := client.ListImages(dockerclient.ListImagesOptions{})
	if err != nil {
		return nil, err
	}

	imagesChan := make(chan *CompleteDockerImage)
	var wg sync.WaitGroup

	for _, shortDescImage := range shortDescImages {
		img := &CompleteDockerImage{}
		img.ID = shortDescImage.ID
		img.RepoTags = shortDescImage.RepoTags
		img.VirtualSize = shortDescImage.VirtualSize
		img.ParentID = shortDescImage.ParentID

		wg.Add(1)

		go func(img *CompleteDockerImage) {
			defer wg.Done()

			fullDescImage, err := client.InspectImage(img.ID)
			if err == nil && fullDescImage != nil {
				img.Parent = fullDescImage.Parent
				img.Comment = fullDescImage.Comment
				img.Created = fullDescImage.Created
				img.Container = fullDescImage.Container
				img.ContainerConfig = fullDescImage.ContainerConfig
				img.DockerVersion = fullDescImage.DockerVersion
				img.Author = fullDescImage.Author
				img.Config = fullDescImage.Config
				img.Architecture = fullDescImage.Architecture
				img.Size = fullDescImage.Size

				imagesChan <- img
			}
		}(img)
	}

	images := make([]*CompleteDockerImage, 0)

	go func() {
		for image := range imagesChan {
			images = append(images, image)
		}
	}()

	wg.Wait()

	return images, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

// +build windows

package libdokan

import (
	"sync"
	"time"

	"github.com/keybase/kbfs/dokan"
	"github.com/keybase/kbfs/libkbfs"
	"golang.org/x/net/context"
)

type fileOpener interface {
	open(ctx context.Context, oc *openContext, path []string) (f dokan.File, isDir bool, err error)
	dokan.File
}

// FolderList is a node that can list all of the logged-in user's
// favorite top-level folders, on either a public or private basis.
type FolderList struct {
	emptyFile
	fs *FS
	// only accept public folders
	public bool

	mu         sync.Mutex
	folders    map[string]fileOpener
	aliasCache map[string]string
}

// GetFileInformation for dokan.
func (*FolderList) GetFileInformation(*dokan.FileInfo) (*dokan.Stat, error) {
	return defaultDirectoryInformation()
}

func (fl *FolderList) reportErr(ctx context.Context,
	mode libkbfs.ErrorModeType, tlfName libkbfs.CanonicalTlfName, err error, cancelFn func()) {
	if cancelFn != nil {
		defer cancelFn()
	}
	if err == nil {
		fl.fs.log.CDebugf(ctx, "Request complete")
		return
	}

	fl.fs.config.Reporter().ReportErr(ctx, tlfName, fl.public, mode, err)
	// We just log the error as debug, rather than error, because it
	// might just indicate an expected error such as an ENOENT.
	//
	// TODO: Classify errors and escalate the logging level of the
	// important ones.
	fl.fs.log.CDebugf(ctx, err.Error())

}

// open tries to open the correct thing. Following aliases and deferring to
// Dir.open as necessary.
func (fl *FolderList) open(ctx context.Context, oc *openContext, path []string) (f dokan.File, isDir bool, err error) {
	fl.fs.log.CDebugf(ctx, "FL Lookup %#v public=%v", path, fl.public)
	if len(path) == 0 {
		return oc.returnDirNoCleanup(fl)
	}

	defer func() {
		fl.reportErr(ctx, libkbfs.ReadMode, libkbfs.CanonicalTlfName(path[0]), err, nil)
	}()

	for oc.reduceRedirectionsLeft() {
		name := path[0]

		if name == "desktop.ini" {
			fl.fs.log.CDebugf(ctx, "FL Lookup ignoring desktop.ini")
			return nil, false, dokan.ErrObjectNameNotFound
		}

		var aliasTarget string
		fl.mu.Lock()
		child, ok := fl.folders[name]
		if !ok {
			aliasTarget = fl.aliasCache[name]
		}
		fl.mu.Unlock()

		if ok {
			fl.fs.log.CDebugf(ctx, "FL Lookup recursing to child %q", name)
			return child.open(ctx, oc, path[1:])
		}

		if len(path) == 1 && oc.isCreateDirectory() && isNewFolderName(name) {
			fl.fs.log.CDebugf(ctx, "FL Lookup creating EmptyFolder for Explorer")
			e := &EmptyFolder{}
			fl.lockedAddChild(name, e)
			return e, true, nil
		}

		if aliasTarget != "" {
			if len(path) == 1 && oc.isOpenReparsePoint() {
				return &Alias{canon: aliasTarget}, true, nil
			}
			path[0] = aliasTarget
			continue
		}

		h, err := libkbfs.ParseTlfHandle(
			ctx, fl.fs.config.KBPKI(), name, fl.public)
		fl.fs.log.CDebugf(ctx, "FL Lookup continuing -> %v,%v", h, err)
		switch err := err.(type) {
		case nil:
			// No error.
			break

		case libkbfs.TlfNameNotCanonical:
			// Only permit Aliases to targets that contain no errors.
			aliasTarget = err.NameToTry
			if !fl.isValidAliasTarget(ctx, aliasTarget) {
				fl.fs.log.CDebugf(ctx, "FL Refusing alias to non-valid target %q", aliasTarget)
				return nil, false, dokan.ErrObjectNameNotFound
			}
			fl.mu.Lock()
			fl.aliasCache[name] = aliasTarget
			fl.mu.Unlock()

			if len(path) == 1 && oc.isOpenReparsePoint() {
				return &Alias{canon: aliasTarget}, true, nil
			}
			path[0] = aliasTarget
			continue

		case libkbfs.NoSuchNameError, libkbfs.BadTLFNameError:
			return nil, false, dokan.ErrObjectNameNotFound

		default:
			// Some other error.
			return nil, false, err
		}

		fl.fs.log.CDebugf(ctx, "FL Lookup adding new child")
		child = newTLF(fl, h)
		fl.lockedAddChild(name, child)
		return child.open(ctx, oc, path[1:])
	}
	return nil, false, dokan.ErrObjectNameNotFound
}

func (fl *FolderList) forgetFolder(folderName string) {
	fl.mu.Lock()
	defer fl.mu.Unlock()
	delete(fl.folders, folderName)
}

// FindFiles for dokan.
func (fl *FolderList) FindFiles(fi *dokan.FileInfo, callback func(*dokan.NamedStat) error) (err error) {
	ctx, cancel := NewContextWithOpID(fl.fs, "FL FindFiles")
	defer func() { fl.fs.reportErr(ctx, libkbfs.ReadMode, err, cancel) }()

	_, _, err = fl.fs.config.KBPKI().GetCurrentUserInfo(ctx)
	isLoggedIn := err == nil

	var favs []libkbfs.Favorite
	if isLoggedIn {
		favs, err = fl.fs.config.KBFSOps().GetFavorites(ctx)
		if err != nil {
			return err
		}
	}
	var ns dokan.NamedStat
	ns.FileAttributes = fileAttributeDirectory
	ns.NumberOfLinks = 1
	empty := true
	for _, fav := range favs {
		if fav.Public != fl.public {
			continue
		}
		empty = false
		ns.Name = fav.Name
		err = callback(&ns)
		if err != nil {
			return err
		}
	}
	if empty {
		return dokan.ErrObjectNameNotFound
	}
	return nil
}

func (fl *FolderList) isValidAliasTarget(ctx context.Context, nameToTry string) bool {
	return libkbfs.CheckTlfHandleOffline(ctx, nameToTry, fl.public) == nil
}

func (fl *FolderList) lockedAddChild(name string, val fileOpener) {
	fl.mu.Lock()
	fl.folders[name] = val
	fl.mu.Unlock()
}

func (fl *FolderList) updateTlfName(ctx context.Context, oldName string,
	newName string) {
	fl.mu.Lock()
	defer fl.mu.Unlock()
	tlf, ok := fl.folders[oldName]
	if !ok {
		return
	}

	fl.fs.log.CDebugf(ctx, "Folder name updated: %s -> %s", oldName, newName)
	delete(fl.folders, oldName)
	fl.folders[newName] = tlf
	// TODO: invalidate kernel cache for this name? (Make sure to
	// do so outside of the lock!)
}

func (fl *FolderList) clearAliasCache() {
	fl.mu.Lock()
	fl.aliasCache = map[string]string{}
	fl.mu.Unlock()
}

func clearFolderListCacheLoop(ctx context.Context, r *Root) {
	t := time.NewTicker(time.Hour)
	for {
		select {
		case <-ctx.Done():
			return
		case <-t.C:
		}
		r.private.clearAliasCache()
		r.public.clearAliasCache()
	}
}
<commit_msg>libdokan: More debug printing for aliases<commit_after>// Copyright 2016 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

// +build windows

package libdokan

import (
	"sync"
	"time"

	"github.com/keybase/kbfs/dokan"
	"github.com/keybase/kbfs/libkbfs"
	"golang.org/x/net/context"
)

type fileOpener interface {
	open(ctx context.Context, oc *openContext, path []string) (f dokan.File, isDir bool, err error)
	dokan.File
}

// FolderList is a node that can list all of the logged-in user's
// favorite top-level folders, on either a public or private basis.
type FolderList struct {
	emptyFile
	fs *FS
	// only accept public folders
	public bool

	mu         sync.Mutex
	folders    map[string]fileOpener
	aliasCache map[string]string
}

// GetFileInformation for dokan.
func (*FolderList) GetFileInformation(*dokan.FileInfo) (*dokan.Stat, error) {
	return defaultDirectoryInformation()
}

func (fl *FolderList) reportErr(ctx context.Context,
	mode libkbfs.ErrorModeType, tlfName libkbfs.CanonicalTlfName, err error, cancelFn func()) {
	if cancelFn != nil {
		defer cancelFn()
	}
	if err == nil {
		fl.fs.log.CDebugf(ctx, "Request complete")
		return
	}

	fl.fs.config.Reporter().ReportErr(ctx, tlfName, fl.public, mode, err)
	// We just log the error as debug, rather than error, because it
	// might just indicate an expected error such as an ENOENT.
	//
	// TODO: Classify errors and escalate the logging level of the
	// important ones.
	fl.fs.log.CDebugf(ctx, err.Error())

}

// open tries to open the correct thing. Following aliases and deferring to
// Dir.open as necessary.
func (fl *FolderList) open(ctx context.Context, oc *openContext, path []string) (f dokan.File, isDir bool, err error) {
	fl.fs.log.CDebugf(ctx, "FL Lookup %#v public=%v", path, fl.public)
	if len(path) == 0 {
		return oc.returnDirNoCleanup(fl)
	}

	defer func() {
		fl.reportErr(ctx, libkbfs.ReadMode, libkbfs.CanonicalTlfName(path[0]), err, nil)
	}()

	for oc.reduceRedirectionsLeft() {
		name := path[0]

		if name == "desktop.ini" {
			fl.fs.log.CDebugf(ctx, "FL Lookup ignoring desktop.ini")
			return nil, false, dokan.ErrObjectNameNotFound
		}

		var aliasTarget string
		fl.mu.Lock()
		child, ok := fl.folders[name]
		if !ok {
			aliasTarget = fl.aliasCache[name]
		}
		fl.mu.Unlock()

		if ok {
			fl.fs.log.CDebugf(ctx, "FL Lookup recursing to child %q", name)
			return child.open(ctx, oc, path[1:])
		}

		if len(path) == 1 && oc.isCreateDirectory() && isNewFolderName(name) {
			fl.fs.log.CDebugf(ctx, "FL Lookup creating EmptyFolder for Explorer")
			e := &EmptyFolder{}
			fl.lockedAddChild(name, e)
			return e, true, nil
		}

		if aliasTarget != "" {
			fl.fs.log.CDebugf(ctx, "FL Lookup aliasCache hit: %q -> %q", name, aliasTarget)
			if len(path) == 1 && oc.isOpenReparsePoint() {
				return &Alias{canon: aliasTarget}, true, nil
			}
			path[0] = aliasTarget
			continue
		}

		h, err := libkbfs.ParseTlfHandle(
			ctx, fl.fs.config.KBPKI(), name, fl.public)
		fl.fs.log.CDebugf(ctx, "FL Lookup continuing -> %v,%v", h, err)
		switch err := err.(type) {
		case nil:
			// No error.
			break

		case libkbfs.TlfNameNotCanonical:
			// Only permit Aliases to targets that contain no errors.
			aliasTarget = err.NameToTry
			fl.fs.log.CDebugf(ctx, "FL Lookup set alias: %q -> %q", name, aliasTarget)
			if !fl.isValidAliasTarget(ctx, aliasTarget) {
				fl.fs.log.CDebugf(ctx, "FL Refusing alias to non-valid target %q", aliasTarget)
				return nil, false, dokan.ErrObjectNameNotFound
			}
			fl.mu.Lock()
			fl.aliasCache[name] = aliasTarget
			fl.mu.Unlock()

			if len(path) == 1 && oc.isOpenReparsePoint() {
				fl.fs.log.CDebugf(ctx, "FL Lookup ret alias, oc: %#v", oc.CreateData)
				return &Alias{canon: aliasTarget}, true, nil
			}
			path[0] = aliasTarget
			continue

		case libkbfs.NoSuchNameError, libkbfs.BadTLFNameError:
			return nil, false, dokan.ErrObjectNameNotFound

		default:
			// Some other error.
			return nil, false, err
		}

		fl.fs.log.CDebugf(ctx, "FL Lookup adding new child")
		child = newTLF(fl, h)
		fl.lockedAddChild(name, child)
		return child.open(ctx, oc, path[1:])
	}
	return nil, false, dokan.ErrObjectNameNotFound
}

func (fl *FolderList) forgetFolder(folderName string) {
	fl.mu.Lock()
	defer fl.mu.Unlock()
	delete(fl.folders, folderName)
}

// FindFiles for dokan.
func (fl *FolderList) FindFiles(fi *dokan.FileInfo, callback func(*dokan.NamedStat) error) (err error) {
	ctx, cancel := NewContextWithOpID(fl.fs, "FL FindFiles")
	defer func() { fl.fs.reportErr(ctx, libkbfs.ReadMode, err, cancel) }()

	_, _, err = fl.fs.config.KBPKI().GetCurrentUserInfo(ctx)
	isLoggedIn := err == nil

	var favs []libkbfs.Favorite
	if isLoggedIn {
		favs, err = fl.fs.config.KBFSOps().GetFavorites(ctx)
		if err != nil {
			return err
		}
	}
	var ns dokan.NamedStat
	ns.FileAttributes = fileAttributeDirectory
	ns.NumberOfLinks = 1
	empty := true
	for _, fav := range favs {
		if fav.Public != fl.public {
			continue
		}
		empty = false
		ns.Name = fav.Name
		err = callback(&ns)
		if err != nil {
			return err
		}
	}
	if empty {
		return dokan.ErrObjectNameNotFound
	}
	return nil
}

func (fl *FolderList) isValidAliasTarget(ctx context.Context, nameToTry string) bool {
	return libkbfs.CheckTlfHandleOffline(ctx, nameToTry, fl.public) == nil
}

func (fl *FolderList) lockedAddChild(name string, val fileOpener) {
	fl.mu.Lock()
	fl.folders[name] = val
	fl.mu.Unlock()
}

func (fl *FolderList) updateTlfName(ctx context.Context, oldName string,
	newName string) {
	fl.mu.Lock()
	defer fl.mu.Unlock()
	tlf, ok := fl.folders[oldName]
	if !ok {
		return
	}

	fl.fs.log.CDebugf(ctx, "Folder name updated: %s -> %s", oldName, newName)
	delete(fl.folders, oldName)
	fl.folders[newName] = tlf
	// TODO: invalidate kernel cache for this name? (Make sure to
	// do so outside of the lock!)
}

func (fl *FolderList) clearAliasCache() {
	fl.mu.Lock()
	fl.aliasCache = map[string]string{}
	fl.mu.Unlock()
}

func clearFolderListCacheLoop(ctx context.Context, r *Root) {
	t := time.NewTicker(time.Hour)
	for {
		select {
		case <-ctx.Done():
			return
		case <-t.C:
		}
		r.private.clearAliasCache()
		r.public.clearAliasCache()
	}
}
<|endoftext|>


<commit_before>/*
 * Copyright © 2017-2018 Aeneas Rekkas <aeneas+oss@aeneas.io>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * @author		Aeneas Rekkas <aeneas+oss@aeneas.io>
 * @Copyright 	2017-2018 Aeneas Rekkas <aeneas+oss@aeneas.io>
 * @license 	Apache-2.0
 *
 */

package openid

import (
	"strconv"
	"time"

	jwtgo "github.com/dgrijalva/jwt-go"
	"github.com/ory/fosite"
	"github.com/ory/fosite/token/jwt"
	"github.com/ory/go-convenience/stringslice"
	"github.com/ory/go-convenience/stringsx"
	"github.com/pkg/errors"
)

type OpenIDConnectRequestValidator struct {
	AllowedPrompt []string
	Strategy      jwt.JWTStrategy
}

func NewOpenIDConnectRequestValidator(prompt []string, strategy jwt.JWTStrategy) *OpenIDConnectRequestValidator {
	if len(prompt) == 0 {
		prompt = []string{"login", "none", "consent", "select_account"}
	}

	return &OpenIDConnectRequestValidator{
		AllowedPrompt: prompt,
		Strategy:      strategy,
	}
}

func (v *OpenIDConnectRequestValidator) ValidatePrompt(req fosite.AuthorizeRequester) error {
	// prompt is case sensitive!
	prompt := stringsx.Splitx(req.GetRequestForm().Get("prompt"), " ")

	if req.GetClient().IsPublic() {
		// Threat: Malicious Client Obtains Existing Authorization by Fraud
		// https://tools.ietf.org/html/rfc6819#section-4.2.3
		//
		//  Authorization servers should not automatically process repeat
		//  authorizations to public clients unless the client is validated
		//  using a pre-registered redirect URI

		// Client Impersonation
		// https://tools.ietf.org/html/rfc8252#section-8.6#
		//
		//  As stated in Section 10.2 of OAuth 2.0 [RFC6749], the authorization
		//  server SHOULD NOT process authorization requests automatically
		//  without user consent or interaction, except when the identity of the
		//  client can be assured.  This includes the case where the user has
		//  previously approved an authorization request for a given client id --
		//  unless the identity of the client can be proven, the request SHOULD
		//  be processed as if no previous request had been approved.

		// To make sure that we are not vulnerable to this type of attack, we will always require consent for public
		// clients.

		// If prompt is none - meaning that no consent should be requested, we must terminate with an error.
		if stringslice.Has(prompt, "none") {
			return errors.WithStack(fosite.ErrConsentRequired.WithHint("OAuth 2.0 Client is marked public and requires end-user consent, but \"prompt=none\" was requested."))
		}
	}

	if !isWhitelisted(prompt, v.AllowedPrompt) {
		return errors.WithStack(fosite.ErrInvalidRequest.WithHintf(`Used unknown value "%s" for prompt parameter`, prompt))
	}

	if stringslice.Has(prompt, "none") && len(prompt) > 1 {
		// If this parameter contains none with any other value, an error is returned.
		return errors.WithStack(fosite.ErrInvalidRequest.WithHint("Parameter \"prompt\" was set to \"none\", but contains other values as well which is not allowed."))
	}

	maxAge, err := strconv.ParseInt(req.GetRequestForm().Get("max_age"), 10, 64)
	if err != nil {
		maxAge = 0
	}

	session, ok := req.GetSession().(Session)
	if !ok {
		return errors.WithStack(fosite.ErrServerError.WithDebug("Failed to validate OpenID Connect request because session is not of type fosite/handler/openid.Session."))
	}

	claims := session.IDTokenClaims()
	if claims.Subject == "" {
		return errors.WithStack(fosite.ErrServerError.WithDebug("Failed to validate OpenID Connect request because session subject is empty."))
	}

	// Adds a bit of wiggle room for timing issues
	if claims.AuthTime.After(time.Now().UTC().Add(time.Second * 5)) {
		return errors.WithStack(fosite.ErrServerError.WithDebug("Failed to validate OpenID Connect request because authentication time is in the future."))
	}

	if maxAge > 0 {
		if claims.AuthTime.IsZero() {
			return errors.WithStack(fosite.ErrServerError.WithDebug("Failed to validate OpenID Connect request because authentication time claim is required when max_age is set."))
		} else if claims.RequestedAt.IsZero() {
			return errors.WithStack(fosite.ErrServerError.WithDebug("Failed to validate OpenID Connect request because requested at claim is required when max_age is set."))
		} else if claims.AuthTime.Add(time.Second * time.Duration(maxAge)).Before(claims.RequestedAt) {
			return errors.WithStack(fosite.ErrLoginRequired.WithDebug("Failed to validate OpenID Connect request because authentication time does not satisfy max_age time."))
		}
	}

	if stringslice.Has(prompt, "none") {
		if claims.AuthTime.IsZero() {
			return errors.WithStack(fosite.ErrServerError.WithDebug("Failed to validate OpenID Connect request because because auth_time is missing from session."))
		}
		if claims.AuthTime.After(claims.RequestedAt) {
			return errors.WithStack(fosite.ErrLoginRequired.WithHint("Failed to validate OpenID Connect request because prompt was set to \"none\" but auth_time happened after the authorization request was registered, indicating that the user was logged in during this request which is not allowed."))
		}
	}

	if stringslice.Has(prompt, "login") {
		if claims.AuthTime.Before(claims.RequestedAt) {
			return errors.WithStack(fosite.ErrLoginRequired.WithHint("Failed to validate OpenID Connect request because prompt was set to \"login\" but auth_time happened before the authorization request was registered, indicating that the user was not re-authenticated which is forbidden."))
		}
	}

	idTokenHint := req.GetRequestForm().Get("id_token_hint")
	if idTokenHint == "" {

		return nil
	}

	tokenHint, err := v.Strategy.Decode(idTokenHint)
	if err != nil {
		return errors.WithStack(fosite.ErrInvalidRequest.WithHintf("Failed to validate OpenID Connect request as decoding id token from id_token_hint parameter failed because %s.", err.Error()))
	}

	if hintClaims, ok := tokenHint.Claims.(jwtgo.MapClaims); !ok {
		return errors.WithStack(fosite.ErrInvalidRequest.WithHint("Failed to validate OpenID Connect request as decoding id token from id_token_hint to *jwt.StandardClaims failed."))
	} else if hintSub, _ := hintClaims["sub"].(string); hintSub == "" {
		return errors.WithStack(fosite.ErrInvalidRequest.WithHint("Failed to validate OpenID Connect request because provided id token from id_token_hint does not have a subject."))
	} else if hintSub != claims.Subject || hintSub != session.GetSubject() {
		return errors.WithStack(fosite.ErrLoginRequired.WithHintf("Failed to validate OpenID Connect request because subject from session does not subject from id_token_hint."))
	}

	return nil
}

func isWhitelisted(items []string, whiteList []string) bool {
	for _, item := range items {
		if !stringslice.Has(whiteList, item) {
			return false
		}
	}
	return true
}
<commit_msg>openid: Validate id_token_hint only via ID claims (#296)<commit_after>/*
 * Copyright © 2017-2018 Aeneas Rekkas <aeneas+oss@aeneas.io>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * @author		Aeneas Rekkas <aeneas+oss@aeneas.io>
 * @Copyright 	2017-2018 Aeneas Rekkas <aeneas+oss@aeneas.io>
 * @license 	Apache-2.0
 *
 */

package openid

import (
	"strconv"
	"time"

	jwtgo "github.com/dgrijalva/jwt-go"
	"github.com/ory/fosite"
	"github.com/ory/fosite/token/jwt"
	"github.com/ory/go-convenience/stringslice"
	"github.com/ory/go-convenience/stringsx"
	"github.com/pkg/errors"
)

type OpenIDConnectRequestValidator struct {
	AllowedPrompt []string
	Strategy      jwt.JWTStrategy
}

func NewOpenIDConnectRequestValidator(prompt []string, strategy jwt.JWTStrategy) *OpenIDConnectRequestValidator {
	if len(prompt) == 0 {
		prompt = []string{"login", "none", "consent", "select_account"}
	}

	return &OpenIDConnectRequestValidator{
		AllowedPrompt: prompt,
		Strategy:      strategy,
	}
}

func (v *OpenIDConnectRequestValidator) ValidatePrompt(req fosite.AuthorizeRequester) error {
	// prompt is case sensitive!
	prompt := stringsx.Splitx(req.GetRequestForm().Get("prompt"), " ")

	if req.GetClient().IsPublic() {
		// Threat: Malicious Client Obtains Existing Authorization by Fraud
		// https://tools.ietf.org/html/rfc6819#section-4.2.3
		//
		//  Authorization servers should not automatically process repeat
		//  authorizations to public clients unless the client is validated
		//  using a pre-registered redirect URI

		// Client Impersonation
		// https://tools.ietf.org/html/rfc8252#section-8.6#
		//
		//  As stated in Section 10.2 of OAuth 2.0 [RFC6749], the authorization
		//  server SHOULD NOT process authorization requests automatically
		//  without user consent or interaction, except when the identity of the
		//  client can be assured.  This includes the case where the user has
		//  previously approved an authorization request for a given client id --
		//  unless the identity of the client can be proven, the request SHOULD
		//  be processed as if no previous request had been approved.

		// To make sure that we are not vulnerable to this type of attack, we will always require consent for public
		// clients.

		// If prompt is none - meaning that no consent should be requested, we must terminate with an error.
		if stringslice.Has(prompt, "none") {
			return errors.WithStack(fosite.ErrConsentRequired.WithHint("OAuth 2.0 Client is marked public and requires end-user consent, but \"prompt=none\" was requested."))
		}
	}

	if !isWhitelisted(prompt, v.AllowedPrompt) {
		return errors.WithStack(fosite.ErrInvalidRequest.WithHintf(`Used unknown value "%s" for prompt parameter`, prompt))
	}

	if stringslice.Has(prompt, "none") && len(prompt) > 1 {
		// If this parameter contains none with any other value, an error is returned.
		return errors.WithStack(fosite.ErrInvalidRequest.WithHint("Parameter \"prompt\" was set to \"none\", but contains other values as well which is not allowed."))
	}

	maxAge, err := strconv.ParseInt(req.GetRequestForm().Get("max_age"), 10, 64)
	if err != nil {
		maxAge = 0
	}

	session, ok := req.GetSession().(Session)
	if !ok {
		return errors.WithStack(fosite.ErrServerError.WithDebug("Failed to validate OpenID Connect request because session is not of type fosite/handler/openid.Session."))
	}

	claims := session.IDTokenClaims()
	if claims.Subject == "" {
		return errors.WithStack(fosite.ErrServerError.WithDebug("Failed to validate OpenID Connect request because session subject is empty."))
	}

	// Adds a bit of wiggle room for timing issues
	if claims.AuthTime.After(time.Now().UTC().Add(time.Second * 5)) {
		return errors.WithStack(fosite.ErrServerError.WithDebug("Failed to validate OpenID Connect request because authentication time is in the future."))
	}

	if maxAge > 0 {
		if claims.AuthTime.IsZero() {
			return errors.WithStack(fosite.ErrServerError.WithDebug("Failed to validate OpenID Connect request because authentication time claim is required when max_age is set."))
		} else if claims.RequestedAt.IsZero() {
			return errors.WithStack(fosite.ErrServerError.WithDebug("Failed to validate OpenID Connect request because requested at claim is required when max_age is set."))
		} else if claims.AuthTime.Add(time.Second * time.Duration(maxAge)).Before(claims.RequestedAt) {
			return errors.WithStack(fosite.ErrLoginRequired.WithDebug("Failed to validate OpenID Connect request because authentication time does not satisfy max_age time."))
		}
	}

	if stringslice.Has(prompt, "none") {
		if claims.AuthTime.IsZero() {
			return errors.WithStack(fosite.ErrServerError.WithDebug("Failed to validate OpenID Connect request because because auth_time is missing from session."))
		}
		if claims.AuthTime.After(claims.RequestedAt) {
			return errors.WithStack(fosite.ErrLoginRequired.WithHint("Failed to validate OpenID Connect request because prompt was set to \"none\" but auth_time happened after the authorization request was registered, indicating that the user was logged in during this request which is not allowed."))
		}
	}

	if stringslice.Has(prompt, "login") {
		if claims.AuthTime.Before(claims.RequestedAt) {
			return errors.WithStack(fosite.ErrLoginRequired.WithHint("Failed to validate OpenID Connect request because prompt was set to \"login\" but auth_time happened before the authorization request was registered, indicating that the user was not re-authenticated which is forbidden."))
		}
	}

	idTokenHint := req.GetRequestForm().Get("id_token_hint")
	if idTokenHint == "" {

		return nil
	}

	tokenHint, err := v.Strategy.Decode(idTokenHint)
	if err != nil {
		return errors.WithStack(fosite.ErrInvalidRequest.WithHintf("Failed to validate OpenID Connect request as decoding id token from id_token_hint parameter failed because %s.", err.Error()))
	}

	if hintClaims, ok := tokenHint.Claims.(jwtgo.MapClaims); !ok {
		return errors.WithStack(fosite.ErrInvalidRequest.WithHint("Failed to validate OpenID Connect request as decoding id token from id_token_hint to *jwt.StandardClaims failed."))
	} else if hintSub, _ := hintClaims["sub"].(string); hintSub == "" {
		return errors.WithStack(fosite.ErrInvalidRequest.WithHint("Failed to validate OpenID Connect request because provided id token from id_token_hint does not have a subject."))
	} else if hintSub != claims.Subject {
		return errors.WithStack(fosite.ErrLoginRequired.WithHintf("Failed to validate OpenID Connect request because subject from ID token session claims does not subject from id_token_hint."))
	}

	return nil
}

func isWhitelisted(items []string, whiteList []string) bool {
	for _, item := range items {
		if !stringslice.Has(whiteList, item) {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package engine

import (
	"errors"
	"fmt"

	"github.com/keybase/go/libkb"
	keybase_1 "github.com/keybase/protocol/go"
)

type Doctor struct {
	user       *libkb.User
	docUI      libkb.DoctorUI
	secretUI   libkb.SecretUI
	logUI      libkb.LogUI
	identifyUI libkb.IdentifyUI

	signingKey libkb.GenericKey
}

func NewDoctor(docUI libkb.DoctorUI, secUI libkb.SecretUI, logUI libkb.LogUI, identifyUI libkb.IdentifyUI) *Doctor {
	return &Doctor{docUI: docUI, secretUI: secUI, logUI: logUI, identifyUI: identifyUI}
}

func (d *Doctor) LoginCheckup(u *libkb.User) error {
	d.user = u

	// This can fail, but we'll warn if it does.
	d.syncSecrets()

	if err := d.checkUID(); err != nil {
		return err
	}

	if err := d.checkKeys(); err != nil {
		return err
	}
	return nil
}

// checkUID makes sure that we've verified our own UID.  This might result in a
// self-tracking operation.
func (d *Doctor) checkUID() (err error) {
	uid := d.user.GetUid()
	if u2 := G.Env.GetVerifiedUID(); u2 != nil && !u2.Eq(uid) {
		err = libkb.UidMismatchError{Msg: fmt.Sprintf("Got wrong uid; wanted %s but got %s", uid, u2)}
	} else if u2 == nil && d.identifyUI != nil {
		d.logUI.Warning("Verifying your UID...")
		err = d.user.IdentifySelf(d.identifyUI)
		if err == nil {
			d.logUI.Warning("Setting UID to %s", uid)
		}
	}
	return err
}

func (d *Doctor) syncSecrets() (err error) {
	if err = G.SecretSyncer.Load(d.user.GetUid()); err != nil {
		G.Log.Warning("Problem syncing secrets from server: %s", err.Error())
	}
	return err
}

func (d *Doctor) checkKeys() error {
	kf := d.user.GetKeyFamily()
	if kf == nil {
		return d.addBasicKeys()
	}
	if kf.GetEldest() == nil {
		return d.addBasicKeys()
	}

	// they have at least one key

	if d.user.HasDeviceInCurrentInstall() {
		// they have a device sibkey for this device
		return nil
	}

	// make sure secretsyncer loaded
	if err := G.SecretSyncer.Load(d.user.GetUid()); err != nil {
		return err
	}

	if G.SecretSyncer.HasActiveDevice() {
		// they have at least one device, just not this device...
		return d.deviceSign()
	}

	// they don't have any devices.  use their detkey to sign a new device.
	dk, err := d.detkey()
	if err != nil {
		return err
	}
	return d.addDeviceKeyWithDetKey(dk)
}

// addBasicKeys is used for accounts that have no device or det
// keys.
func (d *Doctor) addBasicKeys() error {
	if err := d.addDeviceKey(); err != nil {
		return err
	}

	if err := d.addDetKey(); err != nil {
		return err
	}

	return nil
}

func (d *Doctor) addDeviceKey() error {
	// XXX session id...what to put there?
	devname, err := d.docUI.PromptDeviceName(0)
	if err != nil {
		return err
	}
	tk, err := d.tspkey()
	if err != nil {
		return err
	}
	eng := NewDeviceEngine(d.user, d.logUI)
	if err := eng.Run(devname, tk.LksClientHalf()); err != nil {
		return err
	}

	d.signingKey = eng.EldestKey()
	return nil
}

func (d *Doctor) addDeviceKeyWithDetKey(eldest libkb.GenericKey) error {
	// XXX session id...what to put there?
	devname, err := d.docUI.PromptDeviceName(0)
	if err != nil {
		return err
	}
	tk, err := d.tspkey()
	if err != nil {
		return err
	}
	eng := NewDeviceEngine(d.user, d.logUI)
	if err := eng.RunWithDetKey(devname, tk.LksClientHalf(), eldest); err != nil {
		return fmt.Errorf("RunWithDetKey error: %s", err)
	}

	d.signingKey = eng.EldestKey()
	return nil
}

func (d *Doctor) addDetKey() error {
	tk, err := d.tspkey()
	if err != nil {
		return err
	}
	eng := NewDetKeyEngine(d.user, d.signingKey, d.logUI)
	return eng.Run(tk)
}

var ErrNotYetImplemented = errors.New("not yet implemented")

func (d *Doctor) deviceSign() error {
	devs, err := G.SecretSyncer.ActiveDevices()
	if err != nil {
		return err
	}

	var devDescs []keybase_1.DeviceDescription
	for k, v := range devs {
		G.Log.Info("Device %s: %+v", k, v)
		devDescs = append(devDescs, keybase_1.DeviceDescription{Type: v.Type, Name: v.Description})
	}

	_, err = d.docUI.SelectSigner(devDescs)
	if err != nil {
		return err
	}
	return ErrNotYetImplemented
}

func (d *Doctor) tspkey() (*libkb.TSPassKey, error) {
	t := G.LoginState.GetCachedTSPassKey()
	if t != nil {
		return t, nil
	}

	// not cached: get it from the ui
	pp, err := d.secretUI.GetKeybasePassphrase(keybase_1.GetKeybasePassphraseArg{Username: G.Env.GetUsername()})
	if err != nil {
		return nil, err
	}
	err = G.LoginState.StretchKey(pp)
	if err != nil {
		return nil, err
	}
	return G.LoginState.GetCachedTSPassKey(), nil
}

func (d *Doctor) detkey() (libkb.GenericKey, error) {
	// get server half of detkey via ss
	half, err := G.SecretSyncer.FindDetKeySrvHalf(libkb.KEY_TYPE_KB_NACL_EDDSA_SERVER_HALF)
	if err != nil {
		return nil, err
	}

	// regenerate the detkey
	tk, err := d.tspkey()
	if err != nil {
		return nil, err
	}

	detkey, err := GenSigningDetKey(tk, half)
	if err != nil {
		return nil, err
	}
	return detkey, nil
}
<commit_msg>don't self identify if there's no active key<commit_after>package engine

import (
	"errors"
	"fmt"

	"github.com/keybase/go/libkb"
	keybase_1 "github.com/keybase/protocol/go"
)

type Doctor struct {
	user       *libkb.User
	docUI      libkb.DoctorUI
	secretUI   libkb.SecretUI
	logUI      libkb.LogUI
	identifyUI libkb.IdentifyUI

	signingKey libkb.GenericKey
}

func NewDoctor(docUI libkb.DoctorUI, secUI libkb.SecretUI, logUI libkb.LogUI, identifyUI libkb.IdentifyUI) *Doctor {
	return &Doctor{docUI: docUI, secretUI: secUI, logUI: logUI, identifyUI: identifyUI}
}

func (d *Doctor) LoginCheckup(u *libkb.User) error {
	d.user = u

	// This can fail, but we'll warn if it does.
	d.syncSecrets()

	if err := d.checkUID(); err != nil {
		return err
	}

	if err := d.checkKeys(); err != nil {
		return err
	}
	return nil
}

// checkUID makes sure that we've verified our own UID.  This might result in a
// self-tracking operation.
func (d *Doctor) checkUID() (err error) {
	uid := d.user.GetUid()

	if d.user.HasActiveKey() {
		d.logUI.Debug("Skipping checkUID due to no active key")
	} else if u2 := G.Env.GetVerifiedUID(); u2 != nil && !u2.Eq(uid) {
		err = libkb.UidMismatchError{Msg: fmt.Sprintf("Got wrong uid; wanted %s but got %s", uid, u2)}
	} else if u2 == nil && d.identifyUI != nil {
		d.logUI.Warning("Verifying your UID...")
		err = d.user.IdentifySelf(d.identifyUI)
		if err == nil {
			d.logUI.Warning("Setting UID to %s", uid)
		}
	}
	return err
}

func (d *Doctor) syncSecrets() (err error) {
	if err = G.SecretSyncer.Load(d.user.GetUid()); err != nil {
		G.Log.Warning("Problem syncing secrets from server: %s", err.Error())
	}
	return err
}

func (d *Doctor) checkKeys() error {
	kf := d.user.GetKeyFamily()
	if kf == nil {
		return d.addBasicKeys()
	}
	if kf.GetEldest() == nil {
		return d.addBasicKeys()
	}

	// they have at least one key

	if d.user.HasDeviceInCurrentInstall() {
		// they have a device sibkey for this device
		return nil
	}

	// make sure secretsyncer loaded
	if err := G.SecretSyncer.Load(d.user.GetUid()); err != nil {
		return err
	}

	if G.SecretSyncer.HasActiveDevice() {
		// they have at least one device, just not this device...
		return d.deviceSign()
	}

	// they don't have any devices.  use their detkey to sign a new device.
	dk, err := d.detkey()
	if err != nil {
		return err
	}
	return d.addDeviceKeyWithDetKey(dk)
}

// addBasicKeys is used for accounts that have no device or det
// keys.
func (d *Doctor) addBasicKeys() error {
	if err := d.addDeviceKey(); err != nil {
		return err
	}

	if err := d.addDetKey(); err != nil {
		return err
	}

	return nil
}

func (d *Doctor) addDeviceKey() error {
	// XXX session id...what to put there?
	devname, err := d.docUI.PromptDeviceName(0)
	if err != nil {
		return err
	}
	tk, err := d.tspkey()
	if err != nil {
		return err
	}
	eng := NewDeviceEngine(d.user, d.logUI)
	if err := eng.Run(devname, tk.LksClientHalf()); err != nil {
		return err
	}

	d.signingKey = eng.EldestKey()
	return nil
}

func (d *Doctor) addDeviceKeyWithDetKey(eldest libkb.GenericKey) error {
	// XXX session id...what to put there?
	devname, err := d.docUI.PromptDeviceName(0)
	if err != nil {
		return err
	}
	tk, err := d.tspkey()
	if err != nil {
		return err
	}
	eng := NewDeviceEngine(d.user, d.logUI)
	if err := eng.RunWithDetKey(devname, tk.LksClientHalf(), eldest); err != nil {
		return fmt.Errorf("RunWithDetKey error: %s", err)
	}

	d.signingKey = eng.EldestKey()
	return nil
}

func (d *Doctor) addDetKey() error {
	tk, err := d.tspkey()
	if err != nil {
		return err
	}
	eng := NewDetKeyEngine(d.user, d.signingKey, d.logUI)
	return eng.Run(tk)
}

var ErrNotYetImplemented = errors.New("not yet implemented")

func (d *Doctor) deviceSign() error {
	devs, err := G.SecretSyncer.ActiveDevices()
	if err != nil {
		return err
	}

	var devDescs []keybase_1.DeviceDescription
	for k, v := range devs {
		G.Log.Info("Device %s: %+v", k, v)
		devDescs = append(devDescs, keybase_1.DeviceDescription{Type: v.Type, Name: v.Description})
	}

	_, err = d.docUI.SelectSigner(devDescs)
	if err != nil {
		return err
	}
	return ErrNotYetImplemented
}

func (d *Doctor) tspkey() (*libkb.TSPassKey, error) {
	t := G.LoginState.GetCachedTSPassKey()
	if t != nil {
		return t, nil
	}

	// not cached: get it from the ui
	pp, err := d.secretUI.GetKeybasePassphrase(keybase_1.GetKeybasePassphraseArg{Username: G.Env.GetUsername()})
	if err != nil {
		return nil, err
	}
	err = G.LoginState.StretchKey(pp)
	if err != nil {
		return nil, err
	}
	return G.LoginState.GetCachedTSPassKey(), nil
}

func (d *Doctor) detkey() (libkb.GenericKey, error) {
	// get server half of detkey via ss
	half, err := G.SecretSyncer.FindDetKeySrvHalf(libkb.KEY_TYPE_KB_NACL_EDDSA_SERVER_HALF)
	if err != nil {
		return nil, err
	}

	// regenerate the detkey
	tk, err := d.tspkey()
	if err != nil {
		return nil, err
	}

	detkey, err := GenSigningDetKey(tk, half)
	if err != nil {
		return nil, err
	}
	return detkey, nil
}
<|endoftext|>


<commit_before>package cluster

import (
	"context"
	"fmt"
	"math/rand"
	"net/http"
	"os"
	"time"

	"github.com/canonical/go-dqlite/client"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/logger"
	"github.com/pkg/errors"
)

// NotifyUpgradeCompleted sends a notification to all other nodes in the
// cluster that any possible pending database update has been applied, and any
// nodes which was waiting for this node to be upgraded should re-check if it's
// okay to move forward.
func NotifyUpgradeCompleted(state *state.State, networkCert *shared.CertInfo, serverCert *shared.CertInfo) error {
	notifier, err := NewNotifier(state, networkCert, serverCert, NotifyTryAll)
	if err != nil {
		return err
	}

	return notifier(func(client lxd.InstanceServer) error {
		info, err := client.GetConnectionInfo()
		if err != nil {
			return errors.Wrap(err, "failed to get connection info")
		}

		url := fmt.Sprintf("%s%s", info.Addresses[0], databaseEndpoint)
		request, err := http.NewRequest("PATCH", url, nil)
		if err != nil {
			return errors.Wrap(err, "failed to create database notify upgrade request")
		}
		setDqliteVersionHeader(request)

		httpClient, err := client.GetHTTPClient()
		if err != nil {
			return errors.Wrap(err, "failed to get HTTP client")
		}

		httpClient.Timeout = 5 * time.Second
		response, err := httpClient.Do(request)
		if err != nil {
			return errors.Wrap(err, "failed to notify node about completed upgrade")
		}

		if response.StatusCode != http.StatusOK {
			return fmt.Errorf("database upgrade notification failed: %s", response.Status)
		}

		return nil
	})
}

// MaybeUpdate Check this node's version and possibly run LXD_CLUSTER_UPDATE.
func MaybeUpdate(state *state.State) error {
	shouldUpdate := false

	enabled, err := Enabled(state.Node)
	if err != nil {
		return errors.Wrap(err, "Failed to check clustering is enabled")
	}
	if !enabled {
		return nil
	}

	if state.Cluster == nil {
		return fmt.Errorf("Failed checking cluster update, state not initialised yet")
	}

	err = state.Cluster.Transaction(func(tx *db.ClusterTx) error {
		outdated, err := tx.NodeIsOutdated()
		if err != nil {
			return err
		}
		shouldUpdate = outdated
		return nil
	})

	if err != nil {
		// Just log the error and return.
		return errors.Wrap(err, "Failed to check if this node is out-of-date")
	}

	if !shouldUpdate {
		logger.Debugf("Cluster node is up-to-date")
		return nil
	}

	return triggerUpdate()
}

func triggerUpdate() error {
	logger.Infof("Node is out-of-date with respect to other cluster nodes")

	updateExecutable := os.Getenv("LXD_CLUSTER_UPDATE")
	if updateExecutable == "" {
		logger.Debug("No LXD_CLUSTER_UPDATE variable set, skipping auto-update")
		return nil
	}

	// Wait a random amout of seconds (up to 30) in order to avoid
	// restarting all cluster members at the same time, and make the
	// upgrade more graceful.
	wait := time.Duration(rand.Intn(30)) * time.Second
	logger.Infof("Triggering cluster update in %s using: %s", wait, updateExecutable)
	time.Sleep(wait)

	_, err := shared.RunCommand(updateExecutable)
	if err != nil {
		logger.Errorf("Cluster upgrade failed: '%v'", err.Error())
		return err
	}
	return nil
}

// UpgradeMembersWithoutRole assigns the Spare raft role to all cluster members that are not currently part of the
// raft configuration. It's used for upgrading a cluster from a version without roles support.
func UpgradeMembersWithoutRole(gateway *Gateway, members []db.NodeInfo) error {
	nodes, err := gateway.currentRaftNodes()
	if err == ErrNotLeader {
		return nil
	}
	if err != nil {
		return errors.Wrap(err, "Failed to get current raft nodes")
	}

	// Used raft IDs.
	ids := map[uint64]bool{}
	for _, node := range nodes {
		ids[node.ID] = true
	}

	dqliteClient, err := gateway.getClient()
	if err != nil {
		return errors.Wrap(err, "Failed to connect to local dqlite node")
	}
	defer dqliteClient.Close()

	// Check that each member is present in the raft configuration, and add it if not.
	for _, member := range members {
		found := false
		for _, node := range nodes {
			if member.ID == 1 && node.ID == 1 || member.Address == node.Address {
				found = true
				break
			}
		}
		if found {
			continue
		}

		// Try to use the same ID as the node, but it might not be possible if it's use.
		id := uint64(member.ID)
		if _, ok := ids[id]; ok {
			for _, other := range members {
				if _, ok := ids[uint64(other.ID)]; !ok {
					id = uint64(other.ID)
					break
				}
			}

			// This can't really happen (but has in the past) since there are always at least as many
			// members as there are nodes, and all of them have different IDs.
			if id == uint64(member.ID) {
				return fmt.Errorf("No available raft ID for cluster member ID %d", member.ID)
			}
		}
		ids[id] = true

		info := db.RaftNode{
			NodeInfo: client.NodeInfo{
				ID:      id,
				Address: member.Address,
				Role:    db.RaftSpare,
			},
			Name: "",
		}

		logger.Info("Add spare dqlite node", log15.Ctx{"id": info.ID, "address": info.Address})

		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
		defer cancel()
		err = dqliteClient.Add(ctx, info.NodeInfo)
		if err != nil {
			return errors.Wrap(err, "Failed to add dqlite node")
		}
	}

	return nil
}
<commit_msg>lxd/cluster/upgrade: Improve variable naming in UpgradeMembersWithoutRole<commit_after>package cluster

import (
	"context"
	"fmt"
	"math/rand"
	"net/http"
	"os"
	"time"

	"github.com/canonical/go-dqlite/client"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/logger"
	"github.com/pkg/errors"
)

// NotifyUpgradeCompleted sends a notification to all other nodes in the
// cluster that any possible pending database update has been applied, and any
// nodes which was waiting for this node to be upgraded should re-check if it's
// okay to move forward.
func NotifyUpgradeCompleted(state *state.State, networkCert *shared.CertInfo, serverCert *shared.CertInfo) error {
	notifier, err := NewNotifier(state, networkCert, serverCert, NotifyTryAll)
	if err != nil {
		return err
	}

	return notifier(func(client lxd.InstanceServer) error {
		info, err := client.GetConnectionInfo()
		if err != nil {
			return errors.Wrap(err, "failed to get connection info")
		}

		url := fmt.Sprintf("%s%s", info.Addresses[0], databaseEndpoint)
		request, err := http.NewRequest("PATCH", url, nil)
		if err != nil {
			return errors.Wrap(err, "failed to create database notify upgrade request")
		}
		setDqliteVersionHeader(request)

		httpClient, err := client.GetHTTPClient()
		if err != nil {
			return errors.Wrap(err, "failed to get HTTP client")
		}

		httpClient.Timeout = 5 * time.Second
		response, err := httpClient.Do(request)
		if err != nil {
			return errors.Wrap(err, "failed to notify node about completed upgrade")
		}

		if response.StatusCode != http.StatusOK {
			return fmt.Errorf("database upgrade notification failed: %s", response.Status)
		}

		return nil
	})
}

// MaybeUpdate Check this node's version and possibly run LXD_CLUSTER_UPDATE.
func MaybeUpdate(state *state.State) error {
	shouldUpdate := false

	enabled, err := Enabled(state.Node)
	if err != nil {
		return errors.Wrap(err, "Failed to check clustering is enabled")
	}
	if !enabled {
		return nil
	}

	if state.Cluster == nil {
		return fmt.Errorf("Failed checking cluster update, state not initialised yet")
	}

	err = state.Cluster.Transaction(func(tx *db.ClusterTx) error {
		outdated, err := tx.NodeIsOutdated()
		if err != nil {
			return err
		}
		shouldUpdate = outdated
		return nil
	})

	if err != nil {
		// Just log the error and return.
		return errors.Wrap(err, "Failed to check if this node is out-of-date")
	}

	if !shouldUpdate {
		logger.Debugf("Cluster node is up-to-date")
		return nil
	}

	return triggerUpdate()
}

func triggerUpdate() error {
	logger.Infof("Node is out-of-date with respect to other cluster nodes")

	updateExecutable := os.Getenv("LXD_CLUSTER_UPDATE")
	if updateExecutable == "" {
		logger.Debug("No LXD_CLUSTER_UPDATE variable set, skipping auto-update")
		return nil
	}

	// Wait a random amout of seconds (up to 30) in order to avoid
	// restarting all cluster members at the same time, and make the
	// upgrade more graceful.
	wait := time.Duration(rand.Intn(30)) * time.Second
	logger.Infof("Triggering cluster update in %s using: %s", wait, updateExecutable)
	time.Sleep(wait)

	_, err := shared.RunCommand(updateExecutable)
	if err != nil {
		logger.Errorf("Cluster upgrade failed: '%v'", err.Error())
		return err
	}
	return nil
}

// UpgradeMembersWithoutRole assigns the Spare raft role to all cluster members that are not currently part of the
// raft configuration. It's used for upgrading a cluster from a version without roles support.
func UpgradeMembersWithoutRole(gateway *Gateway, members []db.NodeInfo) error {
	nodes, err := gateway.currentRaftNodes()
	if err == ErrNotLeader {
		return nil
	}
	if err != nil {
		return errors.Wrap(err, "Failed to get current raft nodes")
	}

	// Convert raft node list to map keyed on ID.
	raftNodeIDs := map[uint64]bool{}
	for _, node := range nodes {
		raftNodeIDs[node.ID] = true
	}

	dqliteClient, err := gateway.getClient()
	if err != nil {
		return errors.Wrap(err, "Failed to connect to local dqlite node")
	}
	defer dqliteClient.Close()

	// Check that each member is present in the raft configuration, and add it if not.
	for _, member := range members {
		found := false
		for _, node := range nodes {
			if member.ID == 1 && node.ID == 1 || member.Address == node.Address {
				found = true
				break
			}
		}
		if found {
			continue
		}

		// Try to use the same ID as the node, but it might not be possible if it's use.
		id := uint64(member.ID)
		if _, ok := raftNodeIDs[id]; ok {
			for _, other := range members {
				if _, ok := raftNodeIDs[uint64(other.ID)]; !ok {
					id = uint64(other.ID) // Found unused raft ID for member.
					break
				}
			}

			// This can't really happen (but has in the past) since there are always at least as many
			// members as there are nodes, and all of them have different IDs.
			if id == uint64(member.ID) {
				return fmt.Errorf("No available raft ID for cluster member ID %d", member.ID)
			}
		}
		raftNodeIDs[id] = true

		info := db.RaftNode{
			NodeInfo: client.NodeInfo{
				ID:      id,
				Address: member.Address,
				Role:    db.RaftSpare,
			},
			Name: "",
		}

		logger.Info("Add spare dqlite node", log15.Ctx{"id": info.ID, "address": info.Address})

		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
		defer cancel()
		err = dqliteClient.Add(ctx, info.NodeInfo)
		if err != nil {
			return errors.Wrap(err, "Failed to add dqlite node")
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Stuart Glenn, OMRF. All rights reserved.
// Use of this code is governed by a 3 clause BSD style license
// Full license details in LICENSE file distributed with this software

package matcher

import (
	"strings"
	"testing"
)

func TestMatch(t *testing.T) {
	a := &Record{ID: "a", Atts: []Atter{TextAtt{"text"}, NumericAtt{1}, TextAtt{"green"}}}
	b := &Record{ID: "b", Atts: []Atter{TextAtt{"text"}, NumericAtt{1}, TextAtt{"red"}}}
	if a.IsMatch(b) {
		t.Error("A should not match b on all attributes")
	}
	if !a.IsMatch(a) {
		t.Error("A should match itself on all attributes")
	}
	if !a.IsMatch(b, 0) {
		t.Error("A should match b on just first attribute")
	}
	if !b.IsMatch(a, []int{0, 1}...) {
		t.Error("B should match a on first two attributes")
	}
	if a.IsMatch(b, 2) {
		t.Error("A should not match B on third attribute")
	}
	if a.IsMatch(a, 100) {
		t.Error("A should not match itself with attribute out of bounds")
	}
	if !a.IsMatch(a, 0, 1, 2) {
		t.Error("A should match itself with all attributes specified")
	}
}

func TestMatchAtt(t *testing.T) {
	a := &Record{ID: "a", Atts: []Atter{TextAtt{"first"}, NumericAtt{1}, NumericAtt{8}, TextAtt{"yes"}}}
	b := &Record{ID: "b", Atts: []Atter{TextAtt{"second"}, NumericAtt{4.2}, NumericAtt{8}, TextAtt{"yes"}}}
	if !a.isMatchAt(b, NumericAtt{4}, 1) {
		t.Error("A numeric att should match b with an epsilon")
	}
	if !a.isMatchAt(b, NumericAtt{3.2}, 1) {
		t.Error("A numeric att should match b with an epsilon")
	}
	if a.isMatchAt(b, NumericAtt{2}, 1) {
		t.Error("A numeric att should not match b with a small epsilon")
	}
	e := make([]Atter, len(a.Atts))
	e[1] = NumericAtt{2}
	if a.IsMatchWithRanges(b, e, 1) {
		t.Error("A should not match b with small range on one index")
	}
	if a.IsMatchWithRanges(b, e, 1, 2, 3) {
		t.Error("A should not match on multi indices b with small range on one index")
	}
	e[1] = NumericAtt{3.2}
	tests := map[int]bool{
		0: false,
		1: true,
		2: true,
		3: true,
	}
	for index, result := range tests {
		if result != a.isMatchAt(b, e[index], index) {
			t.Errorf("A:%v att:%v match? b:%v with a %v", a, index, b, e[index])
		}
	}
}

func TestMatchWrongSizes(t *testing.T) {
	a := &Record{ID: "a", Atts: []Atter{TextAtt{"first"}, NumericAtt{1}}}
	b := &Record{ID: "b", Atts: []Atter{TextAtt{"second"}, NumericAtt{4.2}, NumericAtt{8}, TextAtt{"yes"}}}
	if a.IsMatchWithRanges(b, make([]Atter, len(a.Atts))) {
		t.Error("A longer record should not match a shorter record")
	}
	if a.IsMatchWithRanges(a, make([]Atter, 0)) {
		t.Error("A record cannot match itself is the []range is too short")
	}
	if !a.IsMatchWithRanges(a, make([]Atter, len(a.Atts))) {
		t.Error("A record should match itself completely")
	}
	if a.isMatchAt(a, TextAtt{}, len(a.Atts)+1) {
		t.Error("A record cannot equal even itself at a position past the record")
	}
	if a.isMatchAt(a, TextAtt{}, -1) {
		t.Error("A record cannot equal even itself at a position before the record")
	}
}

func TestMatchRange(t *testing.T) {
	a := &Record{ID: "a", Atts: []Atter{TextAtt{"first"}, NumericAtt{1}, NumericAtt{8}, TextAtt{"yes"}}}
	b := &Record{ID: "b", Atts: []Atter{TextAtt{"second"}, NumericAtt{4.2}, NumericAtt{8}, TextAtt{"yes"}}}
	if a.IsMatch(b) {
		t.Error("A and b should not match")
	}
	e := make([]Atter, len(a.Atts))
	e[1] = NumericAtt{2}
	if a.IsMatchWithRanges(b, e, 1) {
		t.Error("A should not match b with small range on one index")
	}
	if a.IsMatchWithRanges(b, e, 1, 2, 3) {
		t.Error("A should not match on multi indices b with small range on one index")
	}
	e[1] = NumericAtt{3.2}
	i := []int{1}
	if !a.IsMatchWithRanges(b, e, i...) {
		t.Errorf("A:%v should match b:%v with correct range:%v on %v", a, b, e, i)
	}
	i = []int{1, 2, 3}
	if !a.IsMatchWithRanges(b, e, i...) {
		t.Errorf("A:%v should match b:%v with correct range:%v on %v", a, b, e, i)
	}
	e[1] = NumericAtt{3.0}
	if a.IsMatchWithRanges(b, e, i...) {
		t.Errorf("A:%v should not match b:%v with small range:%v on %v", a, b, e, i)
	}
	i = []int{2, 3}
	if !a.IsMatchWithRanges(b, e, i...) {
		t.Errorf("A:%v should match b:%v with small range:%v on %v", a, b, e, i)
	}
}

func TestMatchesAll(t *testing.T) {
	a := Records{
		Record{ID: "a1", Atts: []Atter{TextAtt{"red"}, NumericAtt{1}}},
		Record{ID: "a2", Atts: []Atter{TextAtt{"red"}, NumericAtt{20}}},
		Record{ID: "a3", Atts: []Atter{TextAtt{"green"}, NumericAtt{30}}},
	}
	b := Records{
		Record{ID: "b1", Atts: []Atter{TextAtt{"green"}, NumericAtt{5}}},
		Record{ID: "b2", Atts: []Atter{TextAtt{"red"}, NumericAtt{25}}},
		Record{ID: "b3", Atts: []Atter{TextAtt{"red"}, NumericAtt{35}}},
		Record{ID: "b4", Atts: []Atter{TextAtt{"green"}, NumericAtt{30}}},
		Record{ID: "b5", Atts: []Atter{TextAtt{"green"}, NumericAtt{31}}},
		Record{ID: "b6", Atts: []Atter{TextAtt{"red"}, NumericAtt{20}}},
		Record{ID: "b7", Atts: []Atter{TextAtt{"red"}, NumericAtt{20}}},
	}

	if m := a[0].MatchesAll(b); len(m) != 0 {
		t.Errorf("%v should not have found any matches in %v", a[0], b)
	}
	m := a[2].MatchesAll(b)
	if len(m) != 1 {
		t.Errorf("%v should have found one match in %v, but found %v", a[2], b, m)
	}
	if 3 != m[0] {
		t.Errorf("%v should have found one at 1, but found it at %v", a[2], m[0])
	}

	m = a[1].MatchesAll(b)
	if len(m) != 2 {
		t.Errorf("%v should have found two matches in %v, but found %v", a[1], b, m)
	}
	if 5 != m[0] && 6 != m[1] {
		t.Errorf("%v should have found one at 5 & 6, but found it at %v", a[1], m)
	}

	e := []Atter{TextAtt{}, NumericAtt{5}}
	m = a[0].MatchesAll(b, e...)
	if 0 != len(m) {
		t.Errorf("%v should not have found any matches in %v using %v", a[0], b, e)
	}
	m = a[1].MatchesAll(b, e...)
	if 3 != len(m) {
		t.Errorf("%v should have found 3 in %v using %v, but found %v", a[0], b, e, m)
	}
	m = a[2].MatchesAll(b, e...)
	if 2 != len(m) {
		t.Errorf("%v should have found 2 in %v using %v, but found %v", a[0], b, e, m)
	}
}

func TestMatchesColumns(t *testing.T) {
	a := Records{
		Record{ID: "a0", Atts: []Atter{TextAtt{"red"}, NumericAtt{1}}},
		Record{ID: "a1", Atts: []Atter{TextAtt{"red"}, NumericAtt{20}}},
		Record{ID: "a2", Atts: []Atter{TextAtt{"green"}, NumericAtt{30}}},
	}
	b := Records{
		Record{ID: "b0", Atts: []Atter{TextAtt{"green"}, NumericAtt{5}}},
		Record{ID: "b1", Atts: []Atter{TextAtt{"red"}, NumericAtt{25}}},
		Record{ID: "b2", Atts: []Atter{TextAtt{"red"}, NumericAtt{35}}},
		Record{ID: "b3", Atts: []Atter{TextAtt{"green"}, NumericAtt{30}}},
		Record{ID: "b4", Atts: []Atter{TextAtt{"green"}, NumericAtt{31}}},
		Record{ID: "b5", Atts: []Atter{TextAtt{"red"}, NumericAtt{20}}},
		Record{ID: "b6", Atts: []Atter{TextAtt{"red"}, NumericAtt{20}}},
	}

	c := []int{10}
	if m := a[0].Matches(b, c); len(m) != 0 {
		t.Errorf("%v should have found 0 matches using column %v in %v, instead found %v", a[0], c, b, m)
	}

	c = []int{0}
	if m := a[0].Matches(b, c); len(m) != 4 {
		t.Errorf("%v should have found 4 matches in %v, instead found %v", a[0], b, m)
	}
	m := a[2].Matches(b, c)
	if len(m) != 3 {
		t.Errorf("%v should have found 3 matches in %v, but found %v", a[2], b, m)
	}
	if 0 != m[0] {
		t.Errorf("%v should have found one at 1, but found it at %v", a[2], m[0])
	}
	if 3 != m[1] {
		t.Errorf("%v should have found one at 1, but found it at %v", a[2], m[1])
	}
	if 4 != m[2] {
		t.Errorf("%v should have found one at 1, but found it at %v", a[2], m[2])
	}
	c = []int{1}
	m = a[2].Matches(b, c)
	if len(m) != 1 {
		t.Errorf("%v should have found 1 matches in %v, but found %v", a[2], b, m)
	}
	if 3 != m[0] {
		t.Errorf("%v should have found one at 1, but found it at %v", a[2], m[0])
	}

	m = a[1].Matches(b, c)
	if len(m) != 2 {
		t.Errorf("%v should have found two matches in %v, but found %v", a[1], b, m)
	}
	if 5 != m[0] && 6 != m[1] {
		t.Errorf("%v should have found one at 5 & 6, but found it at %v", a[1], m)
	}

	e := []Atter{TextAtt{}, NumericAtt{5}}
	m = a[0].Matches(b, c, e...)
	if 1 != len(m) {
		t.Errorf("%v should have found 1 matches in %v using %v, but found %v", a[0], b, e)
	}
	m = a[1].Matches(b, c, e...)
	if 3 != len(m) {
		t.Errorf("%v should have found 3 in %v using %v, but found %v", a[0], b, e, m)
	}
	m = a[2].Matches(b, c, e...)
	if 4 != len(m) {
		t.Errorf("%v should have found 4 in %v using %v, but found %v", a[0], b, e, m)
	}
}

func TestCSVParsing(t *testing.T) {
	csv := `item,type,color,count
a1,m,red,25`
	r, err := NewRecordsFromCSV(strings.NewReader(csv))
	if err != nil {
		t.Error("Expected no error parsing, but got ", err)
	}
	if 1 != len(r) {
		t.Error("Expected 1 record from", r)
	}
	if 3 != len(r[0].Atts) {
		t.Error("Expedted 3 attributes from", r[0].Atts)
	}

	csv = `item,type,color,count
a1,f,red,15
a2,m,red,25`
	r, err = NewRecordsFromCSV(strings.NewReader(csv), 2)
	if err != nil {
		t.Error("Expected no error parsing, but got ", err)
	}
	if 2 != len(r) {
		t.Error("Expected 1 record from", r)
	}
	if !(NumericAtt{15}).Equal(r[0].Atts[2], NumericAtt{}) {
		t.Error("Expected last attribute to be numeric equal to 15, but was not in", r[0].Atts[2])
	}

	r, err = NewRecordsFromCSV(strings.NewReader(csv), 0)
	if err == nil {
		t.Error("Expect error parsing, but got ", err)
	}
	if 0 != len(r) {
		t.Error("Expected 0 record from", r)
	}

	csv = "item,type,color,count"
	r, err = NewRecordsFromCSV(strings.NewReader(csv))
	if err != nil {
		t.Error("Expected no error parsing, but got ", err)
	}
	if 0 != len(r) {
		t.Error("Expected 0 record from", r)
	}
}
<commit_msg>Pointless typo fix in test<commit_after>// Copyright 2015 Stuart Glenn, OMRF. All rights reserved.
// Use of this code is governed by a 3 clause BSD style license
// Full license details in LICENSE file distributed with this software

package matcher

import (
	"strings"
	"testing"
)

func TestMatch(t *testing.T) {
	a := &Record{ID: "a", Atts: []Atter{TextAtt{"text"}, NumericAtt{1}, TextAtt{"green"}}}
	b := &Record{ID: "b", Atts: []Atter{TextAtt{"text"}, NumericAtt{1}, TextAtt{"red"}}}
	if a.IsMatch(b) {
		t.Error("A should not match b on all attributes")
	}
	if !a.IsMatch(a) {
		t.Error("A should match itself on all attributes")
	}
	if !a.IsMatch(b, 0) {
		t.Error("A should match b on just first attribute")
	}
	if !b.IsMatch(a, []int{0, 1}...) {
		t.Error("B should match a on first two attributes")
	}
	if a.IsMatch(b, 2) {
		t.Error("A should not match B on third attribute")
	}
	if a.IsMatch(a, 100) {
		t.Error("A should not match itself with attribute out of bounds")
	}
	if !a.IsMatch(a, 0, 1, 2) {
		t.Error("A should match itself with all attributes specified")
	}
}

func TestMatchAtt(t *testing.T) {
	a := &Record{ID: "a", Atts: []Atter{TextAtt{"first"}, NumericAtt{1}, NumericAtt{8}, TextAtt{"yes"}}}
	b := &Record{ID: "b", Atts: []Atter{TextAtt{"second"}, NumericAtt{4.2}, NumericAtt{8}, TextAtt{"yes"}}}
	if !a.isMatchAt(b, NumericAtt{4}, 1) {
		t.Error("A numeric att should match b with an epsilon")
	}
	if !a.isMatchAt(b, NumericAtt{3.2}, 1) {
		t.Error("A numeric att should match b with an epsilon")
	}
	if a.isMatchAt(b, NumericAtt{2}, 1) {
		t.Error("A numeric att should not match b with a small epsilon")
	}
	e := make([]Atter, len(a.Atts))
	e[1] = NumericAtt{2}
	if a.IsMatchWithRanges(b, e, 1) {
		t.Error("A should not match b with small range on one index")
	}
	if a.IsMatchWithRanges(b, e, 1, 2, 3) {
		t.Error("A should not match on multi indices b with small range on one index")
	}
	e[1] = NumericAtt{3.2}
	tests := map[int]bool{
		0: false,
		1: true,
		2: true,
		3: true,
	}
	for index, result := range tests {
		if result != a.isMatchAt(b, e[index], index) {
			t.Errorf("A:%v att:%v match? b:%v with a %v", a, index, b, e[index])
		}
	}
}

func TestMatchWrongSizes(t *testing.T) {
	a := &Record{ID: "a", Atts: []Atter{TextAtt{"first"}, NumericAtt{1}}}
	b := &Record{ID: "b", Atts: []Atter{TextAtt{"second"}, NumericAtt{4.2}, NumericAtt{8}, TextAtt{"yes"}}}
	if a.IsMatchWithRanges(b, make([]Atter, len(a.Atts))) {
		t.Error("A longer record should not match a shorter record")
	}
	if a.IsMatchWithRanges(a, make([]Atter, 0)) {
		t.Error("A record cannot match itself is the []range is too short")
	}
	if !a.IsMatchWithRanges(a, make([]Atter, len(a.Atts))) {
		t.Error("A record should match itself completely")
	}
	if a.isMatchAt(a, TextAtt{}, len(a.Atts)+1) {
		t.Error("A record cannot equal even itself at a position past the record")
	}
	if a.isMatchAt(a, TextAtt{}, -1) {
		t.Error("A record cannot equal even itself at a position before the record")
	}
}

func TestMatchRange(t *testing.T) {
	a := &Record{ID: "a", Atts: []Atter{TextAtt{"first"}, NumericAtt{1}, NumericAtt{8}, TextAtt{"yes"}}}
	b := &Record{ID: "b", Atts: []Atter{TextAtt{"second"}, NumericAtt{4.2}, NumericAtt{8}, TextAtt{"yes"}}}
	if a.IsMatch(b) {
		t.Error("A and b should not match")
	}
	e := make([]Atter, len(a.Atts))
	e[1] = NumericAtt{2}
	if a.IsMatchWithRanges(b, e, 1) {
		t.Error("A should not match b with small range on one index")
	}
	if a.IsMatchWithRanges(b, e, 1, 2, 3) {
		t.Error("A should not match on multi indices b with small range on one index")
	}
	e[1] = NumericAtt{3.2}
	i := []int{1}
	if !a.IsMatchWithRanges(b, e, i...) {
		t.Errorf("A:%v should match b:%v with correct range:%v on %v", a, b, e, i)
	}
	i = []int{1, 2, 3}
	if !a.IsMatchWithRanges(b, e, i...) {
		t.Errorf("A:%v should match b:%v with correct range:%v on %v", a, b, e, i)
	}
	e[1] = NumericAtt{3.0}
	if a.IsMatchWithRanges(b, e, i...) {
		t.Errorf("A:%v should not match b:%v with small range:%v on %v", a, b, e, i)
	}
	i = []int{2, 3}
	if !a.IsMatchWithRanges(b, e, i...) {
		t.Errorf("A:%v should match b:%v with small range:%v on %v", a, b, e, i)
	}
}

func TestMatchesAll(t *testing.T) {
	a := Records{
		Record{ID: "a1", Atts: []Atter{TextAtt{"red"}, NumericAtt{1}}},
		Record{ID: "a2", Atts: []Atter{TextAtt{"red"}, NumericAtt{20}}},
		Record{ID: "a3", Atts: []Atter{TextAtt{"green"}, NumericAtt{30}}},
	}
	b := Records{
		Record{ID: "b1", Atts: []Atter{TextAtt{"green"}, NumericAtt{5}}},
		Record{ID: "b2", Atts: []Atter{TextAtt{"red"}, NumericAtt{25}}},
		Record{ID: "b3", Atts: []Atter{TextAtt{"red"}, NumericAtt{35}}},
		Record{ID: "b4", Atts: []Atter{TextAtt{"green"}, NumericAtt{30}}},
		Record{ID: "b5", Atts: []Atter{TextAtt{"green"}, NumericAtt{31}}},
		Record{ID: "b6", Atts: []Atter{TextAtt{"red"}, NumericAtt{20}}},
		Record{ID: "b7", Atts: []Atter{TextAtt{"red"}, NumericAtt{20}}},
	}

	if m := a[0].MatchesAll(b); len(m) != 0 {
		t.Errorf("%v should not have found any matches in %v", a[0], b)
	}
	m := a[2].MatchesAll(b)
	if len(m) != 1 {
		t.Errorf("%v should have found one match in %v, but found %v", a[2], b, m)
	}
	if 3 != m[0] {
		t.Errorf("%v should have found one at 1, but found it at %v", a[2], m[0])
	}

	m = a[1].MatchesAll(b)
	if len(m) != 2 {
		t.Errorf("%v should have found two matches in %v, but found %v", a[1], b, m)
	}
	if 5 != m[0] && 6 != m[1] {
		t.Errorf("%v should have found one at 5 & 6, but found it at %v", a[1], m)
	}

	e := []Atter{TextAtt{}, NumericAtt{5}}
	m = a[0].MatchesAll(b, e...)
	if 0 != len(m) {
		t.Errorf("%v should not have found any matches in %v using %v", a[0], b, e)
	}
	m = a[1].MatchesAll(b, e...)
	if 3 != len(m) {
		t.Errorf("%v should have found 3 in %v using %v, but found %v", a[0], b, e, m)
	}
	m = a[2].MatchesAll(b, e...)
	if 2 != len(m) {
		t.Errorf("%v should have found 2 in %v using %v, but found %v", a[0], b, e, m)
	}
}

func TestMatchesColumns(t *testing.T) {
	a := Records{
		Record{ID: "a0", Atts: []Atter{TextAtt{"red"}, NumericAtt{1}}},
		Record{ID: "a1", Atts: []Atter{TextAtt{"red"}, NumericAtt{20}}},
		Record{ID: "a2", Atts: []Atter{TextAtt{"green"}, NumericAtt{30}}},
	}
	b := Records{
		Record{ID: "b0", Atts: []Atter{TextAtt{"green"}, NumericAtt{5}}},
		Record{ID: "b1", Atts: []Atter{TextAtt{"red"}, NumericAtt{25}}},
		Record{ID: "b2", Atts: []Atter{TextAtt{"red"}, NumericAtt{35}}},
		Record{ID: "b3", Atts: []Atter{TextAtt{"green"}, NumericAtt{30}}},
		Record{ID: "b4", Atts: []Atter{TextAtt{"green"}, NumericAtt{31}}},
		Record{ID: "b5", Atts: []Atter{TextAtt{"red"}, NumericAtt{20}}},
		Record{ID: "b6", Atts: []Atter{TextAtt{"red"}, NumericAtt{20}}},
	}

	c := []int{10}
	if m := a[0].Matches(b, c); len(m) != 0 {
		t.Errorf("%v should have found 0 matches using column %v in %v, instead found %v", a[0], c, b, m)
	}

	c = []int{0}
	if m := a[0].Matches(b, c); len(m) != 4 {
		t.Errorf("%v should have found 4 matches in %v, instead found %v", a[0], b, m)
	}
	m := a[2].Matches(b, c)
	if len(m) != 3 {
		t.Errorf("%v should have found 3 matches in %v, but found %v", a[2], b, m)
	}
	if 0 != m[0] {
		t.Errorf("%v should have found one at 1, but found it at %v", a[2], m[0])
	}
	if 3 != m[1] {
		t.Errorf("%v should have found one at 1, but found it at %v", a[2], m[1])
	}
	if 4 != m[2] {
		t.Errorf("%v should have found one at 1, but found it at %v", a[2], m[2])
	}
	c = []int{1}
	m = a[2].Matches(b, c)
	if len(m) != 1 {
		t.Errorf("%v should have found 1 matches in %v, but found %v", a[2], b, m)
	}
	if 3 != m[0] {
		t.Errorf("%v should have found one at 1, but found it at %v", a[2], m[0])
	}

	m = a[1].Matches(b, c)
	if len(m) != 2 {
		t.Errorf("%v should have found two matches in %v, but found %v", a[1], b, m)
	}
	if 5 != m[0] && 6 != m[1] {
		t.Errorf("%v should have found one at 5 & 6, but found it at %v", a[1], m)
	}

	e := []Atter{TextAtt{}, NumericAtt{5}}
	m = a[0].Matches(b, c, e...)
	if 1 != len(m) {
		t.Errorf("%v should have found 1 matches in %v using %v, but found %v", a[0], b, e)
	}
	m = a[1].Matches(b, c, e...)
	if 3 != len(m) {
		t.Errorf("%v should have found 3 in %v using %v, but found %v", a[0], b, e, m)
	}
	m = a[2].Matches(b, c, e...)
	if 4 != len(m) {
		t.Errorf("%v should have found 4 in %v using %v, but found %v", a[0], b, e, m)
	}
}

func TestCSVParsing(t *testing.T) {
	csv := `item,type,color,count
a1,m,red,25`
	r, err := NewRecordsFromCSV(strings.NewReader(csv))
	if err != nil {
		t.Error("Expected no error parsing, but got ", err)
	}
	if 1 != len(r) {
		t.Error("Expected 1 record from", r)
	}
	if 3 != len(r[0].Atts) {
		t.Error("Expedted 3 attributes from", r[0].Atts)
	}

	csv = `item,type,color,count
a1,f,red,15
a2,m,red,25`
	r, err = NewRecordsFromCSV(strings.NewReader(csv), 2)
	if err != nil {
		t.Error("Expected no error parsing, but got ", err)
	}
	if 2 != len(r) {
		t.Error("Expected 2 record from", r)
	}
	if !(NumericAtt{15}).Equal(r[0].Atts[2], NumericAtt{}) {
		t.Error("Expected last attribute to be numeric equal to 15, but was not in", r[0].Atts[2])
	}

	r, err = NewRecordsFromCSV(strings.NewReader(csv), 0)
	if err == nil {
		t.Error("Expect error parsing, but got ", err)
	}
	if 0 != len(r) {
		t.Error("Expected 0 record from", r)
	}

	csv = "item,type,color,count"
	r, err = NewRecordsFromCSV(strings.NewReader(csv))
	if err != nil {
		t.Error("Expected no error parsing, but got ", err)
	}
	if 0 != len(r) {
		t.Error("Expected 0 record from", r)
	}
}
<|endoftext|>