<commit_before>package gopisysfs

import (
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"time"
)

type GPIOFlag struct {
	flag bool
	err  error
}

type GPIOMode int

const (
	GPIOInput GPIOMode = iota
	GPIOOutput
	GPIOOutputLow
	GPIOOutputHigh

	// from https://www.kernel.org/doc/Documentation/gpio/sysfs.txt
	direction_in     = "in"
	direction_out    = "out"
	direction_outlow = "low"
	direction_outhi  = "high"

	// the longest time to wait for an operation to complete
	timelimit = time.Second * 2

	low  = "0"
	high = "1"
)

type GPIOPort interface {
	State() string
	IsEnabled() bool
	Enable() error
	Reset() error
	SetMode(GPIOMode) error
	IsOutput() (bool, error)
	SetValue(bool) error
	Value() (bool, error)
	Values() (<-chan bool, error)
}

type gport struct {
	mu        sync.Mutex
	host      *pi
	port      int
	sport     string
	folder    string
	value     string
	direction string
	edge      string
	export    string
	unexport  string
}

func newGPIO(host *pi, port int) *gport {

	sport := fmt.Sprintf("%d", port)
	gpio := host.gpiodir
	folder := filepath.Join(gpio, fmt.Sprintf("gpio%s", sport))
	export := filepath.Join(gpio, "export")
	unexport := filepath.Join(gpio, "unexport")

	return &gport{
		mu:        sync.Mutex{},
		host:      host,
		port:      port,
		sport:     sport,
		folder:    folder,
		value:     filepath.Join(folder, "value"),
		direction: filepath.Join(folder, "direction"),
		edge:      filepath.Join(folder, "edge"),
		export:    export,
		unexport:  unexport,
	}
}

func pause() {
	<-time.After(pollInterval)
}

func (p *gport) String() string {
	return p.folder
}

func (p *gport) IsEnabled() bool {

	defer p.unlock(p.lock())

	return checkFile(p.folder)
}

func (p *gport) Enable() error {

	defer p.unlock(p.lock())

	if checkFile(p.folder) {
		return nil
	}

	info("GPIO Enabling %v\n", p)

	if err := writeFile(p.export, p.sport); err != nil {
		return err
	}

	start := time.Now()

	pause()

	// wait for folder to arrive....
	ch, err := awaitFileCreate(p.folder, timelimit)
	if err != nil {
		return err
	}
	if err := <-ch; err != nil {
		return err
	}
	// delay a bit.
	pause()
	// and for all control files to exist and be readable
	// there's an issue with timeouts perhaps.... but that's OK.
	for _, fname := range []string{p.direction, p.value, p.edge} {
		for {
			remaining := timelimit - time.Since(start)
			info("GPIO Enabling %v checking file %v state (timeout limit %v)\n", p, fname, remaining)
			if checkFile(fname) {
				// check writable.... invalid data will be ignored, but permissions won't
				if err := writeFile(fname, " "); err == nil || !os.IsPermission(err) {
					info("GPIO Enabling %v file %v state OK\n", p, fname)
					break
				} else {
					info("GPIO Enabling %v file %v state %v\n", p, fname, err)
				}
			}
			select {
			case <-time.After(remaining):
				return fmt.Errorf("Timed out enabling GPIO %v - %v not yet writable", p.sport, fname)
			case <-time.After(pollInterval):
				// next cycle
			}
		}

	}

	info("GPIO Enabled %v\n", p)

	return nil
}

func (p *gport) Reset() error {

	defer p.unlock(p.lock())

	if !checkFile(p.folder) {
		// already reset
		return nil
	}
	info("GPIO Resetting  %v\n", p)
	if err := writeFile(p.unexport, p.sport); err != nil {
		return err
	}
	pause()
	ch, err := awaitFileRemove(p.folder, timelimit)
	if err != nil {
		return err
	}

	if err := <-ch; err != nil {
		return err
	}

	// wait for the file to be removed, and then return
	info("GPIO Reset  %v\n", p)
	return nil

}

// GPIOResetAsync will reset the specified port and only return when it is complete
// Configure will
func (p *gport) SetMode(mode GPIOMode) error {

	defer p.unlock(p.lock())

	err := p.checkEnabled()
	if err != nil {
		return err
	}

	switch mode {
	case GPIOInput:
		return p.writeDirection(direction_in)
	case GPIOOutput:
		return p.writeDirection(direction_out)
	case GPIOOutputHigh:
		return p.writeDirection(direction_outhi)
	case GPIOOutputLow:
		return p.writeDirection(direction_outlow)
	}
	return fmt.Errorf("GPIOMode %v does not exist")
}

func (p *gport) IsOutput() (bool, error) {

	defer p.unlock(p.lock())

	err := p.checkEnabled()
	if err != nil {
		return false, err
	}
	d, err := p.readDirection()
	if err != nil {
		return false, err
	}
	return d != "in", nil
}

func (p *gport) State() string {

	defer p.unlock(p.lock())

	base := fmt.Sprintf("GPIO %v: ", p.sport)
	if !checkFile(p.folder) {
		return base + "Reset"
	}

	dir, err := p.readDirection()
	if err != nil {
		return fmt.Sprintf("%v%v", base, err)
	}
	val, err := p.readValue()
	if err != nil {
		return fmt.Sprintf("%v%v", base, err)
	}

	return fmt.Sprintf("%v %v with value %v", base, dir, val)
}

func (p *gport) Value() (bool, error) {

	defer p.unlock(p.lock())

	err := p.checkEnabled()
	if err != nil {
		return false, err
	}

	d, err := p.readValue()
	if err != nil {
		return false, err
	}

	return d == "1", nil
}

func (p *gport) SetValue(value bool) error {

	defer p.unlock(p.lock())

	err := p.checkEnabled()
	if err != nil {
		return err
	}

	info("GPIO Set Value on %v to %v\n", p, value)

	val := low
	if value {
		val = high
	}

	return p.writeValue(val)

}

func (p *gport) Values() (<-chan bool, error) {
	defer p.unlock(p.lock())
	return nil, nil
}

func (p *gport) writeDirection(direction string) error {
	info("GPIO Setting mode on  %v to %v\n", p, direction)

	return writeFile(p.direction, direction)
}

func (p *gport) readDirection() (string, error) {
	return readFile(p.direction)
}

func (p *gport) writeValue(value string) error {
	return writeFile(p.value, value)
}

func (p *gport) readValue() (string, error) {
	return readFile(p.value)
}

func (p *gport) checkEnabled() error {
	if checkFile(p.folder) {
		return nil
	}
	return fmt.Errorf("GPIO %v is not enabled", p.port)
}

func (p *gport) lock() bool {
	p.mu.Lock()
	return true
}

func (p *gport) unlock(bool) {
	p.mu.Unlock()
}
<commit_msg>Remove delays inserted for debugging<commit_after>package gopisysfs

import (
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"time"
)

type GPIOFlag struct {
	flag bool
	err  error
}

type GPIOMode int

const (
	GPIOInput GPIOMode = iota
	GPIOOutput
	GPIOOutputLow
	GPIOOutputHigh

	// from https://www.kernel.org/doc/Documentation/gpio/sysfs.txt
	direction_in     = "in"
	direction_out    = "out"
	direction_outlow = "low"
	direction_outhi  = "high"

	// the longest time to wait for an operation to complete
	timelimit = time.Second * 2

	low  = "0"
	high = "1"
)

type GPIOPort interface {
	State() string
	IsEnabled() bool
	Enable() error
	Reset() error
	SetMode(GPIOMode) error
	IsOutput() (bool, error)
	SetValue(bool) error
	Value() (bool, error)
	Values() (<-chan bool, error)
}

type gport struct {
	mu        sync.Mutex
	host      *pi
	port      int
	sport     string
	folder    string
	value     string
	direction string
	edge      string
	export    string
	unexport  string
}

func newGPIO(host *pi, port int) *gport {

	sport := fmt.Sprintf("%d", port)
	gpio := host.gpiodir
	folder := filepath.Join(gpio, fmt.Sprintf("gpio%s", sport))
	export := filepath.Join(gpio, "export")
	unexport := filepath.Join(gpio, "unexport")

	return &gport{
		mu:        sync.Mutex{},
		host:      host,
		port:      port,
		sport:     sport,
		folder:    folder,
		value:     filepath.Join(folder, "value"),
		direction: filepath.Join(folder, "direction"),
		edge:      filepath.Join(folder, "edge"),
		export:    export,
		unexport:  unexport,
	}
}

func (p *gport) String() string {
	return p.folder
}

func (p *gport) IsEnabled() bool {

	defer p.unlock(p.lock())

	return checkFile(p.folder)
}

func (p *gport) Enable() error {

	defer p.unlock(p.lock())

	if checkFile(p.folder) {
		return nil
	}

	info("GPIO Enabling %v\n", p)

	if err := writeFile(p.export, p.sport); err != nil {
		return err
	}

	start := time.Now()

	// wait for folder to arrive....
	ch, err := awaitFileCreate(p.folder, timelimit)
	if err != nil {
		return err
	}
	if err := <-ch; err != nil {
		return err
	}

	// and for all control files to exist and be writable
	// there's an issue with timeouts perhaps.... but that's OK.
	for _, fname := range []string{p.direction, p.value, p.edge} {
		for {
			remaining := timelimit - time.Since(start)
			info("GPIO Enabling %v checking file %v state (timeout limit %v)\n", p, fname, remaining)
			if checkFile(fname) {
				// exists, but check writable.... invalid data will be ignored(rejected), but permissions won't
				if err := writeFile(fname, " "); err == nil || !os.IsPermission(err) {
					info("GPIO Enabling %v file %v state OK\n", p, fname)
					break
				} else {
					info("GPIO Enabling %v file %v state %v\n", p, fname, err)
				}
			}
			remaining = timelimit - time.Since(start)
			select {
			case <-time.After(remaining):
				return fmt.Errorf("Timed out enabling GPIO %v - %v not yet writable", p.sport, fname)
			case <-time.After(pollInterval):
				// next cycle
			}
		}

	}

	info("GPIO Enabled %v\n", p)

	return nil
}

func (p *gport) Reset() error {

	defer p.unlock(p.lock())

	if !checkFile(p.folder) {
		// already reset
		return nil
	}
	info("GPIO Resetting  %v\n", p)
	if err := writeFile(p.unexport, p.sport); err != nil {
		return err
	}
	ch, err := awaitFileRemove(p.folder, timelimit)
	if err != nil {
		return err
	}

	if err := <-ch; err != nil {
		return err
	}

	// wait for the file to be removed, and then return
	info("GPIO Reset  %v\n", p)
	return nil

}

// GPIOResetAsync will reset the specified port and only return when it is complete
// Configure will
func (p *gport) SetMode(mode GPIOMode) error {

	defer p.unlock(p.lock())

	err := p.checkEnabled()
	if err != nil {
		return err
	}

	direction := ""

	switch mode {
	case GPIOInput:
		direction = direction_in
	case GPIOOutput:
		direction = direction_out
	case GPIOOutputHigh:
		direction = direction_outhi
	case GPIOOutputLow:
		direction = direction_outlow
	default:
		return fmt.Errorf("GPIOMode %v does not exist")
	}

	info("GPIO Setting mode on  %v to %v\n", p, direction)

	if err := p.writeDirection(direction); err != nil {
		return err
	}
	info("GPIO Set mode on  %v to %v\n", p, direction)
	return nil
}

func (p *gport) IsOutput() (bool, error) {

	defer p.unlock(p.lock())

	err := p.checkEnabled()
	if err != nil {
		return false, err
	}
	d, err := p.readDirection()
	if err != nil {
		return false, err
	}
	return d != "in", nil
}

func (p *gport) State() string {

	defer p.unlock(p.lock())

	base := fmt.Sprintf("GPIO %v: ", p.sport)
	if !checkFile(p.folder) {
		return base + "Reset"
	}

	dir, err := p.readDirection()
	if err != nil {
		return fmt.Sprintf("%v%v", base, err)
	}
	val, err := p.readValue()
	if err != nil {
		return fmt.Sprintf("%v%v", base, err)
	}

	return fmt.Sprintf("%v %v with value %v", base, dir, val)
}

func (p *gport) Value() (bool, error) {

	defer p.unlock(p.lock())

	err := p.checkEnabled()
	if err != nil {
		return false, err
	}

	d, err := p.readValue()
	if err != nil {
		return false, err
	}

	return d == "1", nil
}

func (p *gport) SetValue(value bool) error {

	defer p.unlock(p.lock())

	err := p.checkEnabled()
	if err != nil {
		return err
	}

	info("GPIO Set Value on %v to %v\n", p, value)

	val := low
	if value {
		val = high
	}

	return p.writeValue(val)

}

func (p *gport) Values() (<-chan bool, error) {
	defer p.unlock(p.lock())
	return nil, nil
}

func (p *gport) writeDirection(direction string) error {
	return writeFile(p.direction, direction)
}

func (p *gport) readDirection() (string, error) {
	return readFile(p.direction)
}

func (p *gport) writeValue(value string) error {
	return writeFile(p.value, value)
}

func (p *gport) readValue() (string, error) {
	return readFile(p.value)
}

func (p *gport) checkEnabled() error {
	if checkFile(p.folder) {
		return nil
	}
	return fmt.Errorf("GPIO %v is not enabled", p.port)
}

func (p *gport) lock() bool {
	p.mu.Lock()
	return true
}

func (p *gport) unlock(bool) {
	p.mu.Unlock()
}
<|endoftext|>


<commit_before>// Copyright 2014 Arne Roomann-Kurrik
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package twodee

import (
	"github.com/go-gl/mathgl/mgl32"
	"image"
	"image/color"
)

type Grid struct {
	Width     int32
	Height    int32
	BlockSize float32
	points    []bool
}

func NewGrid(w, h, blocksize int32) *Grid {
	return &Grid{
		points:    make([]bool, w*h),
		Width:     w,
		Height:    h,
		BlockSize: float32(blocksize),
	}
}

func (g *Grid) Index(x, y int32) int32 {
	if x < 0 || y < 0 {
		return -1
	}
	return g.Width*(g.Height-y-1) + x
}

func (g *Grid) Get(x, y int32) bool {
	return g.GetIndex(g.Index(x, y))
}

func (g *Grid) GetIndex(index int32) bool {
	if index < 0 || index > g.Width*g.Height {
		return false
	}
	return g.points[index]
}

func (g *Grid) Set(x, y int32, val bool) {
	g.SetIndex(g.Index(x, y), val)
}

func (g *Grid) SetIndex(index int32, val bool) {
	if index < 0 || index > g.Width*g.Height {
		return
	}
	g.points[index] = val
}

func (g *Grid) GetImage(fg, bg color.Color) *image.NRGBA {
	var img = image.NewNRGBA(image.Rect(0, 0, int(g.Width), int(g.Height)))
	for x := 0; x < int(g.Width); x++ {
		for y := 0; y < int(g.Height); y++ {
			if g.Get(int32(x), int32(y)) {
				img.Set(x, y, fg)
			} else {
				img.Set(x, y, bg)
			}
		}
	}
	return img
}

func (g *Grid) squareCollides(bounds mgl32.Vec4, x, y float32) bool {
	// Bounds are {minx, miny, maxx, maxy}
	// Sizex, sizey are the number of coordinate units a grid entry occupies.
	var (
		size  = g.BlockSize
		fudge = float32(0.001) // Prevents item from sticking to wall when we round its coordinates.
		minx  = int32((bounds[0] + x) / size)
		miny  = int32((bounds[1] + y) / size)
		maxx  = int32((bounds[2] + x - fudge) / size)
		maxy  = int32((bounds[3] + y - fudge) / size)
		i     int32
		j     int32
	)
	for i = minx; i <= maxx; i++ {
		for j = miny; j <= maxy; j++ {
			if g.Get(i, j) {
				return true
			}
		}
	}
	return false
}

func (g *Grid) FixMove(bounds mgl32.Vec4, move mgl32.Vec2) (out mgl32.Vec2) {
	out = move
	if g.squareCollides(bounds, out[0], 0.0) {
		out[0] = g.GridAligned(bounds[0], sizex) - bounds[0]
	}
	if g.squareCollides(bounds, out[0], out[1]) {
		out[1] = g.GridAligned(bounds[1], sizey) - bounds[1]
	}
	return
}

func (g *Grid) GridAligned(x float32) float32 {
	return g.BlockSize * float32(int32((x/sizex)+0.5))
}

func (g *Grid) GridPosition(v float32) int32 {
	return int32(v / g.BlockSize)
}

func (g *Grid) InversePosition(i int32) float32 {
	return float32(i)*g.BlockSize + g.BlockSize/2.0
}

func (g *Grid) CanSee(from, to mgl32.Vec2) bool {
	var (
		size  = g.BlockSize
		minx  = int32(from[0] / size)
		maxx  = int32(to[0] / size)
		miny  = int32(from[1] / size)
		maxy  = int32(to[1] / size)
		slope = float32(maxy-miny) / float32(maxx-minx)
		c     = float32(miny) - (slope * float32(minx))
		x     int32
		y     int32
	)
	for x = minx; x <= maxx; x++ {
		y = int32(slope*float32(x) + c)
		if g.Get(x, y) {
			// Something blocks the way
			return false
		}
	}
	for y = miny; y <= maxy; y++ {
		x = int32((float32(y) - c) / slope)
		if g.Get(x, y) {
			// Something blocks the way
			return false
		}
	}
	return true
}
<commit_msg>Make grid items an interface instead of bool<commit_after>// Copyright 2014 Arne Roomann-Kurrik
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package twodee

import (
	"github.com/go-gl/mathgl/mgl32"
	"image"
	"image/color"
)

type GridItem interface {
	Passable() bool
	Opaque() bool
}

type Grid struct {
	Width     int32
	Height    int32
	BlockSize float32
	points    []GridItem
}

func NewGrid(w, h, blocksize int32) *Grid {
	return &Grid{
		points:    make([]GridItem, w*h),
		Width:     w,
		Height:    h,
		BlockSize: float32(blocksize),
	}
}

func (g *Grid) Index(x, y int32) int32 {
	if x < 0 || y < 0 {
		return -1
	}
	return g.Width*(g.Height-y-1) + x
}

func (g *Grid) Get(x, y int32) GridItem {
	return g.GetIndex(g.Index(x, y))
}

func (g *Grid) GetIndex(index int32) GridItem {
	if index < 0 || index > g.Width*g.Height {
		return false
	}
	return g.points[index]
}

func (g *Grid) Set(x, y int32, val GridItem) {
	g.SetIndex(g.Index(x, y), val)
}

func (g *Grid) SetIndex(index int32, val GridItem) {
	if index < 0 || index > g.Width*g.Height {
		return
	}
	g.points[index] = val
}

func (g *Grid) GetImage(fg, bg color.Color) *image.NRGBA {
	var img = image.NewNRGBA(image.Rect(0, 0, int(g.Width), int(g.Height)))
	for x := 0; x < int(g.Width); x++ {
		for y := 0; y < int(g.Height); y++ {
			if g.Get(int32(x), int32(y)).Passable() {
				img.Set(x, y, fg)
			} else {
				img.Set(x, y, bg)
			}
		}
	}
	return img
}

func (g *Grid) squareCollides(bounds mgl32.Vec4, x, y float32) bool {
	// Bounds are {minx, miny, maxx, maxy}
	// Sizex, sizey are the number of coordinate units a grid entry occupies.
	var (
		size  = g.BlockSize
		fudge = float32(0.001) // Prevents item from sticking to wall when we round its coordinates.
		minx  = int32((bounds[0] + x) / size)
		miny  = int32((bounds[1] + y) / size)
		maxx  = int32((bounds[2] + x - fudge) / size)
		maxy  = int32((bounds[3] + y - fudge) / size)
		i     int32
		j     int32
	)
	for i = minx; i <= maxx; i++ {
		for j = miny; j <= maxy; j++ {
			if g.Get(i, j).Passable() {
				return true
			}
		}
	}
	return false
}

func (g *Grid) FixMove(bounds mgl32.Vec4, move mgl32.Vec2) (out mgl32.Vec2) {
	out = move
	if g.squareCollides(bounds, out[0], 0.0) {
		out[0] = g.GridAligned(bounds[0], sizex) - bounds[0]
	}
	if g.squareCollides(bounds, out[0], out[1]) {
		out[1] = g.GridAligned(bounds[1], sizey) - bounds[1]
	}
	return
}

func (g *Grid) GridAligned(x float32) float32 {
	return g.BlockSize * float32(int32((x/sizex)+0.5))
}

func (g *Grid) GridPosition(v float32) int32 {
	return int32(v / g.BlockSize)
}

func (g *Grid) InversePosition(i int32) float32 {
	return float32(i)*g.BlockSize + g.BlockSize/2.0
}

func (g *Grid) CanSee(from, to mgl32.Vec2) bool {
	var (
		size  = g.BlockSize
		minx  = int32(from[0] / size)
		maxx  = int32(to[0] / size)
		miny  = int32(from[1] / size)
		maxy  = int32(to[1] / size)
		slope = float32(maxy-miny) / float32(maxx-minx)
		c     = float32(miny) - (slope * float32(minx))
		x     int32
		y     int32
	)
	for x = minx; x <= maxx; x++ {
		y = int32(slope*float32(x) + c)
		if g.Get(x, y).Opaque() {
			// Something blocks the way
			return false
		}
	}
	for y = miny; y <= maxy; y++ {
		x = int32((float32(y) - c) / slope)
		if g.Get(x, y).Opaque() {
			// Something blocks the way
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package slack

import (
	"errors"
	"fmt"
	"github.com/nlopes/slack"
)

func getUserWithUsername(username string) (user slack.User, err error) {

	users, err := slackClient.GetUsers()
	if err != nil {
		return slack.User{}, err
	}

	for _, user := range users {
		if user.Name == username {
			return user, nil
		}
	}

	return slack.User{}, errors.New(fmt.Sprintf("No user with username %s", username))
}
<commit_msg>Add method to clean up usernames<commit_after>package slack

import (
	"errors"
	"fmt"
	"github.com/nlopes/slack"
	"strings"
)

func getUserWithUsername(username string) (user slack.User, err error) {

	users, err := slackClient.GetUsers()
	if err != nil {
		return slack.User{}, err
	}

	for _, user := range users {
		if user.Name == username {
			return user, nil
		}
	}

	return slack.User{}, errors.New(fmt.Sprintf("No user with username %s", username))
}

func parseUsername(name string) (username string, err error) {

	username = strings.ToLower(name)

	if strings.Contains(username, " ") {
		return "", errors.New(fmt.Sprintf("%s is not a Slack username", name))
	}

	if strings.Contains(name, "@") {
		username = strings.Replace(username, "@", "", 1)
	}

	return
}
<|endoftext|>


<commit_before>package vcs

import (
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"strings"
)

type vcsInfo struct {
	host     string
	pattern  string
	vcs      Type
	addCheck func(m map[string]string, u *url.URL) (Type, error)
	regex    *regexp.Regexp
}

// scpSyntaxRe matches the SCP-like addresses used by Git to access
// repositories by SSH.
var scpSyntaxRe = regexp.MustCompile(`^([a-zA-Z0-9_]+)@([a-zA-Z0-9._-]+):(.*)$`)

var vcsList = []*vcsInfo{
	{
		host:    "github.com",
		vcs:     Git,
		pattern: `^(github\.com[/|:][A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`,
	},
	{
		host:     "bitbucket.org",
		pattern:  `^(bitbucket\.org/(?P<name>[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`,
		addCheck: checkBitbucket,
	},
	{
		host:    "launchpad.net",
		pattern: `^(launchpad\.net/(([A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)?|~[A-Za-z0-9_.\-]+/(\+junk|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`,
		vcs:     Bzr,
	},
	{
		host:    "git.launchpad.net",
		vcs:     Git,
		pattern: `^(git\.launchpad\.net/(([A-Za-z0-9_.\-]+)|~[A-Za-z0-9_.\-]+/(\+git|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))$`,
	},
	{
		host:    "hub.jazz.net",
		vcs:     Git,
		pattern: `^(hub\.jazz\.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`,
	},
	{
		host:    "go.googlesource.com",
		vcs:     Git,
		pattern: `^(go\.googlesource\.com/[A-Za-z0-9_.\-]+/?)$`,
	},
	{
		host:    "git.openstack.org",
		vcs:     Git,
		pattern: `^(git\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)$`,
	},
	// If none of the previous detect the type they will fall to this looking for the type in a generic sense
	// by the extension to the path.
	{
		addCheck: checkURL,
		pattern:  `\.(?P<type>git|hg|svn|bzr)$`,
	},
}

func init() {
	// Precompile the regular expressions used to check VCS locations.
	for _, v := range vcsList {
		v.regex = regexp.MustCompile(v.pattern)
	}
}

// This function is really a hack around Go redirects rather than around
// something VCS related. Should this be moved to the glide project or a
// helper function?
func detectVcsFromRemote(vcsURL string) (Type, string, error) {
	t, e := detectVcsFromURL(vcsURL)
	if e == nil {
		return t, vcsURL, nil
	} else if e != ErrCannotDetectVCS {
		return NoVCS, "", e
	}

	// Pages like https://golang.org/x/net provide an html document with
	// meta tags containing a location to work with. The go tool uses
	// a meta tag with the name go-import which is what we use here.
	// godoc.org also has one call go-source that we do not need to use.
	// The value of go-import is in the form "prefix vcs repo". The prefix
	// should match the vcsURL and the repo is a location that can be
	// checked out. Note, to get the html document you you need to add
	// ?go-get=1 to the url.
	u, err := url.Parse(vcsURL)
	if err != nil {
		return NoVCS, "", err
	}
	if u.RawQuery == "" {
		u.RawQuery = "go-get=1"
	} else {
		u.RawQuery = u.RawQuery + "+go-get=1"
	}
	checkURL := u.String()
	resp, err := http.Get(checkURL)
	if err != nil {
		return NoVCS, "", ErrCannotDetectVCS
	}
	defer resp.Body.Close()
	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		if resp.StatusCode == 404 {
			return NoVCS, "", NewRemoteError(fmt.Sprintf("%s Not Found", vcsURL), nil, "")
		} else if resp.StatusCode == 401 || resp.StatusCode == 403 {
			return NoVCS, "", NewRemoteError(fmt.Sprintf("%s Access Denied", vcsURL), nil, "")
		}
		return NoVCS, "", ErrCannotDetectVCS
	}

	t, nu, err := parseImportFromBody(u, resp.Body)
	if err != nil {
		// TODO(mattfarina): Log the parsing error
		return NoVCS, "", ErrCannotDetectVCS
	} else if t == "" || nu == "" {
		return NoVCS, "", ErrCannotDetectVCS
	}

	return t, nu, nil
}

// From a remote vcs url attempt to detect the VCS.
func detectVcsFromURL(vcsURL string) (Type, error) {

	var u *url.URL
	var err error

	if m := scpSyntaxRe.FindStringSubmatch(vcsURL); m != nil {
		// Match SCP-like syntax and convert it to a URL.
		// Eg, "git@github.com:user/repo" becomes
		// "ssh://git@github.com/user/repo".
		u = &url.URL{
			Scheme: "ssh",
			User:   url.User(m[1]),
			Host:   m[2],
			Path:   "/" + m[3],
		}
	} else {
		u, err = url.Parse(vcsURL)
		if err != nil {
			return "", err
		}
	}

	// Detect file schemes
	if u.Scheme == "file" {
		return DetectVcsFromFS(u.Path)
	}

	if u.Host == "" {
		return "", ErrCannotDetectVCS
	}

	// Try to detect from the scheme
	switch u.Scheme {
	case "git+ssh":
		return Git, nil
	case "git":
		return Git, nil
	case "bzr+ssh":
		return Bzr, nil
	case "svn+ssh":
		return Svn, nil
	}

	// Try to detect from known hosts, such as Github
	for _, v := range vcsList {
		if v.host != "" && v.host != u.Host {
			continue
		}

		// Make sure the pattern matches for an actual repo location. For example,
		// we should fail if the VCS listed is github.com/masterminds as that's
		// not actually a repo.
		uCheck := u.Host + u.Path
		m := v.regex.FindStringSubmatch(uCheck)
		if m == nil {
			if v.host != "" {
				return "", ErrCannotDetectVCS
			}

			continue
		}

		// If we are here the host matches. If the host has a singular
		// VCS type, such as Github, we can return the type right away.
		if v.vcs != "" {
			return v.vcs, nil
		}

		// Run additional checks to determine try and determine the repo
		// for the matched service.
		info := make(map[string]string)
		for i, name := range v.regex.SubexpNames() {
			if name != "" {
				info[name] = m[i]
			}
		}
		t, err := v.addCheck(info, u)
		if err != nil {
			switch err.(type) {
			case *RemoteError:
				return "", err
			}
			return "", ErrCannotDetectVCS
		}

		return t, nil
	}

	// Attempt to ascertain from the username passed in.
	if u.User != nil {
		un := u.User.Username()
		if un == "git" {
			return Git, nil
		} else if un == "hg" {
			return Hg, nil
		}
	}

	// Unable to determine the vcs from the url.
	return "", ErrCannotDetectVCS
}

// Figure out the type for Bitbucket by the passed in information
// or via the public API.
func checkBitbucket(i map[string]string, ul *url.URL) (Type, error) {

	// Fast path for ssh urls where we may not even be able to
	// anonymously get details from the API.
	if ul.User != nil {
		un := ul.User.Username()
		if un == "git" {
			return Git, nil
		} else if un == "hg" {
			return Hg, nil
		}
	}

	// The part of the response we care about.
	var response struct {
		SCM Type `json:"scm"`
	}

	u := expand(i, "https://api.bitbucket.org/1.0/repositories/{name}")
	data, err := get(u)
	if err != nil {
		return "", err
	}

	if err := json.Unmarshal(data, &response); err != nil {
		return "", fmt.Errorf("Decoding error %s: %v", u, err)
	}

	return response.SCM, nil

}

// Expect a type key on i with the exact type detected from the regex.
func checkURL(i map[string]string, u *url.URL) (Type, error) {
	return Type(i["type"]), nil
}

func get(url string) ([]byte, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		if resp.StatusCode == 404 {
			return nil, NewRemoteError("Not Found", err, resp.Status)
		} else if resp.StatusCode == 401 || resp.StatusCode == 403 {
			return nil, NewRemoteError("Access Denied", err, resp.Status)
		}
		return nil, fmt.Errorf("%s: %s", url, resp.Status)
	}
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("%s: %v", url, err)
	}
	return b, nil
}

func expand(match map[string]string, s string) string {
	for k, v := range match {
		s = strings.Replace(s, "{"+k+"}", v, -1)
	}
	return s
}

func parseImportFromBody(ur *url.URL, r io.ReadCloser) (tp Type, u string, err error) {
	d := xml.NewDecoder(r)
	d.CharsetReader = charsetReader
	d.Strict = false
	var t xml.Token
	for {
		t, err = d.Token()
		if err != nil {
			if err == io.EOF {
				// When the end is reached it could not detect a VCS if it
				// got here.
				err = ErrCannotDetectVCS
			}
			return
		}
		if e, ok := t.(xml.StartElement); ok && strings.EqualFold(e.Name.Local, "body") {
			return
		}
		if e, ok := t.(xml.EndElement); ok && strings.EqualFold(e.Name.Local, "head") {
			return
		}
		e, ok := t.(xml.StartElement)
		if !ok || !strings.EqualFold(e.Name.Local, "meta") {
			continue
		}
		if attrValue(e.Attr, "name") != "go-import" {
			continue
		}
		if f := strings.Fields(attrValue(e.Attr, "content")); len(f) == 3 {
			// If the prefix supplied by the remote system isn't a prefix to the
			// url we're fetching continue to look for other imports.
			// This will work for exact matches and prefixes. For example,
			// golang.org/x/net as a prefix will match for golang.org/x/net and
			// golang.org/x/net/context.
			vcsURL := ur.Host + ur.Path
			if !strings.HasPrefix(vcsURL, f[0]) {
				continue
			} else {
				switch Type(f[1]) {
				case Git:
					tp = Git
				case Svn:
					tp = Svn
				case Bzr:
					tp = Bzr
				case Hg:
					tp = Hg
				}

				u = f[2]
				return
			}
		}
	}
}

func charsetReader(charset string, input io.Reader) (io.Reader, error) {
	switch strings.ToLower(charset) {
	case "ascii":
		return input, nil
	default:
		return nil, fmt.Errorf("can't decode XML document using charset %q", charset)
	}
}

func attrValue(attrs []xml.Attr, name string) string {
	for _, a := range attrs {
		if strings.EqualFold(a.Name.Local, name) {
			return a.Value
		}
	}
	return ""
}
<commit_msg>Allow non-200 remote lookup responses<commit_after>package vcs

import (
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"strings"
)

type vcsInfo struct {
	host     string
	pattern  string
	vcs      Type
	addCheck func(m map[string]string, u *url.URL) (Type, error)
	regex    *regexp.Regexp
}

// scpSyntaxRe matches the SCP-like addresses used by Git to access
// repositories by SSH.
var scpSyntaxRe = regexp.MustCompile(`^([a-zA-Z0-9_]+)@([a-zA-Z0-9._-]+):(.*)$`)

var vcsList = []*vcsInfo{
	{
		host:    "github.com",
		vcs:     Git,
		pattern: `^(github\.com[/|:][A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`,
	},
	{
		host:     "bitbucket.org",
		pattern:  `^(bitbucket\.org/(?P<name>[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`,
		addCheck: checkBitbucket,
	},
	{
		host:    "launchpad.net",
		pattern: `^(launchpad\.net/(([A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)?|~[A-Za-z0-9_.\-]+/(\+junk|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`,
		vcs:     Bzr,
	},
	{
		host:    "git.launchpad.net",
		vcs:     Git,
		pattern: `^(git\.launchpad\.net/(([A-Za-z0-9_.\-]+)|~[A-Za-z0-9_.\-]+/(\+git|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))$`,
	},
	{
		host:    "hub.jazz.net",
		vcs:     Git,
		pattern: `^(hub\.jazz\.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`,
	},
	{
		host:    "go.googlesource.com",
		vcs:     Git,
		pattern: `^(go\.googlesource\.com/[A-Za-z0-9_.\-]+/?)$`,
	},
	{
		host:    "git.openstack.org",
		vcs:     Git,
		pattern: `^(git\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)$`,
	},
	// If none of the previous detect the type they will fall to this looking for the type in a generic sense
	// by the extension to the path.
	{
		addCheck: checkURL,
		pattern:  `\.(?P<type>git|hg|svn|bzr)$`,
	},
}

func init() {
	// Precompile the regular expressions used to check VCS locations.
	for _, v := range vcsList {
		v.regex = regexp.MustCompile(v.pattern)
	}
}

// This function is really a hack around Go redirects rather than around
// something VCS related. Should this be moved to the glide project or a
// helper function?
func detectVcsFromRemote(vcsURL string) (Type, string, error) {
	t, e := detectVcsFromURL(vcsURL)
	if e == nil {
		return t, vcsURL, nil
	} else if e != ErrCannotDetectVCS {
		return NoVCS, "", e
	}

	// Pages like https://golang.org/x/net provide an html document with
	// meta tags containing a location to work with. The go tool uses
	// a meta tag with the name go-import which is what we use here.
	// godoc.org also has one call go-source that we do not need to use.
	// The value of go-import is in the form "prefix vcs repo". The prefix
	// should match the vcsURL and the repo is a location that can be
	// checked out. Note, to get the html document you you need to add
	// ?go-get=1 to the url.
	u, err := url.Parse(vcsURL)
	if err != nil {
		return NoVCS, "", err
	}
	if u.RawQuery == "" {
		u.RawQuery = "go-get=1"
	} else {
		u.RawQuery = u.RawQuery + "+go-get=1"
	}
	checkURL := u.String()
	resp, err := http.Get(checkURL)
	if err != nil {
		return NoVCS, "", ErrCannotDetectVCS
	}
	defer resp.Body.Close()

	t, nu, err := parseImportFromBody(u, resp.Body)
	if err != nil {
		// TODO(mattfarina): Log the parsing error
		return NoVCS, "", ErrCannotDetectVCS
	} else if t == "" || nu == "" {
		return NoVCS, "", ErrCannotDetectVCS
	}

	return t, nu, nil
}

// From a remote vcs url attempt to detect the VCS.
func detectVcsFromURL(vcsURL string) (Type, error) {

	var u *url.URL
	var err error

	if m := scpSyntaxRe.FindStringSubmatch(vcsURL); m != nil {
		// Match SCP-like syntax and convert it to a URL.
		// Eg, "git@github.com:user/repo" becomes
		// "ssh://git@github.com/user/repo".
		u = &url.URL{
			Scheme: "ssh",
			User:   url.User(m[1]),
			Host:   m[2],
			Path:   "/" + m[3],
		}
	} else {
		u, err = url.Parse(vcsURL)
		if err != nil {
			return "", err
		}
	}

	// Detect file schemes
	if u.Scheme == "file" {
		return DetectVcsFromFS(u.Path)
	}

	if u.Host == "" {
		return "", ErrCannotDetectVCS
	}

	// Try to detect from the scheme
	switch u.Scheme {
	case "git+ssh":
		return Git, nil
	case "git":
		return Git, nil
	case "bzr+ssh":
		return Bzr, nil
	case "svn+ssh":
		return Svn, nil
	}

	// Try to detect from known hosts, such as Github
	for _, v := range vcsList {
		if v.host != "" && v.host != u.Host {
			continue
		}

		// Make sure the pattern matches for an actual repo location. For example,
		// we should fail if the VCS listed is github.com/masterminds as that's
		// not actually a repo.
		uCheck := u.Host + u.Path
		m := v.regex.FindStringSubmatch(uCheck)
		if m == nil {
			if v.host != "" {
				return "", ErrCannotDetectVCS
			}

			continue
		}

		// If we are here the host matches. If the host has a singular
		// VCS type, such as Github, we can return the type right away.
		if v.vcs != "" {
			return v.vcs, nil
		}

		// Run additional checks to determine try and determine the repo
		// for the matched service.
		info := make(map[string]string)
		for i, name := range v.regex.SubexpNames() {
			if name != "" {
				info[name] = m[i]
			}
		}
		t, err := v.addCheck(info, u)
		if err != nil {
			switch err.(type) {
			case *RemoteError:
				return "", err
			}
			return "", ErrCannotDetectVCS
		}

		return t, nil
	}

	// Attempt to ascertain from the username passed in.
	if u.User != nil {
		un := u.User.Username()
		if un == "git" {
			return Git, nil
		} else if un == "hg" {
			return Hg, nil
		}
	}

	// Unable to determine the vcs from the url.
	return "", ErrCannotDetectVCS
}

// Figure out the type for Bitbucket by the passed in information
// or via the public API.
func checkBitbucket(i map[string]string, ul *url.URL) (Type, error) {

	// Fast path for ssh urls where we may not even be able to
	// anonymously get details from the API.
	if ul.User != nil {
		un := ul.User.Username()
		if un == "git" {
			return Git, nil
		} else if un == "hg" {
			return Hg, nil
		}
	}

	// The part of the response we care about.
	var response struct {
		SCM Type `json:"scm"`
	}

	u := expand(i, "https://api.bitbucket.org/1.0/repositories/{name}")
	data, err := get(u)
	if err != nil {
		return "", err
	}

	if err := json.Unmarshal(data, &response); err != nil {
		return "", fmt.Errorf("Decoding error %s: %v", u, err)
	}

	return response.SCM, nil

}

// Expect a type key on i with the exact type detected from the regex.
func checkURL(i map[string]string, u *url.URL) (Type, error) {
	return Type(i["type"]), nil
}

func get(url string) ([]byte, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		if resp.StatusCode == 404 {
			return nil, NewRemoteError("Not Found", err, resp.Status)
		} else if resp.StatusCode == 401 || resp.StatusCode == 403 {
			return nil, NewRemoteError("Access Denied", err, resp.Status)
		}
		return nil, fmt.Errorf("%s: %s", url, resp.Status)
	}
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("%s: %v", url, err)
	}
	return b, nil
}

func expand(match map[string]string, s string) string {
	for k, v := range match {
		s = strings.Replace(s, "{"+k+"}", v, -1)
	}
	return s
}

func parseImportFromBody(ur *url.URL, r io.ReadCloser) (tp Type, u string, err error) {
	d := xml.NewDecoder(r)
	d.CharsetReader = charsetReader
	d.Strict = false
	var t xml.Token
	for {
		t, err = d.Token()
		if err != nil {
			if err == io.EOF {
				// When the end is reached it could not detect a VCS if it
				// got here.
				err = ErrCannotDetectVCS
			}
			return
		}
		if e, ok := t.(xml.StartElement); ok && strings.EqualFold(e.Name.Local, "body") {
			return
		}
		if e, ok := t.(xml.EndElement); ok && strings.EqualFold(e.Name.Local, "head") {
			return
		}
		e, ok := t.(xml.StartElement)
		if !ok || !strings.EqualFold(e.Name.Local, "meta") {
			continue
		}
		if attrValue(e.Attr, "name") != "go-import" {
			continue
		}
		if f := strings.Fields(attrValue(e.Attr, "content")); len(f) == 3 {
			// If the prefix supplied by the remote system isn't a prefix to the
			// url we're fetching continue to look for other imports.
			// This will work for exact matches and prefixes. For example,
			// golang.org/x/net as a prefix will match for golang.org/x/net and
			// golang.org/x/net/context.
			vcsURL := ur.Host + ur.Path
			if !strings.HasPrefix(vcsURL, f[0]) {
				continue
			} else {
				switch Type(f[1]) {
				case Git:
					tp = Git
				case Svn:
					tp = Svn
				case Bzr:
					tp = Bzr
				case Hg:
					tp = Hg
				}

				u = f[2]
				return
			}
		}
	}
}

func charsetReader(charset string, input io.Reader) (io.Reader, error) {
	switch strings.ToLower(charset) {
	case "ascii":
		return input, nil
	default:
		return nil, fmt.Errorf("can't decode XML document using charset %q", charset)
	}
}

func attrValue(attrs []xml.Attr, name string) string {
	for _, a := range attrs {
		if strings.EqualFold(a.Name.Local, name) {
			return a.Value
		}
	}
	return ""
}
<|endoftext|>


<commit_before>// Package oidc implements logging in through OpenID Connect providers.
package oidc

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/coreos/go-oidc"
	"golang.org/x/oauth2"

	"github.com/dexidp/dex/connector"
	"github.com/dexidp/dex/pkg/log"
)

// Config holds configuration options for OpenID Connect logins.
type Config struct {
	Issuer       string `json:"issuer"`
	ClientID     string `json:"clientID"`
	ClientSecret string `json:"clientSecret"`
	RedirectURI  string `json:"redirectURI"`

	// Causes client_secret to be passed as POST parameters instead of basic
	// auth. This is specifically "NOT RECOMMENDED" by the OAuth2 RFC, but some
	// providers require it.
	//
	// https://tools.ietf.org/html/rfc6749#section-2.3.1
	BasicAuthUnsupported *bool `json:"basicAuthUnsupported"`

	Scopes []string `json:"scopes"` // defaults to "profile" and "email"

	// Optional list of whitelisted domains when using Google
	// If this field is nonempty, only users from a listed domain will be allowed to log in
	HostedDomains []string `json:"hostedDomains"`

	// Override the value of email_verifed to true in the returned claims
	InsecureSkipEmailVerified bool `json:"insecureSkipEmailVerified"`

	// InsecureEnableGroups enables groups claims. This is disabled by default until https://github.com/dexidp/dex/issues/1065 is resolved
	InsecureEnableGroups bool `json:"insecureEnableGroups"`

	// GetUserInfo uses the userinfo endpoint to get additional claims for
	// the token. This is especially useful where upstreams return "thin"
	// id tokens
	GetUserInfo bool `json:"getUserInfo"`

	UserIDKey string `json:"userIDKey"`

	UserNameKey string `json:"userNameKey"`

	// PromptType will be used fot the prompt parameter (when offline_access, by default prompt=consent)
	PromptType string `json:"promptType"`

	ClaimMapping struct {
		// Configurable key which contains the preferred username claims
		PreferredUsernameKey string `json:"preferred_username"` // defaults to "preferred_username"

		// Configurable key which contains the email claims
		EmailKey string `json:"email"` // defaults to "email"

		// Configurable key which contains the groups claims
		GroupsKey string `json:"groups"` // defaults to "groups"
	} `json:"claimMapping"`
}

// Domains that don't support basic auth. golang.org/x/oauth2 has an internal
// list, but it only matches specific URLs, not top level domains.
var brokenAuthHeaderDomains = []string{
	// See: https://github.com/dexidp/dex/issues/859
	"okta.com",
	"oktapreview.com",
}

// connectorData stores information for sessions authenticated by this connector
type connectorData struct {
	RefreshToken []byte
}

// Detect auth header provider issues for known providers. This lets users
// avoid having to explicitly set "basicAuthUnsupported" in their config.
//
// Setting the config field always overrides values returned by this function.
func knownBrokenAuthHeaderProvider(issuerURL string) bool {
	if u, err := url.Parse(issuerURL); err == nil {
		for _, host := range brokenAuthHeaderDomains {
			if u.Host == host || strings.HasSuffix(u.Host, "."+host) {
				return true
			}
		}
	}
	return false
}

// Open returns a connector which can be used to login users through an upstream
// OpenID Connect provider.
func (c *Config) Open(id string, logger log.Logger) (conn connector.Connector, err error) {
	ctx, cancel := context.WithCancel(context.Background())

	provider, err := oidc.NewProvider(ctx, c.Issuer)
	if err != nil {
		cancel()
		return nil, fmt.Errorf("failed to get provider: %v", err)
	}

	endpoint := provider.Endpoint()

	if c.BasicAuthUnsupported != nil {
		// Setting "basicAuthUnsupported" always overrides our detection.
		if *c.BasicAuthUnsupported {
			endpoint.AuthStyle = oauth2.AuthStyleInParams
		}
	} else if knownBrokenAuthHeaderProvider(c.Issuer) {
		endpoint.AuthStyle = oauth2.AuthStyleInParams
	}

	scopes := []string{oidc.ScopeOpenID}
	if len(c.Scopes) > 0 {
		scopes = append(scopes, c.Scopes...)
	} else {
		scopes = append(scopes, "profile", "email")
	}

	// PromptType should be "consent" by default, if not set
	if c.PromptType == "" {
		c.PromptType = "consent"
	}

	clientID := c.ClientID
	return &oidcConnector{
		provider:    provider,
		redirectURI: c.RedirectURI,
		oauth2Config: &oauth2.Config{
			ClientID:     clientID,
			ClientSecret: c.ClientSecret,
			Endpoint:     endpoint,
			Scopes:       scopes,
			RedirectURL:  c.RedirectURI,
		},
		verifier: provider.Verifier(
			&oidc.Config{ClientID: clientID},
		),
		logger:                    logger,
		cancel:                    cancel,
		hostedDomains:             c.HostedDomains,
		insecureSkipEmailVerified: c.InsecureSkipEmailVerified,
		insecureEnableGroups:      c.InsecureEnableGroups,
		getUserInfo:               c.GetUserInfo,
		promptType:                c.PromptType,
		userIDKey:                 c.UserIDKey,
		userNameKey:               c.UserNameKey,
		preferredUsernameKey:      c.ClaimMapping.PreferredUsernameKey,
		emailKey:                  c.ClaimMapping.EmailKey,
		groupsKey:                 c.ClaimMapping.GroupsKey,
	}, nil
}

var (
	_ connector.CallbackConnector = (*oidcConnector)(nil)
	_ connector.RefreshConnector  = (*oidcConnector)(nil)
)

type oidcConnector struct {
	provider                  *oidc.Provider
	redirectURI               string
	oauth2Config              *oauth2.Config
	verifier                  *oidc.IDTokenVerifier
	cancel                    context.CancelFunc
	logger                    log.Logger
	hostedDomains             []string
	insecureSkipEmailVerified bool
	insecureEnableGroups      bool
	getUserInfo               bool
	promptType                string
	userIDKey                 string
	userNameKey               string
	preferredUsernameKey      string
	emailKey                  string
	groupsKey                 string
}

func (c *oidcConnector) Close() error {
	c.cancel()
	return nil
}

func (c *oidcConnector) LoginURL(s connector.Scopes, callbackURL, state string) (string, error) {
	if c.redirectURI != callbackURL {
		return "", fmt.Errorf("expected callback URL %q did not match the URL in the config %q", callbackURL, c.redirectURI)
	}

	var opts []oauth2.AuthCodeOption
	if len(c.hostedDomains) > 0 {
		preferredDomain := c.hostedDomains[0]
		if len(c.hostedDomains) > 1 {
			preferredDomain = "*"
		}
		opts = append(opts, oauth2.SetAuthURLParam("hd", preferredDomain))
	}

	if s.OfflineAccess {
		opts = append(opts, oauth2.AccessTypeOffline, oauth2.SetAuthURLParam("prompt", c.promptType))
	}
	return c.oauth2Config.AuthCodeURL(state, opts...), nil
}

type oauth2Error struct {
	error            string
	errorDescription string
}

func (e *oauth2Error) Error() string {
	if e.errorDescription == "" {
		return e.error
	}
	return e.error + ": " + e.errorDescription
}

func (c *oidcConnector) HandleCallback(s connector.Scopes, r *http.Request) (identity connector.Identity, err error) {
	q := r.URL.Query()
	if errType := q.Get("error"); errType != "" {
		return identity, &oauth2Error{errType, q.Get("error_description")}
	}
	token, err := c.oauth2Config.Exchange(r.Context(), q.Get("code"))
	if err != nil {
		return identity, fmt.Errorf("oidc: failed to get token: %v", err)
	}

	return c.createIdentity(r.Context(), identity, token)
}

// Refresh is used to refresh a session with the refresh token provided by the IdP
func (c *oidcConnector) Refresh(ctx context.Context, s connector.Scopes, identity connector.Identity) (connector.Identity, error) {
	cd := connectorData{}
	err := json.Unmarshal(identity.ConnectorData, &cd)
	if err != nil {
		return identity, fmt.Errorf("oidc: failed to unmarshal connector data: %v", err)
	}

	t := &oauth2.Token{
		RefreshToken: string(cd.RefreshToken),
		Expiry:       time.Now().Add(-time.Hour),
	}
	token, err := c.oauth2Config.TokenSource(ctx, t).Token()
	if err != nil {
		return identity, fmt.Errorf("oidc: failed to get refresh token: %v", err)
	}

	return c.createIdentity(ctx, identity, token)
}

func (c *oidcConnector) createIdentity(ctx context.Context, identity connector.Identity, token *oauth2.Token) (connector.Identity, error) {
	rawIDToken, ok := token.Extra("id_token").(string)
	if !ok {
		return identity, errors.New("oidc: no id_token in token response")
	}
	idToken, err := c.verifier.Verify(ctx, rawIDToken)
	if err != nil {
		return identity, fmt.Errorf("oidc: failed to verify ID Token: %v", err)
	}

	var claims map[string]interface{}
	if err := idToken.Claims(&claims); err != nil {
		return identity, fmt.Errorf("oidc: failed to decode claims: %v", err)
	}

	// We immediately want to run getUserInfo if configured before we validate the claims
	if c.getUserInfo {
		userInfo, err := c.provider.UserInfo(ctx, oauth2.StaticTokenSource(token))
		if err != nil {
			return identity, fmt.Errorf("oidc: error loading userinfo: %v", err)
		}
		if err := userInfo.Claims(&claims); err != nil {
			return identity, fmt.Errorf("oidc: failed to decode userinfo claims: %v", err)
		}
	}

	userNameKey := "name"
	if c.userNameKey != "" {
		userNameKey = c.userNameKey
	}
	name, found := claims[userNameKey].(string)
	if !found {
		return identity, fmt.Errorf("missing \"%s\" claim", userNameKey)
	}

	preferredUsername, found := claims["preferred_username"].(string)
	if !found {
		preferredUsername, _ = claims[c.preferredUsernameKey].(string)
	}

	hasEmailScope := false
	for _, s := range c.oauth2Config.Scopes {
		if s == "email" {
			hasEmailScope = true
			break
		}
	}

	var email string
	emailKey := "email"
	email, found = claims[emailKey].(string)
	if !found && c.emailKey != "" {
		emailKey = c.emailKey
		email, found = claims[emailKey].(string)
	}

	if !found && hasEmailScope {
		return identity, fmt.Errorf("missing email claim, not found \"%s\" key", emailKey)
	}

	emailVerified, found := claims["email_verified"].(bool)
	if !found {
		if c.insecureSkipEmailVerified {
			emailVerified = true
		} else if hasEmailScope {
			return identity, errors.New("missing \"email_verified\" claim")
		}
	}

	var groups []string
	if c.insecureEnableGroups {
		groupsKey := "groups"
		vs, found := claims[groupsKey].([]interface{})
		if !found {
			groupsKey = c.groupsKey
			vs, found = claims[groupsKey].([]interface{})
		}

		if found {
			for _, v := range vs {
				if s, ok := v.(string); ok {
					groups = append(groups, s)
				} else {
					return identity, fmt.Errorf("malformed \"%v\" claim", groupsKey)
				}
			}
		}
	}

	hostedDomain, _ := claims["hd"].(string)
	if len(c.hostedDomains) > 0 {
		found := false
		for _, domain := range c.hostedDomains {
			if hostedDomain == domain {
				found = true
				break
			}
		}

		if !found {
			return identity, fmt.Errorf("oidc: unexpected hd claim %v", hostedDomain)
		}
	}

	cd := connectorData{
		RefreshToken: []byte(token.RefreshToken),
	}

	connData, err := json.Marshal(&cd)
	if err != nil {
		return identity, fmt.Errorf("oidc: failed to encode connector data: %v", err)
	}

	identity = connector.Identity{
		UserID:            idToken.Subject,
		Username:          name,
		PreferredUsername: preferredUsername,
		Email:             email,
		EmailVerified:     emailVerified,
		Groups:            groups,
		ConnectorData:     connData,
	}

	if c.userIDKey != "" {
		userID, found := claims[c.userIDKey].(string)
		if !found {
			return identity, fmt.Errorf("oidc: not found %v claim", c.userIDKey)
		}
		identity.UserID = userID
	}

	return identity, nil
}
<commit_msg>spelling: verified<commit_after>// Package oidc implements logging in through OpenID Connect providers.
package oidc

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/coreos/go-oidc"
	"golang.org/x/oauth2"

	"github.com/dexidp/dex/connector"
	"github.com/dexidp/dex/pkg/log"
)

// Config holds configuration options for OpenID Connect logins.
type Config struct {
	Issuer       string `json:"issuer"`
	ClientID     string `json:"clientID"`
	ClientSecret string `json:"clientSecret"`
	RedirectURI  string `json:"redirectURI"`

	// Causes client_secret to be passed as POST parameters instead of basic
	// auth. This is specifically "NOT RECOMMENDED" by the OAuth2 RFC, but some
	// providers require it.
	//
	// https://tools.ietf.org/html/rfc6749#section-2.3.1
	BasicAuthUnsupported *bool `json:"basicAuthUnsupported"`

	Scopes []string `json:"scopes"` // defaults to "profile" and "email"

	// Optional list of whitelisted domains when using Google
	// If this field is nonempty, only users from a listed domain will be allowed to log in
	HostedDomains []string `json:"hostedDomains"`

	// Override the value of email_verified to true in the returned claims
	InsecureSkipEmailVerified bool `json:"insecureSkipEmailVerified"`

	// InsecureEnableGroups enables groups claims. This is disabled by default until https://github.com/dexidp/dex/issues/1065 is resolved
	InsecureEnableGroups bool `json:"insecureEnableGroups"`

	// GetUserInfo uses the userinfo endpoint to get additional claims for
	// the token. This is especially useful where upstreams return "thin"
	// id tokens
	GetUserInfo bool `json:"getUserInfo"`

	UserIDKey string `json:"userIDKey"`

	UserNameKey string `json:"userNameKey"`

	// PromptType will be used fot the prompt parameter (when offline_access, by default prompt=consent)
	PromptType string `json:"promptType"`

	ClaimMapping struct {
		// Configurable key which contains the preferred username claims
		PreferredUsernameKey string `json:"preferred_username"` // defaults to "preferred_username"

		// Configurable key which contains the email claims
		EmailKey string `json:"email"` // defaults to "email"

		// Configurable key which contains the groups claims
		GroupsKey string `json:"groups"` // defaults to "groups"
	} `json:"claimMapping"`
}

// Domains that don't support basic auth. golang.org/x/oauth2 has an internal
// list, but it only matches specific URLs, not top level domains.
var brokenAuthHeaderDomains = []string{
	// See: https://github.com/dexidp/dex/issues/859
	"okta.com",
	"oktapreview.com",
}

// connectorData stores information for sessions authenticated by this connector
type connectorData struct {
	RefreshToken []byte
}

// Detect auth header provider issues for known providers. This lets users
// avoid having to explicitly set "basicAuthUnsupported" in their config.
//
// Setting the config field always overrides values returned by this function.
func knownBrokenAuthHeaderProvider(issuerURL string) bool {
	if u, err := url.Parse(issuerURL); err == nil {
		for _, host := range brokenAuthHeaderDomains {
			if u.Host == host || strings.HasSuffix(u.Host, "."+host) {
				return true
			}
		}
	}
	return false
}

// Open returns a connector which can be used to login users through an upstream
// OpenID Connect provider.
func (c *Config) Open(id string, logger log.Logger) (conn connector.Connector, err error) {
	ctx, cancel := context.WithCancel(context.Background())

	provider, err := oidc.NewProvider(ctx, c.Issuer)
	if err != nil {
		cancel()
		return nil, fmt.Errorf("failed to get provider: %v", err)
	}

	endpoint := provider.Endpoint()

	if c.BasicAuthUnsupported != nil {
		// Setting "basicAuthUnsupported" always overrides our detection.
		if *c.BasicAuthUnsupported {
			endpoint.AuthStyle = oauth2.AuthStyleInParams
		}
	} else if knownBrokenAuthHeaderProvider(c.Issuer) {
		endpoint.AuthStyle = oauth2.AuthStyleInParams
	}

	scopes := []string{oidc.ScopeOpenID}
	if len(c.Scopes) > 0 {
		scopes = append(scopes, c.Scopes...)
	} else {
		scopes = append(scopes, "profile", "email")
	}

	// PromptType should be "consent" by default, if not set
	if c.PromptType == "" {
		c.PromptType = "consent"
	}

	clientID := c.ClientID
	return &oidcConnector{
		provider:    provider,
		redirectURI: c.RedirectURI,
		oauth2Config: &oauth2.Config{
			ClientID:     clientID,
			ClientSecret: c.ClientSecret,
			Endpoint:     endpoint,
			Scopes:       scopes,
			RedirectURL:  c.RedirectURI,
		},
		verifier: provider.Verifier(
			&oidc.Config{ClientID: clientID},
		),
		logger:                    logger,
		cancel:                    cancel,
		hostedDomains:             c.HostedDomains,
		insecureSkipEmailVerified: c.InsecureSkipEmailVerified,
		insecureEnableGroups:      c.InsecureEnableGroups,
		getUserInfo:               c.GetUserInfo,
		promptType:                c.PromptType,
		userIDKey:                 c.UserIDKey,
		userNameKey:               c.UserNameKey,
		preferredUsernameKey:      c.ClaimMapping.PreferredUsernameKey,
		emailKey:                  c.ClaimMapping.EmailKey,
		groupsKey:                 c.ClaimMapping.GroupsKey,
	}, nil
}

var (
	_ connector.CallbackConnector = (*oidcConnector)(nil)
	_ connector.RefreshConnector  = (*oidcConnector)(nil)
)

type oidcConnector struct {
	provider                  *oidc.Provider
	redirectURI               string
	oauth2Config              *oauth2.Config
	verifier                  *oidc.IDTokenVerifier
	cancel                    context.CancelFunc
	logger                    log.Logger
	hostedDomains             []string
	insecureSkipEmailVerified bool
	insecureEnableGroups      bool
	getUserInfo               bool
	promptType                string
	userIDKey                 string
	userNameKey               string
	preferredUsernameKey      string
	emailKey                  string
	groupsKey                 string
}

func (c *oidcConnector) Close() error {
	c.cancel()
	return nil
}

func (c *oidcConnector) LoginURL(s connector.Scopes, callbackURL, state string) (string, error) {
	if c.redirectURI != callbackURL {
		return "", fmt.Errorf("expected callback URL %q did not match the URL in the config %q", callbackURL, c.redirectURI)
	}

	var opts []oauth2.AuthCodeOption
	if len(c.hostedDomains) > 0 {
		preferredDomain := c.hostedDomains[0]
		if len(c.hostedDomains) > 1 {
			preferredDomain = "*"
		}
		opts = append(opts, oauth2.SetAuthURLParam("hd", preferredDomain))
	}

	if s.OfflineAccess {
		opts = append(opts, oauth2.AccessTypeOffline, oauth2.SetAuthURLParam("prompt", c.promptType))
	}
	return c.oauth2Config.AuthCodeURL(state, opts...), nil
}

type oauth2Error struct {
	error            string
	errorDescription string
}

func (e *oauth2Error) Error() string {
	if e.errorDescription == "" {
		return e.error
	}
	return e.error + ": " + e.errorDescription
}

func (c *oidcConnector) HandleCallback(s connector.Scopes, r *http.Request) (identity connector.Identity, err error) {
	q := r.URL.Query()
	if errType := q.Get("error"); errType != "" {
		return identity, &oauth2Error{errType, q.Get("error_description")}
	}
	token, err := c.oauth2Config.Exchange(r.Context(), q.Get("code"))
	if err != nil {
		return identity, fmt.Errorf("oidc: failed to get token: %v", err)
	}

	return c.createIdentity(r.Context(), identity, token)
}

// Refresh is used to refresh a session with the refresh token provided by the IdP
func (c *oidcConnector) Refresh(ctx context.Context, s connector.Scopes, identity connector.Identity) (connector.Identity, error) {
	cd := connectorData{}
	err := json.Unmarshal(identity.ConnectorData, &cd)
	if err != nil {
		return identity, fmt.Errorf("oidc: failed to unmarshal connector data: %v", err)
	}

	t := &oauth2.Token{
		RefreshToken: string(cd.RefreshToken),
		Expiry:       time.Now().Add(-time.Hour),
	}
	token, err := c.oauth2Config.TokenSource(ctx, t).Token()
	if err != nil {
		return identity, fmt.Errorf("oidc: failed to get refresh token: %v", err)
	}

	return c.createIdentity(ctx, identity, token)
}

func (c *oidcConnector) createIdentity(ctx context.Context, identity connector.Identity, token *oauth2.Token) (connector.Identity, error) {
	rawIDToken, ok := token.Extra("id_token").(string)
	if !ok {
		return identity, errors.New("oidc: no id_token in token response")
	}
	idToken, err := c.verifier.Verify(ctx, rawIDToken)
	if err != nil {
		return identity, fmt.Errorf("oidc: failed to verify ID Token: %v", err)
	}

	var claims map[string]interface{}
	if err := idToken.Claims(&claims); err != nil {
		return identity, fmt.Errorf("oidc: failed to decode claims: %v", err)
	}

	// We immediately want to run getUserInfo if configured before we validate the claims
	if c.getUserInfo {
		userInfo, err := c.provider.UserInfo(ctx, oauth2.StaticTokenSource(token))
		if err != nil {
			return identity, fmt.Errorf("oidc: error loading userinfo: %v", err)
		}
		if err := userInfo.Claims(&claims); err != nil {
			return identity, fmt.Errorf("oidc: failed to decode userinfo claims: %v", err)
		}
	}

	userNameKey := "name"
	if c.userNameKey != "" {
		userNameKey = c.userNameKey
	}
	name, found := claims[userNameKey].(string)
	if !found {
		return identity, fmt.Errorf("missing \"%s\" claim", userNameKey)
	}

	preferredUsername, found := claims["preferred_username"].(string)
	if !found {
		preferredUsername, _ = claims[c.preferredUsernameKey].(string)
	}

	hasEmailScope := false
	for _, s := range c.oauth2Config.Scopes {
		if s == "email" {
			hasEmailScope = true
			break
		}
	}

	var email string
	emailKey := "email"
	email, found = claims[emailKey].(string)
	if !found && c.emailKey != "" {
		emailKey = c.emailKey
		email, found = claims[emailKey].(string)
	}

	if !found && hasEmailScope {
		return identity, fmt.Errorf("missing email claim, not found \"%s\" key", emailKey)
	}

	emailVerified, found := claims["email_verified"].(bool)
	if !found {
		if c.insecureSkipEmailVerified {
			emailVerified = true
		} else if hasEmailScope {
			return identity, errors.New("missing \"email_verified\" claim")
		}
	}

	var groups []string
	if c.insecureEnableGroups {
		groupsKey := "groups"
		vs, found := claims[groupsKey].([]interface{})
		if !found {
			groupsKey = c.groupsKey
			vs, found = claims[groupsKey].([]interface{})
		}

		if found {
			for _, v := range vs {
				if s, ok := v.(string); ok {
					groups = append(groups, s)
				} else {
					return identity, fmt.Errorf("malformed \"%v\" claim", groupsKey)
				}
			}
		}
	}

	hostedDomain, _ := claims["hd"].(string)
	if len(c.hostedDomains) > 0 {
		found := false
		for _, domain := range c.hostedDomains {
			if hostedDomain == domain {
				found = true
				break
			}
		}

		if !found {
			return identity, fmt.Errorf("oidc: unexpected hd claim %v", hostedDomain)
		}
	}

	cd := connectorData{
		RefreshToken: []byte(token.RefreshToken),
	}

	connData, err := json.Marshal(&cd)
	if err != nil {
		return identity, fmt.Errorf("oidc: failed to encode connector data: %v", err)
	}

	identity = connector.Identity{
		UserID:            idToken.Subject,
		Username:          name,
		PreferredUsername: preferredUsername,
		Email:             email,
		EmailVerified:     emailVerified,
		Groups:            groups,
		ConnectorData:     connData,
	}

	if c.userIDKey != "" {
		userID, found := claims[c.userIDKey].(string)
		if !found {
			return identity, fmt.Errorf("oidc: not found %v claim", c.userIDKey)
		}
		identity.UserID = userID
	}

	return identity, nil
}
<|endoftext|>


<commit_before>package starbound

import (
	"bytes"
	"io"
)

const (
	BTreeDB5HeaderSize = 512
)

var (
	BTreeDB5BlockFree  = []byte("FF")
	BTreeDB5BlockIndex = []byte("II")
	BTreeDB5BlockLeaf  = []byte("LL")

	BTreeDB5Signature = []byte("BTreeDB5")
)

func NewBTreeDB5(r io.ReaderAt) (db *BTreeDB5, err error) {
	db = &BTreeDB5{r: r}
	header := make([]byte, 67)
	n, err := r.ReadAt(header, 0)
	if n != len(header) || err != nil {
		return nil, ErrInvalidHeader
	}
	if !bytes.Equal(header[:8], BTreeDB5Signature) {
		return nil, ErrInvalidHeader
	}
	db.BlockSize = getInt(header, 8)
	db.Name = string(bytes.TrimRight(header[12:28], "\x00"))
	db.KeySize = getInt(header, 28)
	db.Swap = (header[32] == 1)
	db.freeBlock1 = getInt(header, 33)
	// Skip 3 bytes...
	db.unknown1 = getInt(header, 40)
	// Skip 1 byte...
	db.rootBlock1 = getInt(header, 45)
	db.rootBlock1IsLeaf = (header[49] == 1)
	db.freeBlock2 = getInt(header, 50)
	// Skip 3 bytes...
	db.unknown2 = getInt(header, 57)
	// Skip 1 byte...
	db.rootBlock2 = getInt(header, 62)
	db.rootBlock2IsLeaf = (header[66] == 1)
	return
}

type BTreeDB5 struct {
	Name      string
	BlockSize int
	KeySize   int
	Swap      bool

	r io.ReaderAt

	freeBlock1, freeBlock2 int
	rootBlock1, rootBlock2 int
	rootBlock1IsLeaf       bool
	rootBlock2IsLeaf       bool
	unknown1, unknown2     int
}

func (db *BTreeDB5) FreeBlock() int {
	if !db.Swap {
		return db.freeBlock1
	} else {
		return db.freeBlock2
	}
}

func (db *BTreeDB5) Get(key []byte) (data []byte, err error) {
	r, err := db.GetReader(key)
	if err != nil {
		return
	}
	lr := r.(*io.LimitedReader)
	data = make([]byte, lr.N)
	_, err = io.ReadFull(r, data)
	return
}

func (db *BTreeDB5) GetReader(key []byte) (r io.Reader, err error) {
	if len(key) != db.KeySize {
		return nil, ErrInvalidKeyLength
	}
	bufSize := 11
	if db.KeySize > bufSize {
		bufSize = db.KeySize
	}
	buf := make([]byte, bufSize)
	bufBlock := buf[:4]
	bufHead := buf[:11]
	bufKey := buf[:db.KeySize]
	bufType := buf[:2]
	block := db.RootBlock()
	offset := db.blockOffset(block)
	entrySize := db.KeySize + 4
	// Traverse the B-tree until we reach a leaf.
	for {
		if _, err = db.r.ReadAt(bufHead, offset); err != nil {
			return
		}
		if !bytes.Equal(bufType, BTreeDB5BlockIndex) {
			break
		}
		offset += 11
		// Binary search for the key.
		lo, hi := 0, getInt(buf, 3)
		block = getInt(buf, 7)
		for lo < hi {
			mid := (lo + hi) / 2
			if _, err = db.r.ReadAt(bufKey, offset+int64(entrySize*mid)); err != nil {
				return
			}
			if bytes.Compare(key, bufKey) < 0 {
				hi = mid
			} else {
				lo = mid + 1
			}
		}
		if lo > 0 {
			// A candidate leaf/index was found in the current index. Get the block index.
			db.r.ReadAt(bufBlock, offset+int64(entrySize*(lo-1)+db.KeySize))
			block = getInt(buf, 0)
		}
		offset = db.blockOffset(block)
	}
	// Scan leaves for the key, then read the data.
	lr := NewLeafReader(db, block)
	if _, err = lr.Read(bufBlock); err != nil {
		return
	}
	keyCount := getInt(buf, 0)
	for i := 0; i < keyCount; i += 1 {
		if _, err = lr.Read(bufKey); err != nil {
			return
		}
		var n int64
		if n, err = ReadVarint(lr); err != nil {
			return
		}
		// Is this the key you're looking for?
		if bytes.Equal(bufKey, key) {
			// Key found. Return a reader for the value.
			return io.LimitReader(lr, n), nil
		}
		// This isn't the key you're looking for.
		err = lr.Skip(int(n))
		if err != nil {
			return
		}
	}
	return nil, ErrKeyNotFound
}

func (db *BTreeDB5) RootBlock() int {
	if !db.Swap {
		return db.rootBlock1
	} else {
		return db.rootBlock2
	}
}

func (db *BTreeDB5) blockOffset(block int) int64 {
	return BTreeDB5HeaderSize + int64(block*db.BlockSize)
}

func NewLeafReader(db *BTreeDB5, block int) *LeafReader {
	l := &LeafReader{
		db:   db,
		buf4: make([]byte, 4),
		cur:  db.blockOffset(block),
	}
	l.buf2 = l.buf4[:2]
	return l
}

type LeafReader struct {
	db         *BTreeDB5
	buf2, buf4 []byte
	cur, end   int64
}

// Reads n bytes from the LeafReader, jumping to the next leaf when necessary.
func (l *LeafReader) Read(p []byte) (n int, err error) {
	off, n, err := l.step(len(p))
	if err != nil {
		return
	}
	n, err = l.db.r.ReadAt(p[:n], off)
	return
}

// Increments the LeafReader pointer by n bytes. This may require several reads
// from the underlying ReaderAt as the LeafReader reaches the boundary of the
// current leaf.
func (l *LeafReader) Skip(n int) error {
	for n > 0 {
		if _, m, err := l.step(n); err != nil {
			return err
		} else {
			n -= m
		}
	}
	return nil
}

func (l *LeafReader) step(max int) (off int64, n int, err error) {
	// We're at the end of the block – move to the next one.
	if l.cur == l.end {
		l.db.r.ReadAt(l.buf4, l.cur)
		l.cur = l.db.blockOffset(getInt(l.buf4, 0))
		l.end = 0
	}
	// We haven't verified that the current block is a leaf yet.
	if l.end == 0 {
		if _, err = l.db.r.ReadAt(l.buf2, l.cur); err != nil {
			return
		}
		if !bytes.Equal(l.buf2, BTreeDB5BlockLeaf) {
			return 0, 0, ErrDidNotReachLeaf
		}
		l.end = l.cur + int64(l.db.BlockSize-4)
		l.cur += 2
	}
	// Move the current pointer forward.
	off, n = l.cur, max
	if l.cur+int64(n) > l.end {
		n = int(l.end - l.cur)
	}
	l.cur += int64(n)
	return
}
<commit_msg>Remove a few constants<commit_after>package starbound

import (
	"bytes"
	"io"
)

var (
	BTreeDB5BlockFree  = []byte("FF")
	BTreeDB5BlockIndex = []byte("II")
	BTreeDB5BlockLeaf  = []byte("LL")
)

func NewBTreeDB5(r io.ReaderAt) (db *BTreeDB5, err error) {
	db = &BTreeDB5{r: r}
	header := make([]byte, 67)
	n, err := r.ReadAt(header, 0)
	if n != len(header) || err != nil {
		return nil, ErrInvalidHeader
	}
	if !bytes.Equal(header[:8], []byte("BTreeDB5")) {
		return nil, ErrInvalidHeader
	}
	db.BlockSize = getInt(header, 8)
	db.Name = string(bytes.TrimRight(header[12:28], "\x00"))
	db.KeySize = getInt(header, 28)
	db.Swap = (header[32] == 1)
	db.freeBlock1 = getInt(header, 33)
	// Skip 3 bytes...
	db.unknown1 = getInt(header, 40)
	// Skip 1 byte...
	db.rootBlock1 = getInt(header, 45)
	db.rootBlock1IsLeaf = (header[49] == 1)
	db.freeBlock2 = getInt(header, 50)
	// Skip 3 bytes...
	db.unknown2 = getInt(header, 57)
	// Skip 1 byte...
	db.rootBlock2 = getInt(header, 62)
	db.rootBlock2IsLeaf = (header[66] == 1)
	return
}

type BTreeDB5 struct {
	Name      string
	BlockSize int
	KeySize   int
	Swap      bool

	r io.ReaderAt

	freeBlock1, freeBlock2 int
	rootBlock1, rootBlock2 int
	rootBlock1IsLeaf       bool
	rootBlock2IsLeaf       bool
	unknown1, unknown2     int
}

func (db *BTreeDB5) FreeBlock() int {
	if !db.Swap {
		return db.freeBlock1
	} else {
		return db.freeBlock2
	}
}

func (db *BTreeDB5) Get(key []byte) (data []byte, err error) {
	r, err := db.GetReader(key)
	if err != nil {
		return
	}
	lr := r.(*io.LimitedReader)
	data = make([]byte, lr.N)
	_, err = io.ReadFull(r, data)
	return
}

func (db *BTreeDB5) GetReader(key []byte) (r io.Reader, err error) {
	if len(key) != db.KeySize {
		return nil, ErrInvalidKeyLength
	}
	bufSize := 11
	if db.KeySize > bufSize {
		bufSize = db.KeySize
	}
	buf := make([]byte, bufSize)
	bufBlock := buf[:4]
	bufHead := buf[:11]
	bufKey := buf[:db.KeySize]
	bufType := buf[:2]
	block := db.RootBlock()
	offset := db.blockOffset(block)
	entrySize := db.KeySize + 4
	// Traverse the B-tree until we reach a leaf.
	for {
		if _, err = db.r.ReadAt(bufHead, offset); err != nil {
			return
		}
		if !bytes.Equal(bufType, BTreeDB5BlockIndex) {
			break
		}
		offset += 11
		// Binary search for the key.
		lo, hi := 0, getInt(buf, 3)
		block = getInt(buf, 7)
		for lo < hi {
			mid := (lo + hi) / 2
			if _, err = db.r.ReadAt(bufKey, offset+int64(entrySize*mid)); err != nil {
				return
			}
			if bytes.Compare(key, bufKey) < 0 {
				hi = mid
			} else {
				lo = mid + 1
			}
		}
		if lo > 0 {
			// A candidate leaf/index was found in the current index. Get the block index.
			db.r.ReadAt(bufBlock, offset+int64(entrySize*(lo-1)+db.KeySize))
			block = getInt(buf, 0)
		}
		offset = db.blockOffset(block)
	}
	// Scan leaves for the key, then read the data.
	lr := NewLeafReader(db, block)
	if _, err = lr.Read(bufBlock); err != nil {
		return
	}
	keyCount := getInt(buf, 0)
	for i := 0; i < keyCount; i += 1 {
		if _, err = lr.Read(bufKey); err != nil {
			return
		}
		var n int64
		if n, err = ReadVarint(lr); err != nil {
			return
		}
		// Is this the key you're looking for?
		if bytes.Equal(bufKey, key) {
			// Key found. Return a reader for the value.
			return io.LimitReader(lr, n), nil
		}
		// This isn't the key you're looking for.
		err = lr.Skip(int(n))
		if err != nil {
			return
		}
	}
	return nil, ErrKeyNotFound
}

func (db *BTreeDB5) RootBlock() int {
	if !db.Swap {
		return db.rootBlock1
	} else {
		return db.rootBlock2
	}
}

func (db *BTreeDB5) blockOffset(block int) int64 {
	return 512 + int64(block*db.BlockSize)
}

func NewLeafReader(db *BTreeDB5, block int) *LeafReader {
	l := &LeafReader{
		db:   db,
		buf4: make([]byte, 4),
		cur:  db.blockOffset(block),
	}
	l.buf2 = l.buf4[:2]
	return l
}

type LeafReader struct {
	db         *BTreeDB5
	buf2, buf4 []byte
	cur, end   int64
}

// Reads n bytes from the LeafReader, jumping to the next leaf when necessary.
func (l *LeafReader) Read(p []byte) (n int, err error) {
	off, n, err := l.step(len(p))
	if err != nil {
		return
	}
	n, err = l.db.r.ReadAt(p[:n], off)
	return
}

// Increments the LeafReader pointer by n bytes. This may require several reads
// from the underlying ReaderAt as the LeafReader reaches the boundary of the
// current leaf.
func (l *LeafReader) Skip(n int) error {
	for n > 0 {
		if _, m, err := l.step(n); err != nil {
			return err
		} else {
			n -= m
		}
	}
	return nil
}

func (l *LeafReader) step(max int) (off int64, n int, err error) {
	// We're at the end of the block – move to the next one.
	if l.cur == l.end {
		l.db.r.ReadAt(l.buf4, l.cur)
		l.cur = l.db.blockOffset(getInt(l.buf4, 0))
		l.end = 0
	}
	// We haven't verified that the current block is a leaf yet.
	if l.end == 0 {
		if _, err = l.db.r.ReadAt(l.buf2, l.cur); err != nil {
			return
		}
		if !bytes.Equal(l.buf2, BTreeDB5BlockLeaf) {
			return 0, 0, ErrDidNotReachLeaf
		}
		l.end = l.cur + int64(l.db.BlockSize-4)
		l.cur += 2
	}
	// Move the current pointer forward.
	off, n = l.cur, max
	if l.cur+int64(n) > l.end {
		n = int(l.end - l.cur)
	}
	l.cur += int64(n)
	return
}
<|endoftext|>


<commit_before>package config

import (
	"fmt"
	"sort"
)

// Device represents a LXD container device
type Device map[string]string

// Clone returns a copy of the Device.
func (device Device) Clone() Device {
	copy := map[string]string{}

	for k, v := range device {
		copy[k] = v
	}

	return copy
}

// Validate accepts a map of field/validation functions to run against the device's config.
func (device Device) Validate(rules map[string]func(value string) error) error {
	checkedFields := map[string]struct{}{}

	for k, validator := range rules {
		checkedFields[k] = struct{}{} //Mark field as checked.
		err := validator(device[k])
		if err != nil {
			return fmt.Errorf("Invalid value for device option %q: %v", k, err)
		}
	}

	// Look for any unchecked fields, as these are unknown fields and validation should fail.
	for k := range device {
		_, checked := checkedFields[k]
		if checked {
			continue
		}

		// Skip type fields are these are validated by the presence of an implementation.
		if k == "type" {
			continue
		}

		if k == "nictype" && (device["type"] == "nic" || device["type"] == "infiniband") {
			continue
		}

		return fmt.Errorf("Invalid device option %q", k)
	}

	return nil
}

// Devices represents a set of LXD container devices
type Devices map[string]Device

// NewDevices creates a new Devices set from a native map[string]map[string]string set.
func NewDevices(nativeSet map[string]map[string]string) Devices {
	newDevices := Devices{}

	for devName, devConfig := range nativeSet {
		newDev := Device{}
		for k, v := range devConfig {
			newDev[k] = v
		}
		newDevices[devName] = newDev
	}

	return newDevices
}

// Contains checks if a given device exists in the set and if it's
// identical to that provided
func (list Devices) Contains(k string, d Device) bool {
	// If it didn't exist, it's different
	if list[k] == nil {
		return false
	}

	old := list[k]

	return deviceEquals(old, d)
}

// Update returns the difference between two sets
func (list Devices) Update(newlist Devices, updateFields func(Device, Device) []string) (map[string]Device, map[string]Device, map[string]Device, []string) {
	rmlist := map[string]Device{}
	addlist := map[string]Device{}
	updatelist := map[string]Device{}

	for key, d := range list {
		if !newlist.Contains(key, d) {
			rmlist[key] = d
		}
	}

	for key, d := range newlist {
		if !list.Contains(key, d) {
			addlist[key] = d
		}
	}

	updateDiff := []string{}
	for key, d := range addlist {
		srcOldDevice := rmlist[key]
		oldDevice := srcOldDevice.Clone()

		srcNewDevice := newlist[key]
		newDevice := srcNewDevice.Clone()

		updateDiff = deviceEqualsDiffKeys(oldDevice, newDevice)
		for _, k := range updateFields(oldDevice, newDevice) {
			delete(oldDevice, k)
			delete(newDevice, k)
		}

		if deviceEquals(oldDevice, newDevice) {
			delete(rmlist, key)
			delete(addlist, key)
			updatelist[key] = d
		}
	}

	return rmlist, addlist, updatelist, updateDiff
}

// Clone returns a copy of the Devices set.
func (list Devices) Clone() Devices {
	copy := Devices{}

	for deviceName, device := range list {
		copy[deviceName] = device.Clone()
	}

	return copy
}

// CloneNative returns a copy of the Devices set as a native map[string]map[string]string type.
func (list Devices) CloneNative() map[string]map[string]string {
	copy := map[string]map[string]string{}

	for deviceName, device := range list {
		copy[deviceName] = device.Clone()
	}

	return copy
}

// Sorted returns the name of all devices in the set, sorted properly.
func (list Devices) Sorted() DevicesSortable {
	sortable := DevicesSortable{}
	for k, d := range list {
		sortable = append(sortable, DeviceNamed{k, d})
	}

	sort.Sort(sortable)
	return sortable
}

// Reversed returns the name of all devices in the set, sorted reversed.
func (list Devices) Reversed() DevicesSortable {
	sortable := DevicesSortable{}
	for k, d := range list {
		sortable = append(sortable, DeviceNamed{k, d})
	}

	sort.Sort(sort.Reverse(sortable))
	return sortable
}
<commit_msg>lxd/device/config/devices: Adds NICType function on Device type<commit_after>package config

import (
	"fmt"
	"sort"
)

// Device represents a LXD container device
type Device map[string]string

// Clone returns a copy of the Device.
func (device Device) Clone() Device {
	copy := map[string]string{}

	for k, v := range device {
		copy[k] = v
	}

	return copy
}

// NICType returns the derived NIC Type for a NIC device.
// If the "network" property is specified then this implicitly (at least for now) means the nictype is "bridged".
// Otherwise the "nictype" property is returned. If the device type is not a NIC then an empty string is returned.
func (device Device) NICType() string {
	if device["type"] == "nic" {
		if device["network"] != "" {
			return "bridged"
		}

		return device["nictype"]
	}

	return ""
}

// Validate accepts a map of field/validation functions to run against the device's config.
func (device Device) Validate(rules map[string]func(value string) error) error {
	checkedFields := map[string]struct{}{}

	for k, validator := range rules {
		checkedFields[k] = struct{}{} //Mark field as checked.
		err := validator(device[k])
		if err != nil {
			return fmt.Errorf("Invalid value for device option %q: %v", k, err)
		}
	}

	// Look for any unchecked fields, as these are unknown fields and validation should fail.
	for k := range device {
		_, checked := checkedFields[k]
		if checked {
			continue
		}

		// Skip type fields are these are validated by the presence of an implementation.
		if k == "type" {
			continue
		}

		if k == "nictype" && (device["type"] == "nic" || device["type"] == "infiniband") {
			continue
		}

		return fmt.Errorf("Invalid device option %q", k)
	}

	return nil
}

// Devices represents a set of LXD container devices
type Devices map[string]Device

// NewDevices creates a new Devices set from a native map[string]map[string]string set.
func NewDevices(nativeSet map[string]map[string]string) Devices {
	newDevices := Devices{}

	for devName, devConfig := range nativeSet {
		newDev := Device{}
		for k, v := range devConfig {
			newDev[k] = v
		}
		newDevices[devName] = newDev
	}

	return newDevices
}

// Contains checks if a given device exists in the set and if it's
// identical to that provided
func (list Devices) Contains(k string, d Device) bool {
	// If it didn't exist, it's different
	if list[k] == nil {
		return false
	}

	old := list[k]

	return deviceEquals(old, d)
}

// Update returns the difference between two sets
func (list Devices) Update(newlist Devices, updateFields func(Device, Device) []string) (map[string]Device, map[string]Device, map[string]Device, []string) {
	rmlist := map[string]Device{}
	addlist := map[string]Device{}
	updatelist := map[string]Device{}

	for key, d := range list {
		if !newlist.Contains(key, d) {
			rmlist[key] = d
		}
	}

	for key, d := range newlist {
		if !list.Contains(key, d) {
			addlist[key] = d
		}
	}

	updateDiff := []string{}
	for key, d := range addlist {
		srcOldDevice := rmlist[key]
		oldDevice := srcOldDevice.Clone()

		srcNewDevice := newlist[key]
		newDevice := srcNewDevice.Clone()

		updateDiff = deviceEqualsDiffKeys(oldDevice, newDevice)
		for _, k := range updateFields(oldDevice, newDevice) {
			delete(oldDevice, k)
			delete(newDevice, k)
		}

		if deviceEquals(oldDevice, newDevice) {
			delete(rmlist, key)
			delete(addlist, key)
			updatelist[key] = d
		}
	}

	return rmlist, addlist, updatelist, updateDiff
}

// Clone returns a copy of the Devices set.
func (list Devices) Clone() Devices {
	copy := Devices{}

	for deviceName, device := range list {
		copy[deviceName] = device.Clone()
	}

	return copy
}

// CloneNative returns a copy of the Devices set as a native map[string]map[string]string type.
func (list Devices) CloneNative() map[string]map[string]string {
	copy := map[string]map[string]string{}

	for deviceName, device := range list {
		copy[deviceName] = device.Clone()
	}

	return copy
}

// Sorted returns the name of all devices in the set, sorted properly.
func (list Devices) Sorted() DevicesSortable {
	sortable := DevicesSortable{}
	for k, d := range list {
		sortable = append(sortable, DeviceNamed{k, d})
	}

	sort.Sort(sortable)
	return sortable
}

// Reversed returns the name of all devices in the set, sorted reversed.
func (list Devices) Reversed() DevicesSortable {
	sortable := DevicesSortable{}
	for k, d := range list {
		sortable = append(sortable, DeviceNamed{k, d})
	}

	sort.Sort(sort.Reverse(sortable))
	return sortable
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	"os"

	sqlite3 "github.com/mattn/go-sqlite3"
	"github.com/spf13/cobra"

	lxd "github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/lxd/node"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/idmap"
	"github.com/lxc/lxd/shared/logger"
)

func init() {
	sql.Register("dqlite_direct_access", &sqlite3.SQLiteDriver{ConnectHook: sqliteDirectAccess})
}

type cmdActivateifneeded struct {
	global *cmdGlobal
}

func (c *cmdActivateifneeded) Command() *cobra.Command {
	cmd := &cobra.Command{}
	cmd.Use = "activateifneeded"
	cmd.Short = "Check if LXD should be started"
	cmd.Long = `Description:
  Check if LXD should be started

  This command will check if LXD has any auto-started instances,
  instances which were running prior to LXD's last shutdown or if it's
  configured to listen on the network address.

  If at least one of those is true, then a connection will be attempted to the
  LXD socket which will cause a socket-activated LXD to be spawned.
`
	cmd.RunE = c.Run

	return cmd
}

func (c *cmdActivateifneeded) Run(cmd *cobra.Command, args []string) error {
	// Only root should run this
	if os.Geteuid() != 0 {
		return fmt.Errorf("This must be run as root")
	}

	// Don't start a full daemon, we just need DB access
	d := defaultDaemon()

	// Check if either the local database or the legacy local database
	// files exists.
	path := d.os.LocalDatabasePath()
	if !shared.PathExists(d.os.LocalDatabasePath()) {
		path = d.os.LegacyLocalDatabasePath()
		if !shared.PathExists(path) {
			logger.Debugf("No DB, so no need to start the daemon now")
			return nil
		}
	}

	// Open the database directly to avoid triggering any initialization
	// code, in particular the data migration from node to cluster db.
	sqldb, err := sql.Open("sqlite3", path)
	if err != nil {
		return err
	}
	d.db = db.ForLegacyPatches(sqldb)

	// Load the configured address from the database
	address, err := node.HTTPSAddress(d.db)
	if err != nil {
		return err
	}

	// Look for network socket
	if address != "" {
		logger.Debugf("Daemon has core.https_address set, activating...")
		_, err := lxd.ConnectLXDUnix("", nil)
		return err
	}

	// Load the idmap for unprivileged instances
	d.os.IdmapSet, err = idmap.DefaultIdmapSet("", "")
	if err != nil {
		return err
	}

	// Look for auto-started or previously started instances
	path = d.os.GlobalDatabasePath()
	if !shared.PathExists(path) {
		path = d.os.LegacyGlobalDatabasePath()
		if !shared.PathExists(path) {
			logger.Debugf("No DB, so no need to start the daemon now")
			return nil
		}
	}
	sqldb, err = sql.Open("dqlite_direct_access", path+"?mode=ro")
	if err != nil {
		return err
	}
	defer sqldb.Close()

	d.cluster, err = db.ForLocalInspectionWithPreparedStmts(sqldb)
	if err != nil {
		return err
	}

	instances, err := instance.LoadNodeAll(d.State(), instancetype.Any)
	if err != nil {
		return err
	}

	for _, inst := range instances {
		config := inst.ExpandedConfig()
		lastState := config["volatile.last_state.power"]
		autoStart := config["boot.autostart"]

		if inst.IsRunning() {
			logger.Debugf("Daemon has running instances, activating...")
			_, err := lxd.ConnectLXDUnix("", nil)
			return err
		}

		if lastState == "RUNNING" || lastState == "Running" || shared.IsTrue(autoStart) {
			logger.Debugf("Daemon has auto-started instances, activating...")
			_, err := lxd.ConnectLXDUnix("", nil)
			return err
		}

		// Check for scheduled instance snapshots
		if config["snapshots.schedule"] != "" {
			logger.Debugf("Daemon has scheduled instance snapshots, activating...")
			_, err := lxd.ConnectLXDUnix("", nil)
			return err
		}
	}

	// Check for scheduled volume snapshots
	volumes, err := d.cluster.GetStoragePoolVolumesWithType(db.StoragePoolVolumeTypeCustom)
	if err != nil {
		return err
	}

	for _, vol := range volumes {
		if vol.Config["snapshots.schedule"] != "" {
			logger.Debugf("Daemon has scheduled volume snapshots, activating...")
			_, err := lxd.ConnectLXDUnix("", nil)
			return err
		}
	}

	logger.Debugf("No need to start the daemon now")
	return nil
}

// Configure the sqlite connection so that it's safe to access the
// dqlite-managed sqlite file, also without setting up raft.
func sqliteDirectAccess(conn *sqlite3.SQLiteConn) error {
	// Ensure journal mode is set to WAL, as this is a requirement for
	// replication.
	_, err := conn.Exec("PRAGMA journal_mode=wal", nil)
	if err != nil {
		return err
	}

	// Ensure we don't truncate or checkpoint the WAL on exit, as this
	// would bork replication which must be in full control of the WAL
	// file.
	_, err = conn.Exec("PRAGMA journal_size_limit=-1", nil)
	if err != nil {
		return err
	}

	// Ensure WAL autocheckpoint is disabled, since checkpoints are
	// triggered explicitly by dqlite.
	_, err = conn.Exec("PRAGMA wal_autocheckpoint=0", nil)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>lxd/main_activateifneeded: Clarify 'No DB' debug statements<commit_after>package main

import (
	"database/sql"
	"fmt"
	"os"

	sqlite3 "github.com/mattn/go-sqlite3"
	"github.com/spf13/cobra"

	lxd "github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/lxd/node"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/idmap"
	"github.com/lxc/lxd/shared/logger"
)

func init() {
	sql.Register("dqlite_direct_access", &sqlite3.SQLiteDriver{ConnectHook: sqliteDirectAccess})
}

type cmdActivateifneeded struct {
	global *cmdGlobal
}

func (c *cmdActivateifneeded) Command() *cobra.Command {
	cmd := &cobra.Command{}
	cmd.Use = "activateifneeded"
	cmd.Short = "Check if LXD should be started"
	cmd.Long = `Description:
  Check if LXD should be started

  This command will check if LXD has any auto-started instances,
  instances which were running prior to LXD's last shutdown or if it's
  configured to listen on the network address.

  If at least one of those is true, then a connection will be attempted to the
  LXD socket which will cause a socket-activated LXD to be spawned.
`
	cmd.RunE = c.Run

	return cmd
}

func (c *cmdActivateifneeded) Run(cmd *cobra.Command, args []string) error {
	// Only root should run this
	if os.Geteuid() != 0 {
		return fmt.Errorf("This must be run as root")
	}

	// Don't start a full daemon, we just need database access
	d := defaultDaemon()

	// Check if either the local database or the legacy local database
	// files exists.
	path := d.os.LocalDatabasePath()
	if !shared.PathExists(d.os.LocalDatabasePath()) {
		path = d.os.LegacyLocalDatabasePath()
		if !shared.PathExists(path) {
			logger.Debugf("No local database, so no need to start the daemon now")
			return nil
		}
	}

	// Open the database directly to avoid triggering any initialization
	// code, in particular the data migration from node to cluster db.
	sqldb, err := sql.Open("sqlite3", path)
	if err != nil {
		return err
	}
	d.db = db.ForLegacyPatches(sqldb)

	// Load the configured address from the database
	address, err := node.HTTPSAddress(d.db)
	if err != nil {
		return err
	}

	// Look for network socket
	if address != "" {
		logger.Debugf("Daemon has core.https_address set, activating...")
		_, err := lxd.ConnectLXDUnix("", nil)
		return err
	}

	// Load the idmap for unprivileged instances
	d.os.IdmapSet, err = idmap.DefaultIdmapSet("", "")
	if err != nil {
		return err
	}

	// Look for auto-started or previously started instances
	path = d.os.GlobalDatabasePath()
	if !shared.PathExists(path) {
		path = d.os.LegacyGlobalDatabasePath()
		if !shared.PathExists(path) {
			logger.Debugf("No global database, so no need to start the daemon now")
			return nil
		}
	}
	sqldb, err = sql.Open("dqlite_direct_access", path+"?mode=ro")
	if err != nil {
		return err
	}
	defer sqldb.Close()

	d.cluster, err = db.ForLocalInspectionWithPreparedStmts(sqldb)
	if err != nil {
		return err
	}

	instances, err := instance.LoadNodeAll(d.State(), instancetype.Any)
	if err != nil {
		return err
	}

	for _, inst := range instances {
		config := inst.ExpandedConfig()
		lastState := config["volatile.last_state.power"]
		autoStart := config["boot.autostart"]

		if inst.IsRunning() {
			logger.Debugf("Daemon has running instances, activating...")
			_, err := lxd.ConnectLXDUnix("", nil)
			return err
		}

		if lastState == "RUNNING" || lastState == "Running" || shared.IsTrue(autoStart) {
			logger.Debugf("Daemon has auto-started instances, activating...")
			_, err := lxd.ConnectLXDUnix("", nil)
			return err
		}

		// Check for scheduled instance snapshots
		if config["snapshots.schedule"] != "" {
			logger.Debugf("Daemon has scheduled instance snapshots, activating...")
			_, err := lxd.ConnectLXDUnix("", nil)
			return err
		}
	}

	// Check for scheduled volume snapshots
	volumes, err := d.cluster.GetStoragePoolVolumesWithType(db.StoragePoolVolumeTypeCustom)
	if err != nil {
		return err
	}

	for _, vol := range volumes {
		if vol.Config["snapshots.schedule"] != "" {
			logger.Debugf("Daemon has scheduled volume snapshots, activating...")
			_, err := lxd.ConnectLXDUnix("", nil)
			return err
		}
	}

	logger.Debugf("No need to start the daemon now")
	return nil
}

// Configure the sqlite connection so that it's safe to access the
// dqlite-managed sqlite file, also without setting up raft.
func sqliteDirectAccess(conn *sqlite3.SQLiteConn) error {
	// Ensure journal mode is set to WAL, as this is a requirement for
	// replication.
	_, err := conn.Exec("PRAGMA journal_mode=wal", nil)
	if err != nil {
		return err
	}

	// Ensure we don't truncate or checkpoint the WAL on exit, as this
	// would bork replication which must be in full control of the WAL
	// file.
	_, err = conn.Exec("PRAGMA journal_size_limit=-1", nil)
	if err != nil {
		return err
	}

	// Ensure WAL autocheckpoint is disabled, since checkpoints are
	// triggered explicitly by dqlite.
	_, err = conn.Exec("PRAGMA wal_autocheckpoint=0", nil)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"strings"
)

const (
	histBlocks = 70
)

type Bucket struct {
	start float64
	count uint64
}

type Hist struct {
	bucketSize float64
	buckets    []Bucket
}

func (s *Stats) Hist(n int) *Hist {
	h := &Hist{buckets: make([]Bucket, n)}
	rnge := s.max - s.min
	h.bucketSize = rnge / float64(n)
	i := 0
	limit := s.min + h.bucketSize
	h.buckets[0].start = s.min
	for j := 0; j < len(s.sorted); {
		v := s.sorted[j]
		if v > limit && i < len(h.buckets)-1 {
			i++
			h.buckets[i].start = limit
			limit = s.min + float64(i+1)*(rnge/float64(n))
			continue
		}
		h.buckets[i].count++
		j++
	}
	return h
}

func (h *Hist) String() string {
	// TODO: if the range is large, expand the bucketsize and start/end a bit to get integer boundaries.
	labels := make([]string, len(h.buckets))
	labelSpaceBefore := 0
	labelSpaceAfter := 0
	var maxCount float64
	for i, b := range h.buckets {
		s := "<"
		if i == len(h.buckets)-1 {
			s = "≤"
		}
		label := fmt.Sprintf("%.3g ≤ x %s %.3g", b.start, s, b.start+h.bucketSize)
		xPos := runeIndex(label, 'x')
		if xPos > labelSpaceBefore {
			labelSpaceBefore = xPos
		}
		if after := runeLen(label) - xPos - 1; after > labelSpaceAfter {
			labelSpaceAfter = after
		}
		labels[i] = label
		if f := float64(b.count); f > maxCount {
			maxCount = f
		}
	}

	var buf bytes.Buffer
	for i, b := range h.buckets {
		xPos := runeIndex(labels[i], 'x')
		before := labelSpaceBefore - xPos
		after := labelSpaceAfter - runeLen(labels[i]) + xPos + 1
		fmt.Fprintf(&buf, " %*s%s%*s │", before, "", labels[i], after, "")
		fmt.Fprint(&buf, makeBar((float64(b.count)/float64(maxCount))*histBlocks))
		fmt.Fprintf(&buf, " %d\n", b.count)
	}
	b := buf.Bytes()
	return string(b[:len(b)-1]) // drop the \n
}

func runeLen(s string) int { return len([]rune(s)) }

func runeIndex(s string, r rune) int {
	for i, r2 := range []rune(s) {
		if r2 == r {
			return i
		}
	}
	return -1
}

var barEighths = [9]rune{
	' ', // empty
	'▏',
	'▎',
	'▍',
	'▌',
	'▋',
	'▊',
	'▉',
	'█', // full
}

func makeBar(n float64) string {
	eighths := round(n * 8)
	full := eighths / 8
	rem := eighths % 8
	return strings.Repeat(string(barEighths[8]), full) + string(barEighths[rem])
}
<commit_msg>Fix boundary condition bug in histogram<commit_after>package main

import (
	"bytes"
	"fmt"
	"strings"
)

const (
	histBlocks = 70
)

type Bucket struct {
	start float64
	count uint64
}

type Hist struct {
	bucketSize float64
	buckets    []Bucket
}

func (s *Stats) Hist(n int) *Hist {
	h := &Hist{buckets: make([]Bucket, n)}
	rnge := s.max - s.min
	h.bucketSize = rnge / float64(n)
	i := 0
	limit := s.min + h.bucketSize
	h.buckets[0].start = s.min
	for j := 0; j < len(s.sorted); {
		v := s.sorted[j]
		if v >= limit && i < len(h.buckets)-1 {
			i++
			h.buckets[i].start = limit
			limit = s.min + float64(i+1)*(rnge/float64(n))
			continue
		}
		h.buckets[i].count++
		j++
	}
	return h
}

func (h *Hist) String() string {
	// TODO: if the range is large, expand the bucketsize and start/end a bit to get integer boundaries.
	labels := make([]string, len(h.buckets))
	labelSpaceBefore := 0
	labelSpaceAfter := 0
	var maxCount float64
	for i, b := range h.buckets {
		s := "<"
		if i == len(h.buckets)-1 {
			s = "≤"
		}
		label := fmt.Sprintf("%.3g ≤ x %s %.3g", b.start, s, b.start+h.bucketSize)
		xPos := runeIndex(label, 'x')
		if xPos > labelSpaceBefore {
			labelSpaceBefore = xPos
		}
		if after := runeLen(label) - xPos - 1; after > labelSpaceAfter {
			labelSpaceAfter = after
		}
		labels[i] = label
		if f := float64(b.count); f > maxCount {
			maxCount = f
		}
	}

	var buf bytes.Buffer
	for i, b := range h.buckets {
		xPos := runeIndex(labels[i], 'x')
		before := labelSpaceBefore - xPos
		after := labelSpaceAfter - runeLen(labels[i]) + xPos + 1
		fmt.Fprintf(&buf, " %*s%s%*s │", before, "", labels[i], after, "")
		fmt.Fprint(&buf, makeBar((float64(b.count)/float64(maxCount))*histBlocks))
		fmt.Fprintf(&buf, " %d\n", b.count)
	}
	b := buf.Bytes()
	return string(b[:len(b)-1]) // drop the \n
}

func runeLen(s string) int { return len([]rune(s)) }

func runeIndex(s string, r rune) int {
	for i, r2 := range []rune(s) {
		if r2 == r {
			return i
		}
	}
	return -1
}

var barEighths = [9]rune{
	' ', // empty
	'▏',
	'▎',
	'▍',
	'▌',
	'▋',
	'▊',
	'▉',
	'█', // full
}

func makeBar(n float64) string {
	eighths := round(n * 8)
	full := eighths / 8
	rem := eighths % 8
	return strings.Repeat(string(barEighths[8]), full) + string(barEighths[rem])
}
<|endoftext|>