<commit_before>package restserver

/*
Original version copied from: github.com/bitly/oauth2_proxy

MIT License

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

import (
	"crypto/sha1"
	"encoding/base64"
	"encoding/csv"
	"log"
	"os"
	"regexp"
	"sync"
	"time"

	"golang.org/x/crypto/bcrypt"
)

// CheckInterval represents how often we check for changes in htpasswd file.
const CheckInterval = 30 * time.Second

// Lookup passwords in a htpasswd file.  The entries must have been created with -s for SHA encryption.

// HtpasswdFile is a map for usernames to passwords.
type HtpasswdFile struct {
	mutex    sync.Mutex
	path     string
	stat     os.FileInfo
	throttle chan struct{}
	Users    map[string]string
}

// NewHtpasswdFromFile reads the users and passwords from a htpasswd file and returns them.  If an error is encountered,
// it is returned, together with a nil-Pointer for the HtpasswdFile.
func NewHtpasswdFromFile(path string) (*HtpasswdFile, error) {
	stat, err := os.Stat(path)
	if err != nil {
		return nil, err
	}

	h := &HtpasswdFile{
		mutex:    sync.Mutex{},
		path:     path,
		stat:     stat,
		throttle: make(chan struct{}),
	}

	if err := h.Reload(); err != nil {
		return nil, err
	}

	// Start a goroutine that limits reload checks to once per CheckInterval
	go h.throttleTimer()

	return h, nil
}

// throttleTimer sends at most one message per CheckInterval to throttle file change checks.
func (h *HtpasswdFile) throttleTimer() {
	var check struct{}
	for {
		time.Sleep(CheckInterval)
		h.throttle <- check
	}
}

// Reload reloads the htpasswd file. If the reload fails, the Users map is not changed and the error is returned.
func (h *HtpasswdFile) Reload() error {
	r, err := os.Open(h.path)
	if err != nil {
		return err
	}

	cr := csv.NewReader(r)
	cr.Comma = ':'
	cr.Comment = '#'
	cr.TrimLeadingSpace = true

	records, err := cr.ReadAll()
	if err != nil {
		_ = r.Close()
		return err
	}
	users := make(map[string]string)
	for _, record := range records {
		users[record[0]] = record[1]
	}

	// Replace the Users map
	h.mutex.Lock()
	h.Users = users
	h.mutex.Unlock()

	_ = r.Close()
	return nil
}

// ReloadCheck checks at most once per CheckInterval if the file changed and will reload the file if it did.
// It logs errors and successful reloads, and returns an error if any was encountered.
func (h *HtpasswdFile) ReloadCheck() error {
	select {
	case <-h.throttle:
		stat, err := os.Stat(h.path)
		if err != nil {
			log.Printf("Could not stat htpasswd file: %v", err)
			return err
		}

		reload := false

		h.mutex.Lock()
		if stat.ModTime() != h.stat.ModTime() || stat.Size() != h.stat.Size() {
			reload = true
			h.stat = stat
		}
		h.mutex.Unlock()

		if reload {
			err := h.Reload()
			if err == nil {
				log.Printf("Reloaded htpasswd file")
			} else {
				log.Printf("Could not reload htpasswd file: %v", err)
				return err
			}
		}
	default:
		// No need to check
	}
	return nil
}

// Validate returns true if password matches the stored password for user.  If no password for user is stored, or the
// password is wrong, false is returned.
func (h *HtpasswdFile) Validate(user string, password string) bool {
	_ = h.ReloadCheck()

	h.mutex.Lock()
	realPassword, exists := h.Users[user]
	h.mutex.Unlock()

	if !exists {
		return false
	}

	var shaRe = regexp.MustCompile(`^{SHA}`)
	var bcrRe = regexp.MustCompile(`^\$2b\$`)

	switch {
	case shaRe.MatchString(realPassword):
		d := sha1.New()
		_, _ = d.Write([]byte(password))
		if realPassword[5:] == base64.StdEncoding.EncodeToString(d.Sum(nil)) {
			return true
		}
	case bcrRe.MatchString(realPassword):
		err := bcrypt.CompareHashAndPassword([]byte(realPassword), []byte(password))
		if err == nil {
			return true
		}
	}
	log.Printf("Invalid htpasswd entry for %s.", user)
	return false
}
<commit_msg>Check for more bcrypt prefixes<commit_after>package restserver

/*
Original version copied from: github.com/bitly/oauth2_proxy

MIT License

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

import (
	"crypto/sha1"
	"encoding/base64"
	"encoding/csv"
	"log"
	"os"
	"regexp"
	"sync"
	"time"

	"golang.org/x/crypto/bcrypt"
)

// CheckInterval represents how often we check for changes in htpasswd file.
const CheckInterval = 30 * time.Second

// Lookup passwords in a htpasswd file.  The entries must have been created with -s for SHA encryption.

// HtpasswdFile is a map for usernames to passwords.
type HtpasswdFile struct {
	mutex    sync.Mutex
	path     string
	stat     os.FileInfo
	throttle chan struct{}
	Users    map[string]string
}

// NewHtpasswdFromFile reads the users and passwords from a htpasswd file and returns them.  If an error is encountered,
// it is returned, together with a nil-Pointer for the HtpasswdFile.
func NewHtpasswdFromFile(path string) (*HtpasswdFile, error) {
	stat, err := os.Stat(path)
	if err != nil {
		return nil, err
	}

	h := &HtpasswdFile{
		mutex:    sync.Mutex{},
		path:     path,
		stat:     stat,
		throttle: make(chan struct{}),
	}

	if err := h.Reload(); err != nil {
		return nil, err
	}

	// Start a goroutine that limits reload checks to once per CheckInterval
	go h.throttleTimer()

	return h, nil
}

// throttleTimer sends at most one message per CheckInterval to throttle file change checks.
func (h *HtpasswdFile) throttleTimer() {
	var check struct{}
	for {
		time.Sleep(CheckInterval)
		h.throttle <- check
	}
}

// Reload reloads the htpasswd file. If the reload fails, the Users map is not changed and the error is returned.
func (h *HtpasswdFile) Reload() error {
	r, err := os.Open(h.path)
	if err != nil {
		return err
	}

	cr := csv.NewReader(r)
	cr.Comma = ':'
	cr.Comment = '#'
	cr.TrimLeadingSpace = true

	records, err := cr.ReadAll()
	if err != nil {
		_ = r.Close()
		return err
	}
	users := make(map[string]string)
	for _, record := range records {
		users[record[0]] = record[1]
	}

	// Replace the Users map
	h.mutex.Lock()
	h.Users = users
	h.mutex.Unlock()

	_ = r.Close()
	return nil
}

// ReloadCheck checks at most once per CheckInterval if the file changed and will reload the file if it did.
// It logs errors and successful reloads, and returns an error if any was encountered.
func (h *HtpasswdFile) ReloadCheck() error {
	select {
	case <-h.throttle:
		stat, err := os.Stat(h.path)
		if err != nil {
			log.Printf("Could not stat htpasswd file: %v", err)
			return err
		}

		reload := false

		h.mutex.Lock()
		if stat.ModTime() != h.stat.ModTime() || stat.Size() != h.stat.Size() {
			reload = true
			h.stat = stat
		}
		h.mutex.Unlock()

		if reload {
			err := h.Reload()
			if err == nil {
				log.Printf("Reloaded htpasswd file")
			} else {
				log.Printf("Could not reload htpasswd file: %v", err)
				return err
			}
		}
	default:
		// No need to check
	}
	return nil
}

// Validate returns true if password matches the stored password for user.  If no password for user is stored, or the
// password is wrong, false is returned.
func (h *HtpasswdFile) Validate(user string, password string) bool {
	_ = h.ReloadCheck()

	h.mutex.Lock()
	realPassword, exists := h.Users[user]
	h.mutex.Unlock()

	if !exists {
		return false
	}

	var shaRe = regexp.MustCompile(`^{SHA}`)
	var bcrRe = regexp.MustCompile(`^\$2b\$|^\$2a\$|^\$2y\$`)

	switch {
	case shaRe.MatchString(realPassword):
		d := sha1.New()
		_, _ = d.Write([]byte(password))
		if realPassword[5:] == base64.StdEncoding.EncodeToString(d.Sum(nil)) {
			return true
		}
	case bcrRe.MatchString(realPassword):
		err := bcrypt.CompareHashAndPassword([]byte(realPassword), []byte(password))
		if err == nil {
			return true
		}
	}
	log.Printf("Invalid htpasswd entry for %s.", user)
	return false
}
<|endoftext|>


<commit_before>package main

import "fmt"
import "net"
import "os"

import "unicode/utf8"

// Copy everything from the pty master to the socket.
func handleOutputSock(ptym *os.File, conn *net.TCPConn) {
	buf := make([]byte, 512)
	var payload, overflow []byte
	// TODO: more graceful exit on socket close / process exit.
	for {
		n, err := ptym.Read(buf)
		if err != nil {
			fmt.Println("failed to read from pty master: ", err)
			return
		}

		// Empty the overflow from the last read into the payload first.
		payload = append(payload[0:], overflow...)
		overflow = nil
		// Then empty the new buf read into the payload.
		payload = append(payload, buf[:n]...)

		// Strip out any incomplete utf-8 from current payload into overflow.
		for !utf8.Valid(payload) {
			overflow = append(overflow[:0], append(payload[len(payload)-1:], overflow[0:]...)...)
			payload = payload[:len(payload)-1]
		}

		// Send out the finished payload as long as it's not empty.
		if len(payload) >= 1 {
			_, err = conn.Write(payload)
	        if err != nil {
	            fmt.Println("Write: ", err)
	        }
		}

		// Empty the payload.
		payload = nil
	}
}

// Read from the websocket, copying to the pty master.
func handleInputSock(ptym *os.File, conn *net.TCPConn) {
	for {
        buf := make([]byte, 512)
        _, err := conn.Read(buf)
        if err != nil {
            return
        }

		ptym.Write(buf)
	}
}

func ptySetupSock(conn *net.TCPConn, sizeFlag string) {
	ptym, cmd := start()
	setPtySize(ptym, sizeFlag)

	go func() {
		handleOutputSock(ptym, conn)
	}()

	go func() {
		handleInputSock(ptym, conn)
	}()

	// Listen for a new winsize on stdin.
	for {
		var newSize string
		_, scanErr := fmt.Scanln(&newSize)
		if scanErr != nil {
			fmt.Println("scan failed: ", scanErr)
		}

		setPtySize(ptym, newSize)
		fmt.Println("new size: ", newSize)
	}

	stop(ptym, cmd)
	conn.Close()
}<commit_msg>Fixed problem with input overflow in TCP mode<commit_after>package main

import "fmt"
import "net"
import "os"

import "unicode/utf8"

// Copy everything from the pty master to the socket.
func handleOutputSock(ptym *os.File, conn *net.TCPConn) {
	buf := make([]byte, 512)
	var payload, overflow []byte
	// TODO: more graceful exit on socket close / process exit.
	for {
		n, err := ptym.Read(buf)
		if err != nil {
			fmt.Println("failed to read from pty master: ", err)
			return
		}

		// Empty the overflow from the last read into the payload first.
		payload = append(payload[0:], overflow...)
		overflow = nil
		// Then empty the new buf read into the payload.
		payload = append(payload, buf[:n]...)

		// Strip out any incomplete utf-8 from current payload into overflow.
		for !utf8.Valid(payload) {
			overflow = append(overflow[:0], append(payload[len(payload)-1:], overflow[0:]...)...)
			payload = payload[:len(payload)-1]
		}

		// Send out the finished payload as long as it's not empty.
		if len(payload) >= 1 {
			_, err = conn.Write(payload)
	        if err != nil {
	            fmt.Println("Write: ", err)
	        }
		}

		// Empty the payload.
		payload = nil
	}
}

// Read from the websocket, copying to the pty master.
func handleInputSock(ptym *os.File, conn *net.TCPConn) {
	for {
        buf := make([]byte, 512)
        bytes, err := conn.Read(buf)
        if err != nil {
            return
        }

        buf = buf[:bytes]
		ptym.Write(buf)
	}
}

func ptySetupSock(conn *net.TCPConn, sizeFlag string) {
	ptym, cmd := start()
	setPtySize(ptym, sizeFlag)

	go func() {
		handleOutputSock(ptym, conn)
	}()

	go func() {
		handleInputSock(ptym, conn)
	}()

	// Listen for a new winsize on stdin.
	for {
		var newSize string
		_, scanErr := fmt.Scanln(&newSize)
		if scanErr != nil {
			fmt.Println("scan failed: ", scanErr)
		}

		setPtySize(ptym, newSize)
		fmt.Println("new size: ", newSize)
	}

	stop(ptym, cmd)
	conn.Close()
}<|endoftext|>


<commit_before>// +build freebsd

package gopsutil

import (
	"bytes"
	"encoding/binary"
	"syscall"
	"unsafe"
)

func Pids() ([]int32, error) {
	ret := make([]int32, 0)
	procs, err := processes()
	if err != nil {
		return ret, nil
	}

	for _, p := range procs {
		ret = append(ret, p.Pid)
	}

	return ret, nil
}

// Refresh reloads all the data associated with this process.
func (p *Process) Refresh() error {

	mib := []int32{CTL_KERN, KERN_PROC, KERN_PROC_PID, p.Pid}

	buf, length, err := call_syscall(mib)
	if err != nil {
		return err
	}
	proc_k := Kinfo_proc{}
	if length != uint64(unsafe.Sizeof(proc_k)) {
		return err
	}

	k, err := parse_kinfo_proc(buf)
	if err != nil {
		return err
	}

	copy_params(&k, p)
	return nil
}

func copy_params(k *Kinfo_proc, p *Process) error {
	p.Exe = byteToString(k.Ki_comm[:])
	p.Ppid = k.Ki_ppid

	return nil
}

func findProcess(pid int32) (*Process, error) {
	mib := []int32{CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, pid}

	_, _, err := call_syscall(mib)
	if err != nil {
		return &Process{}, err
	}

	return newProcess(pid)
}

func processes() ([]Process, error) {
	results := make([]Process, 0, 50)

	mib := []int32{CTL_KERN, KERN_PROC, KERN_PROC_PROC, 0}
	buf, length, err := call_syscall(mib)
	if err != nil {
		return results, err
	}

	// get kinfo_proc size
	k := Kinfo_proc{}
	procinfo_len := int(unsafe.Sizeof(k))
	count := int(length / uint64(procinfo_len))

	// parse buf to procs
	for i := 0; i < count; i++ {
		b := buf[i*procinfo_len : i*procinfo_len+procinfo_len]
		k, err := parse_kinfo_proc(b)
		if err != nil {
			continue
		}
		p, err := newProcess(int32(k.Ki_pid))
		if err != nil {
			continue
		}
		copy_params(&k, p)

		results = append(results, *p)
	}

	return results, nil
}

func parse_kinfo_proc(buf []byte) (Kinfo_proc, error) {
	var k Kinfo_proc
	br := bytes.NewReader(buf)
	err := binary.Read(br, binary.LittleEndian, &k)
	if err != nil {
		return k, err
	}

	return k, nil
}

func call_syscall(mib []int32) ([]byte, uint64, error) {
	miblen := uint64(len(mib))

	// get required buffer size
	length := uint64(0)
	_, _, err := syscall.Syscall6(
		syscall.SYS___SYSCTL,
		uintptr(unsafe.Pointer(&mib[0])),
		uintptr(miblen),
		0,
		uintptr(unsafe.Pointer(&length)),
		0,
		0)
	if err != 0 {
		b := make([]byte, 0)
		return b, length, err
	}
	if length == 0 {
		b := make([]byte, 0)
		return b, length, err
	}
	// get proc info itself
	buf := make([]byte, length)
	_, _, err = syscall.Syscall6(
		syscall.SYS___SYSCTL,
		uintptr(unsafe.Pointer(&mib[0])),
		uintptr(miblen),
		uintptr(unsafe.Pointer(&buf[0])),
		uintptr(unsafe.Pointer(&length)),
		0,
		0)
	if err != 0 {
		return buf, length, err
	}

	return buf, length, nil
}

func newProcess(pid int32) (*Process, error) {
	p := &Process{Pid: pid}
	return p, p.Refresh()
}
<commit_msg>remove Refresh func.<commit_after>// +build freebsd

package gopsutil

import (
	"bytes"
	"encoding/binary"
	"syscall"
	"unsafe"
)

func Pids() ([]int32, error) {
	ret := make([]int32, 0)
	procs, err := processes()
	if err != nil {
		return ret, nil
	}

	for _, p := range procs {
		ret = append(ret, p.Pid)
	}

	return ret, nil
}

func copy_params(k *Kinfo_proc, p *Process) error {
	p.Exe = byteToString(k.Ki_comm[:])
	p.Ppid = k.Ki_ppid

	return nil
}

func processes() ([]Process, error) {
	results := make([]Process, 0, 50)

	mib := []int32{CTL_KERN, KERN_PROC, KERN_PROC_PROC, 0}
	buf, length, err := call_syscall(mib)
	if err != nil {
		return results, err
	}

	// get kinfo_proc size
	k := Kinfo_proc{}
	procinfo_len := int(unsafe.Sizeof(k))
	count := int(length / uint64(procinfo_len))

	// parse buf to procs
	for i := 0; i < count; i++ {
		b := buf[i*procinfo_len : i*procinfo_len+procinfo_len]
		k, err := parse_kinfo_proc(b)
		if err != nil {
			continue
		}
		p, err := NewProcess(int32(k.Ki_pid))
		if err != nil {
			continue
		}
		copy_params(&k, p)

		results = append(results, *p)
	}

	return results, nil
}

func parse_kinfo_proc(buf []byte) (Kinfo_proc, error) {
	var k Kinfo_proc
	br := bytes.NewReader(buf)
	err := binary.Read(br, binary.LittleEndian, &k)
	if err != nil {
		return k, err
	}

	return k, nil
}

func call_syscall(mib []int32) ([]byte, uint64, error) {
	miblen := uint64(len(mib))

	// get required buffer size
	length := uint64(0)
	_, _, err := syscall.Syscall6(
		syscall.SYS___SYSCTL,
		uintptr(unsafe.Pointer(&mib[0])),
		uintptr(miblen),
		0,
		uintptr(unsafe.Pointer(&length)),
		0,
		0)
	if err != 0 {
		b := make([]byte, 0)
		return b, length, err
	}
	if length == 0 {
		b := make([]byte, 0)
		return b, length, err
	}
	// get proc info itself
	buf := make([]byte, length)
	_, _, err = syscall.Syscall6(
		syscall.SYS___SYSCTL,
		uintptr(unsafe.Pointer(&mib[0])),
		uintptr(miblen),
		uintptr(unsafe.Pointer(&buf[0])),
		uintptr(unsafe.Pointer(&length)),
		0,
		0)
	if err != 0 {
		return buf, length, err
	}

	return buf, length, nil
}

func NewProcess(pid int32) (*Process, error) {
	p := &Process{Pid: pid}
	mib := []int32{CTL_KERN, KERN_PROC, KERN_PROC_PID, p.Pid}

	buf, length, err := call_syscall(mib)
	if err != nil {
		return nil, err
	}
	proc_k := Kinfo_proc{}
	if length != uint64(unsafe.Sizeof(proc_k)) {
		return nil, err
	}

	k, err := parse_kinfo_proc(buf)
	if err != nil {
		return nil, err
	}

	copy_params(&k, p)
	return p, nil
}
<|endoftext|>


<commit_before>package project

import (
	"github.com/monax/relic"
)

// Can be used to set the commit hash version of the binary at build time with:
// `go build -ldflags "-X github.com/monax/hoard/project.commit=$(git rev-parse --short HEAD)" ./cmd/hoard`

var commit = ""
var date = ""

func Commit() string {
	return commit
}

func FullVersion() string {
	version := History.CurrentVersion().String()
	if commit != "" && date != "" {
		return version + "+commit." + commit + "+date." + date
	}
	return version
}

// The releases described by version string and changes, newest release first.
// The current release is taken to be the first release in the slice, and its
// version determines the single authoritative version for the next release.
//
// To cut a new release add a release to the front of this slice then run the
// release tagging script: ./scripts/tag_release.sh
var History relic.ImmutableHistory = relic.NewHistory("Monax Hoard", "https://github.com/monax/hoard").
	MustDeclareReleases("",
		``,
		"3.0.0 - 2019-02-26",
		`### Changed
- [PROTO] Renamed symmetric grant SecretID to PublicID
- [PROTO] Renamed openpgp grant ID to PrivateID

### Fixed
- [GRANTS] Throw an exception if symmetric secret for ID cannot be found

### Added
- [NODEJS] Added integration tests including test for symmetric secrets
- [GRANTS] Added openpgp grants example
- [CLI] Added ability to configure secrets on command line with hoard config <config> --secret
-
`,
		"2.0.0 - 2019-02-21",
		`### Changed
- [PROTO] Upper case field names in protobuf
- [PROTO] Used gogoproto for types
- [STORE] Go-cloud project for s3 store backend
- [STORE] Minimized IPFS configuration

### Added
- [STORE] Initial support for Azure backend thanks to go-cloud
- [GRANTS] Interface and GRPC service
- [GRANTS] Symmetric AES-GCM-based grants
- [GRANTS] Asymmetric OpenPGP support
- [GRANTS] Go-client (hoarctl) tooling
- [NODEJS] Support for using the grant service + examples
- [STORE] IPFS integration test`,

		"1.1.5 - 2018-10-17",
		`Scripted integration tests, better makefile and ci configs, gcs creds read from env var.`,

		"1.1.4",
		`IPFS & GCP Support`,

		"1.1.3",
		`Just create new hasher each call of addresses - we only use SHA256 and this operation is cheap`,

		"1.1.2",
		`Upgrade all Go dependencies`,

		"1.1.1",
		`Bump docker image Alpine Linux version to 3.8 and Go to 1.10.3`,

		"1.1.0",
		`Fix unsafe concurrent access of hash.Hash function in makeAddresser with sync.Pool`,

		"1.0.2",
		`Improve success/failure logging of LoggingStore.`,

		"1.0.1",
		`Add S3 integration test and include ca-certificates to Docker image so TLS (and S3) works.`,

		"1.0.0",
		`Minor breaking change in that 'hoard init' becomes 'hoard config':
	- 'hoard config' adds some niceties for printing JSON config for --env configuration source
	- Added S3 'remote' credentials provider enabling credentials to be sourced from EC2 instance roles (note since [RemoteCredProvider()](https://github.com/aws/aws-sdk-go/blob/5a2026bfb28e86839f9fcc46523850319399006c/aws/defaults/defaults.go#L108) is used it also support ECS configuration via AWS_CONTAINER_CREDENTIALS_RELATIVE_URI and AWS_CONTAINER_CREDENTIALS_FULL_URI)`,

		"0.1.1",
		`Include hoarctl in Docker image`,

		"0.1.0",
		`Release adding environment config and docker image
	- Adds --env flag to read JSON config from HOARD_JSON_CONFIG
	- Add --json and --toml flags to 'hoard init' to generate JSON optionally
	- Added alpine based docker image pushed on releases (that reads config from environment variable)`,

		"0.0.2",
		`Bug fix release for FileSystemStorage:
	- Switch to URL and filesystem compliant base64 alphabet so some addresses do not target non-existent directories
	- Create root directory for FileSystemStorage if it does not exist`,

		"0.0.1",
		`This is the first Hoard open source release and includes:
	- Deterministic encryption scheme
	- GRPC API for encryption, storage, and cleartext
	- Memory, Filesystem, and S3 storage backends
	- Configuration
	- Hoar-Daemon hoard
	- Hoar-Control hoarctl CLI`,
	)
<commit_msg>release with yaml config<commit_after>package project

import (
	"github.com/monax/relic"
)

// Can be used to set the commit hash version of the binary at build time with:
// `go build -ldflags "-X github.com/monax/hoard/project.commit=$(git rev-parse --short HEAD)" ./cmd/hoard`

var commit = ""
var date = ""

func Commit() string {
	return commit
}

func FullVersion() string {
	version := History.CurrentVersion().String()
	if commit != "" && date != "" {
		return version + "+commit." + commit + "+date." + date
	}
	return version
}

// The releases described by version string and changes, newest release first.
// The current release is taken to be the first release in the slice, and its
// version determines the single authoritative version for the next release.
//
// To cut a new release add a release to the front of this slice then run the
// release tagging script: ./scripts/tag_release.sh
var History relic.ImmutableHistory = relic.NewHistory("Monax Hoard", "https://github.com/monax/hoard").
	MustDeclareReleases("",
		``,
		"3.0.1 - 2019-03-01",
		`### Added
		- [CLI] Optional YAML configuration`,

		"3.0.0 - 2019-02-26",
		`### Changed
- [PROTO] Renamed symmetric grant SecretID to PublicID
- [PROTO] Renamed openpgp grant ID to PrivateID

### Fixed
- [GRANTS] Throw an exception if symmetric secret for ID cannot be found

### Added
- [NODEJS] Added integration tests including test for symmetric secrets
- [GRANTS] Added openpgp grants example
- [CLI] Added ability to configure secrets on command line with hoard config <config> --secret`,
		"2.0.0 - 2019-02-21",
		`### Changed
- [PROTO] Upper case field names in protobuf
- [PROTO] Used gogoproto for types
- [STORE] Go-cloud project for s3 store backend
- [STORE] Minimized IPFS configuration

### Added
- [STORE] Initial support for Azure backend thanks to go-cloud
- [GRANTS] Interface and GRPC service
- [GRANTS] Symmetric AES-GCM-based grants
- [GRANTS] Asymmetric OpenPGP support
- [GRANTS] Go-client (hoarctl) tooling
- [NODEJS] Support for using the grant service + examples
- [STORE] IPFS integration test`,

		"1.1.5 - 2018-10-17",
		`Scripted integration tests, better makefile and ci configs, gcs creds read from env var.`,

		"1.1.4",
		`IPFS & GCP Support`,

		"1.1.3",
		`Just create new hasher each call of addresses - we only use SHA256 and this operation is cheap`,

		"1.1.2",
		`Upgrade all Go dependencies`,

		"1.1.1",
		`Bump docker image Alpine Linux version to 3.8 and Go to 1.10.3`,

		"1.1.0",
		`Fix unsafe concurrent access of hash.Hash function in makeAddresser with sync.Pool`,

		"1.0.2",
		`Improve success/failure logging of LoggingStore.`,

		"1.0.1",
		`Add S3 integration test and include ca-certificates to Docker image so TLS (and S3) works.`,

		"1.0.0",
		`Minor breaking change in that 'hoard init' becomes 'hoard config':
	- 'hoard config' adds some niceties for printing JSON config for --env configuration source
	- Added S3 'remote' credentials provider enabling credentials to be sourced from EC2 instance roles (note since [RemoteCredProvider()](https://github.com/aws/aws-sdk-go/blob/5a2026bfb28e86839f9fcc46523850319399006c/aws/defaults/defaults.go#L108) is used it also support ECS configuration via AWS_CONTAINER_CREDENTIALS_RELATIVE_URI and AWS_CONTAINER_CREDENTIALS_FULL_URI)`,

		"0.1.1",
		`Include hoarctl in Docker image`,

		"0.1.0",
		`Release adding environment config and docker image
	- Adds --env flag to read JSON config from HOARD_JSON_CONFIG
	- Add --json and --toml flags to 'hoard init' to generate JSON optionally
	- Added alpine based docker image pushed on releases (that reads config from environment variable)`,

		"0.0.2",
		`Bug fix release for FileSystemStorage:
	- Switch to URL and filesystem compliant base64 alphabet so some addresses do not target non-existent directories
	- Create root directory for FileSystemStorage if it does not exist`,

		"0.0.1",
		`This is the first Hoard open source release and includes:
	- Deterministic encryption scheme
	- GRPC API for encryption, storage, and cleartext
	- Memory, Filesystem, and S3 storage backends
	- Configuration
	- Hoar-Daemon hoard
	- Hoar-Control hoarctl CLI`,
	)
<|endoftext|>


<commit_before>// Package project implements multi-function operations.
package project

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"text/template"

	"github.com/apex/log"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/lambda/lambdaiface"
	"github.com/tj/go-sync/semaphore"
	"gopkg.in/validator.v2"

	"github.com/apex/apex/function"
	"github.com/apex/apex/hooks"
	"github.com/apex/apex/infra"
	"github.com/apex/apex/utils"
	"github.com/apex/apex/vpc"
)

const (
	// DefaultMemory defines default memory value (MB) for every function in a project
	DefaultMemory = 128

	// DefaultTimeout defines default timeout value (s) for every function in a project
	DefaultTimeout = 3
)

// Config for project.
type Config struct {
	Name             string            `json:"name" validate:"nonzero"`
	Description      string            `json:"description"`
	Runtime          string            `json:"runtime"`
	Memory           int64             `json:"memory"`
	Timeout          int64             `json:"timeout"`
	Role             string            `json:"role"`
	Handler          string            `json:"handler"`
	Shim             bool              `json:"shim"`
	NameTemplate     string            `json:"nameTemplate"`
	RetainedVersions int               `json:"retainedVersions"`
	Environment      map[string]string `json:"environment"`
	Hooks            hooks.Hooks       `json:"hooks"`
	VPC              vpc.VPC           `json:"vpc"`
}

// Project represents zero or more Lambda functions.
type Project struct {
	Config
	Path         string
	Concurrency  int
	Log          log.Interface
	Service      lambdaiface.LambdaAPI
	Functions    []*function.Function
	IgnoreFile   []byte
	nameTemplate *template.Template
}

// defaults applies configuration defaults.
func (p *Project) defaults() {
	p.Memory = DefaultMemory
	p.Timeout = DefaultTimeout
	p.IgnoreFile = []byte(".apexignore\nfunction.json\n")
	p.Role = p.readInfraRole()

	if p.Concurrency == 0 {
		p.Concurrency = 5
	}

	if p.Environment == nil {
		p.Environment = make(map[string]string)
	}

	if p.NameTemplate == "" {
		p.NameTemplate = "{{.Project.Name}}_{{.Function.Name}}"
	}

	if p.RetainedVersions == 0 {
		p.RetainedVersions = 10
	}
}

// Open the project.json file and prime the config.
func (p *Project) Open() error {
	p.defaults()

	f, err := os.Open(filepath.Join(p.Path, "project.json"))
	if err != nil {
		return err
	}

	if err := json.NewDecoder(f).Decode(&p.Config); err != nil {
		return err
	}

	if err := validator.Validate(&p.Config); err != nil {
		return err
	}

	t, err := template.New("nameTemplate").Parse(p.NameTemplate)
	if err != nil {
		return err
	}
	p.nameTemplate = t

	ignoreFile, err := utils.ReadIgnoreFile(p.Path)
	if err != nil {
		return err
	}
	p.IgnoreFile = append(p.IgnoreFile, ignoreFile...)

	return nil
}

// LoadFunctions reads the ./functions directory, populating the Functions field.
// If no `names` are specified, all functions are loaded.
func (p *Project) LoadFunctions(names ...string) error {
	dir := filepath.Join(p.Path, functionsDir)
	p.Log.Debugf("loading functions in %s", dir)

	existing, err := p.FunctionDirNames()
	if err != nil {
		return err
	}

	if len(names) == 0 {
		names = existing
	}

	for _, name := range names {
		if !utils.ContainsString(existing, name) {
			p.Log.Warnf("function %q does not exist in project", name)
			continue
		}

		fn, err := p.loadFunction(name)
		if err != nil {
			return err
		}

		p.Functions = append(p.Functions, fn)
	}

	if len(p.Functions) == 0 {
		return errors.New("no function loaded")
	}

	return nil
}

// DeployAndClean deploys functions and then cleans up their build artifacts.
func (p *Project) DeployAndClean() error {
	if err := p.Deploy(); err != nil {
		return err
	}

	return p.Clean()
}

// Deploy functions and their configurations.
func (p *Project) Deploy() error {
	p.Log.Debugf("deploying %d functions", len(p.Functions))

	sem := make(semaphore.Semaphore, p.Concurrency)
	errs := make(chan error)

	go func() {
		for _, fn := range p.Functions {
			fn := fn
			sem.Acquire()

			go func() {
				defer sem.Release()

				err := fn.Deploy()
				if err != nil {
					err = fmt.Errorf("function %s: %s", fn.Name, err)
				}

				errs <- err
			}()
		}

		sem.Wait()
		close(errs)
	}()

	for err := range errs {
		if err != nil {
			return err
		}
	}

	return nil
}

// Clean up function build artifacts.
func (p *Project) Clean() error {
	p.Log.Debugf("cleaning %d functions", len(p.Functions))

	for _, fn := range p.Functions {
		if err := fn.Clean(); err != nil {
			return fmt.Errorf("function %s: %s", fn.Name, err)
		}
	}

	return nil
}

// Delete functions.
func (p *Project) Delete() error {
	p.Log.Debugf("deleting %d functions", len(p.Functions))

	for _, fn := range p.Functions {
		if _, err := fn.GetConfig(); err != nil {
			if awserr, ok := err.(awserr.Error); ok && awserr.Code() == "ResourceNotFoundException" {
				p.Log.Infof("function %q hasn't been deployed yet or has been deleted manually on AWS Lambda", fn.Name)
				continue
			}
			return fmt.Errorf("function %s: %s", fn.Name, err)
		}

		if err := fn.Delete(); err != nil {
			return fmt.Errorf("function %s: %s", fn.Name, err)
		}
	}

	return nil
}

// FunctionDirNames returns a list of function directory names.
func (p *Project) FunctionDirNames() (list []string, err error) {
	dir := filepath.Join(p.Path, functionsDir)

	files, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}

	for _, file := range files {
		if file.IsDir() {
			list = append(list, file.Name())
		}
	}

	return list, nil
}

// Setenv sets environment variable `name` to `value` on every function in project.
func (p *Project) Setenv(name, value string) {
	for _, fn := range p.Functions {
		fn.Setenv(name, value)
	}
}

// loadFunction returns the function in the ./functions/<name> directory.
func (p *Project) loadFunction(name string) (*function.Function, error) {
	return p.LoadFunctionByPath(name, filepath.Join(p.Path, functionsDir, name))
}

// LoadFunctionByPath returns the function in the given directory.
func (p *Project) LoadFunctionByPath(name, path string) (*function.Function, error) {
	dir := filepath.Join(p.Path, functionsDir, name)
	p.Log.Debugf("loading function in %s", dir)

	fn := &function.Function{
		Config: function.Config{
			Runtime:          p.Runtime,
			Memory:           p.Memory,
			Timeout:          p.Timeout,
			Role:             p.Role,
			Handler:          p.Handler,
			Shim:             p.Shim,
			Hooks:            p.Hooks,
			Environment:      copyStringMap(p.Environment),
			RetainedVersions: p.RetainedVersions,
			VPC:              p.VPC,
		},
		Name:       name,
		Path:       dir,
		Service:    p.Service,
		Log:        p.Log,
		IgnoreFile: p.IgnoreFile,
	}

	if name, err := p.name(fn); err == nil {
		fn.FunctionName = name
	} else {
		return nil, err
	}

	if err := fn.Open(); err != nil {
		return nil, err
	}

	return fn, nil
}

// name returns the computed name for `fn`, using the nameTemplate.
func (p *Project) name(fn *function.Function) (string, error) {
	data := struct {
		Project  *Project
		Function *function.Function
	}{
		Project:  p,
		Function: fn,
	}

	name, err := render(p.nameTemplate, data)
	if err != nil {
		return "", err
	}

	return name, nil
}

// readInfraRole reads lambda function IAM role from infrastructure
func (p *Project) readInfraRole() string {
	role, err := infra.ReadRole()
	if err != nil {
		p.Log.Debugf("couldn't read role from infrastructure: %s", err)
		return ""
	}
	return role
}

const functionsDir = "functions"

// render returns a string by executing template `t` against the given value `v`.
func render(t *template.Template, v interface{}) (string, error) {
	buf := new(bytes.Buffer)

	if err := t.Execute(buf, v); err != nil {
		return "", err
	}

	return buf.String(), nil
}

// copyStringMap returns a copy of `in`.
func copyStringMap(in map[string]string) map[string]string {
	out := make(map[string]string)
	for k, v := range in {
		out[k] = v
	}
	return out
}
<commit_msg>add Project.LoadFunction(name)<commit_after>// Package project implements multi-function operations.
package project

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"text/template"

	"github.com/apex/log"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/lambda/lambdaiface"
	"github.com/tj/go-sync/semaphore"
	"gopkg.in/validator.v2"

	"github.com/apex/apex/function"
	"github.com/apex/apex/hooks"
	"github.com/apex/apex/infra"
	"github.com/apex/apex/utils"
	"github.com/apex/apex/vpc"
)

const (
	// DefaultMemory defines default memory value (MB) for every function in a project
	DefaultMemory = 128

	// DefaultTimeout defines default timeout value (s) for every function in a project
	DefaultTimeout = 3
)

// Config for project.
type Config struct {
	Name             string            `json:"name" validate:"nonzero"`
	Description      string            `json:"description"`
	Runtime          string            `json:"runtime"`
	Memory           int64             `json:"memory"`
	Timeout          int64             `json:"timeout"`
	Role             string            `json:"role"`
	Handler          string            `json:"handler"`
	Shim             bool              `json:"shim"`
	NameTemplate     string            `json:"nameTemplate"`
	RetainedVersions int               `json:"retainedVersions"`
	Environment      map[string]string `json:"environment"`
	Hooks            hooks.Hooks       `json:"hooks"`
	VPC              vpc.VPC           `json:"vpc"`
}

// Project represents zero or more Lambda functions.
type Project struct {
	Config
	Path         string
	Concurrency  int
	Log          log.Interface
	Service      lambdaiface.LambdaAPI
	Functions    []*function.Function
	IgnoreFile   []byte
	nameTemplate *template.Template
}

// defaults applies configuration defaults.
func (p *Project) defaults() {
	p.Memory = DefaultMemory
	p.Timeout = DefaultTimeout
	p.IgnoreFile = []byte(".apexignore\nfunction.json\n")
	p.Role = p.readInfraRole()

	if p.Concurrency == 0 {
		p.Concurrency = 5
	}

	if p.Environment == nil {
		p.Environment = make(map[string]string)
	}

	if p.NameTemplate == "" {
		p.NameTemplate = "{{.Project.Name}}_{{.Function.Name}}"
	}

	if p.RetainedVersions == 0 {
		p.RetainedVersions = 10
	}
}

// Open the project.json file and prime the config.
func (p *Project) Open() error {
	p.defaults()

	f, err := os.Open(filepath.Join(p.Path, "project.json"))
	if err != nil {
		return err
	}

	if err := json.NewDecoder(f).Decode(&p.Config); err != nil {
		return err
	}

	if err := validator.Validate(&p.Config); err != nil {
		return err
	}

	t, err := template.New("nameTemplate").Parse(p.NameTemplate)
	if err != nil {
		return err
	}
	p.nameTemplate = t

	ignoreFile, err := utils.ReadIgnoreFile(p.Path)
	if err != nil {
		return err
	}
	p.IgnoreFile = append(p.IgnoreFile, ignoreFile...)

	return nil
}

// LoadFunctions reads the ./functions directory, populating the Functions field.
// If no `names` are specified, all functions are loaded.
func (p *Project) LoadFunctions(names ...string) error {
	dir := filepath.Join(p.Path, functionsDir)
	p.Log.Debugf("loading functions in %s", dir)

	existing, err := p.FunctionDirNames()
	if err != nil {
		return err
	}

	if len(names) == 0 {
		names = existing
	}

	for _, name := range names {
		if !utils.ContainsString(existing, name) {
			p.Log.Warnf("function %q does not exist in project", name)
			continue
		}

		fn, err := p.LoadFunction(name)
		if err != nil {
			return err
		}

		p.Functions = append(p.Functions, fn)
	}

	if len(p.Functions) == 0 {
		return errors.New("no function loaded")
	}

	return nil
}

// DeployAndClean deploys functions and then cleans up their build artifacts.
func (p *Project) DeployAndClean() error {
	if err := p.Deploy(); err != nil {
		return err
	}

	return p.Clean()
}

// Deploy functions and their configurations.
func (p *Project) Deploy() error {
	p.Log.Debugf("deploying %d functions", len(p.Functions))

	sem := make(semaphore.Semaphore, p.Concurrency)
	errs := make(chan error)

	go func() {
		for _, fn := range p.Functions {
			fn := fn
			sem.Acquire()

			go func() {
				defer sem.Release()

				err := fn.Deploy()
				if err != nil {
					err = fmt.Errorf("function %s: %s", fn.Name, err)
				}

				errs <- err
			}()
		}

		sem.Wait()
		close(errs)
	}()

	for err := range errs {
		if err != nil {
			return err
		}
	}

	return nil
}

// Clean up function build artifacts.
func (p *Project) Clean() error {
	p.Log.Debugf("cleaning %d functions", len(p.Functions))

	for _, fn := range p.Functions {
		if err := fn.Clean(); err != nil {
			return fmt.Errorf("function %s: %s", fn.Name, err)
		}
	}

	return nil
}

// Delete functions.
func (p *Project) Delete() error {
	p.Log.Debugf("deleting %d functions", len(p.Functions))

	for _, fn := range p.Functions {
		if _, err := fn.GetConfig(); err != nil {
			if awserr, ok := err.(awserr.Error); ok && awserr.Code() == "ResourceNotFoundException" {
				p.Log.Infof("function %q hasn't been deployed yet or has been deleted manually on AWS Lambda", fn.Name)
				continue
			}
			return fmt.Errorf("function %s: %s", fn.Name, err)
		}

		if err := fn.Delete(); err != nil {
			return fmt.Errorf("function %s: %s", fn.Name, err)
		}
	}

	return nil
}

// FunctionDirNames returns a list of function directory names.
func (p *Project) FunctionDirNames() (list []string, err error) {
	dir := filepath.Join(p.Path, functionsDir)

	files, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}

	for _, file := range files {
		if file.IsDir() {
			list = append(list, file.Name())
		}
	}

	return list, nil
}

// Setenv sets environment variable `name` to `value` on every function in project.
func (p *Project) Setenv(name, value string) {
	for _, fn := range p.Functions {
		fn.Setenv(name, value)
	}
}

// LoadFunction returns the function in the ./functions/<name> directory.
func (p *Project) LoadFunction(name string) (*function.Function, error) {
	return p.LoadFunctionByPath(name, filepath.Join(p.Path, functionsDir, name))
}

// LoadFunctionByPath returns the function in the given directory.
func (p *Project) LoadFunctionByPath(name, path string) (*function.Function, error) {
	dir := filepath.Join(p.Path, functionsDir, name)
	p.Log.Debugf("loading function in %s", dir)

	fn := &function.Function{
		Config: function.Config{
			Runtime:          p.Runtime,
			Memory:           p.Memory,
			Timeout:          p.Timeout,
			Role:             p.Role,
			Handler:          p.Handler,
			Shim:             p.Shim,
			Hooks:            p.Hooks,
			Environment:      copyStringMap(p.Environment),
			RetainedVersions: p.RetainedVersions,
			VPC:              p.VPC,
		},
		Name:       name,
		Path:       dir,
		Service:    p.Service,
		Log:        p.Log,
		IgnoreFile: p.IgnoreFile,
	}

	if name, err := p.name(fn); err == nil {
		fn.FunctionName = name
	} else {
		return nil, err
	}

	if err := fn.Open(); err != nil {
		return nil, err
	}

	return fn, nil
}

// name returns the computed name for `fn`, using the nameTemplate.
func (p *Project) name(fn *function.Function) (string, error) {
	data := struct {
		Project  *Project
		Function *function.Function
	}{
		Project:  p,
		Function: fn,
	}

	name, err := render(p.nameTemplate, data)
	if err != nil {
		return "", err
	}

	return name, nil
}

// readInfraRole reads lambda function IAM role from infrastructure
func (p *Project) readInfraRole() string {
	role, err := infra.ReadRole()
	if err != nil {
		p.Log.Debugf("couldn't read role from infrastructure: %s", err)
		return ""
	}
	return role
}

const functionsDir = "functions"

// render returns a string by executing template `t` against the given value `v`.
func render(t *template.Template, v interface{}) (string, error) {
	buf := new(bytes.Buffer)

	if err := t.Execute(buf, v); err != nil {
		return "", err
	}

	return buf.String(), nil
}

// copyStringMap returns a copy of `in`.
func copyStringMap(in map[string]string) map[string]string {
	out := make(map[string]string)
	for k, v := range in {
		out[k] = v
	}
	return out
}
<|endoftext|>


<commit_before>package fabric

import (
	"context"
	"errors"
	"strings"

	"go.uber.org/zap"
)

var (
	// ErrNoSuchRoute when requests route does not exist
	ErrNoSuchRoute = errors.New("No such route")
	// ErrInvalidCommand when our router doesn't know about this command
	ErrInvalidCommand = errors.New("Invalid command")
)

// RouterProtocol is the selector protocol
type RouterProtocol struct {
	routes map[string][]Protocol
}

// NewRouter returns a new router protocol
func NewRouter() *RouterProtocol {
	return &RouterProtocol{
		routes: map[string][]Protocol{},
	}
}

// Name of the protocol
func (m *RouterProtocol) Name() string {
	return "router"
}

// Handle is the protocol handler for the server
func (m *RouterProtocol) Handle(fn HandlerFunc) HandlerFunc {
	// one time scope setup area for middleware
	return func(ctx context.Context, c Conn) error {
		addr := c.GetAddress()
		lgr := Logger(ctx).With(
			zap.Namespace("protocol:router"),
			zap.String("addr.current", addr.Current()),
			zap.String("addr.params", addr.CurrentParams()),
		)
		lgr.Debug("Reading token")

		// we need to negotiate what they need from us
		// read the next token, which is the request for the next protocol
		pr, err := c.ReadToken()
		if err != nil {
			return err
		}
		lgr.Debug("Read token", zap.String("pr", string(pr)))

		pf := strings.Split(string(pr), " ")
		if len(pf) != 2 {
			return ErrInvalidCommand
		}

		cm := pf[0]
		pm := pf[1]

		switch cm {
		case "SEL":
			lgr.Debug("Handling SEL", zap.String("cm", cm), zap.String("pm", pm))
			return m.handleGet(ctx, c, pm)
		default:
			lgr.Debug("Invalid command", zap.String("cm", cm), zap.String("pm", pm))
			c.Close()
			return ErrInvalidCommand
		}
	}
}

func (m *RouterProtocol) handleGet(ctx context.Context, c Conn, remainingAddrString string) error {
	remainingAddr := strings.Split(remainingAddrString, "/")

	validRoute := ""
	for route := range m.routes {
		if strings.HasPrefix(remainingAddrString, route) {
			validRoute = route
			break
		}
	}

	if validRoute == "" {
		return ErrNoSuchRoute
	}

	// TODO not sure about append, might wanna cut the stack up to our index
	// and the append the new stack
	addr := c.GetAddress()
	addr.stack = append(addr.stack, remainingAddr...)

	if err := c.WriteToken([]byte("ACK " + remainingAddrString)); err != nil {
		return err
	}

	chain := handlerChain(m.routes[validRoute]...)
	return chain(ctx, c)
}

// Negotiate handles the client's side of the nimona protocol
func (m *RouterProtocol) Negotiate(fn NegotiatorFunc) NegotiatorFunc {
	// one time scope setup area for middleware
	return func(ctx context.Context, c Conn) error {
		c.GetAddress().Pop()
		pr := c.GetAddress().RemainingString()
		if err := c.WriteToken([]byte("SEL " + pr)); err != nil {
			return err
		}

		resp, err := c.ReadToken()
		if err != nil {
			return err
		}

		if string(resp) != "ACK "+pr {
			return errors.New("Invalid selector response")
		}

		return fn(ctx, c)
	}
}

// AddRoute adds an allowed route made up of protocols
func (m *RouterProtocol) AddRoute(protocols ...Protocol) error {
	protocolNames := []string{}
	for _, protocol := range protocols {
		protocolNames = append(protocolNames, protocol.Name())
	}
	routeName := strings.Join(protocolNames, "/")
	m.routes[routeName] = protocols
	return nil
}
<commit_msg>Add missing pop in router handler<commit_after>package fabric

import (
	"context"
	"errors"
	"strings"

	"go.uber.org/zap"
)

var (
	// ErrNoSuchRoute when requests route does not exist
	ErrNoSuchRoute = errors.New("No such route")
	// ErrInvalidCommand when our router doesn't know about this command
	ErrInvalidCommand = errors.New("Invalid command")
)

// RouterProtocol is the selector protocol
type RouterProtocol struct {
	routes map[string][]Protocol
}

// NewRouter returns a new router protocol
func NewRouter() *RouterProtocol {
	return &RouterProtocol{
		routes: map[string][]Protocol{},
	}
}

// Name of the protocol
func (m *RouterProtocol) Name() string {
	return "router"
}

// Handle is the protocol handler for the server
func (m *RouterProtocol) Handle(fn HandlerFunc) HandlerFunc {
	// one time scope setup area for middleware
	return func(ctx context.Context, c Conn) error {
		addr := c.GetAddress()
		lgr := Logger(ctx).With(
			zap.Namespace("protocol:router"),
			zap.String("addr.current", addr.Current()),
			zap.String("addr.params", addr.CurrentParams()),
		)
		lgr.Debug("Reading token")

		// we need to negotiate what they need from us
		// read the next token, which is the request for the next protocol
		pr, err := c.ReadToken()
		if err != nil {
			return err
		}
		lgr.Debug("Read token", zap.String("pr", string(pr)))

		pf := strings.Split(string(pr), " ")
		if len(pf) != 2 {
			return ErrInvalidCommand
		}

		cm := pf[0]
		pm := pf[1]

		c.GetAddress().Pop()
		switch cm {
		case "SEL":
			lgr.Debug("Handling SEL", zap.String("cm", cm), zap.String("pm", pm))
			return m.handleGet(ctx, c, pm)
		default:
			lgr.Debug("Invalid command", zap.String("cm", cm), zap.String("pm", pm))
			c.Close()
			return ErrInvalidCommand
		}
	}
}

func (m *RouterProtocol) handleGet(ctx context.Context, c Conn, remainingAddrString string) error {
	remainingAddr := strings.Split(remainingAddrString, "/")

	validRoute := ""
	for route := range m.routes {
		if strings.HasPrefix(remainingAddrString, route) {
			validRoute = route
			break
		}
	}

	if validRoute == "" {
		return ErrNoSuchRoute
	}

	// TODO not sure about append, might wanna cut the stack up to our index
	// and the append the new stack
	addr := c.GetAddress()
	addr.stack = append(addr.stack, remainingAddr...)

	if err := c.WriteToken([]byte("ACK " + remainingAddrString)); err != nil {
		return err
	}

	chain := handlerChain(m.routes[validRoute]...)
	return chain(ctx, c)
}

// Negotiate handles the client's side of the nimona protocol
func (m *RouterProtocol) Negotiate(fn NegotiatorFunc) NegotiatorFunc {
	// one time scope setup area for middleware
	return func(ctx context.Context, c Conn) error {
		c.GetAddress().Pop()
		pr := c.GetAddress().RemainingString()
		if err := c.WriteToken([]byte("SEL " + pr)); err != nil {
			return err
		}

		resp, err := c.ReadToken()
		if err != nil {
			return err
		}

		if string(resp) != "ACK "+pr {
			return errors.New("Invalid selector response")
		}

		return fn(ctx, c)
	}
}

// AddRoute adds an allowed route made up of protocols
func (m *RouterProtocol) AddRoute(protocols ...Protocol) error {
	protocolNames := []string{}
	for _, protocol := range protocols {
		protocolNames = append(protocolNames, protocol.Name())
	}
	routeName := strings.Join(protocolNames, "/")
	m.routes[routeName] = protocols
	return nil
}
<|endoftext|>


<commit_before>package iamy

import (
	"log"
	"regexp"
	"sync"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/iam"
	"github.com/pkg/errors"
)

var cfnResourceRegexp = regexp.MustCompile(`-[A-Z0-9]{10,20}$`)

// AwsFetcher fetches account data from AWS
type AwsFetcher struct {
	// As Policy and Role descriptions are immutable, we can skip fetching them
	// when pushing to AWS
	SkipFetchingPolicyAndRoleDescriptions bool

	Debug *log.Logger

	iam     *iamClient
	s3      *s3Client
	account *Account
	data    AccountData

	descriptionFetchWaitGroup sync.WaitGroup
	descriptionFetchError     error
}

func (a *AwsFetcher) init() error {
	var err error

	s := awsSession()
	a.iam = newIamClient(s)
	a.s3 = newS3Client(s)
	if a.account, err = a.getAccount(); err != nil {
		return err
	}
	a.data = AccountData{
		Account: a.account,
	}

	return nil
}

// Fetch queries AWS for account data
func (a *AwsFetcher) Fetch() (*AccountData, error) {
	if err := a.init(); err != nil {
		return nil, errors.Wrap(err, "Error in init")
	}

	var wg sync.WaitGroup
	var iamErr, s3Err error

	log.Println("Fetching IAM data")
	wg.Add(1)
	go func() {
		defer wg.Done()
		iamErr = a.fetchIamData()
	}()

	log.Println("Fetching S3 data")
	wg.Add(1)
	go func() {
		defer wg.Done()
		s3Err = a.fetchS3Data()
	}()

	wg.Wait()

	if iamErr != nil {
		return nil, errors.Wrap(iamErr, "Error fetching IAM data")
	}
	if s3Err != nil {
		return nil, errors.Wrap(s3Err, "Error fetching S3 data")
	}

	return &a.data, nil
}

func (a *AwsFetcher) fetchS3Data() error {
	buckets, err := a.s3.listAllBuckets()
	if err != nil {
		return errors.Wrap(err, "Error listing buckets")
	}
	for _, b := range buckets {
		if b.policyJson == "" {
			continue
		}

		policyDoc, err := NewPolicyDocumentFromEncodedJson(b.policyJson)
		if err != nil {
			return errors.Wrap(err, "Error creating Policy document")
		}

		bp := BucketPolicy{
			BucketName: b.name,
			Policy:     policyDoc,
		}

		a.data.BucketPolicies = append(a.data.BucketPolicies, &bp)
	}

	return nil
}
func (a *AwsFetcher) fetchIamData() error {
	var populateIamDataErr error
	var populateInstanceProfileErr error
	err := a.iam.GetAccountAuthorizationDetailsPages(
		&iam.GetAccountAuthorizationDetailsInput{
			Filter: aws.StringSlice([]string{
				iam.EntityTypeUser,
				iam.EntityTypeGroup,
				iam.EntityTypeRole,
				iam.EntityTypeLocalManagedPolicy,
			}),
		},
		func(resp *iam.GetAccountAuthorizationDetailsOutput, lastPage bool) bool {
			populateIamDataErr = a.populateIamData(resp)
			if populateIamDataErr != nil {
				return false
			}
			return true
		},
	)
	if populateIamDataErr != nil {
		return err
	}
	if err != nil {
		return err
	}
	// Fetch instance profiles
	err = a.iam.ListInstanceProfilesPages(&iam.ListInstanceProfilesInput{},
		func(resp *iam.ListInstanceProfilesOutput, lastPage bool) bool {
			populateInstanceProfileErr = a.populateInstanceProfileData(resp)
			if populateInstanceProfileErr != nil {
				return false
			}
			return true
		})
	if populateInstanceProfileErr != nil {
		return err
	}
	if err != nil {
		return err
	}
	return nil
}

func (a *AwsFetcher) populateInlinePolicies(source []*iam.PolicyDetail, target *[]InlinePolicy) error {
	for _, ip := range source {
		doc, err := NewPolicyDocumentFromEncodedJson(*ip.PolicyDocument)
		if err != nil {
			return err
		}
		*target = append(*target, InlinePolicy{
			Name:   *ip.PolicyName,
			Policy: doc,
		})
	}

	return nil
}

func (a *AwsFetcher) marshalPolicyDescriptionAsync(policyArn string, target *string) {
	a.descriptionFetchWaitGroup.Add(1)
	go func() {
		defer a.descriptionFetchWaitGroup.Done()
		log.Println("Fetching policy description for", policyArn)

		var err error
		*target, err = a.iam.getPolicyDescription(policyArn)
		if err != nil {
			a.descriptionFetchError = err
		}
	}()
}

func (a *AwsFetcher) marshalRoleDescriptionAsync(roleName string, target *string) {
	a.descriptionFetchWaitGroup.Add(1)
	go func() {
		defer a.descriptionFetchWaitGroup.Done()
		log.Println("Fetching role description for", roleName)

		var err error
		*target, err = a.iam.getRoleDescription(roleName)
		if err != nil {
			a.descriptionFetchError = err
		}
	}()
}

func (a *AwsFetcher) populateInstanceProfileData(resp *iam.ListInstanceProfilesOutput) error {
	for _, profileResp := range resp.InstanceProfiles {
		if cfnResourceRegexp.MatchString(*profileResp.InstanceProfileName) {
			log.Printf("Skipping CloudFormation generated instance profile %s", *profileResp.InstanceProfileName)
			continue
		}
		profile := InstanceProfile{iamService: iamService{
			Name: *profileResp.InstanceProfileName,
			Path: *profileResp.Path,
		}}
		for _, roleResp := range profileResp.Roles {
			role := *(roleResp.RoleName)
			profile.Roles = append(profile.Roles, role)
		}
		a.data.InstanceProfiles = append(a.data.InstanceProfiles, &profile)
	}
	return nil
}

func (a *AwsFetcher) populateIamData(resp *iam.GetAccountAuthorizationDetailsOutput) error {
	for _, userResp := range resp.UserDetailList {
		if cfnResourceRegexp.MatchString(*userResp.UserName) {
			log.Printf("Skipping CloudFormation generated user %s", *userResp.UserName)
			continue
		}

		user := User{iamService: iamService{
			Name: *userResp.UserName,
			Path: *userResp.Path,
		}}

		for _, g := range userResp.GroupList {
			user.Groups = append(user.Groups, *g)
		}
		for _, p := range userResp.AttachedManagedPolicies {
			user.Policies = append(user.Policies, a.account.normalisePolicyArn(*p.PolicyArn))
		}
		if err := a.populateInlinePolicies(userResp.UserPolicyList, &user.InlinePolicies); err != nil {
			return err
		}

		a.data.Users = append(a.data.Users, &user)
	}

	for _, groupResp := range resp.GroupDetailList {
		if cfnResourceRegexp.MatchString(*groupResp.GroupName) {
			log.Printf("Skipping CloudFormation generated group %s", *groupResp.GroupName)
			continue
		}

		group := Group{iamService: iamService{
			Name: *groupResp.GroupName,
			Path: *groupResp.Path,
		}}

		for _, p := range groupResp.AttachedManagedPolicies {
			group.Policies = append(group.Policies, a.account.normalisePolicyArn(*p.PolicyArn))
		}
		if err := a.populateInlinePolicies(groupResp.GroupPolicyList, &group.InlinePolicies); err != nil {
			return err
		}

		a.data.Groups = append(a.data.Groups, &group)
	}

	for _, roleResp := range resp.RoleDetailList {
		if cfnResourceRegexp.MatchString(*roleResp.RoleName) {
			log.Printf("Skipping CloudFormation generated role %s", *roleResp.RoleName)
			continue
		}

		role := Role{iamService: iamService{
			Name: *roleResp.RoleName,
			Path: *roleResp.Path,
		}}

		if !a.SkipFetchingPolicyAndRoleDescriptions {
			a.marshalRoleDescriptionAsync(*roleResp.RoleName, &role.Description)
		}

		var err error
		role.AssumeRolePolicyDocument, err = NewPolicyDocumentFromEncodedJson(*roleResp.AssumeRolePolicyDocument)
		if err != nil {
			return err
		}
		for _, p := range roleResp.AttachedManagedPolicies {
			role.Policies = append(role.Policies, a.account.normalisePolicyArn(*p.PolicyArn))
		}
		if err := a.populateInlinePolicies(roleResp.RolePolicyList, &role.InlinePolicies); err != nil {
			return err
		}

		a.data.addRole(&role)
	}

	for _, policyResp := range resp.Policies {
		if cfnResourceRegexp.MatchString(*policyResp.PolicyName) {
			log.Printf("Skipping CloudFormation generated policy %s", *policyResp.PolicyName)
			continue
		}

		defaultPolicyVersion := findDefaultPolicyVersion(policyResp.PolicyVersionList)
		doc, err := NewPolicyDocumentFromEncodedJson(*defaultPolicyVersion.Document)
		if err != nil {
			return err
		}

		p := Policy{
			iamService: iamService{
				Name: *policyResp.PolicyName,
				Path: *policyResp.Path,
			},
			oldestVersionId:      findOldestPolicyVersionId(policyResp.PolicyVersionList),
			numberOfVersions:     len(policyResp.PolicyVersionList),
			nondefaultVersionIds: findNonDefaultPolicyVersionIds(policyResp.PolicyVersionList),
			Policy:               doc,
		}

		if !a.SkipFetchingPolicyAndRoleDescriptions {
			a.marshalPolicyDescriptionAsync(*policyResp.Arn, &p.Description)
		}

		a.data.addPolicy(&p)
	}

	a.descriptionFetchWaitGroup.Wait()

	return a.descriptionFetchError
}

func findDefaultPolicyVersion(versions []*iam.PolicyVersion) *iam.PolicyVersion {
	for _, version := range versions {
		if *version.IsDefaultVersion {
			return version
		}
	}
	panic("Expected a default policy version")
}

func findNonDefaultPolicyVersionIds(versions []*iam.PolicyVersion) []string {
	ss := []string{}
	for _, version := range versions {
		if !*version.IsDefaultVersion {
			ss = append(ss, *version.VersionId)
		}
	}
	return ss
}

func findOldestPolicyVersionId(versions []*iam.PolicyVersion) string {
	oldest := versions[0]
	for _, version := range versions[1:] {
		if version.CreateDate.Before(*oldest.CreateDate) {
			oldest = version
		}
	}
	return *oldest.VersionId
}

func (a *AwsFetcher) getAccount() (*Account, error) {
	var err error
	acct := Account{}

	acct.Id, err = GetAwsAccountId(awsSession(), a.Debug)
	if err == aws.ErrMissingRegion {
		return nil, errors.New("Error determining the AWS account id - check the AWS_REGION environment variable is set")
	}
	if err != nil {
		return nil, err
	}

	aliasResp, err := a.iam.ListAccountAliases(&iam.ListAccountAliasesInput{})
	if err != nil {
		return nil, err
	}
	if len(aliasResp.AccountAliases) > 0 {
		acct.Alias = *aliasResp.AccountAliases[0]
	}

	return &acct, nil
}
<commit_msg>Consolidate skip matching in `isSkippableManagedResource`<commit_after>package iamy

import (
	"fmt"
	"log"
	"regexp"
	"sync"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/iam"
	"github.com/pkg/errors"
)

var cfnResourceRegexp = regexp.MustCompile(`-[A-Z0-9]{10,20}$`)

// AwsFetcher fetches account data from AWS
type AwsFetcher struct {
	// As Policy and Role descriptions are immutable, we can skip fetching them
	// when pushing to AWS
	SkipFetchingPolicyAndRoleDescriptions bool

	Debug *log.Logger

	iam     *iamClient
	s3      *s3Client
	account *Account
	data    AccountData

	descriptionFetchWaitGroup sync.WaitGroup
	descriptionFetchError     error
}

func (a *AwsFetcher) init() error {
	var err error

	s := awsSession()
	a.iam = newIamClient(s)
	a.s3 = newS3Client(s)
	if a.account, err = a.getAccount(); err != nil {
		return err
	}
	a.data = AccountData{
		Account: a.account,
	}

	return nil
}

// Fetch queries AWS for account data
func (a *AwsFetcher) Fetch() (*AccountData, error) {
	if err := a.init(); err != nil {
		return nil, errors.Wrap(err, "Error in init")
	}

	var wg sync.WaitGroup
	var iamErr, s3Err error

	log.Println("Fetching IAM data")
	wg.Add(1)
	go func() {
		defer wg.Done()
		iamErr = a.fetchIamData()
	}()

	log.Println("Fetching S3 data")
	wg.Add(1)
	go func() {
		defer wg.Done()
		s3Err = a.fetchS3Data()
	}()

	wg.Wait()

	if iamErr != nil {
		return nil, errors.Wrap(iamErr, "Error fetching IAM data")
	}
	if s3Err != nil {
		return nil, errors.Wrap(s3Err, "Error fetching S3 data")
	}

	return &a.data, nil
}

func (a *AwsFetcher) fetchS3Data() error {
	buckets, err := a.s3.listAllBuckets()
	if err != nil {
		return errors.Wrap(err, "Error listing buckets")
	}
	for _, b := range buckets {
		if b.policyJson == "" {
			continue
		}

		policyDoc, err := NewPolicyDocumentFromEncodedJson(b.policyJson)
		if err != nil {
			return errors.Wrap(err, "Error creating Policy document")
		}

		bp := BucketPolicy{
			BucketName: b.name,
			Policy:     policyDoc,
		}

		a.data.BucketPolicies = append(a.data.BucketPolicies, &bp)
	}

	return nil
}
func (a *AwsFetcher) fetchIamData() error {
	var populateIamDataErr error
	var populateInstanceProfileErr error
	err := a.iam.GetAccountAuthorizationDetailsPages(
		&iam.GetAccountAuthorizationDetailsInput{
			Filter: aws.StringSlice([]string{
				iam.EntityTypeUser,
				iam.EntityTypeGroup,
				iam.EntityTypeRole,
				iam.EntityTypeLocalManagedPolicy,
			}),
		},
		func(resp *iam.GetAccountAuthorizationDetailsOutput, lastPage bool) bool {
			populateIamDataErr = a.populateIamData(resp)
			if populateIamDataErr != nil {
				return false
			}
			return true
		},
	)
	if populateIamDataErr != nil {
		return err
	}
	if err != nil {
		return err
	}
	// Fetch instance profiles
	err = a.iam.ListInstanceProfilesPages(&iam.ListInstanceProfilesInput{},
		func(resp *iam.ListInstanceProfilesOutput, lastPage bool) bool {
			populateInstanceProfileErr = a.populateInstanceProfileData(resp)
			if populateInstanceProfileErr != nil {
				return false
			}
			return true
		})
	if populateInstanceProfileErr != nil {
		return err
	}
	if err != nil {
		return err
	}
	return nil
}

func (a *AwsFetcher) populateInlinePolicies(source []*iam.PolicyDetail, target *[]InlinePolicy) error {
	for _, ip := range source {
		doc, err := NewPolicyDocumentFromEncodedJson(*ip.PolicyDocument)
		if err != nil {
			return err
		}
		*target = append(*target, InlinePolicy{
			Name:   *ip.PolicyName,
			Policy: doc,
		})
	}

	return nil
}

func (a *AwsFetcher) marshalPolicyDescriptionAsync(policyArn string, target *string) {
	a.descriptionFetchWaitGroup.Add(1)
	go func() {
		defer a.descriptionFetchWaitGroup.Done()
		log.Println("Fetching policy description for", policyArn)

		var err error
		*target, err = a.iam.getPolicyDescription(policyArn)
		if err != nil {
			a.descriptionFetchError = err
		}
	}()
}

func (a *AwsFetcher) marshalRoleDescriptionAsync(roleName string, target *string) {
	a.descriptionFetchWaitGroup.Add(1)
	go func() {
		defer a.descriptionFetchWaitGroup.Done()
		log.Println("Fetching role description for", roleName)

		var err error
		*target, err = a.iam.getRoleDescription(roleName)
		if err != nil {
			a.descriptionFetchError = err
		}
	}()
}

func (a *AwsFetcher) populateInstanceProfileData(resp *iam.ListInstanceProfilesOutput) error {
	for _, profileResp := range resp.InstanceProfiles {
		if ok, err := isSkippableManagedResource(*profileResp.InstanceProfileName); ok {
			log.Printf(err)
			continue
		}

		profile := InstanceProfile{iamService: iamService{
			Name: *profileResp.InstanceProfileName,
			Path: *profileResp.Path,
		}}
		for _, roleResp := range profileResp.Roles {
			role := *(roleResp.RoleName)
			profile.Roles = append(profile.Roles, role)
		}
		a.data.InstanceProfiles = append(a.data.InstanceProfiles, &profile)
	}
	return nil
}

func (a *AwsFetcher) populateIamData(resp *iam.GetAccountAuthorizationDetailsOutput) error {
	for _, userResp := range resp.UserDetailList {
		if ok, err := isSkippableManagedResource(*userResp.UserName); ok {
			log.Printf(err)
			continue
		}

		user := User{iamService: iamService{
			Name: *userResp.UserName,
			Path: *userResp.Path,
		}}

		for _, g := range userResp.GroupList {
			user.Groups = append(user.Groups, *g)
		}
		for _, p := range userResp.AttachedManagedPolicies {
			user.Policies = append(user.Policies, a.account.normalisePolicyArn(*p.PolicyArn))
		}
		if err := a.populateInlinePolicies(userResp.UserPolicyList, &user.InlinePolicies); err != nil {
			return err
		}

		a.data.Users = append(a.data.Users, &user)
	}

	for _, groupResp := range resp.GroupDetailList {
		if ok, err := isSkippableManagedResource(*groupResp.GroupName); ok {
			log.Printf(err)
			continue
		}

		group := Group{iamService: iamService{
			Name: *groupResp.GroupName,
			Path: *groupResp.Path,
		}}

		for _, p := range groupResp.AttachedManagedPolicies {
			group.Policies = append(group.Policies, a.account.normalisePolicyArn(*p.PolicyArn))
		}
		if err := a.populateInlinePolicies(groupResp.GroupPolicyList, &group.InlinePolicies); err != nil {
			return err
		}

		a.data.Groups = append(a.data.Groups, &group)
	}

	for _, roleResp := range resp.RoleDetailList {
		if ok, err := isSkippableManagedResource(*roleResp.RoleName); ok {
			log.Printf(err)
			continue
		}

		role := Role{iamService: iamService{
			Name: *roleResp.RoleName,
			Path: *roleResp.Path,
		}}

		if !a.SkipFetchingPolicyAndRoleDescriptions {
			a.marshalRoleDescriptionAsync(*roleResp.RoleName, &role.Description)
		}

		var err error
		role.AssumeRolePolicyDocument, err = NewPolicyDocumentFromEncodedJson(*roleResp.AssumeRolePolicyDocument)
		if err != nil {
			return err
		}
		for _, p := range roleResp.AttachedManagedPolicies {
			role.Policies = append(role.Policies, a.account.normalisePolicyArn(*p.PolicyArn))
		}
		if err := a.populateInlinePolicies(roleResp.RolePolicyList, &role.InlinePolicies); err != nil {
			return err
		}

		a.data.addRole(&role)
	}

	for _, policyResp := range resp.Policies {
		if ok, err := isSkippableManagedResource(*policyResp.PolicyName); ok {
			log.Printf(err)
			continue
		}

		defaultPolicyVersion := findDefaultPolicyVersion(policyResp.PolicyVersionList)
		doc, err := NewPolicyDocumentFromEncodedJson(*defaultPolicyVersion.Document)
		if err != nil {
			return err
		}

		p := Policy{
			iamService: iamService{
				Name: *policyResp.PolicyName,
				Path: *policyResp.Path,
			},
			oldestVersionId:      findOldestPolicyVersionId(policyResp.PolicyVersionList),
			numberOfVersions:     len(policyResp.PolicyVersionList),
			nondefaultVersionIds: findNonDefaultPolicyVersionIds(policyResp.PolicyVersionList),
			Policy:               doc,
		}

		if !a.SkipFetchingPolicyAndRoleDescriptions {
			a.marshalPolicyDescriptionAsync(*policyResp.Arn, &p.Description)
		}

		a.data.addPolicy(&p)
	}

	a.descriptionFetchWaitGroup.Wait()

	return a.descriptionFetchError
}

func findDefaultPolicyVersion(versions []*iam.PolicyVersion) *iam.PolicyVersion {
	for _, version := range versions {
		if *version.IsDefaultVersion {
			return version
		}
	}
	panic("Expected a default policy version")
}

func findNonDefaultPolicyVersionIds(versions []*iam.PolicyVersion) []string {
	ss := []string{}
	for _, version := range versions {
		if !*version.IsDefaultVersion {
			ss = append(ss, *version.VersionId)
		}
	}
	return ss
}

func findOldestPolicyVersionId(versions []*iam.PolicyVersion) string {
	oldest := versions[0]
	for _, version := range versions[1:] {
		if version.CreateDate.Before(*oldest.CreateDate) {
			oldest = version
		}
	}
	return *oldest.VersionId
}

func (a *AwsFetcher) getAccount() (*Account, error) {
	var err error
	acct := Account{}

	acct.Id, err = GetAwsAccountId(awsSession(), a.Debug)
	if err == aws.ErrMissingRegion {
		return nil, errors.New("Error determining the AWS account id - check the AWS_REGION environment variable is set")
	}
	if err != nil {
		return nil, err
	}

	aliasResp, err := a.iam.ListAccountAliases(&iam.ListAccountAliasesInput{})
	if err != nil {
		return nil, err
	}
	if len(aliasResp.AccountAliases) > 0 {
		acct.Alias = *aliasResp.AccountAliases[0]
	}

	return &acct, nil
}

// isSkippableResource takes the resource name as a string and checks it
// against known resources that we shouldn't need to manage as it will
// already be managed by another process (such as Cloudformation roles).
// Returns a boolean of whether it can be skipped and a string of the
// reasoning why it was skipped.
func isSkippableManagedResource(resourceName string) (bool, string) {
	if cfnResourceRegexp.MatchString(resourceName) {
		return true, fmt.Sprintf("CloudFormation generated resource %s", resourceName)
	}

	return false, ""
}
<|endoftext|>


<commit_before>package rethinkgo

import (
	"fmt"
	test "launchpad.net/gocheck"
)

func (s *RethinkSuite) TestWriteInsert(c *test.C) {
	query := Db("test").Table("test").Insert(Obj{"num": 1})
	_, err := query.Run(conn)
	c.Assert(err, test.IsNil)
}

func (s *RethinkSuite) TestWriteInsertStruct(c *test.C) {
	o := object{
		Id:   5,
		Name: "Object 3",
		Attrs: []attr{
			attr{
				Name:  "Attr 2",
				Value: "Value",
			},
		},
	}

	query := Db("test").Table("test").Insert(o)
	err := query.RunRow(conn).Scan(&response)

	c.Assert(err, test.IsNil)
	c.Assert(response, JsonEquals, Obj{"inserted": 1})
}

func (s *RethinkSuite) TestWriteUpdate(c *test.C) {
	query := Db("test").Table("test").Insert(Obj{"num": 1})
	_, err := query.Run(conn)
	c.Assert(err, test.IsNil)

	// Update the first row in the table
	query = Db("test").Table("test").Sample(1).Update(Obj{"num": 2})
	_, err = query.Run(conn)
	c.Assert(err, test.IsNil)
}

func (s *RethinkSuite) TestWriteReplace(c *test.C) {
	query := Db("test").Table("test").Insert(Obj{"num": 1})
	_, err := query.Run(conn)
	c.Assert(err, test.IsNil)

	// Replace the first row in the table
	query = Db("test").Table("test").Sample(1).Update(Obj{"num": 2})
	_, err = query.Run(conn)
	c.Assert(err, test.IsNil)
}

func (s *RethinkSuite) TestWriteDelete(c *test.C) {
	query := Db("test").Table("test").Insert(Obj{"num": 1})
	_, err := query.Run(conn)
	c.Assert(err, test.IsNil)

	// Delete the first row in the table
	query = Db("test").Table("test").Sample(1).Delete()
	_, err = query.Run(conn)
	c.Assert(err, test.IsNil)
}
<commit_msg>Fixed broken tests<commit_after>package rethinkgo

import (
	test "launchpad.net/gocheck"
)

func (s *RethinkSuite) TestWriteInsert(c *test.C) {
	query := Db("test").Table("test").Insert(Obj{"num": 1})
	_, err := query.Run(conn)
	c.Assert(err, test.IsNil)
}

func (s *RethinkSuite) TestWriteInsertStruct(c *test.C) {
	var response interface{}
	o := object{
		Id:   5,
		Name: "Object 3",
		Attrs: []attr{
			attr{
				Name:  "Attr 2",
				Value: "Value",
			},
		},
	}

	query := Db("test").Table("test").Insert(o)
	err := query.RunRow(conn).Scan(&response)

	c.Assert(err, test.IsNil)
	c.Assert(response, JsonEquals, Obj{"inserted": 1})
}

func (s *RethinkSuite) TestWriteUpdate(c *test.C) {
	query := Db("test").Table("test").Insert(Obj{"num": 1})
	_, err := query.Run(conn)
	c.Assert(err, test.IsNil)

	// Update the first row in the table
	query = Db("test").Table("test").Sample(1).Update(Obj{"num": 2})
	_, err = query.Run(conn)
	c.Assert(err, test.IsNil)
}

func (s *RethinkSuite) TestWriteReplace(c *test.C) {
	query := Db("test").Table("test").Insert(Obj{"num": 1})
	_, err := query.Run(conn)
	c.Assert(err, test.IsNil)

	// Replace the first row in the table
	query = Db("test").Table("test").Sample(1).Update(Obj{"num": 2})
	_, err = query.Run(conn)
	c.Assert(err, test.IsNil)
}

func (s *RethinkSuite) TestWriteDelete(c *test.C) {
	query := Db("test").Table("test").Insert(Obj{"num": 1})
	_, err := query.Run(conn)
	c.Assert(err, test.IsNil)

	// Delete the first row in the table
	query = Db("test").Table("test").Sample(1).Delete()
	_, err = query.Run(conn)
	c.Assert(err, test.IsNil)
}
<|endoftext|>


<commit_before>package rackspace

import (
	"fmt"

	"github.com/rackspace/gophercloud"
	os "github.com/rackspace/gophercloud/openstack"
	"github.com/rackspace/gophercloud/openstack/utils"
	tokens2 "github.com/rackspace/gophercloud/rackspace/identity/v2/tokens"
)

const (
	// RackspaceUSIdentity is an identity endpoint located in the United States.
	RackspaceUSIdentity = "https://identity.api.rackspacecloud.com/v2.0/"

	// RackspaceUKIdentity is an identity endpoint located in the UK.
	RackspaceUKIdentity = "https://lon.identity.api.rackspacecloud.com/v2.0/"
)

const (
	v20 = "v2.0"
)

// NewClient creates a client that's prepared to communicate with the Rackspace API, but is not
// yet authenticated. Most users will probably prefer using the AuthenticatedClient function
// instead.
//
// Provide the base URL of the identity endpoint you wish to authenticate against as "endpoint".
// Often, this will be either RackspaceUSIdentity or RackspaceUKIdentity.
func NewClient(endpoint string) (*gophercloud.ProviderClient, error) {
	return os.NewClient(endpoint)
}

// AuthenticatedClient logs in to Rackspace with the provided credentials and constructs a
// ProviderClient that's ready to operate.
//
// If the provided AuthOptions does not specify an explicit IdentityEndpoint, it will default to
// the canonical, production Rackspace US identity endpoint.
func AuthenticatedClient(options gophercloud.AuthOptions) (*gophercloud.ProviderClient, error) {
	if options.IdentityEndpoint == "" {
		options.IdentityEndpoint = RackspaceUSIdentity
	}

	client, err := NewClient(options.IdentityEndpoint)
	if err != nil {
		return nil, err
	}

	err = Authenticate(client, options)
	if err != nil {
		return nil, err
	}
	return client, nil
}

// Authenticate or re-authenticate against the most recent identity service supported at the
// provided endpoint.
func Authenticate(client *gophercloud.ProviderClient, options gophercloud.AuthOptions) error {
	versions := []*utils.Version{
		&utils.Version{ID: v20, Priority: 20, Suffix: "/v2.0/"},
	}

	chosen, endpoint, err := utils.ChooseVersion(client.IdentityBase, client.IdentityEndpoint, versions)
	if err != nil {
		return err
	}

	switch chosen.ID {
	case v20:
		return v2auth(client, endpoint, options)
	default:
		// The switch statement must be out of date from the versions list.
		return fmt.Errorf("Unrecognized identity version: %s", chosen.ID)
	}
}

func v2auth(client *gophercloud.ProviderClient, endpoint string, options gophercloud.AuthOptions) error {
	v2Client := NewIdentityV2(client)
	if endpoint != "" {
		v2Client.Endpoint = endpoint
	}

	result := tokens2.Create(v2Client, tokens2.WrapOptions(options))

	token, err := result.ExtractToken()
	if err != nil {
		return err
	}

	catalog, err := result.ExtractServiceCatalog()
	if err != nil {
		return err
	}

	client.TokenID = token.ID
	client.EndpointLocator = func(opts gophercloud.EndpointOpts) (string, error) {
		return os.V2EndpointURL(catalog, opts)
	}

	return nil
}

// NewIdentityV2 creates a ServiceClient that may be used to access the v2 identity service.
func NewIdentityV2(client *gophercloud.ProviderClient) *gophercloud.ServiceClient {
	v2Endpoint := client.IdentityBase + "v2.0/"

	return &gophercloud.ServiceClient{
		Provider: client,
		Endpoint: v2Endpoint,
	}
}
<commit_msg>Default the endpoint in rackspace.NewClient().<commit_after>package rackspace

import (
	"fmt"

	"github.com/rackspace/gophercloud"
	os "github.com/rackspace/gophercloud/openstack"
	"github.com/rackspace/gophercloud/openstack/utils"
	tokens2 "github.com/rackspace/gophercloud/rackspace/identity/v2/tokens"
)

const (
	// RackspaceUSIdentity is an identity endpoint located in the United States.
	RackspaceUSIdentity = "https://identity.api.rackspacecloud.com/v2.0/"

	// RackspaceUKIdentity is an identity endpoint located in the UK.
	RackspaceUKIdentity = "https://lon.identity.api.rackspacecloud.com/v2.0/"
)

const (
	v20 = "v2.0"
)

// NewClient creates a client that's prepared to communicate with the Rackspace API, but is not
// yet authenticated. Most users will probably prefer using the AuthenticatedClient function
// instead.
//
// Provide the base URL of the identity endpoint you wish to authenticate against as "endpoint".
// Often, this will be either RackspaceUSIdentity or RackspaceUKIdentity.
func NewClient(endpoint string) (*gophercloud.ProviderClient, error) {
	if endpoint == "" {
		return os.NewClient(RackspaceUSIdentity)
	}
	return os.NewClient(endpoint)
}

// AuthenticatedClient logs in to Rackspace with the provided credentials and constructs a
// ProviderClient that's ready to operate.
//
// If the provided AuthOptions does not specify an explicit IdentityEndpoint, it will default to
// the canonical, production Rackspace US identity endpoint.
func AuthenticatedClient(options gophercloud.AuthOptions) (*gophercloud.ProviderClient, error) {
	client, err := NewClient(options.IdentityEndpoint)
	if err != nil {
		return nil, err
	}

	err = Authenticate(client, options)
	if err != nil {
		return nil, err
	}
	return client, nil
}

// Authenticate or re-authenticate against the most recent identity service supported at the
// provided endpoint.
func Authenticate(client *gophercloud.ProviderClient, options gophercloud.AuthOptions) error {
	versions := []*utils.Version{
		&utils.Version{ID: v20, Priority: 20, Suffix: "/v2.0/"},
	}

	chosen, endpoint, err := utils.ChooseVersion(client.IdentityBase, client.IdentityEndpoint, versions)
	if err != nil {
		return err
	}

	switch chosen.ID {
	case v20:
		return v2auth(client, endpoint, options)
	default:
		// The switch statement must be out of date from the versions list.
		return fmt.Errorf("Unrecognized identity version: %s", chosen.ID)
	}
}

func v2auth(client *gophercloud.ProviderClient, endpoint string, options gophercloud.AuthOptions) error {
	v2Client := NewIdentityV2(client)
	if endpoint != "" {
		v2Client.Endpoint = endpoint
	}

	result := tokens2.Create(v2Client, tokens2.WrapOptions(options))

	token, err := result.ExtractToken()
	if err != nil {
		return err
	}

	catalog, err := result.ExtractServiceCatalog()
	if err != nil {
		return err
	}

	client.TokenID = token.ID
	client.EndpointLocator = func(opts gophercloud.EndpointOpts) (string, error) {
		return os.V2EndpointURL(catalog, opts)
	}

	return nil
}

// NewIdentityV2 creates a ServiceClient that may be used to access the v2 identity service.
func NewIdentityV2(client *gophercloud.ProviderClient) *gophercloud.ServiceClient {
	v2Endpoint := client.IdentityBase + "v2.0/"

	return &gophercloud.ServiceClient{
		Provider: client,
		Endpoint: v2Endpoint,
	}
}
<|endoftext|>


<commit_before>package socks

import (
	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"strconv"
	"sync"
	"time"

	"github.com/eycorsican/go-tun2socks/core"
	"github.com/eycorsican/go-tun2socks/proxy"
)

type udpHandler struct {
	sync.Mutex

	proxyHost   string
	proxyPort   uint16
	udpConns    map[core.Connection]net.Conn
	tcpConns    map[core.Connection]net.Conn
	targetAddrs map[core.Connection]Addr
	dnsCache    *proxy.DNSCache
	timeout     time.Duration
}

func NewUDPHandler(proxyHost string, proxyPort uint16, timeout time.Duration) core.ConnectionHandler {
	return &udpHandler{
		proxyHost:   proxyHost,
		proxyPort:   proxyPort,
		udpConns:    make(map[core.Connection]net.Conn, 8),
		tcpConns:    make(map[core.Connection]net.Conn, 8),
		targetAddrs: make(map[core.Connection]Addr, 8),
		dnsCache:    proxy.NewDNSCache(),
		timeout:     timeout,
	}
}

func (h *udpHandler) handleTCP(conn core.Connection, c net.Conn) {
	var buf = make([]byte, 1)
	c.SetDeadline(time.Time{})
	c.Read(buf)
	h.Close(conn)
}

func (h *udpHandler) fetchUDPInput(conn core.Connection, input net.Conn) {
	buf := core.NewBytes(core.BufSize)

	defer func() {
		h.Close(conn)
		core.FreeBytes(buf)
	}()

	for {
		input.SetDeadline(time.Now().Add(h.timeout))
		n, err := input.Read(buf)
		if err != nil {
			// log.Printf("read remote failed: %v", err)
			return
		}

		addr := SplitAddr(buf[3:])
		_, err = conn.Write(buf[int(3+len(addr)):n])
		if err != nil {
			log.Printf("write local failed: %v", err)
			return
		}

		h.Lock()
		targetAddr, ok2 := h.targetAddrs[conn]
		h.Unlock()
		if ok2 {
			_, port, err := net.SplitHostPort(targetAddr.String())
			if err != nil {
				log.Fatal("impossible error")
			}
			if port == strconv.Itoa(proxy.COMMON_DNS_PORT) {
				h.dnsCache.Store(buf[int(3+len(addr)):n])
				return // DNS response
			}
		}
	}
}

func (h *udpHandler) Connect(conn core.Connection, target net.Addr) error {
	c, err := net.Dial("tcp", core.ParseTCPAddr(h.proxyHost, h.proxyPort).String())
	if err != nil {
		return err
	}

	// send VER, NMETHODS, METHODS
	c.Write([]byte{5, 1, 0})

	buf := make([]byte, MaxAddrLen)
	// read VER METHOD
	if _, err := io.ReadFull(c, buf[:2]); err != nil {
		return err
	}

	targetAddr := ParseAddr(target.String())
	// write VER CMD RSV ATYP DST.ADDR DST.PORT
	c.Write(append([]byte{5, socks5UDPAssociate, 0}, targetAddr...))

	// read VER REP RSV ATYP BND.ADDR BND.PORT
	if _, err := io.ReadFull(c, buf[:3]); err != nil {
		return err
	}

	rep := buf[1]
	if rep != 0 {
		return errors.New("SOCKS handshake failed")
	}

	remoteAddr, err := readAddr(c, buf)
	if err != nil {
		return err
	}

	go h.handleTCP(conn, c)

	pc, err := net.Dial("udp", remoteAddr.String())
	if err != nil {
		return err
	}

	h.Lock()
	h.tcpConns[conn] = c
	h.udpConns[conn] = pc
	h.targetAddrs[conn] = targetAddr
	h.Unlock()
	go h.fetchUDPInput(conn, pc)
	log.Printf("new proxy connection for target: %s:%s", target.Network(), target.String())
	return nil
}

func (h *udpHandler) DidReceive(conn core.Connection, data []byte) error {
	h.Lock()
	pc, ok1 := h.udpConns[conn]
	targetAddr, ok2 := h.targetAddrs[conn]
	h.Unlock()

	if ok2 {
		_, port, err := net.SplitHostPort(targetAddr.String())
		if err != nil {
			log.Fatal("impossible error")
		}
		if port == strconv.Itoa(proxy.COMMON_DNS_PORT) {
			if answer := h.dnsCache.Query(data); answer != nil {
				var buf [1024]byte
				if dnsAnswer, err := answer.PackBuffer(buf[:]); err == nil {
					_, err = conn.Write(dnsAnswer)
					if err != nil {
						return errors.New(fmt.Sprintf("cache dns answer failed: %v", err))
					}
					h.Close(conn)
					return nil
				}
			}
		}
	}

	if ok1 && ok2 {
		buf := append([]byte{0, 0, 0}, targetAddr...)
		buf = append(buf, data[:]...)
		_, err := pc.Write(buf)
		if err != nil {
			h.Close(conn)
			return errors.New(fmt.Sprintf("write remote failed: %v", err))
		}
		return nil
	} else {
		h.Close(conn)
		return errors.New(fmt.Sprintf("proxy connection %v->%v does not exists", conn.LocalAddr(), conn.RemoteAddr()))
	}
}

func (h *udpHandler) DidSend(conn core.Connection, len uint16) {
	// unused
}

func (h *udpHandler) DidClose(conn core.Connection) {
	// unused
}

func (h *udpHandler) LocalDidClose(conn core.Connection) {
	// unused
}

func (h *udpHandler) Close(conn core.Connection) {
	conn.Close()

	h.Lock()
	defer h.Unlock()

	if c, ok := h.tcpConns[conn]; ok {
		c.Close()
		delete(h.tcpConns, conn)
	}
	if pc, ok := h.udpConns[conn]; ok {
		pc.Close()
		delete(h.udpConns, conn)
	}
}
<commit_msg>ignore unknown data read from tcp conn<commit_after>package socks

import (
	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"strconv"
	"sync"
	"time"

	"github.com/eycorsican/go-tun2socks/core"
	"github.com/eycorsican/go-tun2socks/proxy"
)

type udpHandler struct {
	sync.Mutex

	proxyHost   string
	proxyPort   uint16
	udpConns    map[core.Connection]net.Conn
	tcpConns    map[core.Connection]net.Conn
	targetAddrs map[core.Connection]Addr
	dnsCache    *proxy.DNSCache
	timeout     time.Duration
}

func NewUDPHandler(proxyHost string, proxyPort uint16, timeout time.Duration) core.ConnectionHandler {
	return &udpHandler{
		proxyHost:   proxyHost,
		proxyPort:   proxyPort,
		udpConns:    make(map[core.Connection]net.Conn, 8),
		tcpConns:    make(map[core.Connection]net.Conn, 8),
		targetAddrs: make(map[core.Connection]Addr, 8),
		dnsCache:    proxy.NewDNSCache(),
		timeout:     timeout,
	}
}

func (h *udpHandler) handleTCP(conn core.Connection, c net.Conn) {
	buf := core.NewBytes(core.BufSize)
	defer core.FreeBytes(buf)

	for {
		c.SetDeadline(time.Time{})
		_, err := c.Read(buf)
		if err == io.EOF {
			h.Close(conn)
			return
		} else if err != nil {
			log.Printf("failed to handle TCP connection for UDP associate request: %v", err)
			h.Close(conn)
			return
		}
	}
}

func (h *udpHandler) fetchUDPInput(conn core.Connection, input net.Conn) {
	buf := core.NewBytes(core.BufSize)

	defer func() {
		h.Close(conn)
		core.FreeBytes(buf)
	}()

	for {
		input.SetDeadline(time.Now().Add(h.timeout))
		n, err := input.Read(buf)
		if err != nil {
			// log.Printf("read remote failed: %v", err)
			return
		}

		addr := SplitAddr(buf[3:])
		_, err = conn.Write(buf[int(3+len(addr)):n])
		if err != nil {
			log.Printf("write local failed: %v", err)
			return
		}

		h.Lock()
		targetAddr, ok2 := h.targetAddrs[conn]
		h.Unlock()
		if ok2 {
			_, port, err := net.SplitHostPort(targetAddr.String())
			if err != nil {
				log.Fatal("impossible error")
			}
			if port == strconv.Itoa(proxy.COMMON_DNS_PORT) {
				h.dnsCache.Store(buf[int(3+len(addr)):n])
				return // DNS response
			}
		}
	}
}

func (h *udpHandler) Connect(conn core.Connection, target net.Addr) error {
	c, err := net.Dial("tcp", core.ParseTCPAddr(h.proxyHost, h.proxyPort).String())
	if err != nil {
		return err
	}

	// send VER, NMETHODS, METHODS
	c.Write([]byte{5, 1, 0})

	buf := make([]byte, MaxAddrLen)
	// read VER METHOD
	if _, err := io.ReadFull(c, buf[:2]); err != nil {
		return err
	}

	targetAddr := ParseAddr(target.String())
	// write VER CMD RSV ATYP DST.ADDR DST.PORT
	c.Write(append([]byte{5, socks5UDPAssociate, 0}, targetAddr...))

	// read VER REP RSV ATYP BND.ADDR BND.PORT
	if _, err := io.ReadFull(c, buf[:3]); err != nil {
		return err
	}

	rep := buf[1]
	if rep != 0 {
		return errors.New("SOCKS handshake failed")
	}

	remoteAddr, err := readAddr(c, buf)
	if err != nil {
		return err
	}

	go h.handleTCP(conn, c)

	pc, err := net.Dial("udp", remoteAddr.String())
	if err != nil {
		return err
	}

	h.Lock()
	h.tcpConns[conn] = c
	h.udpConns[conn] = pc
	h.targetAddrs[conn] = targetAddr
	h.Unlock()
	go h.fetchUDPInput(conn, pc)
	log.Printf("new proxy connection for target: %s:%s", target.Network(), target.String())
	return nil
}

func (h *udpHandler) DidReceive(conn core.Connection, data []byte) error {
	h.Lock()
	pc, ok1 := h.udpConns[conn]
	targetAddr, ok2 := h.targetAddrs[conn]
	h.Unlock()

	if ok2 {
		_, port, err := net.SplitHostPort(targetAddr.String())
		if err != nil {
			log.Fatal("impossible error")
		}
		if port == strconv.Itoa(proxy.COMMON_DNS_PORT) {
			if answer := h.dnsCache.Query(data); answer != nil {
				var buf [1024]byte
				if dnsAnswer, err := answer.PackBuffer(buf[:]); err == nil {
					_, err = conn.Write(dnsAnswer)
					if err != nil {
						return errors.New(fmt.Sprintf("cache dns answer failed: %v", err))
					}
					h.Close(conn)
					return nil
				}
			}
		}
	}

	if ok1 && ok2 {
		buf := append([]byte{0, 0, 0}, targetAddr...)
		buf = append(buf, data[:]...)
		_, err := pc.Write(buf)
		if err != nil {
			h.Close(conn)
			return errors.New(fmt.Sprintf("write remote failed: %v", err))
		}
		return nil
	} else {
		h.Close(conn)
		return errors.New(fmt.Sprintf("proxy connection %v->%v does not exists", conn.LocalAddr(), conn.RemoteAddr()))
	}
}

func (h *udpHandler) DidSend(conn core.Connection, len uint16) {
	// unused
}

func (h *udpHandler) DidClose(conn core.Connection) {
	// unused
}

func (h *udpHandler) LocalDidClose(conn core.Connection) {
	// unused
}

func (h *udpHandler) Close(conn core.Connection) {
	conn.Close()

	h.Lock()
	defer h.Unlock()

	if c, ok := h.tcpConns[conn]; ok {
		c.Close()
		delete(h.tcpConns, conn)
	}
	if pc, ok := h.udpConns[conn]; ok {
		pc.Close()
		delete(h.udpConns, conn)
	}
}
<|endoftext|>


<commit_before>package proxyfilters

import (
	e2 "errors"
	"net"
	"net/http"

	"github.com/getlantern/errors"
	"github.com/getlantern/ops"
	"github.com/getlantern/proxy/v2/filters"
)

// RecordOp records the proxy_http op.
var RecordOp = filters.FilterFunc(func(cs *filters.ConnectionState, req *http.Request, next filters.Next) (*http.Response, *filters.ConnectionState, error) {
	name := "proxy_http"
	if req.Method == http.MethodConnect {
		name += "s"
	}
	op := ops.Begin(name)
	resp, nextCtx, err := next(cs, req)
	if err != nil {
		op.FailIf(err)
		// Dumping stack trace is useful but in this case, it would create tons
		// of noises as the filters are called recursively.
		if e, ok := err.(errors.Error); ok {
			// Before we do this, we should check if this is a DNS error, if so we don't
			// want to log it, since these log.Error's get sent up to stack driver and become
			// super noisy!
			var dnsError *net.DNSError
			if !e2.As(e.RootCause(), &dnsError) {
				log.Error(e.RootCause())
			}
		} else {
			log.Error(err)
		}
	}
	op.End()
	return resp, nextCtx, err
})
<commit_msg>Suppress noisy network timeout errors (#138)<commit_after>package proxyfilters

import (
	stderrors "errors"
	"net"
	"net/http"

	"github.com/getlantern/errors"
	"github.com/getlantern/ops"
	"github.com/getlantern/proxy/v2/filters"
)

// RecordOp records the proxy_http op.
var RecordOp = filters.FilterFunc(func(cs *filters.ConnectionState, req *http.Request, next filters.Next) (*http.Response, *filters.ConnectionState, error) {
	name := "proxy_http"
	if req.Method == http.MethodConnect {
		name += "s"
	}
	op := ops.Begin(name)
	resp, nextCtx, err := next(cs, req)
	if err != nil {
		op.FailIf(err)
		logFilterError(err)
	}
	op.End()
	return resp, nextCtx, err
})

func logFilterError(err error) {
	var (
		opErr  *net.OpError
		dnsErr *net.DNSError
	)
	switch {
	case stderrors.As(err, &opErr) && opErr.Timeout():
		// Network timeouts are out of our control and create noise in StackDriver.
		log.Debugf("%s timeout error: %v", opErr.Op, err)
		return
	case stderrors.As(err, &dnsErr):
		// DNS errors are out of our control and create noise in StackDriver.
		log.Debugf("DNS error: %v", err)
		return
	}
	if e, ok := err.(errors.Error); ok {
		// Filters are called recursively. We log only the root to reduce stack trace noise.
		err = e.RootCause()
	}
	log.Error(err)
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/golang/glog"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	restclient "k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	clientapi "k8s.io/client-go/tools/clientcmd/api"
	api "k8s.io/kubernetes/pkg/apis/core"
	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
)

func buildConfigFromEnvs(masterURL, kubeconfigPath string) (*restclient.Config, error) {
	if kubeconfigPath == "" && masterURL == "" {
		kubeconfig, err := restclient.InClusterConfig()
		if err != nil {
			return nil, err
		}

		return kubeconfig, nil
	}

	return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
		&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath},
		&clientcmd.ConfigOverrides{ClusterInfo: clientapi.Cluster{Server: masterURL}}).ClientConfig()
}

func flattenSubsets(subsets []api.EndpointSubset) []string {
	ips := []string{}
	for _, ss := range subsets {
		for _, addr := range ss.Addresses {
			ips = append(ips, fmt.Sprintf(`"%s"`, addr.IP))
		}
	}
	return ips
}

func main() {
	flag.Parse()

	glog.Info("Kubernetes Elasticsearch logging discovery")

	cc, err := buildConfigFromEnvs(os.Getenv("APISERVER_HOST"), os.Getenv("KUBE_CONFIG_FILE"))
	if err != nil {
		glog.Fatalf("Failed to make client: %v", err)
	}
	client, err := clientset.NewForConfig(cc)

	if err != nil {
		glog.Fatalf("Failed to make client: %v", err)
	}
	namespace := metav1.NamespaceSystem
	envNamespace := os.Getenv("NAMESPACE")
	if envNamespace != "" {
		if _, err := client.Core().Namespaces().Get(envNamespace, metav1.GetOptions{}); err != nil {
			glog.Fatalf("%s namespace doesn't exist: %v", envNamespace, err)
		}
		namespace = envNamespace
	}

	var elasticsearch *api.Service
	serviceName := os.Getenv("ELASTICSEARCH_SERVICE_NAME")
	if serviceName == "" {
		serviceName = "elasticsearch-logging"
	}

	// Look for endpoints associated with the Elasticsearch logging service.
	// First wait for the service to become available.
	for t := time.Now(); time.Since(t) < 5*time.Minute; time.Sleep(10 * time.Second) {
		elasticsearch, err = client.Core().Services(namespace).Get(serviceName, metav1.GetOptions{})
		if err == nil {
			break
		}
	}
	// If we did not find an elasticsearch logging service then log a warning
	// and return without adding any unicast hosts.
	if elasticsearch == nil {
		glog.Warningf("Failed to find the elasticsearch-logging service: %v", err)
		return
	}

	var endpoints *api.Endpoints
	addrs := []string{}
	// Wait for some endpoints.
	count, _ := strconv.Atoi(os.Getenv("MINIMUM_MASTER_NODES"))
	for t := time.Now(); time.Since(t) < 5*time.Minute; time.Sleep(10 * time.Second) {
		endpoints, err = client.Core().Endpoints(namespace).Get(serviceName, metav1.GetOptions{})
		if err != nil {
			continue
		}
		addrs = flattenSubsets(endpoints.Subsets)
		glog.Infof("Found %s", addrs)
		if len(addrs) > 0 && len(addrs) == count {
			break
		}
	}
	// If there was an error finding endpoints then log a warning and quit.
	if err != nil {
		glog.Warningf("Error finding endpoints: %v", err)
		return
	}

	glog.Infof("Endpoints = %s", addrs)
	fmt.Printf("discovery.zen.ping.unicast.hosts: [%s]\n", strings.Join(addrs, ", "))
}
<commit_msg>Modify the judgement condtion in elasticsearch_logging_discovery.go<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/golang/glog"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	restclient "k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	clientapi "k8s.io/client-go/tools/clientcmd/api"
	api "k8s.io/kubernetes/pkg/apis/core"
	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
)

func buildConfigFromEnvs(masterURL, kubeconfigPath string) (*restclient.Config, error) {
	if kubeconfigPath == "" && masterURL == "" {
		kubeconfig, err := restclient.InClusterConfig()
		if err != nil {
			return nil, err
		}

		return kubeconfig, nil
	}

	return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
		&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath},
		&clientcmd.ConfigOverrides{ClusterInfo: clientapi.Cluster{Server: masterURL}}).ClientConfig()
}

func flattenSubsets(subsets []api.EndpointSubset) []string {
	ips := []string{}
	for _, ss := range subsets {
		for _, addr := range ss.Addresses {
			ips = append(ips, fmt.Sprintf(`"%s"`, addr.IP))
		}
	}
	return ips
}

func main() {
	flag.Parse()

	glog.Info("Kubernetes Elasticsearch logging discovery")

	cc, err := buildConfigFromEnvs(os.Getenv("APISERVER_HOST"), os.Getenv("KUBE_CONFIG_FILE"))
	if err != nil {
		glog.Fatalf("Failed to make client: %v", err)
	}
	client, err := clientset.NewForConfig(cc)

	if err != nil {
		glog.Fatalf("Failed to make client: %v", err)
	}
	namespace := metav1.NamespaceSystem
	envNamespace := os.Getenv("NAMESPACE")
	if envNamespace != "" {
		if _, err := client.Core().Namespaces().Get(envNamespace, metav1.GetOptions{}); err != nil {
			glog.Fatalf("%s namespace doesn't exist: %v", envNamespace, err)
		}
		namespace = envNamespace
	}

	var elasticsearch *api.Service
	serviceName := os.Getenv("ELASTICSEARCH_SERVICE_NAME")
	if serviceName == "" {
		serviceName = "elasticsearch-logging"
	}

	// Look for endpoints associated with the Elasticsearch logging service.
	// First wait for the service to become available.
	for t := time.Now(); time.Since(t) < 5*time.Minute; time.Sleep(10 * time.Second) {
		elasticsearch, err = client.Core().Services(namespace).Get(serviceName, metav1.GetOptions{})
		if err == nil {
			break
		}
	}
	// If we did not find an elasticsearch logging service then log a warning
	// and return without adding any unicast hosts.
	if elasticsearch == nil {
		glog.Warningf("Failed to find the elasticsearch-logging service: %v", err)
		return
	}

	var endpoints *api.Endpoints
	addrs := []string{}
	// Wait for some endpoints.
	count, _ := strconv.Atoi(os.Getenv("MINIMUM_MASTER_NODES"))
	for t := time.Now(); time.Since(t) < 5*time.Minute; time.Sleep(10 * time.Second) {
		endpoints, err = client.Core().Endpoints(namespace).Get(serviceName, metav1.GetOptions{})
		if err != nil {
			continue
		}
		addrs = flattenSubsets(endpoints.Subsets)
		glog.Infof("Found %s", addrs)
		if len(addrs) > 0 && len(addrs) >= count {
			break
		}
	}
	// If there was an error finding endpoints then log a warning and quit.
	if err != nil {
		glog.Warningf("Error finding endpoints: %v", err)
		return
	}

	glog.Infof("Endpoints = %s", addrs)
	fmt.Printf("discovery.zen.ping.unicast.hosts: [%s]\n", strings.Join(addrs, ", "))
}
<|endoftext|>