<commit_before>package main

import (
	// "bufio"
	"fmt"
	"strconv"
	"strings"
	// "os"
	"github.com/tiborvass/uniline"
)

func tokenize(line string) []string {
	tokens := strings.Split(strings.Replace(strings.Replace(line, "(", " ( ", -1), ")", " ) ", -1), " ")
	noWSTokens := make([]string, 0)
	for _, token := range tokens {
		if token != "" {
			noWSTokens = append(noWSTokens, token)
		}
	}
	return noWSTokens
}

// TODO Use only err as error string
type Atom struct {
	err    bool
	errMsg string
	val    int
}

const (
	// Delimiters
	OpenBracket   string = "("
	ClosedBracket string = ")"

	// Operators
	Add string = "+"
	Sub string = "-"
	Mul string = "*"
	Div string = "/"
)

func add(operand1, operand2 Atom) Atom {
	var retVal Atom
	retVal.val = operand1.val + operand2.val
	return retVal
}

func checkArgLen(operatorName string, operands []Atom, expectedArgs int) Atom {
	var retVal Atom
	if len(operands) != expectedArgs {
		var retVal Atom
		retVal.err = true
		retVal.errMsg = fmt.Sprintf("For operator %s, expected %d args, but got %d.", operatorName, expectedArgs, len(operands))
		return retVal
	}
	return retVal
}

func getHandler(operator string) (int, func([]Atom) Atom) {
	if operator == Add {
		return 2, func(operands []Atom) Atom {
			var retVal Atom
			argCheckErr := checkArgLen(Add, operands, 2)
			if argCheckErr.err {
				return argCheckErr
			}
			// TODO
			// Do type checks

			retVal.val = operands[0].val + operands[1].val
			return retVal
		}
	} else if operator == Sub {
		return 2, func(operands []Atom) Atom {
			var retVal Atom
			argCheckErr := checkArgLen(Sub, operands, 2)
			if argCheckErr.err {
				return argCheckErr
			}
			// TODO
			// Do type checks

			retVal.val = operands[0].val - operands[1].val
			return retVal
		}
	} else if operator == Mul {
		return 2, func(operands []Atom) Atom {
			var retVal Atom
			argCheckErr := checkArgLen(Mul, operands, 2)
			if argCheckErr.err {
				return argCheckErr
			}
			// TODO
			// Do type checks

			retVal.val = operands[0].val * operands[1].val
			return retVal
		}
	} else if operator == Div {
		return 2, func(operands []Atom) Atom {
			var retVal Atom
			argCheckErr := checkArgLen(Div, operands, 2)
			if argCheckErr.err {
				return argCheckErr
			}
			// TODO
			// Do type checks

			retVal.val = operands[0].val / operands[1].val
			return retVal
		}
	}
	return 0, nil
}

func pop(tokens []string) (string, []string) {
	if len(tokens) == 0 {
		return "", tokens
	}
	return tokens[0], tokens[1:]
}

func evalArgs(tokens []string) (Atom, []string) {
	var retVal Atom
	retVal.err = false
	retVal.errMsg = ""

	// Expecting a value, but if a ')' occurs, its an error
	if tokens[0] == ClosedBracket {
		retVal.errMsg = "Unexpected ')'"
		retVal.err = true
		return retVal, tokens
	}

	// This could be a nested expression
	if tokens[0] == OpenBracket {
		return eval(tokens)
	}

	// This is an atomic value
	token := ""
	token, tokens = pop(tokens)
	val, err := strconv.Atoi(token)
	if err != nil {
		retVal.errMsg = "Problem while converting " + token + ": " + err.Error()
		retVal.err = true
		return retVal, tokens
	}
	retVal.val = val
	return retVal, tokens
}

func eval(tokens []string) (Atom, []string) {
	var retVal Atom
	retVal.err = false
	retVal.errMsg = ""

	token := ""
	token, tokens = pop(tokens)
	if token != OpenBracket {
		// TODO
		// Raise an exception
		retVal.errMsg = "Expected a '('"
		retVal.err = true
		return retVal, tokens
	}

	token, tokens = pop(tokens)
	argCount, handler := getHandler(token)
	if handler == nil {
		retVal.errMsg = "Invalid operator, " + token
		retVal.err = true
		return retVal, tokens
	}

	// Read two args.
	// TODO change this to read as many args as the operator wants

	// TODO
	// Reading just two ints/floats here
	// Change to read two nested expressions here too
	operands := make([]Atom, 0)
	for i := 0; i < argCount; i++ {
		var operand Atom
		operand, tokens = evalArgs(tokens)
		if operand.err {
			return operand, tokens
		}
		operands = append(operands, operand)
	}

	token, tokens = pop(tokens)
	if token != ClosedBracket {
		retVal.errMsg = "Expected a ')'"
		retVal.err = true
		return retVal, tokens
	}

	return handler(operands), tokens
}

func process(line string) {
	tokens := tokenize(line)
	if len(tokens) == 0 {
		fmt.Println("Nothing to evaluate")
		return
	}
	// fmt.Printf("%q\n", tokens)
	var retVal Atom
	retVal, tokens = eval(tokens)

	if len(tokens) != 0 {
		retVal.errMsg = strconv.Itoa(len(tokens)) + " extra token(s) in the string."
		retVal.err = true
	}
	if retVal.err {
		fmt.Printf("Error: %s\n", retVal.errMsg)
	} else {
		// fmt.Println("All worked fine!")
		fmt.Printf("%d\n", retVal.val)
	}
	fmt.Printf("\n")
}

func main() {
	prompt := "eclisp> "
	scanner := uniline.DefaultScanner()
	for scanner.Scan(prompt) {
		line := scanner.Text()
		if len(line) > 0 {
			scanner.AddToHistory(line)
			process(line)
		}
	}

	if err := scanner.Err(); err != nil {
		panic(err)
	} else {
		fmt.Println("Goodbye!")
	}
}
<commit_msg>Separating the language processing and shell<commit_after>package main

import (
	// "bufio"
	"fmt"
	"strconv"
	// "os"
	"github.com/tiborvass/uniline"
	l "github.com/reddragon/eclisp/lang"
)

func process(line string) {
	tokens := l.Tokenize(line)
	if len(tokens) == 0 {
		fmt.Println("Nothing to evaluate")
		return
	}
	// fmt.Printf("%q\n", tokens)
	var retVal l.Atom
	retVal, tokens = l.Eval(tokens)

	if len(tokens) != 0 {
		retVal.ErrMsg = strconv.Itoa(len(tokens)) + " extra token(s) in the string."
		retVal.Err = true
	}
	if retVal.Err {
		fmt.Printf("Error: %s\n", retVal.ErrMsg)
	} else {
		// fmt.Println("All worked fine!")
		fmt.Printf("%d\n", retVal.Val)
	}
	fmt.Printf("\n")
}

func main() {
	prompt := "eclisp> "
	scanner := uniline.DefaultScanner()
	for scanner.Scan(prompt) {
		line := scanner.Text()
		if len(line) > 0 {
			scanner.AddToHistory(line)
			process(line)
		}
	}

	if err := scanner.Err(); err != nil {
		panic(err)
	} else {
		fmt.Println("Goodbye!")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"./anaconda"
	"fmt"
	"net/url"
	"strings"
)

func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

func isUserFollowing(userName string) (bool, error) {
	friendships, err := api.GetFriendshipsLookup(url.Values{"screen_name": []string{userName}})
	if err != nil {
		fmt.Println("Error while querying twitter api for friendships", err)
		return false, err
	}

	following := false
	for _, friendship := range friendships {
		if stringInSlice("followed_by", friendship.Connections) {
			following = true
		}
	}

	return following, nil
}

func isUserAcceptable(tweet anaconda.Tweet) bool {
	words := strings.Split(tweet.Text, " ")
	for _, word := range words {
		if stringInSlice(strings.ToLower(word), BANNED_KEYWORDS) {
			return false
		}
	}

	if tweet.User.Description == "" {
		return false
	}

	words = strings.Split(tweet.User.Description, " ")
	for _, word := range words {
		if stringInSlice(strings.ToLower(word), BANNED_KEYWORDS) {
			return false
		}
	}

	return true
}

func generateAPISearchValues(word string) (string, url.Values) {
	searchString := word

	for _, word := range BANNED_KEYWORDS {
		searchString += " -" + word
	}

	for _, account := range BANNED_ACCOUNTS {
		searchString +=
	}

	v := url.Values{}
	v.Add("lang", ACCEPTED_LANGUAGE)
	v.Add("count", "100")
	v.Add("result_type", "recent")

	return url.QueryEscape(searchString), v
}

func isMentionOrRT(tweet anaconda.Tweet) bool {
	return strings.HasPrefix(tweet.Text, "RT") || strings.HasPrefix(tweet.Text, "@")
}

func isMe(tweet anaconda.Tweet) bool {
	return strings.ToLower(tweet.User.ScreenName) == strings.ToLower(USER_NAME)
}
<commit_msg>Remove test code<commit_after>package main

import (
	"./anaconda"
	"fmt"
	"net/url"
	"strings"
)

func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

func isUserFollowing(userName string) (bool, error) {
	friendships, err := api.GetFriendshipsLookup(url.Values{"screen_name": []string{userName}})
	if err != nil {
		fmt.Println("Error while querying twitter api for friendships", err)
		return false, err
	}

	following := false
	for _, friendship := range friendships {
		if stringInSlice("followed_by", friendship.Connections) {
			following = true
		}
	}

	return following, nil
}

func isUserAcceptable(tweet anaconda.Tweet) bool {
	words := strings.Split(tweet.Text, " ")
	for _, word := range words {
		if stringInSlice(strings.ToLower(word), BANNED_KEYWORDS) {
			return false
		}
	}

	if tweet.User.Description == "" {
		return false
	}

	words = strings.Split(tweet.User.Description, " ")
	for _, word := range words {
		if stringInSlice(strings.ToLower(word), BANNED_KEYWORDS) {
			return false
		}
	}

	return true
}

func generateAPISearchValues(word string) (string, url.Values) {
	searchString := word

	for _, word := range BANNED_KEYWORDS {
		searchString += " -" + word
	}

	v := url.Values{}
	v.Add("lang", ACCEPTED_LANGUAGE)
	v.Add("count", "100")
	v.Add("result_type", "recent")

	return url.QueryEscape(searchString), v
}

func isMentionOrRT(tweet anaconda.Tweet) bool {
	return strings.HasPrefix(tweet.Text, "RT") || strings.HasPrefix(tweet.Text, "@")
}

func isMe(tweet anaconda.Tweet) bool {
	return strings.ToLower(tweet.User.ScreenName) == strings.ToLower(USER_NAME)
}
<|endoftext|>


<commit_before>// Copyright (c) 2012 The gocql Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The uuid package can be used to generate and parse universally unique
// identifiers, a standardized format in the form of a 128 bit number.
//
// http://tools.ietf.org/html/rfc4122
package gocql

import (
	"crypto/rand"
	"fmt"
	"io"
	"net"
	"time"
)

type UUID [16]byte

var hardwareAddr []byte

const (
	VariantNCSCompat = 0
	VariantIETF      = 2
	VariantMicrosoft = 6
	VariantFuture    = 7
)

func init() {
	if interfaces, err := net.Interfaces(); err == nil {
		for _, i := range interfaces {
			if i.Flags&net.FlagLoopback == 0 && len(i.HardwareAddr) > 0 {
				hardwareAddr = i.HardwareAddr
				break
			}
		}
	}
	if hardwareAddr == nil {
		// If we failed to obtain the MAC address of the current computer,
		// we will use a randomly generated 6 byte sequence instead and set
		// the multicast bit as recommended in RFC 4122.
		hardwareAddr = make([]byte, 6)
		_, err := io.ReadFull(rand.Reader, hardwareAddr)
		if err != nil {
			panic(err)
		}
		hardwareAddr[0] = hardwareAddr[0] | 0x01
	}
}

// ParseUUID parses a 32 digit hexadecimal number (that might contain hypens)
// represanting an UUID.
func ParseUUID(input string) (UUID, error) {
	var u UUID
	j := 0
	for _, r := range input {
		switch {
		case r == '-' && j&1 == 0:
			continue
		case r >= '0' && r <= '9' && j < 32:
			u[j/2] |= byte(r-'0') << uint(4-j&1*4)
		case r >= 'a' && r <= 'f' && j < 32:
			u[j/2] |= byte(r-'a'+10) << uint(4-j&1*4)
		case r >= 'A' && r <= 'F' && j < 32:
			u[j/2] |= byte(r-'A'+10) << uint(4-j&1*4)
		default:
			return UUID{}, fmt.Errorf("invalid UUID %q", input)
		}
		j += 1
	}
	if j != 32 {
		return UUID{}, fmt.Errorf("invalid UUID %q", input)
	}
	return u, nil
}

// UUIDFromBytes converts a raw byte slice to an UUID. It will panic if the
// slice isn't exactly 16 bytes long.
func UUIDFromBytes(input []byte) UUID {
	var u UUID
	if len(input) != 16 {
		panic("UUIDs must be exactly 16 bytes long")
	}
	copy(u[:], input)
	return u
}

// RandomUUID generates a totally random UUID (version 4) as described in
// RFC 4122.
func RandomUUID() UUID {
	var u UUID
	io.ReadFull(rand.Reader, u[:])
	u[6] &= 0x0F // clear version
	u[6] |= 0x40 // set version to 4 (random uuid)
	u[8] &= 0x3F // clear variant
	u[8] |= 0x80 // set to IETF variant
	return u
}

var timeBase = time.Date(1582, time.October, 15, 0, 0, 0, 0, time.UTC).Unix()

// TimeUUID generates a new time based UUID (version 1) using the current
// time as the timestamp.
func TimeUUID() UUID {
	return UUIDFromTime(time.Now())
}

// UUIDFromTime generates a new time based UUID (version 1) as described in
// RFC 4122. This UUID contains the MAC address of the node that generated
// the UUID, the given timestamp and a sequence number.
func UUIDFromTime(aTime time.Time) UUID {
	var u UUID

	utcTime := aTime.In(time.UTC)
	t := uint64(utcTime.Unix()-timeBase)*10000000 + uint64(utcTime.Nanosecond()/100)
	u[0], u[1], u[2], u[3] = byte(t>>24), byte(t>>16), byte(t>>8), byte(t)
	u[4], u[5] = byte(t>>40), byte(t>>32)
	u[6], u[7] = byte(t>>56)&0x0F, byte(t>>48)

	var clockSeq [2]byte
	io.ReadFull(rand.Reader, clockSeq[:])
	u[8] = clockSeq[1]
	u[9] = clockSeq[0]

	copy(u[10:], hardwareAddr)

	u[6] |= 0x10 // set version to 1 (time based uuid)
	u[8] &= 0x3F // clear variant
	u[8] |= 0x80 // set to IETF variant

	return u
}

// String returns the UUID in it's canonical form, a 32 digit hexadecimal
// number in the form of xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.
func (u UUID) String() string {
	return fmt.Sprintf("%x-%x-%x-%x-%x",
		u[0:4], u[4:6], u[6:8], u[8:10], u[10:16])
}

// Bytes returns the raw byte slice for this UUID. A UUID is always 128 bits
// (16 bytes) long.
func (u UUID) Bytes() []byte {
	return u[:]
}

// Variant returns the variant of this UUID. This package will only generate
// UUIDs in the IETF variant.
func (u UUID) Variant() int {
	x := u[8]
	if x&0x80 == 0 {
		return VariantNCSCompat
	}
	if x&0x40 == 0 {
		return VariantIETF
	}
	if x&0x20 == 0 {
		return VariantMicrosoft
	}
	return VariantFuture
}

// Version extracts the version of this UUID variant. The RFC 4122 describes
// five kinds of UUIDs.
func (u UUID) Version() int {
	return int(u[6] & 0xF0 >> 4)
}

// Node extracts the MAC address of the node who generated this UUID. It will
// return nil if the UUID is not a time based UUID (version 1).
func (u UUID) Node() []byte {
	if u.Version() != 1 {
		return nil
	}
	return u[10:]
}

// Timestamp extracts the timestamp information from a time based UUID
// (version 1).
func (u UUID) Timestamp() int64 {
	if u.Version() != 1 {
		return 0
	}
	return int64(uint64(u[0])<<24|uint64(u[1])<<16|
		uint64(u[2])<<8|uint64(u[3])) +
		int64(uint64(u[4])<<40|uint64(u[5])<<32) +
		int64(uint64(u[6]&0x0F)<<56|uint64(u[7])<<48)
}

// Time is like Timestamp, except that it returns a time.Time.
func (u UUID) Time() time.Time {
	if u.Version() != 1 {
		return time.Time{}
	}
	t := u.Timestamp()
	sec := t / 1e7
	nsec := t % 1e7
	return time.Unix(sec+timeBase, nsec).UTC()
}
<commit_msg>uuid: implement the clock sequence properly<commit_after>// Copyright (c) 2012 The gocql Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The uuid package can be used to generate and parse universally unique
// identifiers, a standardized format in the form of a 128 bit number.
//
// http://tools.ietf.org/html/rfc4122
package gocql

import (
	"crypto/rand"
	"fmt"
	"io"
	"net"
	"sync/atomic"
	"time"
)

type UUID [16]byte

var hardwareAddr []byte
var clockSeq uint32

const (
	VariantNCSCompat = 0
	VariantIETF      = 2
	VariantMicrosoft = 6
	VariantFuture    = 7
)

func init() {
	if interfaces, err := net.Interfaces(); err == nil {
		for _, i := range interfaces {
			if i.Flags&net.FlagLoopback == 0 && len(i.HardwareAddr) > 0 {
				hardwareAddr = i.HardwareAddr
				break
			}
		}
	}
	if hardwareAddr == nil {
		// If we failed to obtain the MAC address of the current computer,
		// we will use a randomly generated 6 byte sequence instead and set
		// the multicast bit as recommended in RFC 4122.
		hardwareAddr = make([]byte, 6)
		_, err := io.ReadFull(rand.Reader, hardwareAddr)
		if err != nil {
			panic(err)
		}
		hardwareAddr[0] = hardwareAddr[0] | 0x01
	}

	// initialize the clock sequence with a random number
	var clockSeqRand [2]byte
	io.ReadFull(rand.Reader, clockSeqRand[:])
	clockSeq = uint32(clockSeqRand[1])<<8 | uint32(clockSeqRand[0])
}

// ParseUUID parses a 32 digit hexadecimal number (that might contain hypens)
// represanting an UUID.
func ParseUUID(input string) (UUID, error) {
	var u UUID
	j := 0
	for _, r := range input {
		switch {
		case r == '-' && j&1 == 0:
			continue
		case r >= '0' && r <= '9' && j < 32:
			u[j/2] |= byte(r-'0') << uint(4-j&1*4)
		case r >= 'a' && r <= 'f' && j < 32:
			u[j/2] |= byte(r-'a'+10) << uint(4-j&1*4)
		case r >= 'A' && r <= 'F' && j < 32:
			u[j/2] |= byte(r-'A'+10) << uint(4-j&1*4)
		default:
			return UUID{}, fmt.Errorf("invalid UUID %q", input)
		}
		j += 1
	}
	if j != 32 {
		return UUID{}, fmt.Errorf("invalid UUID %q", input)
	}
	return u, nil
}

// UUIDFromBytes converts a raw byte slice to an UUID. It will panic if the
// slice isn't exactly 16 bytes long.
func UUIDFromBytes(input []byte) UUID {
	var u UUID
	if len(input) != 16 {
		panic("UUIDs must be exactly 16 bytes long")
	}
	copy(u[:], input)
	return u
}

// RandomUUID generates a totally random UUID (version 4) as described in
// RFC 4122.
func RandomUUID() UUID {
	var u UUID
	io.ReadFull(rand.Reader, u[:])
	u[6] &= 0x0F // clear version
	u[6] |= 0x40 // set version to 4 (random uuid)
	u[8] &= 0x3F // clear variant
	u[8] |= 0x80 // set to IETF variant
	return u
}

var timeBase = time.Date(1582, time.October, 15, 0, 0, 0, 0, time.UTC).Unix()

// TimeUUID generates a new time based UUID (version 1) using the current
// time as the timestamp.
func TimeUUID() UUID {
	return UUIDFromTime(time.Now())
}

// UUIDFromTime generates a new time based UUID (version 1) as described in
// RFC 4122. This UUID contains the MAC address of the node that generated
// the UUID, the given timestamp and a sequence number.
func UUIDFromTime(aTime time.Time) UUID {
	var u UUID

	utcTime := aTime.In(time.UTC)
	t := uint64(utcTime.Unix()-timeBase)*10000000 + uint64(utcTime.Nanosecond()/100)
	u[0], u[1], u[2], u[3] = byte(t>>24), byte(t>>16), byte(t>>8), byte(t)
	u[4], u[5] = byte(t>>40), byte(t>>32)
	u[6], u[7] = byte(t>>56)&0x0F, byte(t>>48)

	clock := atomic.AddUint32(&clockSeq, 1)
	u[8] = byte(clock >> 8)
	u[9] = byte(clock)

	copy(u[10:], hardwareAddr)

	u[6] |= 0x10 // set version to 1 (time based uuid)
	u[8] &= 0x3F // clear variant
	u[8] |= 0x80 // set to IETF variant

	return u
}

// String returns the UUID in it's canonical form, a 32 digit hexadecimal
// number in the form of xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.
func (u UUID) String() string {
	return fmt.Sprintf("%x-%x-%x-%x-%x",
		u[0:4], u[4:6], u[6:8], u[8:10], u[10:16])
}

// Bytes returns the raw byte slice for this UUID. A UUID is always 128 bits
// (16 bytes) long.
func (u UUID) Bytes() []byte {
	return u[:]
}

// Variant returns the variant of this UUID. This package will only generate
// UUIDs in the IETF variant.
func (u UUID) Variant() int {
	x := u[8]
	if x&0x80 == 0 {
		return VariantNCSCompat
	}
	if x&0x40 == 0 {
		return VariantIETF
	}
	if x&0x20 == 0 {
		return VariantMicrosoft
	}
	return VariantFuture
}

// Version extracts the version of this UUID variant. The RFC 4122 describes
// five kinds of UUIDs.
func (u UUID) Version() int {
	return int(u[6] & 0xF0 >> 4)
}

// Node extracts the MAC address of the node who generated this UUID. It will
// return nil if the UUID is not a time based UUID (version 1).
func (u UUID) Node() []byte {
	if u.Version() != 1 {
		return nil
	}
	return u[10:]
}

// Timestamp extracts the timestamp information from a time based UUID
// (version 1).
func (u UUID) Timestamp() int64 {
	if u.Version() != 1 {
		return 0
	}
	return int64(uint64(u[0])<<24|uint64(u[1])<<16|
		uint64(u[2])<<8|uint64(u[3])) +
		int64(uint64(u[4])<<40|uint64(u[5])<<32) +
		int64(uint64(u[6]&0x0F)<<56|uint64(u[7])<<48)
}

// Time is like Timestamp, except that it returns a time.Time.
func (u UUID) Time() time.Time {
	if u.Version() != 1 {
		return time.Time{}
	}
	t := u.Timestamp()
	sec := t / 1e7
	nsec := t % 1e7
	return time.Unix(sec+timeBase, nsec).UTC()
}
<|endoftext|>


<commit_before>package obj

import (
	"bytes"
	"crypto/x509"
	"encoding/base64"
	"encoding/pem"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudfront/sign"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/aws/aws-sdk-go/service/storagegateway"
	"github.com/cenkalti/backoff"
	"go.pedge.io/lion"
)

type amazonClient struct {
	bucket       string
	distribution string
	s3           *s3.S3
	uploader     *s3manager.Uploader
}

func newAmazonClient(bucket string, distribution string, id string, secret string, token string, region string) (*amazonClient, error) {
	session := session.New(&aws.Config{
		Credentials: credentials.NewStaticCredentials(id, secret, token),
		Region:      aws.String(region),
	})
	return &amazonClient{
		bucket:       bucket,
		distribution: strings.TrimSpace(distribution),
		s3:           s3.New(session),
		uploader:     s3manager.NewUploader(session),
	}, nil
}

func (c *amazonClient) Writer(name string) (io.WriteCloser, error) {
	return newBackoffWriteCloser(c, newWriter(c, name)), nil
}

func (c *amazonClient) Walk(name string, fn func(name string) error) error {
	var fnErr error
	if err := c.s3.ListObjectsPages(
		&s3.ListObjectsInput{
			Bucket: aws.String(c.bucket),
			Prefix: aws.String(name),
		},
		func(listObjectsOutput *s3.ListObjectsOutput, lastPage bool) bool {
			for _, object := range listObjectsOutput.Contents {
				if err := fn(*object.Key); err != nil {
					fnErr = err
					return false
				}
			}
			return true
		},
	); err != nil {
		return err
	}
	return fnErr
}

func (c *amazonClient) Reader(name string, offset uint64, size uint64) (io.ReadCloser, error) {
	byteRange := byteRange(offset, size)
	if byteRange != "" {
		byteRange = fmt.Sprintf("bytes=%s", byteRange)
	}

	var reader io.ReadCloser
	if c.distribution != "" {
		var resp *http.Response
		var connErr error
		url := fmt.Sprintf("http://%v.cloudfront.net/%v", c.distribution, name)

		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			return nil, err
		}
		req.Header.Add("Range", byteRange)

		fmt.Println("Checking for cloudfront private key")
		rawCloudfrontPrivateKey, err := ioutil.ReadFile("/amazon-secret/cloudfrontPrivateKey")
		if err == nil {
			// If cloudfront security credentials are present, use them
			fmt.Printf("got cf private key secret: (%v)\n", string(rawCloudfrontPrivateKey))

			rawCloudfrontKeyPairId, err := ioutil.ReadFile("/amazon-secret/cloudfrontKeyPairId")
			if err != nil {
				return nil, fmt.Errorf("cloudfront private key provided, but missing cloudfront key pair id")
			}
			fmt.Printf("got cf keypaird id (%v)\n", rawCloudfrontKeyPairId)
			decodedCloudfrontKeyPairId, err := base64.StdEncoding.DecodeString(string(rawCloudfrontKeyPairId))
			if err != nil {
				return nil, err
			}
			decodedCloudfrontKeyPairId = bytes.TrimSpace(decodedCloudfrontKeyPairId)
			fmt.Printf("decoded keypair id (%v)\n", string(decodedCloudfrontKeyPairId))

			decodedCloudfrontPrivateKey, err := base64.StdEncoding.DecodeString(string(rawCloudfrontPrivateKey))
			if err != nil {
				return nil, err
			}
			decodedCloudfrontPrivateKey = bytes.TrimSpace(decodedCloudfrontPrivateKey)
			fmt.Printf("decoded private key (%v)\n", string(decodedCloudfrontPrivateKey))
			block, _ := pem.Decode(decodedCloudfrontPrivateKey)
			if block == nil || block.Type != "RSA PRIVATE KEY" {
				return nil, fmt.Errorf("block undefined or wrong type: type is (%v) should be (RSA PRIVATE KEY)", block.Type)
			}

			cloudfrontPrivateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes)
			if err != nil {
				return nil, err
			}
			signer := sign.NewURLSigner(string(decodedCloudfrontKeyPairId), cloudfrontPrivateKey)
			signedURL, err := signer.Sign(url, time.Now().Add(1*time.Hour))
			fmt.Printf("orig url (%v), signed url (%v)\n", url, signedURL)
			if err != nil {
				return nil, err
			}
			url = signedURL
		}

		backoff.RetryNotify(func() error {
			resp, connErr = http.DefaultClient.Do(req)
			if connErr != nil && isNetRetryable(connErr) {
				return connErr
			}
			return nil
		}, backoff.NewExponentialBackOff(), func(err error, d time.Duration) {
			lion.Infof("Error connecting to (%v); retrying in %s: %#v", url, d, err)
		})
		if connErr != nil {
			return nil, connErr
		}
		if resp.StatusCode >= 300 {
			// Cloudfront returns 200s, and 206s as success codes
			return nil, fmt.Errorf("cloudfront returned HTTP error code %v for url %v", resp.Status, url)
		}
		reader = resp.Body
	} else {
		getObjectOutput, err := c.s3.GetObject(&s3.GetObjectInput{
			Bucket: aws.String(c.bucket),
			Key:    aws.String(name),
			Range:  aws.String(byteRange),
		})
		if err != nil {
			return nil, err
		}
		reader = getObjectOutput.Body
	}
	return newBackoffReadCloser(c, reader), nil
}

func (c *amazonClient) Delete(name string) error {
	_, err := c.s3.DeleteObject(&s3.DeleteObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err
}

func (c *amazonClient) Exists(name string) bool {
	_, err := c.s3.HeadObject(&s3.HeadObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err == nil
}

func (c *amazonClient) isRetryable(err error) (retVal bool) {
	if strings.Contains(err.Error(), "unexpected EOF") {
		return true
	}

	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	for _, c := range []string{
		storagegateway.ErrorCodeServiceUnavailable,
		storagegateway.ErrorCodeInternalError,
		storagegateway.ErrorCodeGatewayInternalError,
	} {
		if c == awsErr.Code() {
			return true
		}
	}
	return false
}

func (c *amazonClient) IsIgnorable(err error) bool {
	return false
}

func (c *amazonClient) IsNotExist(err error) bool {
	if c.distribution != "" {
		// cloudfront returns forbidden error for nonexisting data
		if strings.Contains(err.Error(), "error code 403") {
			return true
		}
	}
	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	if awsErr.Code() == "NoSuchKey" {
		return true
	}
	return false
}

type amazonWriter struct {
	errChan chan error
	pipe    *io.PipeWriter
}

func newWriter(client *amazonClient, name string) *amazonWriter {
	reader, writer := io.Pipe()
	w := &amazonWriter{
		errChan: make(chan error),
		pipe:    writer,
	}
	go func() {
		_, err := client.uploader.Upload(&s3manager.UploadInput{
			Body:            reader,
			Bucket:          aws.String(client.bucket),
			Key:             aws.String(name),
			ContentEncoding: aws.String("application/octet-stream"),
		})
		w.errChan <- err
	}()
	return w
}

func (w *amazonWriter) Write(p []byte) (int, error) {
	return w.pipe.Write(p)
}

func (w *amazonWriter) Close() error {
	if err := w.pipe.Close(); err != nil {
		return err
	}
	return <-w.errChan
}
<commit_msg>Secrets are not base64 encoded when mounted as files<commit_after>package obj

import (
	"bytes"
	"crypto/x509"
	"encoding/base64"
	"encoding/pem"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudfront/sign"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/aws/aws-sdk-go/service/storagegateway"
	"github.com/cenkalti/backoff"
	"go.pedge.io/lion"
)

type amazonClient struct {
	bucket       string
	distribution string
	s3           *s3.S3
	uploader     *s3manager.Uploader
}

func newAmazonClient(bucket string, distribution string, id string, secret string, token string, region string) (*amazonClient, error) {
	session := session.New(&aws.Config{
		Credentials: credentials.NewStaticCredentials(id, secret, token),
		Region:      aws.String(region),
	})
	return &amazonClient{
		bucket:       bucket,
		distribution: strings.TrimSpace(distribution),
		s3:           s3.New(session),
		uploader:     s3manager.NewUploader(session),
	}, nil
}

func (c *amazonClient) Writer(name string) (io.WriteCloser, error) {
	return newBackoffWriteCloser(c, newWriter(c, name)), nil
}

func (c *amazonClient) Walk(name string, fn func(name string) error) error {
	var fnErr error
	if err := c.s3.ListObjectsPages(
		&s3.ListObjectsInput{
			Bucket: aws.String(c.bucket),
			Prefix: aws.String(name),
		},
		func(listObjectsOutput *s3.ListObjectsOutput, lastPage bool) bool {
			for _, object := range listObjectsOutput.Contents {
				if err := fn(*object.Key); err != nil {
					fnErr = err
					return false
				}
			}
			return true
		},
	); err != nil {
		return err
	}
	return fnErr
}

func (c *amazonClient) Reader(name string, offset uint64, size uint64) (io.ReadCloser, error) {
	byteRange := byteRange(offset, size)
	if byteRange != "" {
		byteRange = fmt.Sprintf("bytes=%s", byteRange)
	}
	fmt.Printf("in amazon.Reader()\n")
	var reader io.ReadCloser
	if c.distribution != "" {
		var resp *http.Response
		var connErr error
		url := fmt.Sprintf("http://%v.cloudfront.net/%v", c.distribution, name)

		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			return nil, err
		}
		req.Header.Add("Range", byteRange)

		fmt.Println("Checking for cloudfront private key")
		rawCloudfrontPrivateKey, err := ioutil.ReadFile("/amazon-secret/cloudfrontPrivateKey")
		if err == nil {
			// If cloudfront security credentials are present, use them
			fmt.Printf("got cf private key secret: (%v)\n", string(rawCloudfrontPrivateKey))

			rawCloudfrontKeyPairId, err := ioutil.ReadFile("/amazon-secret/cloudfrontKeyPairId")
			if err != nil {
				return nil, fmt.Errorf("cloudfront private key provided, but missing cloudfront key pair id")
			}
			fmt.Printf("keypair id: %v\n", string(rawcloudfrontKeyPairId))
			/*
					fmt.Printf("got cf keypaird id (%v)\n", string(rawCloudfrontKeyPairId))
					decodedCloudfrontKeyPairId, err := base64.StdEncoding.DecodeString(string(rawCloudfrontKeyPairId))
					if err != nil {
						return nil, err
					}
					decodedCloudfrontKeyPairId = bytes.TrimSpace(decodedCloudfrontKeyPairId)
					fmt.Printf("decoded keypair id (%v)\n", string(decodedCloudfrontKeyPairId))

				decodedCloudfrontPrivateKey, err := base64.StdEncoding.DecodeString(string(rawCloudfrontPrivateKey))
				if err != nil {
					return nil, err
				}
				decodedCloudfrontPrivateKey = bytes.TrimSpace(decodedCloudfrontPrivateKey)
				fmt.Printf("decoded private key (%v)\n", string(decodedCloudfrontPrivateKey))
			*/
			//			block, _ := pem.Decode(bytes.TrimSpace(decodedCloudfrontPrivateKey))
			block, _ := pem.Decode(bytes.TrimSpace(rawCloudfrontPrivateKey))
			if block == nil || block.Type != "RSA PRIVATE KEY" {
				return nil, fmt.Errorf("block undefined or wrong type: type is (%v) should be (RSA PRIVATE KEY)", block.Type)
			}

			cloudfrontPrivateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes)
			if err != nil {
				return nil, err
			}
			signer := sign.NewURLSigner(string(decodedCloudfrontKeyPairId), cloudfrontPrivateKey)
			signedURL, err := signer.Sign(url, time.Now().Add(1*time.Hour))
			fmt.Printf("orig url (%v), signed url (%v)\n", url, signedURL)
			if err != nil {
				return nil, err
			}
			url = signedURL
		}

		backoff.RetryNotify(func() error {
			resp, connErr = http.DefaultClient.Do(req)
			if connErr != nil && isNetRetryable(connErr) {
				return connErr
			}
			return nil
		}, backoff.NewExponentialBackOff(), func(err error, d time.Duration) {
			lion.Infof("Error connecting to (%v); retrying in %s: %#v", url, d, err)
		})
		if connErr != nil {
			return nil, connErr
		}
		if resp.StatusCode >= 300 {
			// Cloudfront returns 200s, and 206s as success codes
			return nil, fmt.Errorf("cloudfront returned HTTP error code %v for url %v", resp.Status, url)
		}
		reader = resp.Body
	} else {
		getObjectOutput, err := c.s3.GetObject(&s3.GetObjectInput{
			Bucket: aws.String(c.bucket),
			Key:    aws.String(name),
			Range:  aws.String(byteRange),
		})
		if err != nil {
			return nil, err
		}
		reader = getObjectOutput.Body
	}
	return newBackoffReadCloser(c, reader), nil
}

func (c *amazonClient) Delete(name string) error {
	_, err := c.s3.DeleteObject(&s3.DeleteObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err
}

func (c *amazonClient) Exists(name string) bool {
	_, err := c.s3.HeadObject(&s3.HeadObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err == nil
}

func (c *amazonClient) isRetryable(err error) (retVal bool) {
	if strings.Contains(err.Error(), "unexpected EOF") {
		return true
	}

	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	for _, c := range []string{
		storagegateway.ErrorCodeServiceUnavailable,
		storagegateway.ErrorCodeInternalError,
		storagegateway.ErrorCodeGatewayInternalError,
	} {
		if c == awsErr.Code() {
			return true
		}
	}
	return false
}

func (c *amazonClient) IsIgnorable(err error) bool {
	return false
}

func (c *amazonClient) IsNotExist(err error) bool {
	if c.distribution != "" {
		// cloudfront returns forbidden error for nonexisting data
		if strings.Contains(err.Error(), "error code 403") {
			return true
		}
	}
	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	if awsErr.Code() == "NoSuchKey" {
		return true
	}
	return false
}

type amazonWriter struct {
	errChan chan error
	pipe    *io.PipeWriter
}

func newWriter(client *amazonClient, name string) *amazonWriter {
	reader, writer := io.Pipe()
	w := &amazonWriter{
		errChan: make(chan error),
		pipe:    writer,
	}
	go func() {
		_, err := client.uploader.Upload(&s3manager.UploadInput{
			Body:            reader,
			Bucket:          aws.String(client.bucket),
			Key:             aws.String(name),
			ContentEncoding: aws.String("application/octet-stream"),
		})
		w.errChan <- err
	}()
	return w
}

func (w *amazonWriter) Write(p []byte) (int, error) {
	return w.pipe.Write(p)
}

func (w *amazonWriter) Close() error {
	if err := w.pipe.Close(); err != nil {
		return err
	}
	return <-w.errChan
}
<|endoftext|>


<commit_before>package mirror

import (
	"reflect"
)

// VisitType
type VisitType func(t *reflect.StructField, typeIndex, depth int) error

type typeWalker struct {
	indices map[reflect.Type]int
	types   []reflect.Type
}

func (walker *typeWalker) index(t reflect.Type) (int, bool) {
	idx, known := walker.indices[t]
	if known {
		return idx, true
	}
	idx = len(walker.types)
	walker.indices[t] = idx
	walker.types = append(walker.types, t)
	return idx, false
}

func (walker *typeWalker) walk(t *reflect.StructField, visit VisitType) error {
	if t == nil || visit == nil {
		return nil
	}
	type stackNode struct {
		field      *reflect.StructField
		currentIdx int
		parentIdx  int
		depth      int
		known      bool
	}
	idx, known := walker.index(t.Type)
	stack := []stackNode{{t, idx, idx, 0, known}}
	var node stackNode
	for len(stack) > 0 {
		node, stack = stack[0], stack[1:]
		err := visit(node.field, node.currentIdx, node.depth)
		if err != nil {
			return err
		}
		if node.known {
			// to prevent endless loops, don't follow known types
			continue
		}
		// follow container types
		parentIdx := node.currentIdx
		depth := node.depth + 1
		switch t := node.field.Type; t.Kind() {
		case reflect.Struct:
			fields := t.NumField()
			newNodes := make([]stackNode, 0, len(stack)+fields)
			for i := 0; i < fields; i++ {
				// add each field from struct to stack
				field := t.Field(i)
				typeIdx, known := walker.index(field.Type)
				newNodes = append(newNodes, stackNode{&field, typeIdx, parentIdx, depth, known})
			}
			newNodes = append(newNodes, stack...)
		case reflect.Ptr, reflect.Array, reflect.Slice, reflect.Map, reflect.Chan:
			// add element to stack
			field := &reflect.StructField{Type: t.Elem()}
			typeIdx, known := walker.index(field.Type)
			stack = append([]stackNode{stackNode{field, typeIdx, parentIdx, depth, known}}, stack...)
		}
	}
	return nil
}

// Walk will call visit on t and each type in t.
// It will follow struct fields, pointers, arrays, slices, maps and chans
// and call visit on each element.
// For struct fields, Walk will only follow into fields at the first occurence of their type.
// Function handling for reflect.Interface and reflect.Method has to be done by visit.
func Walk(t reflect.Type, visit VisitType) error {
	walker := &typeWalker{indices: make(map[reflect.Type]int)}
	return walker.walk(&reflect.StructField{Type: t}, visit)
}
<commit_msg>at it...<commit_after>package mirror

import (
	"reflect"
)

// VisitType
type VisitType func(t *reflect.StructField, typeIndex, depth int) error

type typeWalker struct {
	indices map[reflect.Type]int
	types   []reflect.Type
}

func (walker *typeWalker) index(t reflect.Type) (int, bool) {
	idx, known := walker.indices[t]
	if known {
		return idx, true
	}
	idx = len(walker.types)
	walker.indices[t] = idx
	walker.types = append(walker.types, t)
	return idx, false
}

func (walker *typeWalker) walk(t *reflect.StructField, visit VisitType) error {
	if t == nil || visit == nil {
		return nil
	}
	type stackNode struct {
		field      *reflect.StructField
		currentIdx int
		parentIdx  int
		depth      int
		known      bool
	}
	idx, known := walker.index(t.Type)
	stack := []stackNode{{t, idx, idx, 0, known}}
	var node stackNode
	for len(stack) > 0 {
		stack, node = stack[:len(stack)-2], stack[len(stack)-1]
		err := visit(node.field, node.currentIdx, node.depth)
		if err != nil {
			return err
		}
		if node.known {
			// to prevent endless loops, don't follow known types
			continue
		}
		// follow container types
		parentIdx := node.currentIdx
		depth := node.depth + 1
		switch t := node.field.Type; t.Kind() {
		case reflect.Struct:
			fields := t.NumField()
			for i := fields - 1; i >= 0; i-- {
				// add each field from struct to stack (hightest first, visiting order is reversed)
				field := t.Field(i)
				typeIdx, known := walker.index(field.Type)
				stack = append(stack, stackNode{&field, typeIdx, parentIdx, depth, known})
			}
		case reflect.Ptr, reflect.Array, reflect.Slice, reflect.Map, reflect.Chan:
			// add element to stack
			field := &reflect.StructField{Type: t.Elem()}
			typeIdx, known := walker.index(field.Type)
			stack = append(stack, stackNode{field, typeIdx, parentIdx, depth, known})
		}
	}
	return nil
}

// Walk will call visit on t and each type in t.
// It will follow struct fields, pointers, arrays, slices, maps and chans
// and call visit on each element.
// For struct fields, Walk will only follow into fields at the first occurence of their type.
// Function handling for reflect.Interface and reflect.Method has to be done by visit.
func Walk(t reflect.Type, visit VisitType) error {
	walker := &typeWalker{indices: make(map[reflect.Type]int)}
	return walker.walk(&reflect.StructField{Type: t}, visit)
}
<|endoftext|>


<commit_before>// +build windows

package storage

import (
	"os"
	"sync"
	"time"
)

func getLockFile(path string, ro bool) (Locker, error) {
	return &lockfile{}, nil
}

type lockfile struct {
	mu   sync.Mutex
	file string
}

func (l *lockfile) Lock() {
}
func (l *lockfile) Unlock() {
}
func (l *lockfile) Modified() (bool, error) {
	return false, nil
}
func (l *lockfile) Touch() error {
	return nil
}
func (l *lockfile) IsReadWrite() bool {
	return false
}

func (l *lockfile) TouchedSince(when time.Time) bool {
	stat, err := os.Stat(l.file)
	if err != nil {
		return true
	}
	return when.Before(stat.ModTime())
}
<commit_msg>Implement Locker.Locked() for windows<commit_after>// +build windows

package storage

import (
	"os"
	"sync"
	"time"
)

func getLockFile(path string, ro bool) (Locker, error) {
	return &lockfile{locked: false}, nil
}

type lockfile struct {
	mu     sync.Mutex
	file   string
	locked bool
}

func (l *lockfile) Lock() {
	l.mu.Lock()
	l.locked = true
}

func (l *lockfile) Unlock() {
	l.locked = false
	l.mu.Unlock()
}

func (l *lockfile) Locked() bool {
	return l.locked
}

func (l *lockfile) Modified() (bool, error) {
	return false, nil
}
func (l *lockfile) Touch() error {
	return nil
}
func (l *lockfile) IsReadWrite() bool {
	return false
}

func (l *lockfile) TouchedSince(when time.Time) bool {
	stat, err := os.Stat(l.file)
	if err != nil {
		return true
	}
	return when.Before(stat.ModTime())
}
<|endoftext|>


<commit_before>package tnetstring

import (
	"errors"
	"reflect"
	"strconv"
)

func Marshal(v interface{}) (s string, err error) {
	defer func() {
		if s, ok := recover().(string); ok {
			err = errors.New(s)
		}
	}()
	val := reflect.ValueOf(v)
	var b outbuf
	b.n = 20
	b.buf = make([]byte, b.n)
	lookupEncode(val.Kind())(&b, val)
	return string(b.buf[b.n:]), nil
}

var encodeTable []func(*outbuf, reflect.Value)

func init() {
	encodeTable = []func(*outbuf, reflect.Value){
		reflect.Invalid:   encodeNull,
		reflect.Bool:      encodeBool,
		reflect.Int:       encodeInt,
		reflect.Int8:      encodeInt,
		reflect.Int16:     encodeInt,
		reflect.Int32:     encodeInt,
		reflect.Int64:     encodeInt,
		reflect.Uint:      encodeUint,
		reflect.Uint8:     encodeUint,
		reflect.Uint16:    encodeUint,
		reflect.Uint32:    encodeUint,
		reflect.Uint64:    encodeUint,
		reflect.Uintptr:   encodeUint,
		reflect.String:    encodeString,
		reflect.Array:     encodeArray,
		reflect.Slice:     encodeArray,
		reflect.Map:       encodeMap,
		reflect.Struct:    encodeStruct,
		reflect.Ptr:       encodeIndirect,
		reflect.Interface: encodeIndirect,
		// include last item so the array has the right length
		reflect.UnsafePointer: nil,
	}
}

func encodeNull(b *outbuf, v reflect.Value) {
	if b.n < 3 {
		b.grow(3)
	}
	b.n -= 3
	copy(b.buf[b.n:], "0:~")
}

func encodeBool(b *outbuf, v reflect.Value) {
	if v.Bool() {
		if b.n < 7 {
			b.grow(7)
		}
		b.n -= 7
		copy(b.buf[b.n:], "4:true!")
		return
	}
	if b.n < 8 {
		b.grow(8)
	}
	b.n -= 8
	copy(b.buf[b.n:], "5:false!")
}

func encodeInt(b *outbuf, v reflect.Value) {
	b.writeTString('#', strconv.FormatInt(v.Int(), 10))
}

func encodeUint(b *outbuf, v reflect.Value) {
	b.writeTString('#', strconv.FormatUint(v.Uint(), 10))
}

func encodeString(b *outbuf, v reflect.Value) {
	b.writeTString(',', v.String())
}

func encodeArray(b *outbuf, v reflect.Value) {
	mark := b.mark(']')
	encodeFunc := lookupEncode(v.Type().Elem().Kind())
	for i := v.Len() - 1; i >= 0; i-- {
		encodeFunc(b, v.Index(i))
	}
	b.writeLen(mark)
}

func encodeMap(b *outbuf, v reflect.Value) {
	mark := b.mark('}')
	mapType := v.Type()
	if mapType.Key().Kind() != reflect.String {
		panic("tnetstring: only maps with string keys can be encoded")
	}
	encodeFunc := lookupEncode(mapType.Elem().Kind())
	for _, key := range v.MapKeys() {
		encodeFunc(b, v.MapIndex(key))
		b.writeTString(',', key.String())
	}
	b.writeLen(mark)
}

func encodeStruct(b *outbuf, v reflect.Value) {
	mark := b.mark('}')
	t := v.Type()
	l := t.NumField()
	for i := l - 1; i >= 0; i-- {
		field := t.Field(i)
		str := field.Tag.Get("tnetstring")
		if str == "" {
			str = field.Name
		}
		lookupEncode(field.Type.Kind())(b, v.Field(i))
		b.writeTString(',', str)
	}
	b.writeLen(mark)
}

func encodeIndirect(b *outbuf, v reflect.Value) {
	for {
		switch kind := v.Kind(); kind {
		case reflect.Ptr, reflect.Interface:
			v = v.Elem()
		default:
			lookupEncode(kind)(b, v)
			return
		}
	}
}

func lookupEncode(k reflect.Kind) func(*outbuf, reflect.Value) {
	if f := encodeTable[k]; f != nil {
		return f
	}
	panic("tnetstring: unsupported type")
}

type outbuf struct {
	buf []byte
	// n is the index we last wrote to or the amount of space left,
	// depending on how you look at it
	n int
}

func (buf *outbuf) writeByte(b byte) {
	if buf.n <= 0 {
		buf.grow(1)
	}
	buf.n--
	buf.buf[buf.n] = b
}

func (b *outbuf) mark(typ byte) int {
	b.writeByte(typ)
	return len(b.buf) - b.n
}

func (b *outbuf) writeTString(typ byte, s string) {
	l := len(s)
	lstr := strconv.Itoa(l)
	n := len(lstr) + 1 + l + 1
	if b.n < n {
		b.grow(n)
	}
	b.n--
	b.buf[b.n] = typ
	b.n -= l
	copy(b.buf[b.n:], s)
	b.n--
	b.buf[b.n] = ':'
	b.n -= len(lstr)
	copy(b.buf[b.n:], lstr)
}

func (b *outbuf) writeLen(mark int) {
	l := len(b.buf) - b.n - mark
	str := strconv.Itoa(l)
	b.writeByte(':')
	if b.n < len(str) {
		b.grow(len(str))
	}
	b.n -= len(str)
	copy(b.buf[b.n:], str)
}

func (b *outbuf) grow(n int) {
	l := len(b.buf)
	need := 2 * l
	if need < l+n-b.n {
		need = l + n - b.n
	}
	buf := make([]byte, need)
	copy(buf[need-l+b.n:], b.buf[b.n:])
	b.buf = buf
	b.n = need - l + b.n
}
<commit_msg>Cleanup from last two commits<commit_after>package tnetstring

import (
	"errors"
	"reflect"
	"strconv"
)

func Marshal(v interface{}) (s string, err error) {
	defer func() {
		if s, ok := recover().(string); ok {
			err = errors.New(s)
		}
	}()
	val := reflect.ValueOf(v)
	var b outbuf
	b.n = 20
	b.buf = make([]byte, b.n)
	lookupEncode(val.Kind())(&b, val)
	return string(b.buf[b.n:]), nil
}

var encodeTable []func(*outbuf, reflect.Value)

func init() {
	encodeTable = []func(*outbuf, reflect.Value){
		reflect.Invalid:   encodeNull,
		reflect.Bool:      encodeBool,
		reflect.Int:       encodeInt,
		reflect.Int8:      encodeInt,
		reflect.Int16:     encodeInt,
		reflect.Int32:     encodeInt,
		reflect.Int64:     encodeInt,
		reflect.Uint:      encodeUint,
		reflect.Uint8:     encodeUint,
		reflect.Uint16:    encodeUint,
		reflect.Uint32:    encodeUint,
		reflect.Uint64:    encodeUint,
		reflect.Uintptr:   encodeUint,
		reflect.String:    encodeString,
		reflect.Array:     encodeArray,
		reflect.Slice:     encodeArray,
		reflect.Map:       encodeMap,
		reflect.Struct:    encodeStruct,
		reflect.Ptr:       encodeIndirect,
		reflect.Interface: encodeIndirect,
		// include last item so the array has the right length
		reflect.UnsafePointer: nil,
	}
}

func encodeNull(b *outbuf, v reflect.Value) {
	b.writeRawString("0:~")
}

func encodeBool(b *outbuf, v reflect.Value) {
	if v.Bool() {
		b.writeRawString("4:true!")
		return
	}
	b.writeRawString("5:false!")
}

func encodeInt(b *outbuf, v reflect.Value) {
	b.writeTString('#', strconv.FormatInt(v.Int(), 10))
}

func encodeUint(b *outbuf, v reflect.Value) {
	b.writeTString('#', strconv.FormatUint(v.Uint(), 10))
}

func encodeString(b *outbuf, v reflect.Value) {
	b.writeTString(',', v.String())
}

func encodeArray(b *outbuf, v reflect.Value) {
	mark := b.mark(']')
	encodeFunc := lookupEncode(v.Type().Elem().Kind())
	for i := v.Len() - 1; i >= 0; i-- {
		encodeFunc(b, v.Index(i))
	}
	b.writeLen(mark)
}

func encodeMap(b *outbuf, v reflect.Value) {
	mark := b.mark('}')
	mapType := v.Type()
	if mapType.Key().Kind() != reflect.String {
		panic("tnetstring: only maps with string keys can be encoded")
	}
	encodeFunc := lookupEncode(mapType.Elem().Kind())
	for _, key := range v.MapKeys() {
		encodeFunc(b, v.MapIndex(key))
		b.writeTString(',', key.String())
	}
	b.writeLen(mark)
}

func encodeStruct(b *outbuf, v reflect.Value) {
	mark := b.mark('}')
	t := v.Type()
	l := t.NumField()
	for i := l - 1; i >= 0; i-- {
		field := t.Field(i)
		str := field.Tag.Get("tnetstring")
		if str == "" {
			str = field.Name
		}
		lookupEncode(field.Type.Kind())(b, v.Field(i))
		b.writeTString(',', str)
	}
	b.writeLen(mark)
}

func encodeIndirect(b *outbuf, v reflect.Value) {
	for {
		switch kind := v.Kind(); kind {
		case reflect.Ptr, reflect.Interface:
			v = v.Elem()
		default:
			lookupEncode(kind)(b, v)
			return
		}
	}
}

func lookupEncode(k reflect.Kind) func(*outbuf, reflect.Value) {
	if f := encodeTable[k]; f != nil {
		return f
	}
	panic("tnetstring: unsupported type")
}

type outbuf struct {
	buf []byte
	// n is the index we last wrote to or the amount of space left,
	// depending on how you look at it
	n int
}

func (b *outbuf) writeRawString(s string) {
	n := len(s)
	if b.n < n {
		b.grow(n)
	}
	b.n -= n
	copy(b.buf[b.n:], s)
}

func (buf *outbuf) writeByte(b byte) {
	if buf.n <= 0 {
		buf.grow(1)
	}
	buf.n--
	buf.buf[buf.n] = b
}

func (b *outbuf) mark(typ byte) int {
	b.writeByte(typ)
	return len(b.buf) - b.n
}

func (b *outbuf) writeTString(typ byte, s string) {
	l := len(s)
	lstr := strconv.Itoa(l)
	n := len(lstr) + 1 + l + 1
	if b.n < n {
		b.grow(n)
	}
	b.n--
	b.buf[b.n] = typ
	b.n -= l
	copy(b.buf[b.n:], s)
	b.n--
	b.buf[b.n] = ':'
	b.n -= len(lstr)
	copy(b.buf[b.n:], lstr)
}

func (b *outbuf) writeLen(mark int) {
	l := len(b.buf) - b.n - mark
	str := strconv.Itoa(l)
	b.writeByte(':')
	if b.n < len(str) {
		b.grow(len(str))
	}
	b.n -= len(str)
	copy(b.buf[b.n:], str)
}

func (b *outbuf) grow(n int) {
	l := len(b.buf)
	need := 2 * l
	if need < l+n-b.n {
		need = l + n - b.n
	}
	buf := make([]byte, need)
	copy(buf[need-l+b.n:], b.buf[b.n:])
	b.buf = buf
	b.n = need - l + b.n
}
<|endoftext|>


<commit_before>package godb

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"reflect"
	"syscall"
	"unsafe"
)

// database engine
type engine struct {
	file *os.File
	data []byte
	page int
	zero []byte
	//maxKey = 24
	//maxVal = page - maxKey - 1 // (-1 is for EOF) 4071
}

func createEmptyFile(path string, size int) error {
	fd, err := os.Create(path)
	if err != nil {
		return err
	}
	if err = fd.Truncate(int64(size)); err != nil {
		return err
	}
	if err = fd.Close(); err != nil {
		return err
	}
	return nil
}

func (e *engine) open(path string) (bool, error) {
	// check to make sure engine is not already open
	if e.file != nil {
		// return an error if it is
		return true, fmt.Errorf("engine[open]: engine is already open at path %q\n", path)
	}
	_, err := os.Stat(path + `.db`)
	var fdstat bool
	// new instance
	if err != nil && !os.IsExist(err) {
		fdstat = true
		dirs, _ := filepath.Split(path)
		err = os.MkdirAll(dirs, 0755) // 0800
		if err != nil {
			return fdstat, err
		}
		// create new database file with initial size of 4MB
		err = createEmptyFile(path+`.db`, (1 << 22))
		if err != nil {
			return fdstat, err
		}
		// create new meta file with initial record size of 4KB
		err = createEmptyFile(path+`.ix`, (1 << 12))
		if err != nil {
			return fdstat, err
		}
	}
	// existing
	fd, err := os.OpenFile(path+`.db`, os.O_RDWR|os.O_APPEND, 0666) // 0800
	if err != nil {
		return fdstat, err
	}
	info, err := fd.Stat()
	if err != nil {
		return fdstat, err
	}
	// map file into virtual address space and set up engine
	e.file = fd
	e.data = mmap(fd, 0, int(info.Size()))
	// read meta file and set page/block size
	info, err = os.Stat(path + `.ix`)
	if err != nil {
		return fdstat, err
	}
	// set / reassign page size
	e.page = int(info.Size())
	// set / reassign maxVal size
	maxVal = page - maxKey - 1
	// set / reassign empty block
	e.zero = make([]byte, e.page)
	// there were no errors, so return mapped size and a nil error
	return fdstat, nil
}

// create and return a new record, returning a non-nil error if
// there is an issue with the key or val check
func (e *engine) newRecord() *record {

}

// add a new record to the engine at the first available slot
// return a non-nil error if there is an issue growing the file
func (e *engine) addRecord(r *record) (int, error) {
	// initialize block position k at beginning of mapped file, as well as future byte offset
	var k, o int
	// start iterating through mapped file reigon one page at a time
	for o < len(e.data) {
		// checking for empty page
		if bytes.Equal(e.data[o:o+e.page], e.zero) {
			// found an empty page, re-use it; copy data into it
			copy(e.data[o:o+e.page], r.data)
			// return location of block in page offset
			return o / e.page, nil
		}
		// go to next page offset
		k++
		o = k * e.page
	}
	// haven't found any empty pages, so let's grow the file
	if err := e.grow(); err != nil {
		return -1, err
	}
	// write.data to page
	copy(e.data[o:o+e.page], r.data)
	// return location of block in page offset
	return o / e.page, nil
}

// update a record at provided offset, assuming one exists
// return a non-nil error if offset is outside of mapped reigon
func (e *engine) setRecord(k int, r *record) error {
	// get byte offset from block position k
	o := k * e.page
	// do a bounds check; if outside of mapped reigon...
	if o+e.page > len(e.data) {
		// do not grow, return an error
		return fmt.Errorf("engine[set]: cannot update record at block %d (offset %d)\n", k, o)
	}
	// wipe page in case updated data is smaller than original dataset
	copy(e.data[o:o+e.page], e.zero)
	// write updated data to page
	copy(e.data[o:o+e.page], r.data)
	// there were no errors, so return nil
	return nil
}

var ErrEmptyRecord error = errors.New("engine: empty record found")
var ErrEngineEOF error = io.EOF

// return a record at provided offset, assuming one exists
// return a non-nil error if offset is outside of mapped reigon
func (e *engine) getRecord(k int) (*record, error) {
	// get byte offset from block position k
	o := k * e.page
	// do a bounds check; if outside of mapped reigon...
	if o+e.page > len(e.data) {
		// ...return an error
		return nil, ErrEngineEOF //fmt.Errorf("engine[get]: cannot return record at block %d (offset %d)\n", k, o)
	}
	// create record to return
	r := new(record)
	// fill out record data if not empty, returning no error
	if n := bytes.LastIndexByte(e.data[o+maxKey-1:o+e.page], eofVal); n > 0 {
		r.data = e.data[o : o+n]
		return r, nil
	}
	// otherwise, return empty record, with an error
	return r, ErrEmptyRecord //fmt.Errorf("engine[get]: empty record found at block %d (offset %d)", k, o)
}

func (e *engine) getRecordKey(k int) ([]byte, error) {
	// get byte offset from block position k
	o := k * e.page
	// do a bounds check; if outside of mapped reigon...
	if o+e.page > len(e.data) {
		// ...return an error
		return nil, fmt.Errorf("engine[getKey]: cannot return key at block %d (offset %d)\n", k, o)
	}
	if !bytes.Equal(e.data[o:o+page], e.zero) {
		return e.data[o : o+maxKey], nil
	}
	// otherwise, return empty record, with an error
	return nil, fmt.Errorf("engine[getKey]: empty key found at block %d (offset %d)", k, o)
}

func (e *engine) getRecordVal(k int) ([]byte, error) {
	// get byte offset from block position k
	o := k * e.page
	// do a bounds check; if outside of mapped reigon...
	if o+e.page > len(e.data) {
		// ...return an error
		return nil, fmt.Errorf("engine[getVal]: cannot return val at block %d (offset %d)\n", k, o)
	}
	// fill out record data if not empty, returning no error
	if n := bytes.LastIndexByte(e.data[o+maxKey:o+e.page], eofVal); n > 0 {
		v := e.data[o+maxKey : o+maxKey+n]
		return v, nil
	}
	// otherwise, return empty record, with an error
	return nil, fmt.Errorf("engine[getVal]: empty val found at block %d (offset %d)", k, o)
}

// delete a record at provided offset, assuming one exists
// return a non-nil error if offset is outside of mapped reigon
func (e *engine) delRecord(k int) error {
	// get byte offset from block position k
	o := k * e.page
	// do a bounds check; if outside of mapped reigon...
	if o+e.page > len(e.data) {
		// ...return an error
		return fmt.Errorf("engine[del]: cannot delete record at block %d (offset %d)\n", k, o)
	}
	// otherwise, wipe page block at offset
	copy(e.data[o:o+page], e.zero)
	// there were no errors, so return nil
	return nil
}

// grow the underlying mapped file
func (e *engine) grow() error {
	// resize the size to double the current, ie. len * 2
	size := ((len(e.data) * 2) + e.page - 1) &^ (e.page - 1)
	// unmap current mapping before growing underlying file...
	e.munmap()
	// truncate underlying file to updated size, check for errors
	if err := syscall.Ftruncate(int(e.file.Fd()), int64(size)); err != nil {
		return err
	}
	// remap underlying file now that it has grown
	e.data = mmap(e.file, 0, size)
	// there were no errors, so return nil
	return nil
}

// close the engine, return any errors encountered
func (e *engine) close() error {
	e.munmap()                             // unmap memory mappings (Munmap automatically flushes)
	if err := e.file.Close(); err != nil { // close underlying file
		return err
	}
	e.file = nil // set file descriptor to nil
	// there were no errors, so return nil
	return nil
}

// temp structure
type payload struct {
	key []byte
	pos int
}

// get all of the record data payloads from the engine
func (e *engine) loadAllRecords() <-chan payload {
	// initialize the channels to return the keys and blocks on
	loader := make(chan payload)
	go func() {
		var o, k int
		// start iterating through mapped file reigon one page at a time
		for o < len(e.data) {
			// checking for non-empty page
			if !bytes.Equal(e.data[o:o+e.page], e.zero) {
				// found one; return key and block offset
				loader <- payload{e.data[o : o+maxKey], (o / e.page)}
			}
			k++
			o = k * e.page
		}
		close(loader)
	}()
	return loader
}

/* +--------------------+ //
// | MEMORY MAPPED FILE | //
// +--------------------+ */

const (
	PROT  uint = syscall.PROT_READ | syscall.PROT_WRITE
	FLAGS uint = syscall.MAP_SHARED
)

func mmap(fd *os.File, off, len int) []byte {
	mm, err := mmapat(0, fd.Fd(), int64(off), int64(len), PROT, FLAGS)
	if err != nil {
		panic(err)
	}
	return mm
}

func mmapat(addr uintptr, fd uintptr, offset, length int64, prot uint, flags uint) ([]byte, error) {
	if length == -1 {
		var stat syscall.Stat_t
		if err := syscall.Fstat(int(fd), &stat); err != nil {
			return nil, err
		}
		length = stat.Size
	}
	addr, err := mmap_syscall(addr, uintptr(length), uintptr(prot), uintptr(flags), fd, offset)
	if err != syscall.Errno(0) {
		return nil, err
	}
	mm := *new([]byte)
	dh := (*reflect.SliceHeader)(unsafe.Pointer(&mm))
	dh.Data = addr
	dh.Len = int(length) // umm... truncating here feels like trouble??
	dh.Cap = dh.Len
	return mm, nil
}

func (e *engine) munmap() {
	dh := *(*reflect.SliceHeader)(unsafe.Pointer(&e.data))
	_, _, err := syscall.Syscall(syscall.SYS_MUNMAP, uintptr(dh.Data), uintptr(dh.Len), 0)
	if err != 0 {
		panic(err)
	}
}

func (e *engine) msync() {
	rh := *(*reflect.SliceHeader)(unsafe.Pointer(&e.data))
	_, _, err := syscall.Syscall(syscall.SYS_MSYNC, uintptr(rh.Data), uintptr(rh.Len), uintptr(syscall.MS_ASYNC))
	if err != 0 {
		panic(err)
	}
}

func (e *engine) misresident() ([]bool, error) {
	sz := os.Getpagesize()                                 // page size
	re := make([]bool, (len(e.data)+sz-1)/sz)              // result
	dh := *(*reflect.SliceHeader)(unsafe.Pointer(&e.data)) // result data ptr
	rh := *(*reflect.SliceHeader)(unsafe.Pointer(&re))     // result data header ptr
	_, _, err := syscall.Syscall(syscall.SYS_MINCORE, uintptr(dh.Data), uintptr(dh.Len), uintptr(rh.Data))
	for i := range re {
		*(*uint8)(unsafe.Pointer(&re[i])) &= 1
	}
	if err != 0 {
		return nil, err
	}
	return re, nil
}
<commit_msg>this is the CORRECT ONE<commit_after>package godb

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"reflect"
	"syscall"
	"unsafe"
)

// database engine
type engine struct {
	file *os.File
	data []byte
	page int
	zero []byte
	//maxKey = 24
	//maxVal = page - maxKey - 1 // (-1 is for EOF) 4071
}

func createEmptyFile(path string, size int) error {
	fd, err := os.Create(path)
	if err != nil {
		return err
	}
	if err = fd.Truncate(int64(size)); err != nil {
		return err
	}
	if err = fd.Close(); err != nil {
		return err
	}
	return nil
}

func (e *engine) open(path string) (bool, error) {
	// check to make sure engine is not already open
	if e.file != nil {
		// return an error if it is
		return true, fmt.Errorf("engine[open]: engine is already open at path %q\n", path)
	}
	_, err := os.Stat(path + `.db`)
	var fdstat bool
	// new instance
	if err != nil && !os.IsExist(err) {
		fdstat = true
		dirs, _ := filepath.Split(path)
		err = os.MkdirAll(dirs, 0755) // 0800
		if err != nil {
			return fdstat, err
		}
		// create new database file with initial size of 2MB
		err = createEmptyFile(path+`.db`, (1 << 21))
		if err != nil {
			return fdstat, err
		}
		// create new meta file with initial record size of 16KB
		err = createEmptyFile(path+`.ix`, (1 << 14))
		if err != nil {
			return fdstat, err
		}
	}
	// existing
	fd, err := os.OpenFile(path+`.db`, os.O_RDWR|os.O_APPEND, 0666) // 0800
	if err != nil {
		return fdstat, err
	}
	info, err := fd.Stat()
	if err != nil {
		return fdstat, err
	}
	// map file into virtual address space and set up engine
	e.file = fd
	e.data = mmap(fd, 0, int(info.Size()))
	// read meta file and set page/block size
	info, err = os.Stat(path + `.ix`)
	if err != nil {
		return fdstat, err
	}
	// set / reassign page size
	e.page = int(info.Size())
	// set / reassign maxVal size
	maxVal = page - maxKey - 1
	// set / reassign empty block
	e.zero = make([]byte, e.page)
	// there were no errors, so return mapped size and a nil error
	return fdstat, nil
}

// create and return a new record, returning a non-nil error if
// there is an issue with the key or val check
func (e *engine) newRecord() *record {

}

// add a new record to the engine at the first available slot
// return a non-nil error if there is an issue growing the file
func (e *engine) addRecord(r *record) (int, error) {
	// initialize block position k at beginning of mapped file, as well as future byte offset
	var k, o int
	// start iterating through mapped file reigon one page at a time
	for o < len(e.data) {
		// checking for empty page
		if bytes.Equal(e.data[o:o+e.page], e.zero) {
			// found an empty page, re-use it; copy data into it
			copy(e.data[o:o+e.page], r.data)
			// return location of block in page offset
			return o / e.page, nil
		}
		// go to next page offset
		k++
		o = k * e.page
	}
	// haven't found any empty pages, so let's grow the file
	if err := e.grow(); err != nil {
		return -1, err
	}
	// write.data to page
	copy(e.data[o:o+e.page], r.data)
	// return location of block in page offset
	return o / e.page, nil
}

// update a record at provided offset, assuming one exists
// return a non-nil error if offset is outside of mapped reigon
func (e *engine) setRecord(k int, r *record) error {
	// get byte offset from block position k
	o := k * e.page
	// do a bounds check; if outside of mapped reigon...
	if o+e.page > len(e.data) {
		// do not grow, return an error
		return fmt.Errorf("engine[set]: cannot update record at block %d (offset %d)\n", k, o)
	}
	// wipe page in case updated data is smaller than original dataset
	copy(e.data[o:o+e.page], e.zero)
	// write updated data to page
	copy(e.data[o:o+e.page], r.data)
	// there were no errors, so return nil
	return nil
}

var ErrEmptyRecord error = errors.New("engine: empty record found")
var ErrEngineEOF error = io.EOF

// return a record at provided offset, assuming one exists
// return a non-nil error if offset is outside of mapped reigon
func (e *engine) getRecord(k int) (*record, error) {
	// get byte offset from block position k
	o := k * e.page
	// do a bounds check; if outside of mapped reigon...
	if o+e.page > len(e.data) {
		// ...return an error
		return nil, ErrEngineEOF //fmt.Errorf("engine[get]: cannot return record at block %d (offset %d)\n", k, o)
	}
	// create record to return
	r := new(record)
	// fill out record data if not empty, returning no error
	if n := bytes.LastIndexByte(e.data[o+maxKey-1:o+e.page], eofVal); n > 0 {
		r.data = e.data[o : o+n]
		return r, nil
	}
	// otherwise, return empty record, with an error
	return r, ErrEmptyRecord //fmt.Errorf("engine[get]: empty record found at block %d (offset %d)", k, o)
}

func (e *engine) getRecordKey(k int) ([]byte, error) {
	// get byte offset from block position k
	o := k * e.page
	// do a bounds check; if outside of mapped reigon...
	if o+e.page > len(e.data) {
		// ...return an error
		return nil, fmt.Errorf("engine[getKey]: cannot return key at block %d (offset %d)\n", k, o)
	}
	if !bytes.Equal(e.data[o:o+page], e.zero) {
		return e.data[o : o+maxKey], nil
	}
	// otherwise, return empty record, with an error
	return nil, fmt.Errorf("engine[getKey]: empty key found at block %d (offset %d)", k, o)
}

func (e *engine) getRecordVal(k int) ([]byte, error) {
	// get byte offset from block position k
	o := k * e.page
	// do a bounds check; if outside of mapped reigon...
	if o+e.page > len(e.data) {
		// ...return an error
		return nil, fmt.Errorf("engine[getVal]: cannot return val at block %d (offset %d)\n", k, o)
	}
	// fill out record data if not empty, returning no error
	if n := bytes.LastIndexByte(e.data[o+maxKey:o+e.page], eofVal); n > 0 {
		v := e.data[o+maxKey : o+maxKey+n]
		return v, nil
	}
	// otherwise, return empty record, with an error
	return nil, fmt.Errorf("engine[getVal]: empty val found at block %d (offset %d)", k, o)
}

// delete a record at provided offset, assuming one exists
// return a non-nil error if offset is outside of mapped reigon
func (e *engine) delRecord(k int) error {
	// get byte offset from block position k
	o := k * e.page
	// do a bounds check; if outside of mapped reigon...
	if o+e.page > len(e.data) {
		// ...return an error
		return fmt.Errorf("engine[del]: cannot delete record at block %d (offset %d)\n", k, o)
	}
	// otherwise, wipe page block at offset
	copy(e.data[o:o+page], e.zero)
	// there were no errors, so return nil
	return nil
}

// grow the underlying mapped file
func (e *engine) grow() error {
	// resize the size to double the current, ie. len * 2
	size := ((len(e.data) * 2) + e.page - 1) &^ (e.page - 1)
	// unmap current mapping before growing underlying file...
	e.munmap()
	// truncate underlying file to updated size, check for errors
	if err := syscall.Ftruncate(int(e.file.Fd()), int64(size)); err != nil {
		return err
	}
	// remap underlying file now that it has grown
	e.data = mmap(e.file, 0, size)
	// there were no errors, so return nil
	return nil
}

// close the engine, return any errors encountered
func (e *engine) close() error {
	e.munmap()                             // unmap memory mappings (Munmap automatically flushes)
	if err := e.file.Close(); err != nil { // close underlying file
		return err
	}
	e.file = nil // set file descriptor to nil
	// there were no errors, so return nil
	return nil
}

// temp structure
type payload struct {
	key []byte
	pos int
}

// get all of the record data payloads from the engine
func (e *engine) loadAllRecords() <-chan payload {
	// initialize the channels to return the keys and blocks on
	loader := make(chan payload)
	go func() {
		var o, k int
		// start iterating through mapped file reigon one page at a time
		for o < len(e.data) {
			// checking for non-empty page
			if !bytes.Equal(e.data[o:o+e.page], e.zero) {
				// found one; return key and block offset
				loader <- payload{e.data[o : o+maxKey], (o / e.page)}
			}
			k++
			o = k * e.page
		}
		close(loader)
	}()
	return loader
}

/* +--------------------+ //
// | MEMORY MAPPED FILE | //
// +--------------------+ */

const (
	PROT  uint = syscall.PROT_READ | syscall.PROT_WRITE
	FLAGS uint = syscall.MAP_SHARED
)

func mmap(fd *os.File, off, len int) []byte {
	mm, err := mmapat(0, fd.Fd(), int64(off), int64(len), PROT, FLAGS)
	if err != nil {
		panic(err)
	}
	return mm
}

func mmapat(addr uintptr, fd uintptr, offset, length int64, prot uint, flags uint) ([]byte, error) {
	if length == -1 {
		var stat syscall.Stat_t
		if err := syscall.Fstat(int(fd), &stat); err != nil {
			return nil, err
		}
		length = stat.Size
	}
	addr, err := mmap_syscall(addr, uintptr(length), uintptr(prot), uintptr(flags), fd, offset)
	if err != syscall.Errno(0) {
		return nil, err
	}
	mm := *new([]byte)
	dh := (*reflect.SliceHeader)(unsafe.Pointer(&mm))
	dh.Data = addr
	dh.Len = int(length) // umm... truncating here feels like trouble??
	dh.Cap = dh.Len
	return mm, nil
}

func (e *engine) munmap() {
	dh := *(*reflect.SliceHeader)(unsafe.Pointer(&e.data))
	_, _, err := syscall.Syscall(syscall.SYS_MUNMAP, uintptr(dh.Data), uintptr(dh.Len), 0)
	if err != 0 {
		panic(err)
	}
}

func (e *engine) msync() {
	rh := *(*reflect.SliceHeader)(unsafe.Pointer(&e.data))
	_, _, err := syscall.Syscall(syscall.SYS_MSYNC, uintptr(rh.Data), uintptr(rh.Len), uintptr(syscall.MS_ASYNC))
	if err != 0 {
		panic(err)
	}
}

func (e *engine) misresident() ([]bool, error) {
	sz := os.Getpagesize()                                 // page size
	re := make([]bool, (len(e.data)+sz-1)/sz)              // result
	dh := *(*reflect.SliceHeader)(unsafe.Pointer(&e.data)) // result data ptr
	rh := *(*reflect.SliceHeader)(unsafe.Pointer(&re))     // result data header ptr
	_, _, err := syscall.Syscall(syscall.SYS_MINCORE, uintptr(dh.Data), uintptr(dh.Len), uintptr(rh.Data))
	for i := range re {
		*(*uint8)(unsafe.Pointer(&re[i])) &= 1
	}
	if err != 0 {
		return nil, err
	}
	return re, nil
}
<|endoftext|>


<commit_before>package fuse

import (
	"net/http"
	"os"
	"path/filepath"

	"github.com/julienschmidt/httprouter"
)

type Handler func(c *Context)

type Engine struct {
	NotFound  Handler
	PublicDir string

	router     *httprouter.Router
	middleware []Handler
}

func New() *Engine {
	engine := &Engine{
		PublicDir:  "public",
		middleware: make([]Handler, 0),
	}

	engine.router = &httprouter.Router{
		RedirectTrailingSlash: true,
		NotFound:              &notFound{engine},
	}

	engine.NotFound = func(c *Context) {
		http.NotFound(c.ResponseWriter, c.Request)
	}

	return engine
}

func (e *Engine) Run(addr string) {
	http.ListenAndServe(addr, e.router)
}

func (e *Engine) Use(handler Handler) {
	e.middleware = append(e.middleware, handler)
}

func (e *Engine) GET(path string, handler Handler) {
	e.router.GET(path, func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
		c := e.makeContext(w, r, p, handler)
		c.Next()
	})
}

func (e *Engine) makeContext(w http.ResponseWriter, r *http.Request, p httprouter.Params, handler Handler) *Context {
	r.ParseForm()

	params := make(map[string]string)
	for _, param := range p {
		params[param.Key] = param.Value
	}

	return &Context{
		Request:        r,
		ResponseWriter: responseWriter{w},
		Params:         params,
		Form:           r.Form,
		PostForm:       r.PostForm,
		engine:         e,
		handler:        handler,
	}
}

func (e *Engine) notFoundHandler(w http.ResponseWriter, r *http.Request) {
	filename := filepath.Join(e.PublicDir, r.URL.Path)
	if _, err := os.Stat(filename); err == nil {
		c := e.makeContext(w, r, nil, func(c *Context) {
			http.ServeFile(c.ResponseWriter, c.Request, filename)
		})
		c.Next()
	} else {
		c := e.makeContext(w, r, nil, e.NotFound)
		c.Next()
	}
}
<commit_msg>Added POST, PUT, DELETE and HEAD<commit_after>package fuse

import (
	"net/http"
	"os"
	"path/filepath"

	"github.com/julienschmidt/httprouter"
)

type Handler func(c *Context)

type Engine struct {
	NotFound  Handler
	PublicDir string

	router     *httprouter.Router
	middleware []Handler
}

func New() *Engine {
	engine := &Engine{
		PublicDir:  "public",
		middleware: make([]Handler, 0),
	}

	engine.router = &httprouter.Router{
		RedirectTrailingSlash: true,
		NotFound:              &notFound{engine},
	}

	engine.NotFound = func(c *Context) {
		http.NotFound(c.ResponseWriter, c.Request)
	}

	return engine
}

func (e *Engine) Run(addr string) {
	http.ListenAndServe(addr, e.router)
}

func (e *Engine) Use(handler Handler) {
	e.middleware = append(e.middleware, handler)
}

func (e *Engine) GET(path string, handler Handler) {
	e.router.GET(path, func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
		c := e.makeContext(w, r, p, handler)
		c.Next()
	})
}

func (e *Engine) POST(path string, handler Handler) {
	e.router.POST(path, func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
		c := e.makeContext(w, r, p, handler)
		c.Next()
	})
}

func (e *Engine) PUT(path string, handler Handler) {
	e.router.PUT(path, func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
		c := e.makeContext(w, r, p, handler)
		c.Next()
	})
}

func (e *Engine) DELETE(path string, handler Handler) {
	e.router.DELETE(path, func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
		c := e.makeContext(w, r, p, handler)
		c.Next()
	})
}

func (e *Engine) HEAD(path string, handler Handler) {
	e.router.HEAD(path, func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
		c := e.makeContext(w, r, p, handler)
		c.Next()
	})
}

func (e *Engine) makeContext(w http.ResponseWriter, r *http.Request, p httprouter.Params, handler Handler) *Context {
	r.ParseForm()

	params := make(map[string]string)
	for _, param := range p {
		params[param.Key] = param.Value
	}

	return &Context{
		Request:        r,
		ResponseWriter: responseWriter{w},
		Params:         params,
		Form:           r.Form,
		PostForm:       r.PostForm,
		engine:         e,
		handler:        handler,
	}
}

func (e *Engine) notFoundHandler(w http.ResponseWriter, r *http.Request) {
	filename := filepath.Join(e.PublicDir, r.URL.Path)
	if _, err := os.Stat(filename); err == nil {
		c := e.makeContext(w, r, nil, func(c *Context) {
			http.ServeFile(c.ResponseWriter, c.Request, filename)
		})
		c.Next()
	} else {
		c := e.makeContext(w, r, nil, e.NotFound)
		c.Next()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"os/exec"
	"strings"
        "bytes"
)

type SslCheck struct {
	Url string
}

func (sslCheck *SslCheck) CheckSync() (bool, error) {
        cmd := exec.Command("python", "hb.py", sslCheck.Url)
	var out bytes.Buffer
	cmd.Stdout = &out
	err := cmd.Run()

	if err != nil {
		logger.Errorf("error running python script: %v", err)
		return false, err
	}

	outLines := strings.Split(out.String(), "\n")
	lastLine := outLines[len(outLines)-2]

	return strings.Contains(lastLine, "server is vulnerable"), nil
}
<commit_msg>debugging<commit_after>package main

import (
	"os/exec"
	"strings"
        "bytes"
)

type SslCheck struct {
	Url string
}

func (sslCheck *SslCheck) CheckSync() (bool, error) {
        cmd := exec.Command("python", "./hb.py", sslCheck.Url)
	var out bytes.Buffer
	cmd.Stdout = &out
	err := cmd.Run()

	if err != nil {
		logger.Errorf("error running python script: %v, %s", err, out.String())
		return false, err
	}

	outLines := strings.Split(out.String(), "\n")
	lastLine := outLines[len(outLines)-2]

	return strings.Contains(lastLine, "server is vulnerable"), nil
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package app does all of the work necessary to create a Kubernetes
// APIServer by binding together the API, master and APIServer infrastructure.
// It can be configured and called directly or via the hyperkube cache.
package app

import (
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/pborman/uuid"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"

	"k8s.io/kubernetes/federation/cmd/federation-apiserver/app/options"
	"k8s.io/kubernetes/pkg/admission"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/apiserver/authenticator"
	"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
	"k8s.io/kubernetes/pkg/controller/informers"
	"k8s.io/kubernetes/pkg/generated/openapi"
	"k8s.io/kubernetes/pkg/genericapiserver"
	"k8s.io/kubernetes/pkg/genericapiserver/authorizer"
	genericoptions "k8s.io/kubernetes/pkg/genericapiserver/options"
	"k8s.io/kubernetes/pkg/registry/cachesize"
	"k8s.io/kubernetes/pkg/registry/generic"
	genericregistry "k8s.io/kubernetes/pkg/registry/generic/registry"
	"k8s.io/kubernetes/pkg/routes"
	"k8s.io/kubernetes/pkg/runtime/schema"
	utilerrors "k8s.io/kubernetes/pkg/util/errors"
	"k8s.io/kubernetes/pkg/util/wait"
	"k8s.io/kubernetes/pkg/version"
)

// NewAPIServerCommand creates a *cobra.Command object with default parameters
func NewAPIServerCommand() *cobra.Command {
	s := options.NewServerRunOptions()
	s.AddFlags(pflag.CommandLine)
	cmd := &cobra.Command{
		Use: "federation-apiserver",
		Long: `The Kubernetes federation API server validates and configures data
for the api objects which include pods, services, replicationcontrollers, and
others. The API Server services REST operations and provides the frontend to the
cluster's shared state through which all other components interact.`,
		Run: func(cmd *cobra.Command, args []string) {
		},
	}
	return cmd
}

// Run runs the specified APIServer.  This should never exit.
func Run(s *options.ServerRunOptions) error {
	if errs := s.Etcd.Validate(); len(errs) > 0 {
		utilerrors.NewAggregate(errs)
	}
	if err := s.GenericServerRunOptions.DefaultExternalAddress(s.SecureServing, s.InsecureServing); err != nil {
		return err
	}

	genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions)
	genericConfig := genericapiserver.NewConfig(). // create the new config
							ApplyOptions(s.GenericServerRunOptions). // apply the options selected
							ApplySecureServingOptions(s.SecureServing).
							ApplyInsecureServingOptions(s.InsecureServing).
							ApplyAuthenticationOptions(s.Authentication).
							ApplyRBACSuperUser(s.Authorization.RBACSuperUser)

	if err := genericConfig.MaybeGenerateServingCerts(); err != nil {
		glog.Fatalf("Failed to generate service certificate: %v", err)
	}

	// TODO: register cluster federation resources here.
	resourceConfig := genericapiserver.NewResourceConfig()

	if s.Etcd.StorageConfig.DeserializationCacheSize == 0 {
		// When size of cache is not explicitly set, set it to 50000
		s.Etcd.StorageConfig.DeserializationCacheSize = 50000
	}
	storageGroupsToEncodingVersion, err := s.GenericServerRunOptions.StorageGroupsToEncodingVersion()
	if err != nil {
		glog.Fatalf("error generating storage version map: %s", err)
	}
	storageFactory, err := genericapiserver.BuildDefaultStorageFactory(
		s.Etcd.StorageConfig, s.GenericServerRunOptions.DefaultStorageMediaType, api.Codecs,
		genericapiserver.NewDefaultResourceEncodingConfig(), storageGroupsToEncodingVersion,
		[]schema.GroupVersionResource{}, resourceConfig, s.GenericServerRunOptions.RuntimeConfig)
	if err != nil {
		glog.Fatalf("error in initializing storage factory: %s", err)
	}

	for _, override := range s.Etcd.EtcdServersOverrides {
		tokens := strings.Split(override, "#")
		if len(tokens) != 2 {
			glog.Errorf("invalid value of etcd server overrides: %s", override)
			continue
		}

		apiresource := strings.Split(tokens[0], "/")
		if len(apiresource) != 2 {
			glog.Errorf("invalid resource definition: %s", tokens[0])
			continue
		}
		group := apiresource[0]
		resource := apiresource[1]
		groupResource := schema.GroupResource{Group: group, Resource: resource}

		servers := strings.Split(tokens[1], ";")
		storageFactory.SetEtcdLocation(groupResource, servers)
	}

	apiAuthenticator, securityDefinitions, err := authenticator.New(s.Authentication.ToAuthenticationConfig(s.SecureServing.ClientCA))
	if err != nil {
		glog.Fatalf("Invalid Authentication Config: %v", err)
	}

	privilegedLoopbackToken := uuid.NewRandom().String()
	selfClientConfig, err := genericoptions.NewSelfClientConfig(s.SecureServing, s.InsecureServing, privilegedLoopbackToken)
	if err != nil {
		glog.Fatalf("Failed to create clientset: %v", err)
	}
	client, err := internalclientset.NewForConfig(selfClientConfig)
	if err != nil {
		glog.Errorf("Failed to create clientset: %v", err)
	}
	sharedInformers := informers.NewSharedInformerFactory(nil, client, 10*time.Minute)

	authorizerconfig := s.Authorization.ToAuthorizationConfig(sharedInformers)
	apiAuthorizer, err := authorizer.NewAuthorizerFromAuthorizationConfig(authorizerconfig)
	if err != nil {
		glog.Fatalf("Invalid Authorization Config: %v", err)
	}

	admissionControlPluginNames := strings.Split(s.GenericServerRunOptions.AdmissionControl, ",")
	pluginInitializer := admission.NewPluginInitializer(sharedInformers, apiAuthorizer)
	admissionController, err := admission.NewFromPlugins(client, admissionControlPluginNames, s.GenericServerRunOptions.AdmissionControlConfigFile, pluginInitializer)
	if err != nil {
		glog.Fatalf("Failed to initialize plugins: %v", err)
	}

	kubeVersion := version.Get()
	genericConfig.Version = &kubeVersion
	genericConfig.LoopbackClientConfig = selfClientConfig
	genericConfig.Authenticator = apiAuthenticator
	genericConfig.Authorizer = apiAuthorizer
	genericConfig.AdmissionControl = admissionController
	genericConfig.APIResourceConfigSource = storageFactory.APIResourceConfigSource
	genericConfig.OpenAPIConfig.Definitions = openapi.OpenAPIDefinitions
	genericConfig.EnableOpenAPISupport = true
	genericConfig.OpenAPIConfig.SecurityDefinitions = securityDefinitions

	// TODO: Move this to generic api server (Need to move the command line flag).
	if s.GenericServerRunOptions.EnableWatchCache {
		cachesize.InitializeWatchCacheSizes(s.GenericServerRunOptions.TargetRAMMB)
		cachesize.SetWatchCacheSizes(s.GenericServerRunOptions.WatchCacheSizes)
	}

	m, err := genericConfig.Complete().New()
	if err != nil {
		return err
	}

	routes.UIRedirect{}.Install(m.HandlerContainer)
	routes.Logs{}.Install(m.HandlerContainer)

	// TODO: Refactor this code to share it with kube-apiserver rather than duplicating it here.
	restOptionsFactory := restOptionsFactory{
		storageFactory:          storageFactory,
		enableGarbageCollection: s.GenericServerRunOptions.EnableGarbageCollection,
		deleteCollectionWorkers: s.GenericServerRunOptions.DeleteCollectionWorkers,
	}
	if s.GenericServerRunOptions.EnableWatchCache {
		restOptionsFactory.storageDecorator = genericregistry.StorageWithCacher
	} else {
		restOptionsFactory.storageDecorator = generic.UndecoratedStorage
	}

	installFederationAPIs(m, restOptionsFactory)
	installCoreAPIs(s, m, restOptionsFactory)
	installExtensionsAPIs(m, restOptionsFactory)

	sharedInformers.Start(wait.NeverStop)
	m.PrepareRun().Run(wait.NeverStop)
	return nil
}

type restOptionsFactory struct {
	storageFactory          genericapiserver.StorageFactory
	storageDecorator        generic.StorageDecorator
	deleteCollectionWorkers int
	enableGarbageCollection bool
}

func (f restOptionsFactory) NewFor(resource schema.GroupResource) generic.RESTOptions {
	config, err := f.storageFactory.NewConfig(resource)
	if err != nil {
		glog.Fatalf("Unable to find storage config for %v, due to %v", resource, err.Error())
	}
	return generic.RESTOptions{
		StorageConfig:           config,
		Decorator:               f.storageDecorator,
		DeleteCollectionWorkers: f.deleteCollectionWorkers,
		EnableGarbageCollection: f.enableGarbageCollection,
		ResourcePrefix:          f.storageFactory.ResourcePrefix(resource),
	}
}
<commit_msg>Stratify certificate loading and self-sign cert generation<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package app does all of the work necessary to create a Kubernetes
// APIServer by binding together the API, master and APIServer infrastructure.
// It can be configured and called directly or via the hyperkube cache.
package app

import (
	"fmt"
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/pborman/uuid"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"

	"k8s.io/kubernetes/federation/cmd/federation-apiserver/app/options"
	"k8s.io/kubernetes/pkg/admission"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/apiserver/authenticator"
	"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
	"k8s.io/kubernetes/pkg/controller/informers"
	"k8s.io/kubernetes/pkg/generated/openapi"
	"k8s.io/kubernetes/pkg/genericapiserver"
	"k8s.io/kubernetes/pkg/genericapiserver/authorizer"
	"k8s.io/kubernetes/pkg/registry/cachesize"
	"k8s.io/kubernetes/pkg/registry/generic"
	genericregistry "k8s.io/kubernetes/pkg/registry/generic/registry"
	"k8s.io/kubernetes/pkg/routes"
	"k8s.io/kubernetes/pkg/runtime/schema"
	utilerrors "k8s.io/kubernetes/pkg/util/errors"
	"k8s.io/kubernetes/pkg/util/wait"
	"k8s.io/kubernetes/pkg/version"
)

// NewAPIServerCommand creates a *cobra.Command object with default parameters
func NewAPIServerCommand() *cobra.Command {
	s := options.NewServerRunOptions()
	s.AddFlags(pflag.CommandLine)
	cmd := &cobra.Command{
		Use: "federation-apiserver",
		Long: `The Kubernetes federation API server validates and configures data
for the api objects which include pods, services, replicationcontrollers, and
others. The API Server services REST operations and provides the frontend to the
cluster's shared state through which all other components interact.`,
		Run: func(cmd *cobra.Command, args []string) {
		},
	}
	return cmd
}

// Run runs the specified APIServer.  This should never exit.
func Run(s *options.ServerRunOptions) error {
	if errs := s.Etcd.Validate(); len(errs) > 0 {
		utilerrors.NewAggregate(errs)
	}
	if err := s.GenericServerRunOptions.DefaultExternalAddress(s.SecureServing, s.InsecureServing); err != nil {
		return err
	}

	if err := s.SecureServing.MaybeDefaultWithSelfSignedCerts(s.GenericServerRunOptions.AdvertiseAddress.String()); err != nil {
		return fmt.Errorf("error creating self-signed certificates: %v", err)
	}

	genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions)
	genericConfig, err := genericapiserver.NewConfig(). // create the new config
								ApplyOptions(s.GenericServerRunOptions). // apply the options selected
								ApplyInsecureServingOptions(s.InsecureServing).
								ApplyAuthenticationOptions(s.Authentication).
								ApplyRBACSuperUser(s.Authorization.RBACSuperUser).
								ApplySecureServingOptions(s.SecureServing)
	if err != nil {
		return fmt.Errorf("failed to configure https: %s", err)
	}

	// TODO: register cluster federation resources here.
	resourceConfig := genericapiserver.NewResourceConfig()

	if s.Etcd.StorageConfig.DeserializationCacheSize == 0 {
		// When size of cache is not explicitly set, set it to 50000
		s.Etcd.StorageConfig.DeserializationCacheSize = 50000
	}
	storageGroupsToEncodingVersion, err := s.GenericServerRunOptions.StorageGroupsToEncodingVersion()
	if err != nil {
		glog.Fatalf("error generating storage version map: %s", err)
	}
	storageFactory, err := genericapiserver.BuildDefaultStorageFactory(
		s.Etcd.StorageConfig, s.GenericServerRunOptions.DefaultStorageMediaType, api.Codecs,
		genericapiserver.NewDefaultResourceEncodingConfig(), storageGroupsToEncodingVersion,
		[]schema.GroupVersionResource{}, resourceConfig, s.GenericServerRunOptions.RuntimeConfig)
	if err != nil {
		glog.Fatalf("error in initializing storage factory: %s", err)
	}

	for _, override := range s.Etcd.EtcdServersOverrides {
		tokens := strings.Split(override, "#")
		if len(tokens) != 2 {
			glog.Errorf("invalid value of etcd server overrides: %s", override)
			continue
		}

		apiresource := strings.Split(tokens[0], "/")
		if len(apiresource) != 2 {
			glog.Errorf("invalid resource definition: %s", tokens[0])
			continue
		}
		group := apiresource[0]
		resource := apiresource[1]
		groupResource := schema.GroupResource{Group: group, Resource: resource}

		servers := strings.Split(tokens[1], ";")
		storageFactory.SetEtcdLocation(groupResource, servers)
	}

	apiAuthenticator, securityDefinitions, err := authenticator.New(s.Authentication.ToAuthenticationConfig(s.SecureServing.ClientCA))
	if err != nil {
		glog.Fatalf("Invalid Authentication Config: %v", err)
	}

	privilegedLoopbackToken := uuid.NewRandom().String()
	selfClientConfig, err := genericapiserver.NewSelfClientConfig(genericConfig.SecureServingInfo, genericConfig.InsecureServingInfo, privilegedLoopbackToken)
	if err != nil {
		glog.Fatalf("Failed to create clientset: %v", err)
	}
	client, err := internalclientset.NewForConfig(selfClientConfig)
	if err != nil {
		glog.Errorf("Failed to create clientset: %v", err)
	}
	sharedInformers := informers.NewSharedInformerFactory(nil, client, 10*time.Minute)

	authorizerconfig := s.Authorization.ToAuthorizationConfig(sharedInformers)
	apiAuthorizer, err := authorizer.NewAuthorizerFromAuthorizationConfig(authorizerconfig)
	if err != nil {
		glog.Fatalf("Invalid Authorization Config: %v", err)
	}

	admissionControlPluginNames := strings.Split(s.GenericServerRunOptions.AdmissionControl, ",")
	pluginInitializer := admission.NewPluginInitializer(sharedInformers, apiAuthorizer)
	admissionController, err := admission.NewFromPlugins(client, admissionControlPluginNames, s.GenericServerRunOptions.AdmissionControlConfigFile, pluginInitializer)
	if err != nil {
		glog.Fatalf("Failed to initialize plugins: %v", err)
	}

	kubeVersion := version.Get()
	genericConfig.Version = &kubeVersion
	genericConfig.LoopbackClientConfig = selfClientConfig
	genericConfig.Authenticator = apiAuthenticator
	genericConfig.Authorizer = apiAuthorizer
	genericConfig.AdmissionControl = admissionController
	genericConfig.APIResourceConfigSource = storageFactory.APIResourceConfigSource
	genericConfig.OpenAPIConfig.Definitions = openapi.OpenAPIDefinitions
	genericConfig.EnableOpenAPISupport = true
	genericConfig.OpenAPIConfig.SecurityDefinitions = securityDefinitions

	// TODO: Move this to generic api server (Need to move the command line flag).
	if s.GenericServerRunOptions.EnableWatchCache {
		cachesize.InitializeWatchCacheSizes(s.GenericServerRunOptions.TargetRAMMB)
		cachesize.SetWatchCacheSizes(s.GenericServerRunOptions.WatchCacheSizes)
	}

	m, err := genericConfig.Complete().New()
	if err != nil {
		return err
	}

	routes.UIRedirect{}.Install(m.HandlerContainer)
	routes.Logs{}.Install(m.HandlerContainer)

	// TODO: Refactor this code to share it with kube-apiserver rather than duplicating it here.
	restOptionsFactory := restOptionsFactory{
		storageFactory:          storageFactory,
		enableGarbageCollection: s.GenericServerRunOptions.EnableGarbageCollection,
		deleteCollectionWorkers: s.GenericServerRunOptions.DeleteCollectionWorkers,
	}
	if s.GenericServerRunOptions.EnableWatchCache {
		restOptionsFactory.storageDecorator = genericregistry.StorageWithCacher
	} else {
		restOptionsFactory.storageDecorator = generic.UndecoratedStorage
	}

	installFederationAPIs(m, restOptionsFactory)
	installCoreAPIs(s, m, restOptionsFactory)
	installExtensionsAPIs(m, restOptionsFactory)

	sharedInformers.Start(wait.NeverStop)
	m.PrepareRun().Run(wait.NeverStop)
	return nil
}

type restOptionsFactory struct {
	storageFactory          genericapiserver.StorageFactory
	storageDecorator        generic.StorageDecorator
	deleteCollectionWorkers int
	enableGarbageCollection bool
}

func (f restOptionsFactory) NewFor(resource schema.GroupResource) generic.RESTOptions {
	config, err := f.storageFactory.NewConfig(resource)
	if err != nil {
		glog.Fatalf("Unable to find storage config for %v, due to %v", resource, err.Error())
	}
	return generic.RESTOptions{
		StorageConfig:           config,
		Decorator:               f.storageDecorator,
		DeleteCollectionWorkers: f.deleteCollectionWorkers,
		EnableGarbageCollection: f.enableGarbageCollection,
		ResourcePrefix:          f.storageFactory.ResourcePrefix(resource),
	}
}
<|endoftext|>