<commit_before>package kinesis

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"github.com/AdRoll/goamz/aws"
	kin "github.com/AdRoll/goamz/kinesis"
	"github.com/mozilla-services/heka/pipeline"
	"time"
)

type KinesisOutput struct {
	auth   aws.Auth
	config *KinesisOutputConfig
	Client *kin.Kinesis
}

type KinesisOutputConfig struct {
	Region          string `toml:"region"`
	Stream          string `toml:"stream"`
	AccessKeyID     string `toml:"access_key_id"`
	SecretAccessKey string `toml:"secret_access_key"`
	Token           string `toml:"token"`
	PayloadOnly     bool   `toml:"payload_only"`
}

func (k *KinesisOutput) ConfigStruct() interface{} {
	return &KinesisOutputConfig{
		Region:          "us-east-1",
		Stream:          "",
		AccessKeyID:     "",
		SecretAccessKey: "",
		Token:           "",
	}
}

func (k *KinesisOutput) Init(config interface{}) error {
	k.config = config.(*KinesisOutputConfig)
	a, err := aws.GetAuth(k.config.AccessKeyID, k.config.SecretAccessKey, k.config.Token, time.Now())
	if err != nil {
		return fmt.Errorf("Error authenticating: %s", err)
	}
	k.auth = a

	region, ok := aws.Regions[k.config.Region]
	if !ok {
		return fmt.Errorf("Region does not exist: %s", k.config.Region)
	}

	k.Client = kin.New(k.auth, region)

	return nil
}

func (k *KinesisOutput) Run(or pipeline.OutputRunner, helper pipeline.PluginHelper) error {
	var (
		pack     *pipeline.PipelinePack
		contents []byte
		msg      []byte
		err      error
	)

	for pack = range or.InChan() {
		msg, err = or.Encode(pack)
		if err != nil {
			or.LogError(fmt.Errorf("Error encoding message: %s", err))
			pack.Recycle()
			continue
		}
		if contents, err = json.Marshal(msg); err != nil {
			or.LogError(fmt.Errorf("Error marshalling: %s", err))
			pack.Recycle()
			continue
		} else {
			contents, err = base64.StdEncoding.DecodeString(string(contents))
			if err != nil {
				or.LogError(fmt.Errorf("Error decoding: %s", err))
				pack.Recycle()
				continue
			}
			pk := fmt.Sprintf("%d-%s", pack.Message.Timestamp, pack.Message.Hostname)
			_, err = k.Client.PutRecord(k.config.Stream, pk, contents, "", "")
			if err != nil {
				or.LogError(fmt.Errorf("Error pushing message to Kinesis: %s", err))
				pack.Recycle()
				continue
			}
		}
		pack.Recycle()
	}

	return nil
}

func init() {
	pipeline.RegisterPlugin("KinesisOutput", func() interface{} { return new(KinesisOutput) })
}
<commit_msg>what is going on here<commit_after>package kinesis

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"github.com/AdRoll/goamz/aws"
	kin "github.com/AdRoll/goamz/kinesis"
	"github.com/mozilla-services/heka/pipeline"
	"time"
)

type KinesisOutput struct {
	auth   aws.Auth
	config *KinesisOutputConfig
	Client *kin.Kinesis
}

type KinesisOutputConfig struct {
	Region          string `toml:"region"`
	Stream          string `toml:"stream"`
	AccessKeyID     string `toml:"access_key_id"`
	SecretAccessKey string `toml:"secret_access_key"`
	Token           string `toml:"token"`
	PayloadOnly     bool   `toml:"payload_only"`
}

func (k *KinesisOutput) ConfigStruct() interface{} {
	return &KinesisOutputConfig{
		Region:          "us-east-1",
		Stream:          "",
		AccessKeyID:     "",
		SecretAccessKey: "",
		Token:           "",
	}
}

func (k *KinesisOutput) Init(config interface{}) error {
	k.config = config.(*KinesisOutputConfig)
	a, err := aws.GetAuth(k.config.AccessKeyID, k.config.SecretAccessKey, k.config.Token, time.Now())
	if err != nil {
		return fmt.Errorf("Error authenticating: %s", err)
	}
	k.auth = a

	region, ok := aws.Regions[k.config.Region]
	if !ok {
		return fmt.Errorf("Region does not exist: %s", k.config.Region)
	}

	k.Client = kin.New(k.auth, region)

	return nil
}

func (k *KinesisOutput) Run(or pipeline.OutputRunner, helper pipeline.PluginHelper) error {
	var (
		pack     *pipeline.PipelinePack
		contents []byte
		msg      []byte
		err      error
	)

	for pack = range or.InChan() {
		msg, err = or.Encode(pack)
		if err != nil {
			or.LogError(fmt.Errorf("Error encoding message: %s", err))
			pack.Recycle()
			continue
		}
		if contents, err = json.Marshal(msg); err != nil {
			or.LogError(fmt.Errorf("Error marshalling: %s", err))
			pack.Recycle()
			continue
		} else {
			or.LogMessage(string(contents))
			pk := fmt.Sprintf("%d-%s", pack.Message.Timestamp, pack.Message.Hostname)
			_, err = k.Client.PutRecord(k.config.Stream, pk, contents, "", "")
			if err != nil {
				or.LogError(fmt.Errorf("Error pushing message to Kinesis: %s", err))
				pack.Recycle()
				continue
			}
		}
		pack.Recycle()
	}

	return nil
}

func init() {
	pipeline.RegisterPlugin("KinesisOutput", func() interface{} { return new(KinesisOutput) })
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Unmarshal's a Containers description json file. The json file contains
// an array of ContainerHint structs, each with a container's id and networkInterface
// This allows collecting stats about network interfaces configured outside docker
// and lxc
package common

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"os"
)

var ArgContainerHints = flag.String("container_hints", "/etc/cadvisor/container_hints.json", "location of the container hints file")

type containerHints struct {
	AllHosts []containerHint `json:"all_hosts,omitempty"`
}

type containerHint struct {
	FullName         string            `json:"full_path,omitempty"`
	NetworkInterface *networkInterface `json:"network_interface,omitempty"`
	Mounts           []Mount           `json:"mounts,omitempty"`
}

type Mount struct {
	HostDir      string `json:"host_dir,omitempty"`
	ContainerDir string `json:"container_dir,omitempty"`
}

type networkInterface struct {
	VethHost  string `json:"veth_host,omitempty"`
	VethChild string `json:"veth_child,omitempty"`
}

func GetContainerHintsFromFile(containerHintsFile string) (containerHints, error) {
	dat, err := ioutil.ReadFile(containerHintsFile)
	if os.IsNotExist(err) {
		return containerHints{}, nil
	}
	var cHints containerHints
	if err == nil {
		err = json.Unmarshal(dat, &cHints)
	}

	return cHints, err
}
<commit_msg>Exporting unexported struct<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Unmarshal's a Containers description json file. The json file contains
// an array of ContainerHint structs, each with a container's id and networkInterface
// This allows collecting stats about network interfaces configured outside docker
// and lxc
package common

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"os"
)

var ArgContainerHints = flag.String("container_hints", "/etc/cadvisor/container_hints.json", "location of the container hints file")

type ContainerHints struct {
	AllHosts []containerHint `json:"all_hosts,omitempty"`
}

type containerHint struct {
	FullName         string            `json:"full_path,omitempty"`
	NetworkInterface *networkInterface `json:"network_interface,omitempty"`
	Mounts           []Mount           `json:"mounts,omitempty"`
}

type Mount struct {
	HostDir      string `json:"host_dir,omitempty"`
	ContainerDir string `json:"container_dir,omitempty"`
}

type networkInterface struct {
	VethHost  string `json:"veth_host,omitempty"`
	VethChild string `json:"veth_child,omitempty"`
}

func GetContainerHintsFromFile(containerHintsFile string) (ContainerHints, error) {
	dat, err := ioutil.ReadFile(containerHintsFile)
	if os.IsNotExist(err) {
		return ContainerHints{}, nil
	}
	var cHints ContainerHints
	if err == nil {
		err = json.Unmarshal(dat, &cHints)
	}

	return cHints, err
}
<|endoftext|>


<commit_before><commit_msg>increase min manifest version to 4<commit_after><|endoftext|>


<commit_before>package processorcommand

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"

	"github.com/trustmaster/go-aspell"
)

type OCRResult struct {
	Type string
	Text string
}

func newOCRResult(ocrType string, result string) *OCRResult {
	return &OCRResult{
		ocrType,
		result,
	}
}

func (this *OCRResult) removeNonWords(blob string) string {
	speller, err := aspell.NewSpeller(map[string]string{
		"lang": "en_US",
	})
	if err != nil {
		fmt.Printf("Error: %s", err.Error())
		return ""
	}
	defer speller.Delete()

	singleCharWords := regexp.MustCompile("(a|i)")
	numberRegex := regexp.MustCompile("\\d{3,}")
	wordRegexp := regexp.MustCompile("\\b(\\w+)\\b")
	words := wordRegexp.FindAllString(blob, -1)

	str := ""

	for _, word := range words {
		if numberRegex.MatchString(word) {
			str += " " + word
		} else if len(word) == 1 {
			if singleCharWords.MatchString(word) {
				str += " " + word
			}
		} else if speller.Check(word) {
			str += " " + word
		}
	}

	return strings.TrimSpace(str)
}

func (this *OCRResult) wordCount(blob string) int {
	word_regexp := regexp.MustCompile("\\b(\\w+)\\b")
	words := word_regexp.FindAllString(blob, -1)

	// don't let single char words count towards the overal word count. Gets thrown off by poor OCR results
	count := 0
	for _, word := range words {
		if len(word) > 1 {
			count++
		}
	}

	return count
}

type MultiOCRCommand []OCRCommand

func (this MultiOCRCommand) Run(image string) (*OCRResult, error) {
	results := make(chan *OCRResult, len(this))
	errs := make(chan error, len(this))

	for _, command := range this {
		go func(c OCRCommand) {
			k, err := c.Run(image)
			if err != nil {
				errs <- err
				return
			}
			results <- k
		}(command)
	}

	max := -1
	var best *OCRResult

	for i := 0; i < len(this); i++ {
		select {
		case result := <-results:
			blob := result.removeNonWords(result.Text)
			count := result.wordCount(blob)

			if count > max {
				best = result
				max = count
			}

		case err := <-errs:
			return nil, err
		}
	}

	// Return the average, same as before.
	return best, nil
}

type OCRCommand interface {
	Run(image string) (*OCRResult, error)
}

type MemeOCR struct {
	name string
}

func NewMemeOCR() *MemeOCR {
	return &MemeOCR{
		"MemeOCR",
	}
}

func (this *MemeOCR) Run(image string) (*OCRResult, error) {
	imageTif := fmt.Sprintf("%s_meme.jpg", image)
	outText := fmt.Sprintf("%s_meme", image)
	preprocessingArgs := []string{image, "-resize", "400%", "-fill", "black", "-fuzz", "10%", "+opaque", "#FFFFFF", imageTif}
	tesseractArgs := []string{"-l", "meme", imageTif, outText}

	err := runProcessorCommand(GM_COMMAND, preprocessingArgs)
	if err != nil {
		return nil, errors.New(fmt.Sprintf("Meme preprocessing command failed with error = %v", err))
	}
	defer os.Remove(imageTif)

	err = runProcessorCommand("tesseract", tesseractArgs)
	if err != nil {
		return nil, errors.New(fmt.Sprintf("Meme tesseract command failed with error = %v", err))
	}
	defer os.Remove(outText + ".txt")

	text, err := ioutil.ReadFile(outText + ".txt")
	if err != nil {
		return nil, err
	}

	result := strings.ToLower(strings.TrimSpace(string(text[:])))

	return newOCRResult(this.name, result), nil
}

type StandardOCR struct {
	name string
}

func NewStandardOCR() *StandardOCR {
	return &StandardOCR{
		"StandardOCR",
	}
}

func (this *StandardOCR) Run(image string) (*OCRResult, error) {
	imageTif := fmt.Sprintf("%s_standard.jpg", image)
	outText := fmt.Sprintf("%s_standard", image)

	preprocessingArgs := []string{image, "-resize", "400%", "-type", "Grayscale", imageTif}
	tesseractArgs := []string{"-l", "eng", imageTif, outText}

	err := runProcessorCommand(GM_COMMAND, preprocessingArgs)
	if err != nil {
		return nil, errors.New(fmt.Sprintf("Standard preprocessing command failed with error = %v", err))
	}
	defer os.Remove(imageTif)

	err = runProcessorCommand("tesseract", tesseractArgs)
	if err != nil {
		return nil, errors.New(fmt.Sprintf("Standard tesseract command failed with error = %v", err))
	}
	defer os.Remove(outText + ".txt")

	text, err := ioutil.ReadFile(outText + ".txt")
	if err != nil {
		return nil, err
	}

	result := strings.ToLower(strings.TrimSpace(string(text[:])))

	return newOCRResult(this.name, result), nil
}
<commit_msg>Ocr clean text (#104)<commit_after>package processorcommand

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"

	"github.com/trustmaster/go-aspell"
)

type OCRResult struct {
	Type string
	Text string
}

func newOCRResult(ocrType string, result string) *OCRResult {
	return &OCRResult{
		ocrType,
		result,
	}
}

func (this *OCRResult) removeNonWords() {
	blob := this.Text

	speller, err := aspell.NewSpeller(map[string]string{
		"lang": "en_US",
	})
	if err != nil {
		fmt.Printf("Error: %s", err.Error())
		return
	}
	defer speller.Delete()

	singleCharWords := regexp.MustCompile("(a|i)")
	numberRegex := regexp.MustCompile("\\d{3,}")
	wordRegexp := regexp.MustCompile("\\b(\\w+)\\b")
	words := wordRegexp.FindAllString(blob, -1)

	str := ""

	for _, word := range words {
		if numberRegex.MatchString(word) {
			str += " " + word
		} else if len(word) == 1 {
			if singleCharWords.MatchString(word) {
				str += " " + word
			}
		} else if speller.Check(word) {
			str += " " + word
		}
	}

	this.Text = strings.TrimSpace(str)
}

func (this *OCRResult) wordCount(blob string) int {
	word_regexp := regexp.MustCompile("\\b(\\w+)\\b")
	words := word_regexp.FindAllString(blob, -1)

	// don't let single char words count towards the overal word count. Gets thrown off by poor OCR results
	count := 0
	for _, word := range words {
		if len(word) > 1 {
			count++
		}
	}

	return count
}

type MultiOCRCommand []OCRCommand

func (this MultiOCRCommand) Run(image string) (*OCRResult, error) {
	results := make(chan *OCRResult, len(this))
	errs := make(chan error, len(this))

	for _, command := range this {
		go func(c OCRCommand) {
			k, err := c.Run(image)
			if err != nil {
				errs <- err
				return
			}
			results <- k
		}(command)
	}

	max := -1
	var best *OCRResult

	for i := 0; i < len(this); i++ {
		select {
		case result := <-results:
			result.removeNonWords()
			count := result.wordCount(result.Text)

			if count > max {
				best = result
				max = count
			}

		case err := <-errs:
			return nil, err
		}
	}

	// Return the average, same as before.
	return best, nil
}

type OCRCommand interface {
	Run(image string) (*OCRResult, error)
}

type MemeOCR struct {
	name string
}

func NewMemeOCR() *MemeOCR {
	return &MemeOCR{
		"MemeOCR",
	}
}

func (this *MemeOCR) Run(image string) (*OCRResult, error) {
	imageTif := fmt.Sprintf("%s_meme.jpg", image)
	outText := fmt.Sprintf("%s_meme", image)
	preprocessingArgs := []string{image, "-resize", "400%", "-fill", "black", "-fuzz", "10%", "+opaque", "#FFFFFF", imageTif}
	tesseractArgs := []string{"-l", "meme", imageTif, outText}

	err := runProcessorCommand(GM_COMMAND, preprocessingArgs)
	if err != nil {
		return nil, errors.New(fmt.Sprintf("Meme preprocessing command failed with error = %v", err))
	}
	defer os.Remove(imageTif)

	err = runProcessorCommand("tesseract", tesseractArgs)
	if err != nil {
		return nil, errors.New(fmt.Sprintf("Meme tesseract command failed with error = %v", err))
	}
	defer os.Remove(outText + ".txt")

	text, err := ioutil.ReadFile(outText + ".txt")
	if err != nil {
		return nil, err
	}

	result := strings.ToLower(strings.TrimSpace(string(text[:])))

	return newOCRResult(this.name, result), nil
}

type StandardOCR struct {
	name string
}

func NewStandardOCR() *StandardOCR {
	return &StandardOCR{
		"StandardOCR",
	}
}

func (this *StandardOCR) Run(image string) (*OCRResult, error) {
	imageTif := fmt.Sprintf("%s_standard.jpg", image)
	outText := fmt.Sprintf("%s_standard", image)

	preprocessingArgs := []string{image, "-resize", "400%", "-type", "Grayscale", imageTif}
	tesseractArgs := []string{"-l", "eng", imageTif, outText}

	err := runProcessorCommand(GM_COMMAND, preprocessingArgs)
	if err != nil {
		return nil, errors.New(fmt.Sprintf("Standard preprocessing command failed with error = %v", err))
	}
	defer os.Remove(imageTif)

	err = runProcessorCommand("tesseract", tesseractArgs)
	if err != nil {
		return nil, errors.New(fmt.Sprintf("Standard tesseract command failed with error = %v", err))
	}
	defer os.Remove(outText + ".txt")

	text, err := ioutil.ReadFile(outText + ".txt")
	if err != nil {
		return nil, err
	}

	result := strings.ToLower(strings.TrimSpace(string(text[:])))

	return newOCRResult(this.name, result), nil
}
<|endoftext|>


<commit_before>package qtype

import "fmt"

/*
   +----------+----------------------------------+
   | Low Bits | Stream Type                      |
   +----------+----------------------------------+
   | 0x0      | Client-Initiated, Bidirectional  |
   |          |                                  |
   | 0x1      | Server-Initiated, Bidirectional  |
   |          |                                  |
   | 0x2      | Client-Initiated, Unidirectional |
   |          |                                  |
   | 0x3      | Server-Initiated, Unidirectional |
   +----------+----------------------------------+
*/

type StreamID QuicInt

const (
	BidirectionalStream  = 0x0
	UnidirectionalStream = 0x2
)

func NewStreamID(id uint64) (StreamID, error) {
	sid, err := NewQuicInt(id)
	if err != nil {
		return StreamID(QuicInt{0, 0, 0}), err
	}
	sss := StreamID(sid)
	return sss, err
}

func (s *StreamID) Increment() error {
	// add 0b100
	st, err := NewStreamID(s.GetValue() + 4)
	s = &st
	return err
}

func (s StreamID) GetValue() uint64 {
	qint := QuicInt(s)
	return qint.GetValue()
}

func (s StreamID) PutWire(wire []byte) int {
	return QuicInt(s).PutWire(wire)
}

func (s StreamID) String() string {
	return fmt.Sprintf("%s ID:%d", []string{
		"Client-Initiated, Bidirectional",
		"Server-Initiated, Bidirectional",
		"Client-Initiated, Unidirectional",
		"Server-Initiated, Unidirectional",
	}[QuicInt(s).Value&0x03], s.GetValue())
}
<commit_msg>fix StreamID's Increment method bug<commit_after>package qtype

import "fmt"

/*
   +----------+----------------------------------+
   | Low Bits | Stream Type                      |
   +----------+----------------------------------+
   | 0x0      | Client-Initiated, Bidirectional  |
   |          |                                  |
   | 0x1      | Server-Initiated, Bidirectional  |
   |          |                                  |
   | 0x2      | Client-Initiated, Unidirectional |
   |          |                                  |
   | 0x3      | Server-Initiated, Unidirectional |
   +----------+----------------------------------+
*/

type StreamID QuicInt

const (
	BidirectionalStream  = 0x0
	UnidirectionalStream = 0x2
)

func NewStreamID(id uint64) (StreamID, error) {
	sid, err := NewQuicInt(id)
	if err != nil {
		return StreamID(QuicInt{0, 0, 0}), err
	}
	sss := StreamID(sid)
	return sss, err
}

func (s *StreamID) Increment() error {
	// add 0b100
	st, err := NewStreamID(s.GetValue() + 4)
	s.Value = st.Value
	s.ByteLen = st.ByteLen
	s.Flag = st.Flag
	return err
}

func (s StreamID) GetValue() uint64 {
	qint := QuicInt(s)
	return qint.GetValue()
}

func (s StreamID) PutWire(wire []byte) int {
	return QuicInt(s).PutWire(wire)
}

func (s StreamID) String() string {
	return fmt.Sprintf("%s ID:%d", []string{
		"Client-Initiated, Bidirectional",
		"Server-Initiated, Bidirectional",
		"Client-Initiated, Unidirectional",
		"Server-Initiated, Unidirectional",
	}[QuicInt(s).Value&0x03], s.GetValue())
}
<|endoftext|>


<commit_before>/*
 * Minimalist Object Storage, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package quota

import (
	"net"
	"net/http"
	"sync"
	"time"
)

// map[minute][address] = current quota
type quotaMap struct {
	sync.RWMutex
	data     map[int64]map[uint32]uint64
	limit    uint64
	duration int64
}

func (q *quotaMap) Add(ip uint32, size uint64) bool {
	q.Lock()
	defer q.Unlock()
	currentMinute := time.Now().Unix() / q.duration
	expiredQuotas := (time.Now().Unix() / q.duration) - 5
	for time := range q.data {
		if time < expiredQuotas {
			delete(q.data, time)
		}
	}
	if _, ok := q.data[currentMinute]; !ok {
		q.data[currentMinute] = make(map[uint32]uint64)
	}
	currentData, _ := q.data[currentMinute][ip]
	proposedDataSize := currentData + size
	if proposedDataSize > q.limit {
		return false
	}
	q.data[currentMinute][ip] = proposedDataSize
	return true
}

// HttpQuotaHandler
type httpQuotaHandler struct {
	handler http.Handler
	quotas  *quotaMap
	adder   func(uint64) uint64
}

type longIP struct {
	net.IP
}

// []byte to uint32 representation
func (p longIP) IptoUint32() (result uint32) {
	ip := p.To4()
	if ip == nil {
		return 0
	}
	q0 := uint32(ip[0]) << 24
	q1 := uint32(ip[1]) << 16
	q2 := uint32(ip[2]) << 8
	q3 := uint32(ip[3])
	result = q0 + q1 + q2 + q3
	return
}

// ServeHTTP is an http.Handler ServeHTTP method
func (h *httpQuotaHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	host, _, _ := net.SplitHostPort(req.RemoteAddr)
	longIP := longIP{net.ParseIP(host)}.IptoUint32()
	if h.quotas.Add(longIP, h.adder(uint64(req.ContentLength))) {
		h.handler.ServeHTTP(w, req)
	}
}

// BandwidthCap sets a quote based upon bandwidth used
func BandwidthCap(h http.Handler, limit int64) http.Handler {
	return &httpQuotaHandler{
		handler: h,
		quotas: &quotaMap{
			data:     make(map[int64]map[uint32]uint64),
			limit:    uint64(limit),
			duration: int64(60),
		},
		adder: func(count uint64) uint64 { return count },
	}
}

// RequestLimit sets a quota based upon request count
func RequestLimit(h http.Handler, limit int64) http.Handler {
	return &httpQuotaHandler{
		handler: h,
		quotas: &quotaMap{
			data:     make(map[int64]map[uint32]uint64),
			limit:    uint64(limit),
			duration: int64(60),
		},
		adder: func(count uint64) uint64 { return 1 },
	}
}
<commit_msg>Switching back to BigEndian<commit_after>/*
 * Minimalist Object Storage, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package quota

import (
	"encoding/binary"
	"net"
	"net/http"
	"sync"
	"time"
)

// map[minute][address] = current quota
type quotaMap struct {
	sync.RWMutex
	data     map[int64]map[uint32]uint64
	limit    uint64
	duration int64
}

func (q *quotaMap) Add(ip uint32, size uint64) bool {
	q.Lock()
	defer q.Unlock()
	currentMinute := time.Now().Unix() / q.duration
	expiredQuotas := (time.Now().Unix() / q.duration) - 5
	for time := range q.data {
		if time < expiredQuotas {
			delete(q.data, time)
		}
	}
	if _, ok := q.data[currentMinute]; !ok {
		q.data[currentMinute] = make(map[uint32]uint64)
	}
	currentData, _ := q.data[currentMinute][ip]
	proposedDataSize := currentData + size
	if proposedDataSize > q.limit {
		return false
	}
	q.data[currentMinute][ip] = proposedDataSize
	return true
}

// HttpQuotaHandler
type httpQuotaHandler struct {
	handler http.Handler
	quotas  *quotaMap
	adder   func(uint64) uint64
}

type longIP struct {
	net.IP
}

// []byte to uint32 representation
func (p longIP) IptoUint32() (result uint32) {
	ip := p.To4()
	if ip == nil {
		return 0
	}
	return binary.BigEndian.Uint32(ip)
}

// ServeHTTP is an http.Handler ServeHTTP method
func (h *httpQuotaHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	host, _, _ := net.SplitHostPort(req.RemoteAddr)
	longIP := longIP{net.ParseIP(host)}.IptoUint32()
	if h.quotas.Add(longIP, h.adder(uint64(req.ContentLength))) {
		h.handler.ServeHTTP(w, req)
	}
}

// BandwidthCap sets a quote based upon bandwidth used
func BandwidthCap(h http.Handler, limit int64) http.Handler {
	return &httpQuotaHandler{
		handler: h,
		quotas: &quotaMap{
			data:     make(map[int64]map[uint32]uint64),
			limit:    uint64(limit),
			duration: int64(60),
		},
		adder: func(count uint64) uint64 { return count },
	}
}

// RequestLimit sets a quota based upon request count
func RequestLimit(h http.Handler, limit int64) http.Handler {
	return &httpQuotaHandler{
		handler: h,
		quotas: &quotaMap{
			data:     make(map[int64]map[uint32]uint64),
			limit:    uint64(limit),
			duration: int64(60),
		},
		adder: func(count uint64) uint64 { return 1 },
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiserver

import (
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"reflect"
	"regexp"
	"strings"
	"sync"
	"testing"
	"time"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/errors"
	"k8s.io/kubernetes/pkg/api/testapi"
	"k8s.io/kubernetes/pkg/util/sets"
)

type fakeRL bool

func (fakeRL) Stop()             {}
func (f fakeRL) CanAccept() bool { return bool(f) }
func (f fakeRL) Accept()         {}

func expectHTTP(url string, code int, t *testing.T) {
	r, err := http.Get(url)
	if err != nil {
		t.Errorf("unexpected error: %v", err)
		return
	}
	if r.StatusCode != code {
		t.Errorf("unexpected response: %v", r.StatusCode)
	}
}

func getPath(resource, namespace, name string) string {
	return testapi.Default.ResourcePath(resource, namespace, name)
}

func pathWithPrefix(prefix, resource, namespace, name string) string {
	return testapi.Default.ResourcePathWithPrefix(prefix, resource, namespace, name)
}

func TestMaxInFlight(t *testing.T) {
	const Iterations = 3
	block := sync.WaitGroup{}
	block.Add(1)
	sem := make(chan bool, Iterations)

	re := regexp.MustCompile("[.*\\/watch][^\\/proxy.*]")

	// Calls verifies that the server is actually blocked up before running the rest of the test
	calls := &sync.WaitGroup{}
	calls.Add(Iterations * 3)

	server := httptest.NewServer(MaxInFlightLimit(sem, re, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if strings.Contains(r.URL.Path, "dontwait") {
			return
		}
		if calls != nil {
			calls.Done()
		}
		block.Wait()
	})))
	defer server.Close()

	// These should hang, but not affect accounting.
	for i := 0; i < Iterations; i++ {
		// These should hang waiting on block...
		go func() {
			expectHTTP(server.URL+"/foo/bar/watch", http.StatusOK, t)
		}()
	}

	for i := 0; i < Iterations; i++ {
		// These should hang waiting on block...
		go func() {
			expectHTTP(server.URL+"/proxy/foo/bar", http.StatusOK, t)
		}()
	}
	expectHTTP(server.URL+"/dontwait", http.StatusOK, t)

	for i := 0; i < Iterations; i++ {
		// These should hang waiting on block...
		go func() {
			expectHTTP(server.URL, http.StatusOK, t)
		}()
	}
	calls.Wait()
	calls = nil

	// Do this multiple times to show that it rate limit rejected requests don't block.
	for i := 0; i < 2; i++ {
		expectHTTP(server.URL, errors.StatusTooManyRequests, t)
	}

	// Validate that non-accounted URLs still work
	expectHTTP(server.URL+"/dontwait/watch", http.StatusOK, t)

	block.Done()

	// Show that we recover from being blocked up.
	expectHTTP(server.URL, http.StatusOK, t)
}

func TestReadOnly(t *testing.T) {
	server := httptest.NewServer(ReadOnly(http.HandlerFunc(
		func(w http.ResponseWriter, req *http.Request) {
			if req.Method != "GET" {
				t.Errorf("Unexpected call: %v", req.Method)
			}
		},
	)))
	defer server.Close()
	for _, verb := range []string{"GET", "POST", "PUT", "DELETE", "CREATE"} {
		req, err := http.NewRequest(verb, server.URL, nil)
		if err != nil {
			t.Fatalf("Couldn't make request: %v", err)
		}
		http.DefaultClient.Do(req)
	}
}

func TestTimeout(t *testing.T) {
	sendResponse := make(chan struct{}, 1)
	writeErrors := make(chan error, 1)
	timeout := make(chan time.Time, 1)
	resp := "test response"
	timeoutResp := "test timeout"

	ts := httptest.NewServer(TimeoutHandler(http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			<-sendResponse
			_, err := w.Write([]byte(resp))
			writeErrors <- err
		}),
		func(*http.Request) (<-chan time.Time, string) {
			return timeout, timeoutResp
		}))
	defer ts.Close()

	// No timeouts
	sendResponse <- struct{}{}
	res, err := http.Get(ts.URL)
	if err != nil {
		t.Error(err)
	}
	if res.StatusCode != http.StatusOK {
		t.Errorf("got res.StatusCode %d; expected %d", res.StatusCode, http.StatusOK)
	}
	body, _ := ioutil.ReadAll(res.Body)
	if string(body) != resp {
		t.Errorf("got body %q; expected %q", string(body), resp)
	}
	if err := <-writeErrors; err != nil {
		t.Errorf("got unexpected Write error on first request: %v", err)
	}

	// Times out
	timeout <- time.Time{}
	res, err = http.Get(ts.URL)
	if err != nil {
		t.Error(err)
	}
	if res.StatusCode != http.StatusGatewayTimeout {
		t.Errorf("got res.StatusCode %d; expected %d", res.StatusCode, http.StatusServiceUnavailable)
	}
	body, _ = ioutil.ReadAll(res.Body)
	if string(body) != timeoutResp {
		t.Errorf("got body %q; expected %q", string(body), timeoutResp)
	}

	// Now try to send a response
	sendResponse <- struct{}{}
	if err := <-writeErrors; err != http.ErrHandlerTimeout {
		t.Errorf("got Write error of %v; expected %v", err, http.ErrHandlerTimeout)
	}
}

func TestGetAPIRequestInfo(t *testing.T) {
	successCases := []struct {
		method              string
		url                 string
		expectedVerb        string
		expectedAPIVersion  string
		expectedNamespace   string
		expectedResource    string
		expectedSubresource string
		expectedKind        string
		expectedName        string
		expectedParts       []string
	}{

		// resource paths
		{"GET", "/namespaces", "list", "", "", "namespaces", "", "Namespace", "", []string{"namespaces"}},
		{"GET", "/namespaces/other", "get", "", "other", "namespaces", "", "Namespace", "other", []string{"namespaces", "other"}},

		{"GET", "/namespaces/other/pods", "list", "", "other", "pods", "", "Pod", "", []string{"pods"}},
		{"GET", "/namespaces/other/pods/foo", "get", "", "other", "pods", "", "Pod", "foo", []string{"pods", "foo"}},
		{"GET", "/pods", "list", "", api.NamespaceAll, "pods", "", "Pod", "", []string{"pods"}},
		{"GET", "/namespaces/other/pods/foo", "get", "", "other", "pods", "", "Pod", "foo", []string{"pods", "foo"}},
		{"GET", "/namespaces/other/pods", "list", "", "other", "pods", "", "Pod", "", []string{"pods"}},

		// special verbs
		{"GET", "/proxy/namespaces/other/pods/foo", "proxy", "", "other", "pods", "", "Pod", "foo", []string{"pods", "foo"}},
		{"GET", "/redirect/namespaces/other/pods/foo", "redirect", "", "other", "pods", "", "Pod", "foo", []string{"pods", "foo"}},
		{"GET", "/watch/pods", "watch", "", api.NamespaceAll, "pods", "", "Pod", "", []string{"pods"}},
		{"GET", "/watch/namespaces/other/pods", "watch", "", "other", "pods", "", "Pod", "", []string{"pods"}},

		// fully-qualified paths
		{"GET", getPath("pods", "other", ""), "list", testapi.Default.Version(), "other", "pods", "", "Pod", "", []string{"pods"}},
		{"GET", getPath("pods", "other", "foo"), "get", testapi.Default.Version(), "other", "pods", "", "Pod", "foo", []string{"pods", "foo"}},
		{"GET", getPath("pods", "", ""), "list", testapi.Default.Version(), api.NamespaceAll, "pods", "", "Pod", "", []string{"pods"}},
		{"POST", getPath("pods", "", ""), "create", testapi.Default.Version(), api.NamespaceAll, "pods", "", "Pod", "", []string{"pods"}},
		{"GET", getPath("pods", "", "foo"), "get", testapi.Default.Version(), api.NamespaceAll, "pods", "", "Pod", "foo", []string{"pods", "foo"}},
		{"GET", pathWithPrefix("proxy", "pods", "", "foo"), "proxy", testapi.Default.Version(), api.NamespaceAll, "pods", "", "Pod", "foo", []string{"pods", "foo"}},
		{"GET", pathWithPrefix("watch", "pods", "", ""), "watch", testapi.Default.Version(), api.NamespaceAll, "pods", "", "Pod", "", []string{"pods"}},
		{"GET", pathWithPrefix("redirect", "pods", "", ""), "redirect", testapi.Default.Version(), api.NamespaceAll, "pods", "", "Pod", "", []string{"pods"}},
		{"GET", pathWithPrefix("watch", "pods", "other", ""), "watch", testapi.Default.Version(), "other", "pods", "", "Pod", "", []string{"pods"}},

		// subresource identification
		{"GET", "/namespaces/other/pods/foo/status", "get", "", "other", "pods", "status", "Pod", "foo", []string{"pods", "foo", "status"}},
		{"PUT", "/namespaces/other/finalize", "update", "", "other", "finalize", "", "", "", []string{"finalize"}},

		// verb identification
		{"PATCH", "/namespaces/other/pods/foo", "patch", "", "other", "pods", "", "Pod", "foo", []string{"pods", "foo"}},
		{"DELETE", "/namespaces/other/pods/foo", "delete", "", "other", "pods", "", "Pod", "foo", []string{"pods", "foo"}},
	}

	apiRequestInfoResolver := &APIRequestInfoResolver{sets.NewString("api"), testapi.Default.RESTMapper()}

	for _, successCase := range successCases {
		req, _ := http.NewRequest(successCase.method, successCase.url, nil)

		apiRequestInfo, err := apiRequestInfoResolver.GetAPIRequestInfo(req)
		if err != nil {
			t.Errorf("Unexpected error for url: %s %v", successCase.url, err)
		}
		if successCase.expectedVerb != apiRequestInfo.Verb {
			t.Errorf("Unexpected verb for url: %s, expected: %s, actual: %s", successCase.url, successCase.expectedVerb, apiRequestInfo.Verb)
		}
		if successCase.expectedAPIVersion != apiRequestInfo.APIVersion {
			t.Errorf("Unexpected apiVersion for url: %s, expected: %s, actual: %s", successCase.url, successCase.expectedAPIVersion, apiRequestInfo.APIVersion)
		}
		if successCase.expectedNamespace != apiRequestInfo.Namespace {
			t.Errorf("Unexpected namespace for url: %s, expected: %s, actual: %s", successCase.url, successCase.expectedNamespace, apiRequestInfo.Namespace)
		}
		if successCase.expectedKind != apiRequestInfo.Kind {
			t.Errorf("Unexpected kind for url: %s, expected: %s, actual: %s", successCase.url, successCase.expectedKind, apiRequestInfo.Kind)
		}
		if successCase.expectedResource != apiRequestInfo.Resource {
			t.Errorf("Unexpected resource for url: %s, expected: %s, actual: %s", successCase.url, successCase.expectedResource, apiRequestInfo.Resource)
		}
		if successCase.expectedSubresource != apiRequestInfo.Subresource {
			t.Errorf("Unexpected resource for url: %s, expected: %s, actual: %s", successCase.url, successCase.expectedSubresource, apiRequestInfo.Subresource)
		}
		if successCase.expectedName != apiRequestInfo.Name {
			t.Errorf("Unexpected name for url: %s, expected: %s, actual: %s", successCase.url, successCase.expectedName, apiRequestInfo.Name)
		}
		if !reflect.DeepEqual(successCase.expectedParts, apiRequestInfo.Parts) {
			t.Errorf("Unexpected parts for url: %s, expected: %v, actual: %v", successCase.url, successCase.expectedParts, apiRequestInfo.Parts)
		}
	}

	errorCases := map[string]string{
		"no resource path":            "/",
		"just apiversion":             "/api/version/",
		"apiversion with no resource": "/api/version/",
	}
	for k, v := range errorCases {
		req, err := http.NewRequest("GET", v, nil)
		if err != nil {
			t.Errorf("Unexpected error %v", err)
		}
		_, err = apiRequestInfoResolver.GetAPIRequestInfo(req)
		if err == nil {
			t.Errorf("Expected error for key: %s", k)
		}
	}
}
<commit_msg>Fix TestMaxInFlight<commit_after>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiserver

import (
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"reflect"
	"regexp"
	"strings"
	"sync"
	"testing"
	"time"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/errors"
	"k8s.io/kubernetes/pkg/api/testapi"
	"k8s.io/kubernetes/pkg/util/sets"
)

type fakeRL bool

func (fakeRL) Stop()             {}
func (f fakeRL) CanAccept() bool { return bool(f) }
func (f fakeRL) Accept()         {}

func expectHTTP(url string, code int, t *testing.T) {
	r, err := http.Get(url)
	if err != nil {
		t.Errorf("unexpected error: %v", err)
		return
	}
	if r.StatusCode != code {
		t.Errorf("unexpected response: %v", r.StatusCode)
	}
}

func getPath(resource, namespace, name string) string {
	return testapi.Default.ResourcePath(resource, namespace, name)
}

func pathWithPrefix(prefix, resource, namespace, name string) string {
	return testapi.Default.ResourcePathWithPrefix(prefix, resource, namespace, name)
}

func TestMaxInFlight(t *testing.T) {
	const Iterations = 3
	block := sync.WaitGroup{}
	block.Add(1)
	oneFinished := sync.WaitGroup{}
	oneFinished.Add(1)
	var once sync.Once
	sem := make(chan bool, Iterations)

	re := regexp.MustCompile("[.*\\/watch][^\\/proxy.*]")

	// Calls verifies that the server is actually blocked up before running the rest of the test
	calls := &sync.WaitGroup{}
	calls.Add(Iterations * 3)

	server := httptest.NewServer(MaxInFlightLimit(sem, re, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if strings.Contains(r.URL.Path, "dontwait") {
			return
		}
		if calls != nil {
			calls.Done()
		}
		block.Wait()
	})))
	defer server.Close()

	// These should hang, but not affect accounting.
	for i := 0; i < Iterations; i++ {
		// These should hang waiting on block...
		go func() {
			expectHTTP(server.URL+"/foo/bar/watch", http.StatusOK, t)
			once.Do(oneFinished.Done)
		}()
	}

	for i := 0; i < Iterations; i++ {
		// These should hang waiting on block...
		go func() {
			expectHTTP(server.URL+"/proxy/foo/bar", http.StatusOK, t)
			once.Do(oneFinished.Done)
		}()
	}
	expectHTTP(server.URL+"/dontwait", http.StatusOK, t)

	for i := 0; i < Iterations; i++ {
		// These should hang waiting on block...
		go func() {
			expectHTTP(server.URL, http.StatusOK, t)
			once.Do(oneFinished.Done)
		}()
	}
	calls.Wait()
	calls = nil

	// Do this multiple times to show that it rate limit rejected requests don't block.
	for i := 0; i < 2; i++ {
		expectHTTP(server.URL, errors.StatusTooManyRequests, t)
	}

	// Validate that non-accounted URLs still work
	expectHTTP(server.URL+"/dontwait/watch", http.StatusOK, t)

	block.Done()

	// Show that we recover from being blocked up.
	// However, we should until at least one of the requests really finishes.
	oneFinished.Wait()
	expectHTTP(server.URL, http.StatusOK, t)
}

func TestReadOnly(t *testing.T) {
	server := httptest.NewServer(ReadOnly(http.HandlerFunc(
		func(w http.ResponseWriter, req *http.Request) {
			if req.Method != "GET" {
				t.Errorf("Unexpected call: %v", req.Method)
			}
		},
	)))
	defer server.Close()
	for _, verb := range []string{"GET", "POST", "PUT", "DELETE", "CREATE"} {
		req, err := http.NewRequest(verb, server.URL, nil)
		if err != nil {
			t.Fatalf("Couldn't make request: %v", err)
		}
		http.DefaultClient.Do(req)
	}
}

func TestTimeout(t *testing.T) {
	sendResponse := make(chan struct{}, 1)
	writeErrors := make(chan error, 1)
	timeout := make(chan time.Time, 1)
	resp := "test response"
	timeoutResp := "test timeout"

	ts := httptest.NewServer(TimeoutHandler(http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			<-sendResponse
			_, err := w.Write([]byte(resp))
			writeErrors <- err
		}),
		func(*http.Request) (<-chan time.Time, string) {
			return timeout, timeoutResp
		}))
	defer ts.Close()

	// No timeouts
	sendResponse <- struct{}{}
	res, err := http.Get(ts.URL)
	if err != nil {
		t.Error(err)
	}
	if res.StatusCode != http.StatusOK {
		t.Errorf("got res.StatusCode %d; expected %d", res.StatusCode, http.StatusOK)
	}
	body, _ := ioutil.ReadAll(res.Body)
	if string(body) != resp {
		t.Errorf("got body %q; expected %q", string(body), resp)
	}
	if err := <-writeErrors; err != nil {
		t.Errorf("got unexpected Write error on first request: %v", err)
	}

	// Times out
	timeout <- time.Time{}
	res, err = http.Get(ts.URL)
	if err != nil {
		t.Error(err)
	}
	if res.StatusCode != http.StatusGatewayTimeout {
		t.Errorf("got res.StatusCode %d; expected %d", res.StatusCode, http.StatusServiceUnavailable)
	}
	body, _ = ioutil.ReadAll(res.Body)
	if string(body) != timeoutResp {
		t.Errorf("got body %q; expected %q", string(body), timeoutResp)
	}

	// Now try to send a response
	sendResponse <- struct{}{}
	if err := <-writeErrors; err != http.ErrHandlerTimeout {
		t.Errorf("got Write error of %v; expected %v", err, http.ErrHandlerTimeout)
	}
}

func TestGetAPIRequestInfo(t *testing.T) {
	successCases := []struct {
		method              string
		url                 string
		expectedVerb        string
		expectedAPIVersion  string
		expectedNamespace   string
		expectedResource    string
		expectedSubresource string
		expectedKind        string
		expectedName        string
		expectedParts       []string
	}{

		// resource paths
		{"GET", "/namespaces", "list", "", "", "namespaces", "", "Namespace", "", []string{"namespaces"}},
		{"GET", "/namespaces/other", "get", "", "other", "namespaces", "", "Namespace", "other", []string{"namespaces", "other"}},

		{"GET", "/namespaces/other/pods", "list", "", "other", "pods", "", "Pod", "", []string{"pods"}},
		{"GET", "/namespaces/other/pods/foo", "get", "", "other", "pods", "", "Pod", "foo", []string{"pods", "foo"}},
		{"GET", "/pods", "list", "", api.NamespaceAll, "pods", "", "Pod", "", []string{"pods"}},
		{"GET", "/namespaces/other/pods/foo", "get", "", "other", "pods", "", "Pod", "foo", []string{"pods", "foo"}},
		{"GET", "/namespaces/other/pods", "list", "", "other", "pods", "", "Pod", "", []string{"pods"}},

		// special verbs
		{"GET", "/proxy/namespaces/other/pods/foo", "proxy", "", "other", "pods", "", "Pod", "foo", []string{"pods", "foo"}},
		{"GET", "/redirect/namespaces/other/pods/foo", "redirect", "", "other", "pods", "", "Pod", "foo", []string{"pods", "foo"}},
		{"GET", "/watch/pods", "watch", "", api.NamespaceAll, "pods", "", "Pod", "", []string{"pods"}},
		{"GET", "/watch/namespaces/other/pods", "watch", "", "other", "pods", "", "Pod", "", []string{"pods"}},

		// fully-qualified paths
		{"GET", getPath("pods", "other", ""), "list", testapi.Default.Version(), "other", "pods", "", "Pod", "", []string{"pods"}},
		{"GET", getPath("pods", "other", "foo"), "get", testapi.Default.Version(), "other", "pods", "", "Pod", "foo", []string{"pods", "foo"}},
		{"GET", getPath("pods", "", ""), "list", testapi.Default.Version(), api.NamespaceAll, "pods", "", "Pod", "", []string{"pods"}},
		{"POST", getPath("pods", "", ""), "create", testapi.Default.Version(), api.NamespaceAll, "pods", "", "Pod", "", []string{"pods"}},
		{"GET", getPath("pods", "", "foo"), "get", testapi.Default.Version(), api.NamespaceAll, "pods", "", "Pod", "foo", []string{"pods", "foo"}},
		{"GET", pathWithPrefix("proxy", "pods", "", "foo"), "proxy", testapi.Default.Version(), api.NamespaceAll, "pods", "", "Pod", "foo", []string{"pods", "foo"}},
		{"GET", pathWithPrefix("watch", "pods", "", ""), "watch", testapi.Default.Version(), api.NamespaceAll, "pods", "", "Pod", "", []string{"pods"}},
		{"GET", pathWithPrefix("redirect", "pods", "", ""), "redirect", testapi.Default.Version(), api.NamespaceAll, "pods", "", "Pod", "", []string{"pods"}},
		{"GET", pathWithPrefix("watch", "pods", "other", ""), "watch", testapi.Default.Version(), "other", "pods", "", "Pod", "", []string{"pods"}},

		// subresource identification
		{"GET", "/namespaces/other/pods/foo/status", "get", "", "other", "pods", "status", "Pod", "foo", []string{"pods", "foo", "status"}},
		{"PUT", "/namespaces/other/finalize", "update", "", "other", "finalize", "", "", "", []string{"finalize"}},

		// verb identification
		{"PATCH", "/namespaces/other/pods/foo", "patch", "", "other", "pods", "", "Pod", "foo", []string{"pods", "foo"}},
		{"DELETE", "/namespaces/other/pods/foo", "delete", "", "other", "pods", "", "Pod", "foo", []string{"pods", "foo"}},
	}

	apiRequestInfoResolver := &APIRequestInfoResolver{sets.NewString("api"), testapi.Default.RESTMapper()}

	for _, successCase := range successCases {
		req, _ := http.NewRequest(successCase.method, successCase.url, nil)

		apiRequestInfo, err := apiRequestInfoResolver.GetAPIRequestInfo(req)
		if err != nil {
			t.Errorf("Unexpected error for url: %s %v", successCase.url, err)
		}
		if successCase.expectedVerb != apiRequestInfo.Verb {
			t.Errorf("Unexpected verb for url: %s, expected: %s, actual: %s", successCase.url, successCase.expectedVerb, apiRequestInfo.Verb)
		}
		if successCase.expectedAPIVersion != apiRequestInfo.APIVersion {
			t.Errorf("Unexpected apiVersion for url: %s, expected: %s, actual: %s", successCase.url, successCase.expectedAPIVersion, apiRequestInfo.APIVersion)
		}
		if successCase.expectedNamespace != apiRequestInfo.Namespace {
			t.Errorf("Unexpected namespace for url: %s, expected: %s, actual: %s", successCase.url, successCase.expectedNamespace, apiRequestInfo.Namespace)
		}
		if successCase.expectedKind != apiRequestInfo.Kind {
			t.Errorf("Unexpected kind for url: %s, expected: %s, actual: %s", successCase.url, successCase.expectedKind, apiRequestInfo.Kind)
		}
		if successCase.expectedResource != apiRequestInfo.Resource {
			t.Errorf("Unexpected resource for url: %s, expected: %s, actual: %s", successCase.url, successCase.expectedResource, apiRequestInfo.Resource)
		}
		if successCase.expectedSubresource != apiRequestInfo.Subresource {
			t.Errorf("Unexpected resource for url: %s, expected: %s, actual: %s", successCase.url, successCase.expectedSubresource, apiRequestInfo.Subresource)
		}
		if successCase.expectedName != apiRequestInfo.Name {
			t.Errorf("Unexpected name for url: %s, expected: %s, actual: %s", successCase.url, successCase.expectedName, apiRequestInfo.Name)
		}
		if !reflect.DeepEqual(successCase.expectedParts, apiRequestInfo.Parts) {
			t.Errorf("Unexpected parts for url: %s, expected: %v, actual: %v", successCase.url, successCase.expectedParts, apiRequestInfo.Parts)
		}
	}

	errorCases := map[string]string{
		"no resource path":            "/",
		"just apiversion":             "/api/version/",
		"apiversion with no resource": "/api/version/",
	}
	for k, v := range errorCases {
		req, err := http.NewRequest("GET", v, nil)
		if err != nil {
			t.Errorf("Unexpected error %v", err)
		}
		_, err = apiRequestInfoResolver.GetAPIRequestInfo(req)
		if err == nil {
			t.Errorf("Expected error for key: %s", k)
		}
	}
}
<|endoftext|>


<commit_before>package cli

import (
	"bufio"
	"context"
	"fmt"
	"net"
	"time"

	"github.com/spf13/cobra"
	"github.com/telepresenceio/telepresence/rpc/v2/connector"
	"github.com/telepresenceio/telepresence/rpc/v2/daemon"
	"github.com/telepresenceio/telepresence/v2/pkg/client/cli/cliutil"
	"github.com/telepresenceio/telepresence/v2/pkg/client/scout"
	"github.com/telepresenceio/telepresence/v2/pkg/iputil"
	"github.com/telepresenceio/telepresence/v2/pkg/log"
	"github.com/telepresenceio/telepresence/v2/pkg/vif/routing"
	empty "google.golang.org/protobuf/types/known/emptypb"
)

type vpnDiagInfo struct {
}

func vpnDiagCommand() *cobra.Command {
	di := vpnDiagInfo{}
	cmd := &cobra.Command{
		Use:   "test-vpn",
		Args:  cobra.NoArgs,
		Short: "Test VPN configuration for compatibility with telepresence",
		RunE:  di.run,
	}
	return cmd
}

func waitForNetwork(ctx context.Context) error {
	publicIP := net.ParseIP("8.8.8.8")
	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
	defer cancel()
	for ctx.Err() == nil {
		_, err := routing.GetRoute(ctx, &net.IPNet{IP: publicIP, Mask: net.CIDRMask(32, 32)})
		if err == nil {
			return nil
		}
	}
	return fmt.Errorf("timed out waiting for a route to %s; this usually means your VPN client is misconfigured", publicIP)
}

const (
	good    = "✅"
	bad     = "❌"
	podType = "pod"
	svcType = "svc"
)

func getLiveInterfaces() ([]net.Interface, error) {
	ifaces, err := net.Interfaces()
	if err != nil {
		return nil, fmt.Errorf("unable to list network interfaces: %w", err)
	}
	retval := []net.Interface{}
	for _, iface := range ifaces {
		if iface.Flags&net.FlagUp != 0 {
			retval = append(retval, iface)
		}
	}
	return retval, nil
}

func getVPNInterfaces(interfacesConnected, interfacesDisconnected []net.Interface) map[string]struct{} {
	vpnIfaces := map[string]struct{}{}
ifaces:
	for _, ifaceC := range interfacesConnected {
		for _, ifaceD := range interfacesDisconnected {
			if ifaceD.Name == ifaceC.Name {
				continue ifaces
			}
		}
		vpnIfaces[ifaceC.Name] = struct{}{}
	}
	return vpnIfaces
}

func (di *vpnDiagInfo) run(cmd *cobra.Command, _ []string) (err error) {
	var (
		ctx          = cmd.Context()
		sc           = scout.NewScout(ctx, "cli")
		configIssues = false
		vpnMasks     = false
		clusterMasks = false
		reader       = bufio.NewReader(cmd.InOrStdin())
	)
	cliutil.QuitDaemon(ctx)

	defer func() {
		if err != nil {
			sc.Report(log.WithDiscardingLogger(ctx), "vpn_diag_error", scout.ScoutMeta{Key: "error", Value: err.Error()})
		} else {
			if configIssues {
				sc.Report(log.WithDiscardingLogger(ctx), "vpn_diag_fail",
					scout.ScoutMeta{Key: "vpn_masks", Value: vpnMasks},
					scout.ScoutMeta{Key: "cluster_masks", Value: clusterMasks},
				)
			} else {
				sc.Report(log.WithDiscardingLogger(ctx), "vpn_diag_pass")
			}
		}
	}()

	fmt.Fprintln(cmd.OutOrStdout(), "Please disconnect from your VPN now and hit enter once you're disconnected...")
	reader.ReadString('\n')
	err = waitForNetwork(ctx)
	if err != nil {
		return err
	}
	interfacesDisconnected, err := getLiveInterfaces()
	if err != nil {
		return err
	}

	fmt.Fprintln(cmd.OutOrStdout(), "Please connect to your VPN now and hit enter once you're connected...")
	reader.ReadString('\n')

	err = waitForNetwork(ctx)
	if err != nil {
		return err
	}

	interfacesConnected, err := getLiveInterfaces()
	if err != nil {
		return err
	}
	vpnIfaces := getVPNInterfaces(interfacesConnected, interfacesDisconnected)

	routeTable, err := routing.GetRoutingTable(ctx)
	if err != nil {
		return fmt.Errorf("failed to get routing table: %w", err)
	}
	subnets := map[string][]*net.IPNet{podType: {}, svcType: {}}
	err = withConnector(cmd, false, func(ctx context.Context, cc connector.ConnectorClient, _ *connector.ConnectInfo, dc daemon.DaemonClient) error {
		// If this times out, it's likely to be because the traffic manager never gave us the subnets;
		// this could happen for all kinds of reasons, but it makes no sense to go on if it does.
		ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
		defer cancel()
		clusterSubnets, err := dc.GetClusterSubnets(ctx, &empty.Empty{})
		if err != nil {
			return err
		}
		for _, sn := range clusterSubnets.GetPodSubnets() {
			ipsn := iputil.IPNetFromRPC(sn)
			subnets[podType] = append(subnets[podType], ipsn)
		}
		for _, sn := range clusterSubnets.GetSvcSubnets() {
			ipsn := iputil.IPNetFromRPC(sn)
			subnets[svcType] = append(subnets[svcType], ipsn)
		}
		return nil
	})
	if err != nil {
		return err
	}

	instructions := []string{}
	for _, tp := range []string{podType, svcType} {
		for _, sn := range subnets[tp] {
			ok := true
			for _, rt := range routeTable {
				if _, inVPN := vpnIfaces[rt.Interface.Name]; !inVPN {
					continue
				}
				if rt.Routes(sn.IP) || sn.Contains(rt.RoutedNet.IP) {
					ok = false
					configIssues = true
					snSz, _ := sn.Mask.Size()
					rtSz, _ := rt.RoutedNet.Mask.Size()
					if rtSz > snSz {
						vpnMasks = true
						instructions = append(instructions,
							fmt.Sprintf("%s %s subnet %s being masked by VPN-routed CIDR %s. This usually means that Telepresence will not be able to connect to your cluster. To resolve:", bad, tp, sn, rt.RoutedNet),
							fmt.Sprintf("\t* Move %s subnet %s to a subnet not mapped by the VPN", tp, sn),
							fmt.Sprintf("\t\t* If this is not possible, consider shrinking the mask of the %s CIDR (e.g. from /16 to /8), or disabling split-tunneling", rt.RoutedNet),
						)
					} else {
						clusterMasks = true
						instructions = append(instructions,
							fmt.Sprintf("%s %s subnet %s is masking VPN-routed CIDR %s. This usually means Telepresence will be able to connect to your cluster, but hosts on your VPN may be inaccessible while telepresence is connected; to resolve:", bad, tp, sn, rt.RoutedNet),
							fmt.Sprintf("\t* Move %s subnet %s to a subnet not mapped by the VPN", tp, sn),
							fmt.Sprintf("\t\t* If this is not possible, ensure that any hosts in CIDR %s are placed in the never-proxy list", rt.RoutedNet),
						)
					}
				}
			}
			if ok {
				instructions = append(instructions, fmt.Sprintf("%s %s subnet %s is clear of VPN", good, tp, sn))
			}
		}
	}

	fmt.Fprintln(cmd.OutOrStdout(), "\n---------- Test Results:")
	for _, instruction := range instructions {
		fmt.Fprintln(cmd.OutOrStdout(), instruction)
	}
	if configIssues {
		fmt.Fprintln(cmd.OutOrStdout(), "\nPlease see https://www.telepresence.io/docs/latest/reference/vpn for more info on these corrective actions, as well as examples")
	}
	fmt.Fprintln(cmd.OutOrStdout(), "\nStill having issues? Please create a new github issue at https://github.com/telepresenceio/telepresence/issues/new?template=Bug_report.md\n",
		"Please make sure to add the following to your issue:\n",
		"* Attach the zipfile resulting from running `telepresence gather_logs`\n",
		"* Which VPN client are you using?\n",
		"* Which VPN server are you using?\n",
		"* How is your VPN pushing DNS configuration? It may be useful to add the contents of /etc/resolv.conf")

	return nil
}
<commit_msg>Tell the user to set the loglevel to debug before submitting a VPN issue<commit_after>package cli

import (
	"bufio"
	"context"
	"fmt"
	"net"
	"time"

	"github.com/spf13/cobra"
	"github.com/telepresenceio/telepresence/rpc/v2/connector"
	"github.com/telepresenceio/telepresence/rpc/v2/daemon"
	"github.com/telepresenceio/telepresence/v2/pkg/client/cli/cliutil"
	"github.com/telepresenceio/telepresence/v2/pkg/client/scout"
	"github.com/telepresenceio/telepresence/v2/pkg/iputil"
	"github.com/telepresenceio/telepresence/v2/pkg/log"
	"github.com/telepresenceio/telepresence/v2/pkg/vif/routing"
	empty "google.golang.org/protobuf/types/known/emptypb"
)

type vpnDiagInfo struct {
}

func vpnDiagCommand() *cobra.Command {
	di := vpnDiagInfo{}
	cmd := &cobra.Command{
		Use:   "test-vpn",
		Args:  cobra.NoArgs,
		Short: "Test VPN configuration for compatibility with telepresence",
		RunE:  di.run,
	}
	return cmd
}

func waitForNetwork(ctx context.Context) error {
	publicIP := net.ParseIP("8.8.8.8")
	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
	defer cancel()
	for ctx.Err() == nil {
		_, err := routing.GetRoute(ctx, &net.IPNet{IP: publicIP, Mask: net.CIDRMask(32, 32)})
		if err == nil {
			return nil
		}
	}
	return fmt.Errorf("timed out waiting for a route to %s; this usually means your VPN client is misconfigured", publicIP)
}

const (
	good    = "✅"
	bad     = "❌"
	podType = "pod"
	svcType = "svc"
)

func getLiveInterfaces() ([]net.Interface, error) {
	ifaces, err := net.Interfaces()
	if err != nil {
		return nil, fmt.Errorf("unable to list network interfaces: %w", err)
	}
	retval := []net.Interface{}
	for _, iface := range ifaces {
		if iface.Flags&net.FlagUp != 0 {
			retval = append(retval, iface)
		}
	}
	return retval, nil
}

func getVPNInterfaces(interfacesConnected, interfacesDisconnected []net.Interface) map[string]struct{} {
	vpnIfaces := map[string]struct{}{}
ifaces:
	for _, ifaceC := range interfacesConnected {
		for _, ifaceD := range interfacesDisconnected {
			if ifaceD.Name == ifaceC.Name {
				continue ifaces
			}
		}
		vpnIfaces[ifaceC.Name] = struct{}{}
	}
	return vpnIfaces
}

func (di *vpnDiagInfo) run(cmd *cobra.Command, _ []string) (err error) {
	var (
		ctx          = cmd.Context()
		sc           = scout.NewScout(ctx, "cli")
		configIssues = false
		vpnMasks     = false
		clusterMasks = false
		reader       = bufio.NewReader(cmd.InOrStdin())
	)
	cliutil.QuitDaemon(ctx)

	defer func() {
		if err != nil {
			sc.Report(log.WithDiscardingLogger(ctx), "vpn_diag_error", scout.ScoutMeta{Key: "error", Value: err.Error()})
		} else {
			if configIssues {
				sc.Report(log.WithDiscardingLogger(ctx), "vpn_diag_fail",
					scout.ScoutMeta{Key: "vpn_masks", Value: vpnMasks},
					scout.ScoutMeta{Key: "cluster_masks", Value: clusterMasks},
				)
			} else {
				sc.Report(log.WithDiscardingLogger(ctx), "vpn_diag_pass")
			}
		}
	}()

	fmt.Fprintln(cmd.OutOrStdout(), "Please disconnect from your VPN now and hit enter once you're disconnected...")
	reader.ReadString('\n')
	err = waitForNetwork(ctx)
	if err != nil {
		return err
	}
	interfacesDisconnected, err := getLiveInterfaces()
	if err != nil {
		return err
	}

	fmt.Fprintln(cmd.OutOrStdout(), "Please connect to your VPN now and hit enter once you're connected...")
	reader.ReadString('\n')

	err = waitForNetwork(ctx)
	if err != nil {
		return err
	}

	interfacesConnected, err := getLiveInterfaces()
	if err != nil {
		return err
	}
	vpnIfaces := getVPNInterfaces(interfacesConnected, interfacesDisconnected)

	routeTable, err := routing.GetRoutingTable(ctx)
	if err != nil {
		return fmt.Errorf("failed to get routing table: %w", err)
	}
	subnets := map[string][]*net.IPNet{podType: {}, svcType: {}}
	err = withConnector(cmd, false, func(ctx context.Context, cc connector.ConnectorClient, _ *connector.ConnectInfo, dc daemon.DaemonClient) error {
		// If this times out, it's likely to be because the traffic manager never gave us the subnets;
		// this could happen for all kinds of reasons, but it makes no sense to go on if it does.
		ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
		defer cancel()
		clusterSubnets, err := dc.GetClusterSubnets(ctx, &empty.Empty{})
		if err != nil {
			return err
		}
		for _, sn := range clusterSubnets.GetPodSubnets() {
			ipsn := iputil.IPNetFromRPC(sn)
			subnets[podType] = append(subnets[podType], ipsn)
		}
		for _, sn := range clusterSubnets.GetSvcSubnets() {
			ipsn := iputil.IPNetFromRPC(sn)
			subnets[svcType] = append(subnets[svcType], ipsn)
		}
		return nil
	})
	if err != nil {
		return err
	}

	instructions := []string{}
	for _, tp := range []string{podType, svcType} {
		for _, sn := range subnets[tp] {
			ok := true
			for _, rt := range routeTable {
				if _, inVPN := vpnIfaces[rt.Interface.Name]; !inVPN {
					continue
				}
				if rt.Routes(sn.IP) || sn.Contains(rt.RoutedNet.IP) {
					ok = false
					configIssues = true
					snSz, _ := sn.Mask.Size()
					rtSz, _ := rt.RoutedNet.Mask.Size()
					if rtSz > snSz {
						vpnMasks = true
						instructions = append(instructions,
							fmt.Sprintf("%s %s subnet %s being masked by VPN-routed CIDR %s. This usually means that Telepresence will not be able to connect to your cluster. To resolve:", bad, tp, sn, rt.RoutedNet),
							fmt.Sprintf("\t* Move %s subnet %s to a subnet not mapped by the VPN", tp, sn),
							fmt.Sprintf("\t\t* If this is not possible, consider shrinking the mask of the %s CIDR (e.g. from /16 to /8), or disabling split-tunneling", rt.RoutedNet),
						)
					} else {
						clusterMasks = true
						instructions = append(instructions,
							fmt.Sprintf("%s %s subnet %s is masking VPN-routed CIDR %s. This usually means Telepresence will be able to connect to your cluster, but hosts on your VPN may be inaccessible while telepresence is connected; to resolve:", bad, tp, sn, rt.RoutedNet),
							fmt.Sprintf("\t* Move %s subnet %s to a subnet not mapped by the VPN", tp, sn),
							fmt.Sprintf("\t\t* If this is not possible, ensure that any hosts in CIDR %s are placed in the never-proxy list", rt.RoutedNet),
						)
					}
				}
			}
			if ok {
				instructions = append(instructions, fmt.Sprintf("%s %s subnet %s is clear of VPN", good, tp, sn))
			}
		}
	}

	fmt.Fprintln(cmd.OutOrStdout(), "\n---------- Test Results:")
	for _, instruction := range instructions {
		fmt.Fprintln(cmd.OutOrStdout(), instruction)
	}
	if configIssues {
		fmt.Fprintln(cmd.OutOrStdout(), "\nPlease see https://www.telepresence.io/docs/latest/reference/vpn for more info on these corrective actions, as well as examples")
	}
	fmt.Fprintln(cmd.OutOrStdout(), "\nStill having issues? Please create a new github issue at https://github.com/telepresenceio/telepresence/issues/new?template=Bug_report.md\n",
		"Please make sure to add the following to your issue:\n",
		"* Run `telepresence loglevel debug`, try to connect, then run `telepresence gather_logs`. It will produce a zipfile that you should attach to the issue.\n",
		"* Which VPN client are you using?\n",
		"* Which VPN server are you using?\n",
		"* How is your VPN pushing DNS configuration? It may be useful to add the contents of /etc/resolv.conf")

	return nil
}
<|endoftext|>


<commit_before>package typesys

import (
  "fmt"
  "strings"
  "bitbucket.org/yyuu/bs/core"
)

type TypeTable struct {
  charSize int
  shortSize int
  intSize int
  longSize int
  pointerSize int
  typeTable map[string]core.IType
  refTable map[string]core.ITypeRef
}

func NewTypeTable(charSize, shortSize, intSize, longSize, ptrSize int) *TypeTable {
  loc := core.NewLocation("[builtin:typesys]", 0, 0)
  tt := TypeTable { charSize, shortSize, intSize, longSize, ptrSize, make(map[string]core.IType), make(map[string]core.ITypeRef) }
  tt.PutType(NewVoidTypeRef(loc), NewVoidType())
  tt.PutType(NewCharTypeRef(loc), NewCharType(charSize))
  tt.PutType(NewShortTypeRef(loc), NewShortType(shortSize))
  tt.PutType(NewIntTypeRef(loc), NewIntType(intSize))
  tt.PutType(NewLongTypeRef(loc), NewLongType(longSize))
  tt.PutType(NewUnsignedCharTypeRef(loc), NewUnsignedCharType(charSize))
  tt.PutType(NewUnsignedShortTypeRef(loc), NewUnsignedShortType(shortSize))
  tt.PutType(NewUnsignedIntTypeRef(loc), NewUnsignedIntType(intSize))
  tt.PutType(NewUnsignedLongTypeRef(loc), NewUnsignedLongType(longSize))
  return &tt
}

func NewTypeTableILP32() *TypeTable {
  return NewTypeTable(1, 2, 4, 4, 4)
}

func NewTypeTableILP64() *TypeTable {
  return NewTypeTable(1, 2, 8, 8, 8)
}

func NewTypeTableLP64() *TypeTable {
  return NewTypeTable(1, 2, 4, 8, 8)
}

func NewTypeTableLLP64() *TypeTable {
  return NewTypeTable(1, 2, 4, 4, 8)
}

func NewTypeTableFor(platformId int) *TypeTable {
  switch platformId {
    case core.PLATFORM_LINUX_X86: {
      return NewTypeTableILP32()
    }
    default: {
      panic(fmt.Errorf("unknown platformId: %d", platformId))
    }
  }
}

func (self *TypeTable) PutType(ref core.ITypeRef, t core.IType) {
  self.typeTable[ref.Key()] = t
  self.refTable[ref.Key()] = ref
}

func (self TypeTable) GetType(ref core.ITypeRef) core.IType {
  t := self.typeTable[ref.Key()]
  if t == nil {
    switch typed := ref.(type) {
      case *UserTypeRef: {
        panic(fmt.Errorf("undefined type: %s", typed.GetName()))
      }
      case *PointerTypeRef: {
        t = NewPointerType(self.pointerSize, self.GetType(typed.GetBaseType()))
        self.PutType(typed, t)
      }
      case *ArrayTypeRef: {
        t = NewArrayType(self.GetType(typed.GetBaseType()), typed.GetLength(), self.pointerSize)
        self.PutType(typed, t)
      }
      case *FunctionTypeRef: {
        params := typed.GetParams()
        paramRefs := params.GetParamDescs()
        paramTypes := make([]core.IType, len(paramRefs))
        for i := range paramRefs {
          paramTypes[i] = self.GetParamType(paramRefs[i])
        }
        t = NewFunctionType(
          self.GetType(typed.GetReturnType()),
          NewParamTypes(typed.GetLocation(), paramTypes, params.IsVararg()),
        )
        self.PutType(typed, t)
      }
      default: {
        panic(fmt.Errorf("unregistered type: %s", ref))
      }
    }
  }
  return t
}

func (self TypeTable) GetTypeRef(target core.IType) core.ITypeRef {
  for key, t := range self.typeTable {
    if t == target {
      return self.refTable[key]
    }
  }
  return nil
}

func (self TypeTable) GetCharSize() int {
  return self.charSize
}

func (self TypeTable) GetShortSize() int {
  return self.shortSize
}

func (self TypeTable) GetIntSize() int {
  return self.intSize
}

func (self TypeTable) GetLongSize() int {
  return self.longSize
}

func (self TypeTable) GetPointerSize() int {
  return self.pointerSize
}

func (self TypeTable) IsTypeTable() bool {
  return true
}

func (self TypeTable) String() string {
  xs := make([]string, len(self.typeTable))
  for key, _ := range self.typeTable {
    xs = append(xs, fmt.Sprintf("%s", key))
  }
  return fmt.Sprintf("(%s)", strings.Join(xs, "\n"))
}

func (self TypeTable) IsDefined(ref core.ITypeRef) bool {
  _, ok := self.typeTable[ref.Key()]
  return ok
}

// array is really a pointer on parameters.
func (self TypeTable) GetParamType(ref core.ITypeRef) core.IType {
  t := self.GetType(ref)
  if t == nil {
    panic(fmt.Errorf("unknown parameter type: %s", ref))
  }
  if t.IsArray() {
    return NewPointerType(self.pointerSize, t.(*ArrayType).GetBaseType())
  } else {
    return t
  }
}

func (self TypeTable) NumTypes() int {
  return len(self.typeTable)
}

func (self TypeTable) GetTypes() []core.IType {
  ts := []core.IType { }
  for _, t := range self.typeTable {
    ts = append(ts, t)
  }
  return ts
}

func (self *TypeTable) SemanticCheck(errorHandler *core.ErrorHandler) {
  ts := self.GetTypes()
  for i := range ts {
    t := ts[i]
    if t.IsCompositeType() {
      self.checkCompositeVoidMembers(t.(core.ICompositeType), errorHandler)
      self.checkDuplicatedMembers(t.(core.ICompositeType), errorHandler)
    } else {
      if t.IsArray() {
        self.checkArrayVoidMembers(t.(*ArrayType), errorHandler)
      }
    }
    self.checkRecursiveDefinition(t, errorHandler)
  }
}

func (self TypeTable) checkCompositeVoidMembers(t core.ICompositeType, errorHandler *core.ErrorHandler) {
  members := t.GetMembers()
  for i := range members {
    slot := members[i]
    if slot.GetType().IsVoid() {
      errorHandler.Fatal("struct/union cannot contain void")
    }
  }
}

func (self TypeTable) checkArrayVoidMembers(t *ArrayType, errorHandler *core.ErrorHandler) {
  if t.GetBaseType().IsVoid() {
    errorHandler.Fatal("array cannot contain void")
  }
}

func (self TypeTable) checkDuplicatedMembers(t core.ICompositeType, errorHandler *core.ErrorHandler) {
  seen := make(map[string]core.ISlot)
  members := t.GetMembers()
  for i := range members {
    slot := members[i]
    name := slot.GetName()
    _, found := seen[name]
    if found {
      errorHandler.Fatalf("%s has duplicated member: %s", t.GetName(), name)
    }
    seen[name] = slot
  }
}

func (self TypeTable) checkRecursiveDefinition(t core.IType, errorHandler *core.ErrorHandler) {
  errorHandler.Warnf("FIXME: TypeTable#checkRecursiveDefinition: not implemented for %q", t)
}

func (self TypeTable) VoidType() *VoidType {
  loc := core.NewLocation("[typesys:builtin]", 0, 0)
  ref := NewVoidTypeRef(loc)
  return self.GetType(ref).(*VoidType)
}

func (self TypeTable) SignedChar() *IntegerType {
  loc := core.NewLocation("[typesys:builtin]", 0, 0)
  ref := NewCharTypeRef(loc)
  return self.GetType(ref).(*IntegerType)
}

func (self TypeTable) SignedShort() *IntegerType {
  loc := core.NewLocation("[typesys:builtin]", 0, 0)
  ref := NewShortTypeRef(loc)
  return self.GetType(ref).(*IntegerType)
}

func (self TypeTable) SignedInt() *IntegerType {
  loc := core.NewLocation("[typesys:builtin]", 0, 0)
  ref := NewIntTypeRef(loc)
  return self.GetType(ref).(*IntegerType)
}

func (self TypeTable) SignedLong() *IntegerType {
  loc := core.NewLocation("[typesys:builtin]", 0, 0)
  ref := NewLongTypeRef(loc)
  return self.GetType(ref).(*IntegerType)
}

func (self TypeTable) UnsignedChar() *IntegerType {
  loc := core.NewLocation("[typesys:builtin]", 0, 0)
  ref := NewUnsignedCharTypeRef(loc)
  return self.GetType(ref).(*IntegerType)
}

func (self TypeTable) UnsignedShort() *IntegerType {
  loc := core.NewLocation("[typesys:builtin]", 0, 0)
  ref := NewUnsignedShortTypeRef(loc)
  return self.GetType(ref).(*IntegerType)
}

func (self TypeTable) UnsignedInt() *IntegerType {
  loc := core.NewLocation("[typesys:builtin]", 0, 0)
  ref := NewUnsignedIntTypeRef(loc)
  return self.GetType(ref).(*IntegerType)
}

func (self TypeTable) UnsignedLong() *IntegerType {
  loc := core.NewLocation("[typesys:builtin]", 0, 0)
  ref := NewUnsignedLongTypeRef(loc)
  return self.GetType(ref).(*IntegerType)
}

func (self TypeTable) PointerTo(baseType core.IType) *PointerType {
  return NewPointerType(self.pointerSize, baseType)
}

func (self TypeTable) PtrDiffType() core.IType {
  return self.GetType(self.PtrDiffTypeRef())
}

func (self TypeTable) PtrDiffTypeRef() core.ITypeRef {
  loc := core.NewLocation("[builtin:typesys]", 0, 0)
  return NewIntegerTypeRef(loc, self.PtrDiffTypeName())
}

func (self TypeTable) PtrDiffTypeName() string {
  if self.SignedLong().Size() == self.pointerSize {
    return "long"
  } else {
    if self.SignedInt().Size() == self.pointerSize {
      return "int"
    } else {
      if self.SignedShort().Size() == self.pointerSize {
        return "short"
      } else {
        panic("must not happen: interger.size != pointer.size")
      }
    }
  }
}

func (self TypeTable) SignedStackType() *IntegerType {
  return self.SignedLong()
}

func (self TypeTable) UnsignedStackType() *IntegerType {
  return self.UnsignedLong()
}
<commit_msg>Implement TypeTable#checkRecursiveDefinition<commit_after>package typesys

import (
  "fmt"
  "strings"
  "bitbucket.org/yyuu/bs/core"
)

type TypeTable struct {
  charSize int
  shortSize int
  intSize int
  longSize int
  pointerSize int
  typeTable map[string]core.IType
  refTable map[string]core.ITypeRef
}

func NewTypeTable(charSize, shortSize, intSize, longSize, ptrSize int) *TypeTable {
  loc := core.NewLocation("[builtin:typesys]", 0, 0)
  tt := TypeTable { charSize, shortSize, intSize, longSize, ptrSize, make(map[string]core.IType), make(map[string]core.ITypeRef) }
  tt.PutType(NewVoidTypeRef(loc), NewVoidType())
  tt.PutType(NewCharTypeRef(loc), NewCharType(charSize))
  tt.PutType(NewShortTypeRef(loc), NewShortType(shortSize))
  tt.PutType(NewIntTypeRef(loc), NewIntType(intSize))
  tt.PutType(NewLongTypeRef(loc), NewLongType(longSize))
  tt.PutType(NewUnsignedCharTypeRef(loc), NewUnsignedCharType(charSize))
  tt.PutType(NewUnsignedShortTypeRef(loc), NewUnsignedShortType(shortSize))
  tt.PutType(NewUnsignedIntTypeRef(loc), NewUnsignedIntType(intSize))
  tt.PutType(NewUnsignedLongTypeRef(loc), NewUnsignedLongType(longSize))
  return &tt
}

func NewTypeTableILP32() *TypeTable {
  return NewTypeTable(1, 2, 4, 4, 4)
}

func NewTypeTableILP64() *TypeTable {
  return NewTypeTable(1, 2, 8, 8, 8)
}

func NewTypeTableLP64() *TypeTable {
  return NewTypeTable(1, 2, 4, 8, 8)
}

func NewTypeTableLLP64() *TypeTable {
  return NewTypeTable(1, 2, 4, 4, 8)
}

func NewTypeTableFor(platformId int) *TypeTable {
  switch platformId {
    case core.PLATFORM_LINUX_X86: {
      return NewTypeTableILP32()
    }
    default: {
      panic(fmt.Errorf("unknown platformId: %d", platformId))
    }
  }
}

func (self *TypeTable) PutType(ref core.ITypeRef, t core.IType) {
  self.typeTable[ref.Key()] = t
  self.refTable[ref.Key()] = ref
}

func (self TypeTable) GetType(ref core.ITypeRef) core.IType {
  t := self.typeTable[ref.Key()]
  if t == nil {
    switch typed := ref.(type) {
      case *UserTypeRef: {
        panic(fmt.Errorf("undefined type: %s", typed.GetName()))
      }
      case *PointerTypeRef: {
        t = NewPointerType(self.pointerSize, self.GetType(typed.GetBaseType()))
        self.PutType(typed, t)
      }
      case *ArrayTypeRef: {
        t = NewArrayType(self.GetType(typed.GetBaseType()), typed.GetLength(), self.pointerSize)
        self.PutType(typed, t)
      }
      case *FunctionTypeRef: {
        params := typed.GetParams()
        paramRefs := params.GetParamDescs()
        paramTypes := make([]core.IType, len(paramRefs))
        for i := range paramRefs {
          paramTypes[i] = self.GetParamType(paramRefs[i])
        }
        t = NewFunctionType(
          self.GetType(typed.GetReturnType()),
          NewParamTypes(typed.GetLocation(), paramTypes, params.IsVararg()),
        )
        self.PutType(typed, t)
      }
      default: {
        panic(fmt.Errorf("unregistered type: %s", ref))
      }
    }
  }
  return t
}

func (self TypeTable) GetTypeRef(target core.IType) core.ITypeRef {
  for key, t := range self.typeTable {
    if t == target {
      return self.refTable[key]
    }
  }
  return nil
}

func (self TypeTable) GetCharSize() int {
  return self.charSize
}

func (self TypeTable) GetShortSize() int {
  return self.shortSize
}

func (self TypeTable) GetIntSize() int {
  return self.intSize
}

func (self TypeTable) GetLongSize() int {
  return self.longSize
}

func (self TypeTable) GetPointerSize() int {
  return self.pointerSize
}

func (self TypeTable) IsTypeTable() bool {
  return true
}

func (self TypeTable) String() string {
  xs := make([]string, len(self.typeTable))
  for key, _ := range self.typeTable {
    xs = append(xs, fmt.Sprintf("%s", key))
  }
  return fmt.Sprintf("(%s)", strings.Join(xs, "\n"))
}

func (self TypeTable) IsDefined(ref core.ITypeRef) bool {
  _, ok := self.typeTable[ref.Key()]
  return ok
}

// array is really a pointer on parameters.
func (self TypeTable) GetParamType(ref core.ITypeRef) core.IType {
  t := self.GetType(ref)
  if t == nil {
    panic(fmt.Errorf("unknown parameter type: %s", ref))
  }
  if t.IsArray() {
    return NewPointerType(self.pointerSize, t.(*ArrayType).GetBaseType())
  } else {
    return t
  }
}

func (self TypeTable) NumTypes() int {
  return len(self.typeTable)
}

func (self TypeTable) GetTypes() []core.IType {
  ts := []core.IType { }
  for _, t := range self.typeTable {
    ts = append(ts, t)
  }
  return ts
}

func (self *TypeTable) SemanticCheck(errorHandler *core.ErrorHandler) {
  ts := self.GetTypes()
  for i := range ts {
    t := ts[i]
    if t.IsCompositeType() {
      self.checkCompositeVoidMembers(t.(core.ICompositeType), errorHandler)
      self.checkDuplicatedMembers(t.(core.ICompositeType), errorHandler)
    } else {
      if t.IsArray() {
        self.checkArrayVoidMembers(t.(*ArrayType), errorHandler)
      }
    }
    self.checkRecursiveDefinition(t, errorHandler)
  }
}

func (self TypeTable) checkCompositeVoidMembers(t core.ICompositeType, errorHandler *core.ErrorHandler) {
  members := t.GetMembers()
  for i := range members {
    slot := members[i]
    if slot.GetType().IsVoid() {
      errorHandler.Fatal("struct/union cannot contain void")
    }
  }
}

func (self TypeTable) checkArrayVoidMembers(t *ArrayType, errorHandler *core.ErrorHandler) {
  if t.GetBaseType().IsVoid() {
    errorHandler.Fatal("array cannot contain void")
  }
}

func (self TypeTable) checkDuplicatedMembers(t core.ICompositeType, errorHandler *core.ErrorHandler) {
  seen := make(map[string]core.ISlot)
  members := t.GetMembers()
  for i := range members {
    slot := members[i]
    name := slot.GetName()
    _, found := seen[name]
    if found {
      errorHandler.Fatalf("%s has duplicated member: %s", t.GetName(), name)
    }
    seen[name] = slot
  }
}

func (self TypeTable) checkRecursiveDefinition(t core.IType, h *core.ErrorHandler) {
  self._checkRecursiveDefinition(t, make(map[core.IType]int), h)
}

const (
  checking = 1 << iota
  checked
)

func (self TypeTable) _checkRecursiveDefinition(t core.IType, marks map[core.IType]int, h *core.ErrorHandler) {
  switch {
    case marks[t] == checking: {
      h.Fatalf("recursive type definition: %s", t)
      return
    }
    case marks[t] == checked: {
      return
    }
    default: {
      marks[t] = checking
      ct, ok := t.(core.ICompositeType)
      if ok {
        members := ct.GetMembers()
        for i := range members {
          self._checkRecursiveDefinition(members[i].GetType(), marks, h)
        }
      } else {
        switch xt := t.(type) {
          case *ArrayType: self._checkRecursiveDefinition(xt.GetBaseType(), marks, h)
          case *UserType:  self._checkRecursiveDefinition(xt.GetRealType(), marks, h)
        }
      }
      marks[t] = checked
    }
  }
}

func (self TypeTable) VoidType() *VoidType {
  loc := core.NewLocation("[typesys:builtin]", 0, 0)
  ref := NewVoidTypeRef(loc)
  return self.GetType(ref).(*VoidType)
}

func (self TypeTable) SignedChar() *IntegerType {
  loc := core.NewLocation("[typesys:builtin]", 0, 0)
  ref := NewCharTypeRef(loc)
  return self.GetType(ref).(*IntegerType)
}

func (self TypeTable) SignedShort() *IntegerType {
  loc := core.NewLocation("[typesys:builtin]", 0, 0)
  ref := NewShortTypeRef(loc)
  return self.GetType(ref).(*IntegerType)
}

func (self TypeTable) SignedInt() *IntegerType {
  loc := core.NewLocation("[typesys:builtin]", 0, 0)
  ref := NewIntTypeRef(loc)
  return self.GetType(ref).(*IntegerType)
}

func (self TypeTable) SignedLong() *IntegerType {
  loc := core.NewLocation("[typesys:builtin]", 0, 0)
  ref := NewLongTypeRef(loc)
  return self.GetType(ref).(*IntegerType)
}

func (self TypeTable) UnsignedChar() *IntegerType {
  loc := core.NewLocation("[typesys:builtin]", 0, 0)
  ref := NewUnsignedCharTypeRef(loc)
  return self.GetType(ref).(*IntegerType)
}

func (self TypeTable) UnsignedShort() *IntegerType {
  loc := core.NewLocation("[typesys:builtin]", 0, 0)
  ref := NewUnsignedShortTypeRef(loc)
  return self.GetType(ref).(*IntegerType)
}

func (self TypeTable) UnsignedInt() *IntegerType {
  loc := core.NewLocation("[typesys:builtin]", 0, 0)
  ref := NewUnsignedIntTypeRef(loc)
  return self.GetType(ref).(*IntegerType)
}

func (self TypeTable) UnsignedLong() *IntegerType {
  loc := core.NewLocation("[typesys:builtin]", 0, 0)
  ref := NewUnsignedLongTypeRef(loc)
  return self.GetType(ref).(*IntegerType)
}

func (self TypeTable) PointerTo(baseType core.IType) *PointerType {
  return NewPointerType(self.pointerSize, baseType)
}

func (self TypeTable) PtrDiffType() core.IType {
  return self.GetType(self.PtrDiffTypeRef())
}

func (self TypeTable) PtrDiffTypeRef() core.ITypeRef {
  loc := core.NewLocation("[builtin:typesys]", 0, 0)
  return NewIntegerTypeRef(loc, self.PtrDiffTypeName())
}

func (self TypeTable) PtrDiffTypeName() string {
  if self.SignedLong().Size() == self.pointerSize {
    return "long"
  } else {
    if self.SignedInt().Size() == self.pointerSize {
      return "int"
    } else {
      if self.SignedShort().Size() == self.pointerSize {
        return "short"
      } else {
        panic("must not happen: interger.size != pointer.size")
      }
    }
  }
}

func (self TypeTable) SignedStackType() *IntegerType {
  return self.SignedLong()
}

func (self TypeTable) UnsignedStackType() *IntegerType {
  return self.UnsignedLong()
}
<|endoftext|>


<commit_before>package node

import (
	"fmt"

	"encoding/json"
	"errors"
	"github.com/docker/machine/libmachine/drivers"
	"github.com/docker/machine/libmachine/state"
	nodehelper "github.com/kube-node/kube-machine/pkg/node"
	"github.com/kube-node/kube-machine/pkg/options"
	"k8s.io/client-go/pkg/api/v1"
)

const (
	noExecuteTaintKey = "node.k8s.io/not-up"
)

func (c *Controller) syncPendingNode(node *v1.Node) (changedN *v1.Node, err error) {

	changedN, err = c.pendingCreateTaint(node)
	if err != nil || changedN != nil {
		return changedN, err
	}

	changedN, err = c.pendingCreateFinalizer(node)
	if err != nil || changedN != nil {
		return changedN, err
	}

	changedN, err = c.pendingCreateInstance(node)
	if err != nil || changedN != nil {
		return changedN, err
	}

	changedN, err = c.pendingCreateInstanceDetails(node)
	if err != nil || changedN != nil {
		return changedN, err
	}

	//Will set the phase to provisioning
	changedN, err = c.pendingWaitUntilInstanceIsRunning(node)
	if err != nil || changedN != nil {
		return changedN, err
	}

	return nil, nil
}

func (c *Controller) pendingCreateTaint(node *v1.Node) (*v1.Node, error) {
	if !nodehelper.HasTaint(node, noExecuteTaintKey) {
		node.Spec.Taints = append(node.Spec.Taints, v1.Taint{
			Key:    noExecuteTaintKey,
			Effect: v1.TaintEffectNoExecute,
			Value:  "kube-machine",
		})

		return node, nil
	}

	return nil, nil
}

func (c *Controller) pendingCreateFinalizer(node *v1.Node) (*v1.Node, error) {
	if !nodehelper.HasFinalizer(node, deleteFinalizerName) {
		node.Finalizers = append(node.Finalizers, deleteFinalizerName)
		return node, nil
	}

	return nil, nil
}

func (c *Controller) pendingCreateInstance(node *v1.Node) (*v1.Node, error) {
	if node.Annotations[driverDataAnnotationKey] != "" {
		return nil, nil
	}

	class, config, err := c.getNodeClass(node.Annotations[classAnnotationKey])
	if err != nil {
		return nil, fmt.Errorf("could not get nodeclass %q for node %s: %v", node.Annotations[classAnnotationKey], node.Name, err)
	}

	rawDriver, err := json.Marshal(&drivers.BaseDriver{MachineName: node.Name})
	if err != nil {
		return nil, fmt.Errorf("error attempting to marshal bare driver data: %s", err)
	}

	mhost, err := c.mapi.NewHost(config.Provider, rawDriver)
	if err != nil {
		return nil, fmt.Errorf("failed to create docker machine host for node %q: %v", node.Name, err)
	}

	opts := options.New(config.DockerMachineFlags)
	mcnFlags := mhost.Driver.GetCreateFlags()
	driverOpts := options.GetDriverOpts(opts, mcnFlags, class.Resources)

	mhost.Driver.SetConfigFromFlags(driverOpts)
	err = c.mapi.Create(mhost)
	if err != nil {
		mhost.Driver.Remove()
		return nil, fmt.Errorf("failed to create node %q on cloud provider: %v. Deleted eventually created node on cloud provider", node.Name, err)
	}

	data, err := json.Marshal(mhost)
	if err != nil {
		return nil, err
	}
	node.Annotations[driverDataAnnotationKey] = string(data)
	return node, nil
}

func (c *Controller) pendingCreateInstanceDetails(node *v1.Node) (*v1.Node, error) {
	if node.Annotations[publicIPAnnotationKey] != "" {
		return nil, nil
	}

	h, err := c.mapi.Load(node)
	if err != nil {
		return nil, err
	}

	ip, err := h.Driver.GetIP()
	if err != nil {
		return nil, errors.New("could not get public ip")
	}
	node.Annotations[publicIPAnnotationKey] = ip

	hostname, err := h.Driver.GetSSHHostname()
	if err != nil {
		return nil, errors.New("could not get hostname")
	}
	node.Annotations[hostnameAnnotationKey] = hostname

	return node, nil

}

func (c *Controller) pendingWaitUntilInstanceIsRunning(node *v1.Node) (*v1.Node, error) {
	h, err := c.mapi.Load(node)
	if err != nil {
		return nil, err
	}

	s, err := h.Driver.GetState()
	if err != nil {
		return nil, fmt.Errorf("failed getting instance state: %v", err)
	}
	if s == state.Running {
		node.Annotations[phaseAnnotationKey] = phaseProvisioning
		return node, nil
	}

	return nil, nil
}
<commit_msg>Add support for engine install url<commit_after>package node

import (
	"fmt"

	"encoding/json"
	"errors"
	"github.com/docker/machine/libmachine/drivers"
	"github.com/docker/machine/libmachine/state"
	nodehelper "github.com/kube-node/kube-machine/pkg/node"
	"github.com/kube-node/kube-machine/pkg/options"
	"k8s.io/client-go/pkg/api/v1"
)

const (
	noExecuteTaintKey = "node.k8s.io/not-up"
)

func (c *Controller) syncPendingNode(node *v1.Node) (changedN *v1.Node, err error) {

	changedN, err = c.pendingCreateTaint(node)
	if err != nil || changedN != nil {
		return changedN, err
	}

	changedN, err = c.pendingCreateFinalizer(node)
	if err != nil || changedN != nil {
		return changedN, err
	}

	changedN, err = c.pendingCreateInstance(node)
	if err != nil || changedN != nil {
		return changedN, err
	}

	changedN, err = c.pendingCreateInstanceDetails(node)
	if err != nil || changedN != nil {
		return changedN, err
	}

	//Will set the phase to provisioning
	changedN, err = c.pendingWaitUntilInstanceIsRunning(node)
	if err != nil || changedN != nil {
		return changedN, err
	}

	return nil, nil
}

func (c *Controller) pendingCreateTaint(node *v1.Node) (*v1.Node, error) {
	if !nodehelper.HasTaint(node, noExecuteTaintKey) {
		node.Spec.Taints = append(node.Spec.Taints, v1.Taint{
			Key:    noExecuteTaintKey,
			Effect: v1.TaintEffectNoExecute,
			Value:  "kube-machine",
		})

		return node, nil
	}

	return nil, nil
}

func (c *Controller) pendingCreateFinalizer(node *v1.Node) (*v1.Node, error) {
	if !nodehelper.HasFinalizer(node, deleteFinalizerName) {
		node.Finalizers = append(node.Finalizers, deleteFinalizerName)
		return node, nil
	}

	return nil, nil
}

func (c *Controller) pendingCreateInstance(node *v1.Node) (*v1.Node, error) {
	if node.Annotations[driverDataAnnotationKey] != "" {
		return nil, nil
	}

	class, config, err := c.getNodeClass(node.Annotations[classAnnotationKey])
	if err != nil {
		return nil, fmt.Errorf("could not get nodeclass %q for node %s: %v", node.Annotations[classAnnotationKey], node.Name, err)
	}

	rawDriver, err := json.Marshal(&drivers.BaseDriver{MachineName: node.Name})
	if err != nil {
		return nil, fmt.Errorf("error attempting to marshal bare driver data: %s", err)
	}

	mhost, err := c.mapi.NewHost(config.Provider, rawDriver)
	if err != nil {
		return nil, fmt.Errorf("failed to create docker machine host for node %q: %v", node.Name, err)
	}

	opts := options.New(config.DockerMachineFlags)
	mcnFlags := mhost.Driver.GetCreateFlags()
	driverOpts := options.GetDriverOpts(opts, mcnFlags, class.Resources)

	if url, exists := config.DockerMachineFlags["engine-install-url"]; exists {
		mhost.HostOptions.EngineOptions.InstallURL = url
	}

	mhost.Driver.SetConfigFromFlags(driverOpts)

	err = c.mapi.Create(mhost)
	if err != nil {
		mhost.Driver.Remove()
		return nil, fmt.Errorf("failed to create node %q on cloud provider: %v. Deleted eventually created node on cloud provider", node.Name, err)
	}

	data, err := json.Marshal(mhost)
	if err != nil {
		return nil, err
	}
	node.Annotations[driverDataAnnotationKey] = string(data)
	return node, nil
}

func (c *Controller) pendingCreateInstanceDetails(node *v1.Node) (*v1.Node, error) {
	if node.Annotations[publicIPAnnotationKey] != "" {
		return nil, nil
	}

	h, err := c.mapi.Load(node)
	if err != nil {
		return nil, err
	}

	ip, err := h.Driver.GetIP()
	if err != nil {
		return nil, errors.New("could not get public ip")
	}
	node.Annotations[publicIPAnnotationKey] = ip

	hostname, err := h.Driver.GetSSHHostname()
	if err != nil {
		return nil, errors.New("could not get hostname")
	}
	node.Annotations[hostnameAnnotationKey] = hostname

	return node, nil

}

func (c *Controller) pendingWaitUntilInstanceIsRunning(node *v1.Node) (*v1.Node, error) {
	h, err := c.mapi.Load(node)
	if err != nil {
		return nil, err
	}

	s, err := h.Driver.GetState()
	if err != nil {
		return nil, fmt.Errorf("failed getting instance state: %v", err)
	}
	if s == state.Running {
		node.Annotations[phaseAnnotationKey] = phaseProvisioning
		return node, nil
	}

	return nil, nil
}
<|endoftext|>