<commit_before>package handlers_test

import (
	"encoding/json"
	"errors"
	"net"
	"net/http"
	"net/http/httptest"

	"github.com/appc/cni/pkg/types"
	"github.com/cloudfoundry-incubator/ducati-daemon/fakes"
	"github.com/cloudfoundry-incubator/ducati-daemon/handlers"
	"github.com/cloudfoundry-incubator/ducati-daemon/ipam"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/tedsuo/rata"
)

func rataWrap(handler http.Handler, method, path string, params rata.Params) (http.Handler, *http.Request) {
	testRoutes := rata.Routes{
		{Name: "wicked_smat", Method: method, Path: path},
	}
	requestGenerator := rata.NewRequestGenerator("", testRoutes)
	testHandlers := rata.Handlers{
		"wicked_smat": handler,
	}

	router, err := rata.NewRouter(testRoutes, testHandlers)
	Expect(err).NotTo(HaveOccurred())

	request, err := requestGenerator.CreateRequest("wicked_smat", params, nil)
	Expect(err).NotTo(HaveOccurred())

	return router, request
}

var _ = Describe("Allocate IP", func() {
	var request *http.Request
	var marshaler *fakes.Marshaler
	var logger *fakes.Logger
	var ipAllocator *fakes.IPAllocator
	var expectedResultBytes []byte
	var handler http.Handler

	BeforeEach(func() {
		logger = &fakes.Logger{}

		marshaler = &fakes.Marshaler{}
		marshaler.MarshalStub = json.Marshal

		ipAllocator = &fakes.IPAllocator{}

		allocateIPHandler := &handlers.AllocateIPHandler{
			Marshaler:   marshaler,
			IPAllocator: ipAllocator,
			Logger:      logger,
		}

		handler, request = rataWrap(allocateIPHandler, "POST", "/ipam/:network_id/:container_id", rata.Params{
			"network_id":   "some-network",
			"container_id": "some-container",
		})

		expectedResult := &types.Result{
			IP4: &types.IPConfig{
				IP: net.IPNet{
					IP:   net.ParseIP("192.168.100.1"),
					Mask: net.ParseIP("192.168.100.1").DefaultMask(),
				},
				Gateway: net.ParseIP("192.168.100.1"),
				Routes: []types.Route{
					{
						Dst: net.IPNet{
							IP:   net.ParseIP("192.168.1.5"),
							Mask: net.ParseIP("192.168.1.5").DefaultMask(),
						},
						GW: net.ParseIP("192.168.1.1"),
					},
				},
			},
		}
		var err error
		expectedResultBytes, err = json.Marshal(expectedResult)
		Expect(err).NotTo(HaveOccurred())

		ipAllocator.AllocateIPReturns(expectedResult, nil)
	})

	It("allocates an IP and returns the result", func() {
		resp := httptest.NewRecorder()
		handler.ServeHTTP(resp, request)

		Expect(ipAllocator.AllocateIPCallCount()).To(Equal(1))
		Expect(resp.Body.String()).To(MatchJSON(expectedResultBytes))

		Expect(marshaler.MarshalCallCount()).To(Equal(1))
		Expect(resp.Code).To(Equal(http.StatusCreated))

		networkID, containerID := ipAllocator.AllocateIPArgsForCall(0)
		Expect(networkID).To(Equal("some-network"))
		Expect(containerID).To(Equal("some-container"))
	})

	Context("when things go wrong", func() {
		Context("when the allocator returns a NoMoreAddressesError", func() {
			BeforeEach(func() {
				ipAllocator.AllocateIPReturns(nil, ipam.NoMoreAddressesError)
			})

			It("should log and return a 409 status with JSON body encoding the error message", func() {
				resp := httptest.NewRecorder()
				handler.ServeHTTP(resp, request)

				Expect(marshaler.MarshalCallCount()).To(Equal(1))
				Expect(resp.Code).To(Equal(http.StatusConflict))

				Expect(resp.Body.String()).To(MatchJSON(`{ "error": "no addresses available" }`))

				Expect(logger.ErrorCallCount()).To(Equal(1))
				action, err, _ := logger.ErrorArgsForCall(0)
				Expect(action).To(Equal("allocate-ip"))
				Expect(err).To(Equal(ipam.NoMoreAddressesError))
			})

			Context("when marshaling the error fails", func() {
				BeforeEach(func() {
					ipAllocator.AllocateIPReturns(nil, ipam.NoMoreAddressesError)
					marshaler.MarshalReturns([]byte(`bad`), errors.New("banana"))
				})

				It("should log the error", func() {
					resp := httptest.NewRecorder()
					handler.ServeHTTP(resp, request)

					Expect(resp.Code).To(Equal(http.StatusConflict))

					Expect(logger.ErrorCallCount()).To(Equal(2))
					action, err, _ := logger.ErrorArgsForCall(0)
					Expect(action).To(Equal("allocate-ip"))
					Expect(err).To(Equal(ipam.NoMoreAddressesError))

					action, err, _ = logger.ErrorArgsForCall(1)
					Expect(action).To(Equal("allocate-ip-error-marshaling"))
					Expect(err).To(MatchError("banana"))

					Expect(resp.Body.String()).To(BeEmpty())
				})
			})
		})

		Context("when the allocator errors in some other fashion", func() {
			BeforeEach(func() {
				ipAllocator.AllocateIPReturns(&types.Result{}, errors.New(`{"boom":"bang"}`))
			})

			It("should return 500 and log the error", func() {
				resp := httptest.NewRecorder()
				handler.ServeHTTP(resp, request)

				Expect(marshaler.MarshalCallCount()).To(Equal(1))
				Expect(resp.Code).To(Equal(http.StatusInternalServerError))

				Expect(resp.Body.String()).To(MatchJSON(`{ "error": "{\"boom\":\"bang\"}" }`))

				Expect(logger.ErrorCallCount()).To(Equal(1))
				action, err, _ := logger.ErrorArgsForCall(0)
				Expect(action).To(Equal("allocate-ip"))
				Expect(err).To(MatchError(`{"boom":"bang"}`))
			})
		})

	})

	Context("when marshaling the result fails", func() {
		It("should return 500 and log the error", func() {
			marshaler.MarshalReturns([]byte(`bad`), errors.New("banana"))

			resp := httptest.NewRecorder()
			handler.ServeHTTP(resp, request)

			Expect(resp.Code).To(Equal(http.StatusInternalServerError))

			Expect(logger.ErrorCallCount()).To(Equal(1))
			action, err, _ := logger.ErrorArgsForCall(0)
			Expect(action).To(Equal("allocate-ip"))
			Expect(err).To(MatchError("banana"))

			Expect(resp.Body.String()).To(BeEmpty())
		})
	})

	Context("when writing the response body fails", func() {
		It("should log the error", func() {
			badResponseWriter := &badResponseWriter{}
			handler.ServeHTTP(badResponseWriter, request)

			Expect(logger.ErrorCallCount()).To(Equal(1))
			action, err, _ := logger.ErrorArgsForCall(0)
			Expect(action).To(Equal("allocate-ip"))
			Expect(err).To(MatchError("failed writing body: some bad writer"))
		})
	})
})
<commit_msg>Return nil for result on error path<commit_after>package handlers_test

import (
	"encoding/json"
	"errors"
	"net"
	"net/http"
	"net/http/httptest"

	"github.com/appc/cni/pkg/types"
	"github.com/cloudfoundry-incubator/ducati-daemon/fakes"
	"github.com/cloudfoundry-incubator/ducati-daemon/handlers"
	"github.com/cloudfoundry-incubator/ducati-daemon/ipam"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/tedsuo/rata"
)

func rataWrap(handler http.Handler, method, path string, params rata.Params) (http.Handler, *http.Request) {
	testRoutes := rata.Routes{
		{Name: "wicked_smat", Method: method, Path: path},
	}
	requestGenerator := rata.NewRequestGenerator("", testRoutes)
	testHandlers := rata.Handlers{
		"wicked_smat": handler,
	}

	router, err := rata.NewRouter(testRoutes, testHandlers)
	Expect(err).NotTo(HaveOccurred())

	request, err := requestGenerator.CreateRequest("wicked_smat", params, nil)
	Expect(err).NotTo(HaveOccurred())

	return router, request
}

var _ = Describe("Allocate IP", func() {
	var request *http.Request
	var marshaler *fakes.Marshaler
	var logger *fakes.Logger
	var ipAllocator *fakes.IPAllocator
	var expectedResultBytes []byte
	var handler http.Handler

	BeforeEach(func() {
		logger = &fakes.Logger{}

		marshaler = &fakes.Marshaler{}
		marshaler.MarshalStub = json.Marshal

		ipAllocator = &fakes.IPAllocator{}

		allocateIPHandler := &handlers.AllocateIPHandler{
			Marshaler:   marshaler,
			IPAllocator: ipAllocator,
			Logger:      logger,
		}

		handler, request = rataWrap(allocateIPHandler, "POST", "/ipam/:network_id/:container_id", rata.Params{
			"network_id":   "some-network",
			"container_id": "some-container",
		})

		expectedResult := &types.Result{
			IP4: &types.IPConfig{
				IP: net.IPNet{
					IP:   net.ParseIP("192.168.100.1"),
					Mask: net.ParseIP("192.168.100.1").DefaultMask(),
				},
				Gateway: net.ParseIP("192.168.100.1"),
				Routes: []types.Route{
					{
						Dst: net.IPNet{
							IP:   net.ParseIP("192.168.1.5"),
							Mask: net.ParseIP("192.168.1.5").DefaultMask(),
						},
						GW: net.ParseIP("192.168.1.1"),
					},
				},
			},
		}
		var err error
		expectedResultBytes, err = json.Marshal(expectedResult)
		Expect(err).NotTo(HaveOccurred())

		ipAllocator.AllocateIPReturns(expectedResult, nil)
	})

	It("allocates an IP and returns the result", func() {
		resp := httptest.NewRecorder()
		handler.ServeHTTP(resp, request)

		Expect(ipAllocator.AllocateIPCallCount()).To(Equal(1))
		Expect(resp.Body.String()).To(MatchJSON(expectedResultBytes))

		Expect(marshaler.MarshalCallCount()).To(Equal(1))
		Expect(resp.Code).To(Equal(http.StatusCreated))

		networkID, containerID := ipAllocator.AllocateIPArgsForCall(0)
		Expect(networkID).To(Equal("some-network"))
		Expect(containerID).To(Equal("some-container"))
	})

	Context("when things go wrong", func() {
		Context("when the allocator returns a NoMoreAddressesError", func() {
			BeforeEach(func() {
				ipAllocator.AllocateIPReturns(nil, ipam.NoMoreAddressesError)
			})

			It("should log and return a 409 status with JSON body encoding the error message", func() {
				resp := httptest.NewRecorder()
				handler.ServeHTTP(resp, request)

				Expect(marshaler.MarshalCallCount()).To(Equal(1))
				Expect(resp.Code).To(Equal(http.StatusConflict))

				Expect(resp.Body.String()).To(MatchJSON(`{ "error": "no addresses available" }`))

				Expect(logger.ErrorCallCount()).To(Equal(1))
				action, err, _ := logger.ErrorArgsForCall(0)
				Expect(action).To(Equal("allocate-ip"))
				Expect(err).To(Equal(ipam.NoMoreAddressesError))
			})

			Context("when marshaling the error fails", func() {
				BeforeEach(func() {
					ipAllocator.AllocateIPReturns(nil, ipam.NoMoreAddressesError)
					marshaler.MarshalReturns([]byte(`bad`), errors.New("banana"))
				})

				It("should log the error", func() {
					resp := httptest.NewRecorder()
					handler.ServeHTTP(resp, request)

					Expect(resp.Code).To(Equal(http.StatusConflict))

					Expect(logger.ErrorCallCount()).To(Equal(2))
					action, err, _ := logger.ErrorArgsForCall(0)
					Expect(action).To(Equal("allocate-ip"))
					Expect(err).To(Equal(ipam.NoMoreAddressesError))

					action, err, _ = logger.ErrorArgsForCall(1)
					Expect(action).To(Equal("allocate-ip-error-marshaling"))
					Expect(err).To(MatchError("banana"))

					Expect(resp.Body.String()).To(BeEmpty())
				})
			})
		})

		Context("when the allocator errors in some other fashion", func() {
			BeforeEach(func() {
				ipAllocator.AllocateIPReturns(nil, errors.New(`{"boom":"bang"}`))
			})

			It("should return 500 and log the error", func() {
				resp := httptest.NewRecorder()
				handler.ServeHTTP(resp, request)

				Expect(marshaler.MarshalCallCount()).To(Equal(1))
				Expect(resp.Code).To(Equal(http.StatusInternalServerError))

				Expect(resp.Body.String()).To(MatchJSON(`{ "error": "{\"boom\":\"bang\"}" }`))

				Expect(logger.ErrorCallCount()).To(Equal(1))
				action, err, _ := logger.ErrorArgsForCall(0)
				Expect(action).To(Equal("allocate-ip"))
				Expect(err).To(MatchError(`{"boom":"bang"}`))
			})
		})

	})

	Context("when marshaling the result fails", func() {
		It("should return 500 and log the error", func() {
			marshaler.MarshalReturns([]byte(`bad`), errors.New("banana"))

			resp := httptest.NewRecorder()
			handler.ServeHTTP(resp, request)

			Expect(resp.Code).To(Equal(http.StatusInternalServerError))

			Expect(logger.ErrorCallCount()).To(Equal(1))
			action, err, _ := logger.ErrorArgsForCall(0)
			Expect(action).To(Equal("allocate-ip"))
			Expect(err).To(MatchError("banana"))

			Expect(resp.Body.String()).To(BeEmpty())
		})
	})

	Context("when writing the response body fails", func() {
		It("should log the error", func() {
			badResponseWriter := &badResponseWriter{}
			handler.ServeHTTP(badResponseWriter, request)

			Expect(logger.ErrorCallCount()).To(Equal(1))
			action, err, _ := logger.ErrorArgsForCall(0)
			Expect(action).To(Equal("allocate-ip"))
			Expect(err).To(MatchError("failed writing body: some bad writer"))
		})
	})
})
<|endoftext|>


<commit_before>package core

import (
	"fmt"
	"os"
	"syscall"
	"unsafe"
)

const (
	SWIRQ_START   = 0x201
	SWIRQ_STOP    = 0x202
	SWIRQ_SETPID  = 0x203
	SWIRQ_ENABLE  = 0x204
	SWIRQ_DISABLE = 0x205
)

const swirq_dev = "/dev/swirq"

type SWIrq_Config struct {
	channel   uint8
	mode, pid int
} // TODO: Ioctl

type userFunc func()

func AttachInterrupt(irqno uint8, fn userFunc, modec int) {
	hwmode := 0
	if irqno < EXTERNAL_NUM_INTERRUPTS && fn != nil {
		switch modec {
		case LOW:
			hwmode = 0x3
		case FALLING:
			hwmode = 0x1
		case RISING:
			hwmode = 0x0
		case CHANGE:
			hwmode = 0x4
		default:
			fmt.Fprintf(os.Stderr, "attachInterrupt error: set interrupt %d to invalid mode\n", irqno)
			return
		}
		switch irqno { //TODO: impl signal
		case 0:
			// signal(SIGUSR1, (void (*) (int))userFunc);
		case 1:
			// signal(SIGUSR2, (void (*) (int))userFunc);
		}
		pid := os.Getpid()
		irqconfig := SWIrq_Config{irqno, hwmode, pid}
		fd, err := syscall.Open(swirq_dev, os.O_RDONLY, 0666)
		if err != nil {
			fmt.Fprintf(os.Stderr, "open swirq device fail: %v\n", err)
			os.Exit(-1)
		}
		if err = Ioctl(fd, SWIRQ_STOP, uintptr(unsafe.Pointer(&irqno))); err != nil {
			fmt.Fprintf(os.Stderr, "%v", err)
			panic("can't set SWIRQ_STOP")
		}
		if err = Ioctl(fd, SWIRQ_SETPID, uintptr(unsafe.Pointer(&irqconfig))); err != nil {
			fmt.Fprintf(os.Stderr, "%v", err)
			panic("can't set SWIRQ_SETPID")
		}
		if err = Ioctl(fd, SWIRQ_START, uintptr(unsafe.Pointer(&irqno))); err != nil {
			fmt.Fprintf(os.Stderr, "%v", err)
			panic("can't set SWIRQ_START")
		}
		syscall.Close(fd)
	}
}

func DetachInterrupt(irqno uint8) {
	if irqno < EXTERNAL_NUM_INTERRUPTS {
		fd, err := syscall.Open(swirq_dev, os.O_RDONLY, 0666)
		if err != nil {
			fmt.Fprintf(os.Stderr, "open swirq device fail: %v\n", err)
			os.Exit(-1)
		}
		if err = Ioctl(fd, SWIRQ_STOP, uintptr(unsafe.Pointer(&irqno))); err != nil {
			fmt.Fprintf(os.Stderr, "%v", err)
			panic("can't set SWIRQ_STOP")
		}
		syscall.Close(fd)
	}
}

func Interrupts() {
	var irqno uint8 = 0
	fd, err := syscall.Open(swirq_dev, os.O_RDONLY, 0666)
	if err != nil {
		fmt.Fprintf(os.Stderr, "open swirq device fail: %v\n", err)
		os.Exit(-1)
	}
	if err = Ioctl(fd, SWIRQ_ENABLE, uintptr(unsafe.Pointer(&irqno))); err != nil {
		fmt.Fprintf(os.Stderr, "%v", err)
		panic("can't set interrupt_0 SWIRQ_ENABLE")
	}
	irqno = 1
	if err = Ioctl(fd, SWIRQ_ENABLE, uintptr(unsafe.Pointer(&irqno))); err != nil {
		fmt.Fprintf(os.Stderr, "%v", err)
		panic("can't set interrupt_1 SWIRQ_ENABLE")
	}
	syscall.Close(fd)
}

func NoInterrupts() {
	var irqno uint8 = 0
	fd, err := syscall.Open(swirq_dev, os.O_RDONLY, 0666)
	if err != nil {
		fmt.Fprintf(os.Stderr, "open swirq device fail: %v\n", err)
		os.Exit(-1)
	}
	if err = Ioctl(fd, SWIRQ_DISABLE, uintptr(unsafe.Pointer(&irqno))); err != nil {
		fmt.Fprintf(os.Stderr, "%v", err)
		panic("can't set interrupt_0 SWIRQ_DISABLE")
	}
	irqno = 1
	if err = Ioctl(fd, SWIRQ_DISABLE, uintptr(unsafe.Pointer(&irqno))); err != nil {
		fmt.Fprintf(os.Stderr, "%v", err)
		panic("can't set interrupt_1 SWIRQ_DISABLE")
	}
	syscall.Close(fd)
}
<commit_msg>refine WInterrupts<commit_after>package core

import (
	"fmt"
	"os"
	"os/signal"
	"syscall"
	"unsafe"
)

const (
	SWIRQ_START   = 0x201
	SWIRQ_STOP    = 0x202
	SWIRQ_SETPID  = 0x203
	SWIRQ_ENABLE  = 0x204
	SWIRQ_DISABLE = 0x205
)

const swirq_dev = "/dev/swirq"

type SWIrq_Config struct {
	channel   uint8
	mode, pid int
}

type userFunc func()

var user1fn, user2fn userFunc
var c1, c2 chan os.Signal
var b1, b2 chan bool

func init() {
	c1 = make(chan os.Signal, 1)
	c2 = make(chan os.Signal, 1)
	b1 = make(chan bool, 1)
	b2 = make(chan bool, 1)
	signal.Notify(c1, syscall.SIGUSR1)
	signal.Notify(c2, syscall.SIGUSR2)
}

func AttachInterrupt(irqno uint8, fn userFunc, modec int) {
	hwmode := 0
	if irqno < EXTERNAL_NUM_INTERRUPTS && fn != nil {
		switch modec {
		case LOW:
			hwmode = 0x3
		case FALLING:
			hwmode = 0x1
		case RISING:
			hwmode = 0x0
		case CHANGE:
			hwmode = 0x4
		default:
			fmt.Fprintf(os.Stderr, "attachInterrupt error: set interrupt %d to invalid mode\n", irqno)
			return
		}
		switch irqno {
		case 0:
			if user1fn != nil {
				b1 <- true
			}
			user1fn = fn
			go func() {
				for {
					select {
					case <-c1:
						if user1fn != nil {
							user1fn()
						}
					case <-b1:
						user1fn = nil
						return
					}
				}
			}()
		case 1:
			if user2fn != nil {
				b2 <- true
			}
			user2fn = fn
			go func() {
				for {
					select {
					case <-c2:
						if user2fn != nil {
							user2fn()
						}
					case <-b2:
						user2fn = nil
						return
					}
				}
			}()
		}
		pid := os.Getpid()
		irqconfig := SWIrq_Config{irqno, hwmode, pid}
		fd, err := syscall.Open(swirq_dev, os.O_RDONLY, 0666)
		if err != nil {
			fmt.Fprintf(os.Stderr, "open swirq device fail: %v\n", err)
			os.Exit(-1)
		}
		if err = Ioctl(fd, SWIRQ_STOP, uintptr(unsafe.Pointer(&irqno))); err != nil {
			fmt.Fprintf(os.Stderr, "%v", err)
			panic("can't set SWIRQ_STOP")
		}
		if err = Ioctl(fd, SWIRQ_SETPID, uintptr(unsafe.Pointer(&irqconfig))); err != nil {
			fmt.Fprintf(os.Stderr, "%v", err)
			panic("can't set SWIRQ_SETPID")
		}
		if err = Ioctl(fd, SWIRQ_START, uintptr(unsafe.Pointer(&irqno))); err != nil {
			fmt.Fprintf(os.Stderr, "%v", err)
			panic("can't set SWIRQ_START")
		}
		syscall.Close(fd)
	}
}

func DetachInterrupt(irqno uint8) {
	if irqno < EXTERNAL_NUM_INTERRUPTS {
		fd, err := syscall.Open(swirq_dev, os.O_RDONLY, 0666)
		if err != nil {
			fmt.Fprintf(os.Stderr, "open swirq device fail: %v\n", err)
			os.Exit(-1)
		}
		if err = Ioctl(fd, SWIRQ_STOP, uintptr(unsafe.Pointer(&irqno))); err != nil {
			fmt.Fprintf(os.Stderr, "%v", err)
			panic("can't set SWIRQ_STOP")
		}
		syscall.Close(fd)
		switch irqno {
		case 0:
			b1 <- true
		case 1:
			b2 <- true
		}
	}
}

func Interrupts() {
	var irqno uint8 = 0
	fd, err := syscall.Open(swirq_dev, os.O_RDONLY, 0666)
	if err != nil {
		fmt.Fprintf(os.Stderr, "open swirq device fail: %v\n", err)
		os.Exit(-1)
	}
	if err = Ioctl(fd, SWIRQ_ENABLE, uintptr(unsafe.Pointer(&irqno))); err != nil {
		fmt.Fprintf(os.Stderr, "%v", err)
		panic("can't set interrupt_0 SWIRQ_ENABLE")
	}
	irqno = 1
	if err = Ioctl(fd, SWIRQ_ENABLE, uintptr(unsafe.Pointer(&irqno))); err != nil {
		fmt.Fprintf(os.Stderr, "%v", err)
		panic("can't set interrupt_1 SWIRQ_ENABLE")
	}
	syscall.Close(fd)
}

func NoInterrupts() {
	var irqno uint8 = 0
	fd, err := syscall.Open(swirq_dev, os.O_RDONLY, 0666)
	if err != nil {
		fmt.Fprintf(os.Stderr, "open swirq device fail: %v\n", err)
		os.Exit(-1)
	}
	if err = Ioctl(fd, SWIRQ_DISABLE, uintptr(unsafe.Pointer(&irqno))); err != nil {
		fmt.Fprintf(os.Stderr, "%v", err)
		panic("can't set interrupt_0 SWIRQ_DISABLE")
	}
	irqno = 1
	if err = Ioctl(fd, SWIRQ_DISABLE, uintptr(unsafe.Pointer(&irqno))); err != nil {
		fmt.Fprintf(os.Stderr, "%v", err)
		panic("can't set interrupt_1 SWIRQ_DISABLE")
	}
	syscall.Close(fd)
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package norm contains types and functions for normalizing Unicode strings.
package norm

import "unicode/utf8"

// A Form denotes a canonical representation of Unicode code points.
// The Unicode-defined normalization and equivalence forms are:
//
//   NFC   Unicode Normalization Form C
//   NFD   Unicode Normalization Form D
//   NFKC  Unicode Normalization Form KC
//   NFKD  Unicode Normalization Form KD
//
// For a Form f, this documentation uses the notation f(x) to mean
// the bytes or string x converted to the given form.
// A position n in x is called a boundary if conversion to the form can
// proceed independently on both sides:
//   f(x) == append(f(x[0:n]), f(x[n:])...)
//
// References: http://unicode.org/reports/tr15/ and
// http://unicode.org/notes/tn5/.
type Form int

const (
	NFC Form = iota
	NFD
	NFKC
	NFKD
)

// Bytes returns f(b). May return b if f(b) = b.
func (f Form) Bytes(b []byte) []byte {
	rb := reorderBuffer{}
	rb.init(f, b)
	n := quickSpan(&rb, 0)
	if n == len(b) {
		return b
	}
	out := make([]byte, n, len(b))
	copy(out, b[0:n])
	return doAppend(&rb, out, n)
}

// String returns f(s).
func (f Form) String(s string) string {
	rb := reorderBuffer{}
	rb.initString(f, s)
	n := quickSpan(&rb, 0)
	if n == len(s) {
		return s
	}
	out := make([]byte, n, len(s))
	copy(out, s[0:n])
	return string(doAppend(&rb, out, n))
}

// IsNormal returns true if b == f(b).
func (f Form) IsNormal(b []byte) bool {
	rb := reorderBuffer{}
	rb.init(f, b)
	bp := quickSpan(&rb, 0)
	if bp == len(b) {
		return true
	}
	for bp < len(b) {
		decomposeSegment(&rb, bp)
		if rb.f.composing {
			rb.compose()
		}
		for i := 0; i < rb.nrune; i++ {
			info := rb.rune[i]
			if bp+int(info.size) > len(b) {
				return false
			}
			p := info.pos
			pe := p + info.size
			for ; p < pe; p++ {
				if b[bp] != rb.byte[p] {
					return false
				}
				bp++
			}
		}
		rb.reset()
		bp = quickSpan(&rb, bp)
	}
	return true
}

// IsNormalString returns true if s == f(s).
func (f Form) IsNormalString(s string) bool {
	rb := reorderBuffer{}
	rb.initString(f, s)
	bp := quickSpan(&rb, 0)
	if bp == len(s) {
		return true
	}
	for bp < len(s) {
		decomposeSegment(&rb, bp)
		if rb.f.composing {
			rb.compose()
		}
		for i := 0; i < rb.nrune; i++ {
			info := rb.rune[i]
			if bp+int(info.size) > len(s) {
				return false
			}
			p := info.pos
			pe := p + info.size
			for ; p < pe; p++ {
				if s[bp] != rb.byte[p] {
					return false
				}
				bp++
			}
		}
		rb.reset()
		bp = quickSpan(&rb, bp)
	}
	return true
}

// patchTail fixes a case where a rune may be incorrectly normalized
// if it is followed by illegal continuation bytes. It returns the
// patched buffer and whether there were trailing continuation bytes.
func patchTail(rb *reorderBuffer, buf []byte) ([]byte, bool) {
	info, p := lastRuneStart(&rb.f, buf)
	if p == -1 || info.size == 0 {
		return buf, false
	}
	end := p + int(info.size)
	extra := len(buf) - end
	if extra > 0 {
		// Potentially allocating memory. However, this only
		// happens with ill-formed UTF-8.
		x := make([]byte, 0)
		x = append(x, buf[len(buf)-extra:]...)
		buf = decomposeToLastBoundary(rb, buf[:end])
		if rb.f.composing {
			rb.compose()
		}
		buf = rb.flush(buf)
		return append(buf, x...), true
	}
	return buf, false
}

func appendQuick(rb *reorderBuffer, dst []byte, i int) ([]byte, int) {
	if rb.nsrc == i {
		return dst, i
	}
	end := quickSpan(rb, i)
	return rb.src.appendSlice(dst, i, end), end
}

// Append returns f(append(out, b...)).
// The buffer out must be nil, empty, or equal to f(out).
func (f Form) Append(out []byte, src ...byte) []byte {
	if len(src) == 0 {
		return out
	}
	rb := reorderBuffer{}
	rb.init(f, src)
	return doAppend(&rb, out, 0)
}

func doAppend(rb *reorderBuffer, out []byte, p int) []byte {
	src, n := rb.src, rb.nsrc
	doMerge := len(out) > 0
	if q := src.skipNonStarter(p); q > p {
		// Move leading non-starters to destination.
		out = src.appendSlice(out, p, q)
		buf, endsInError := patchTail(rb, out)
		if endsInError {
			out = buf
			doMerge = false // no need to merge, ends with illegal UTF-8
		} else {
			out = decomposeToLastBoundary(rb, buf) // force decomposition
		}
		p = q
	}
	fd := &rb.f
	if doMerge {
		var info Properties
		if p < n {
			info = fd.info(src, p)
			if p == 0 && !info.BoundaryBefore() {
				out = decomposeToLastBoundary(rb, out)
			}
		}
		if info.size == 0 || info.BoundaryBefore() {
			if fd.composing {
				rb.compose()
			}
			out = rb.flush(out)
			if info.size == 0 {
				// Append incomplete UTF-8 encoding.
				return src.appendSlice(out, p, n)
			}
		}
	}
	if rb.nrune == 0 {
		out, p = appendQuick(rb, out, p)
	}
	for p < n {
		p = decomposeSegment(rb, p)
		if fd.composing {
			rb.compose()
		}
		out = rb.flush(out)
		out, p = appendQuick(rb, out, p)
	}
	return out
}

// AppendString returns f(append(out, []byte(s))).
// The buffer out must be nil, empty, or equal to f(out).
func (f Form) AppendString(out []byte, src string) []byte {
	if len(src) == 0 {
		return out
	}
	rb := reorderBuffer{}
	rb.initString(f, src)
	return doAppend(&rb, out, 0)
}

// QuickSpan returns a boundary n such that b[0:n] == f(b[0:n]).
// It is not guaranteed to return the largest such n.
func (f Form) QuickSpan(b []byte) int {
	rb := reorderBuffer{}
	rb.init(f, b)
	n := quickSpan(&rb, 0)
	return n
}

func quickSpan(rb *reorderBuffer, i int) int {
	var lastCC uint8
	var nc int
	lastSegStart := i
	src, n := rb.src, rb.nsrc
	for i < n {
		if j := src.skipASCII(i, n); i != j {
			i = j
			lastSegStart = i - 1
			lastCC = 0
			nc = 0
			continue
		}
		info := rb.f.info(src, i)
		if info.size == 0 {
			// include incomplete runes
			return n
		}
		cc := info.ccc
		if rb.f.composing {
			if !info.isYesC() {
				break
			}
		} else {
			if !info.isYesD() {
				break
			}
		}
		if cc == 0 {
			lastSegStart = i
			nc = 0
		} else {
			if nc >= maxCombiningChars {
				lastSegStart = i
				lastCC = cc
				nc = 1
			} else {
				if lastCC > cc {
					return lastSegStart
				}
				nc++
			}
		}
		lastCC = cc
		i += int(info.size)
	}
	if i == n {
		return n
	}
	if rb.f.composing {
		return lastSegStart
	}
	return i
}

// QuickSpanString returns a boundary n such that b[0:n] == f(s[0:n]).
// It is not guaranteed to return the largest such n.
func (f Form) QuickSpanString(s string) int {
	rb := reorderBuffer{}
	rb.initString(f, s)
	return quickSpan(&rb, 0)
}

// FirstBoundary returns the position i of the first boundary in b
// or -1 if b contains no boundary.
func (f Form) FirstBoundary(b []byte) int {
	rb := reorderBuffer{}
	rb.init(f, b)
	return firstBoundary(&rb)
}

func firstBoundary(rb *reorderBuffer) int {
	src, nsrc := rb.src, rb.nsrc
	i := src.skipNonStarter(0)
	if i >= nsrc {
		return -1
	}
	fd := &rb.f
	info := fd.info(src, i)
	for n := 0; info.size != 0 && !info.BoundaryBefore(); {
		i += int(info.size)
		if n++; n >= maxCombiningChars {
			return i
		}
		if i >= nsrc {
			if !info.BoundaryAfter() {
				return -1
			}
			return nsrc
		}
		info = fd.info(src, i)
	}
	if info.size == 0 {
		return -1
	}
	return i
}

// FirstBoundaryInString returns the position i of the first boundary in s
// or -1 if s contains no boundary.
func (f Form) FirstBoundaryInString(s string) int {
	rb := reorderBuffer{}
	rb.initString(f, s)
	return firstBoundary(&rb)
}

// LastBoundary returns the position i of the last boundary in b
// or -1 if b contains no boundary.
func (f Form) LastBoundary(b []byte) int {
	return lastBoundary(formTable[f], b)
}

func lastBoundary(fd *formInfo, b []byte) int {
	i := len(b)
	info, p := lastRuneStart(fd, b)
	if p == -1 {
		return -1
	}
	if info.size == 0 { // ends with incomplete rune
		if p == 0 { // starts with incomplete rune
			return -1
		}
		i = p
		info, p = lastRuneStart(fd, b[:i])
		if p == -1 { // incomplete UTF-8 encoding or non-starter bytes without a starter
			return i
		}
	}
	if p+int(info.size) != i { // trailing non-starter bytes: illegal UTF-8
		return i
	}
	if info.BoundaryAfter() {
		return i
	}
	i = p
	for n := 0; i >= 0 && !info.BoundaryBefore(); {
		info, p = lastRuneStart(fd, b[:i])
		if n++; n >= maxCombiningChars {
			return len(b)
		}
		if p+int(info.size) != i {
			if p == -1 { // no boundary found
				return -1
			}
			return i // boundary after an illegal UTF-8 encoding
		}
		i = p
	}
	return i
}

// decomposeSegment scans the first segment in src into rb.
// It returns the number of bytes consumed from src.
// TODO(mpvl): consider inserting U+034f (Combining Grapheme Joiner)
// when we detect a sequence of 30+ non-starter chars.
func decomposeSegment(rb *reorderBuffer, sp int) int {
	// Force one character to be consumed.
	info := rb.f.info(rb.src, sp)
	if info.size == 0 {
		return 0
	}
	for rb.insert(rb.src, sp, info) {
		sp += int(info.size)
		if sp >= rb.nsrc {
			break
		}
		info = rb.f.info(rb.src, sp)
		bound := info.BoundaryBefore()
		if bound || info.size == 0 {
			break
		}
	}
	return sp
}

// lastRuneStart returns the runeInfo and position of the last
// rune in buf or the zero runeInfo and -1 if no rune was found.
func lastRuneStart(fd *formInfo, buf []byte) (Properties, int) {
	p := len(buf) - 1
	for ; p >= 0 && !utf8.RuneStart(buf[p]); p-- {
	}
	if p < 0 {
		return Properties{}, -1
	}
	return fd.info(inputBytes(buf), p), p
}

// decomposeToLastBoundary finds an open segment at the end of the buffer
// and scans it into rb. Returns the buffer minus the last segment.
func decomposeToLastBoundary(rb *reorderBuffer, buf []byte) []byte {
	fd := &rb.f
	info, i := lastRuneStart(fd, buf)
	if int(info.size) != len(buf)-i {
		// illegal trailing continuation bytes
		return buf
	}
	if info.BoundaryAfter() {
		return buf
	}
	var add [maxBackRunes]Properties // stores runeInfo in reverse order
	add[0] = info
	padd := 1
	n := 1
	p := len(buf) - int(info.size)
	for ; p >= 0 && !info.BoundaryBefore(); p -= int(info.size) {
		info, i = lastRuneStart(fd, buf[:p])
		if int(info.size) != p-i {
			break
		}
		// Check that decomposition doesn't result in overflow.
		if info.hasDecomposition() {
			if isHangul(buf) {
				i += int(info.size)
				n++
			} else {
				dcomp := info.Decomposition()
				for i := 0; i < len(dcomp); {
					inf := rb.f.info(inputBytes(dcomp), i)
					i += int(inf.size)
					n++
				}
			}
		} else {
			n++
		}
		if n > maxBackRunes {
			break
		}
		add[padd] = info
		padd++
	}
	pp := p
	for padd--; padd >= 0; padd-- {
		info = add[padd]
		rb.insert(inputBytes(buf), pp, info)
		pp += int(info.size)
	}
	return buf[:p]
}
<commit_msg>Norm32 package comment now says it's for unicode 3.2<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package norm32 contains types and functions for normalizing Unicode 3.2 strings.
package norm

import "unicode/utf8"

// A Form denotes a canonical representation of Unicode code points.
// The Unicode-defined normalization and equivalence forms are:
//
//   NFC   Unicode Normalization Form C
//   NFD   Unicode Normalization Form D
//   NFKC  Unicode Normalization Form KC
//   NFKD  Unicode Normalization Form KD
//
// For a Form f, this documentation uses the notation f(x) to mean
// the bytes or string x converted to the given form.
// A position n in x is called a boundary if conversion to the form can
// proceed independently on both sides:
//   f(x) == append(f(x[0:n]), f(x[n:])...)
//
// References: http://unicode.org/reports/tr15/ and
// http://unicode.org/notes/tn5/.
type Form int

const (
	NFC Form = iota
	NFD
	NFKC
	NFKD
)

// Bytes returns f(b). May return b if f(b) = b.
func (f Form) Bytes(b []byte) []byte {
	rb := reorderBuffer{}
	rb.init(f, b)
	n := quickSpan(&rb, 0)
	if n == len(b) {
		return b
	}
	out := make([]byte, n, len(b))
	copy(out, b[0:n])
	return doAppend(&rb, out, n)
}

// String returns f(s).
func (f Form) String(s string) string {
	rb := reorderBuffer{}
	rb.initString(f, s)
	n := quickSpan(&rb, 0)
	if n == len(s) {
		return s
	}
	out := make([]byte, n, len(s))
	copy(out, s[0:n])
	return string(doAppend(&rb, out, n))
}

// IsNormal returns true if b == f(b).
func (f Form) IsNormal(b []byte) bool {
	rb := reorderBuffer{}
	rb.init(f, b)
	bp := quickSpan(&rb, 0)
	if bp == len(b) {
		return true
	}
	for bp < len(b) {
		decomposeSegment(&rb, bp)
		if rb.f.composing {
			rb.compose()
		}
		for i := 0; i < rb.nrune; i++ {
			info := rb.rune[i]
			if bp+int(info.size) > len(b) {
				return false
			}
			p := info.pos
			pe := p + info.size
			for ; p < pe; p++ {
				if b[bp] != rb.byte[p] {
					return false
				}
				bp++
			}
		}
		rb.reset()
		bp = quickSpan(&rb, bp)
	}
	return true
}

// IsNormalString returns true if s == f(s).
func (f Form) IsNormalString(s string) bool {
	rb := reorderBuffer{}
	rb.initString(f, s)
	bp := quickSpan(&rb, 0)
	if bp == len(s) {
		return true
	}
	for bp < len(s) {
		decomposeSegment(&rb, bp)
		if rb.f.composing {
			rb.compose()
		}
		for i := 0; i < rb.nrune; i++ {
			info := rb.rune[i]
			if bp+int(info.size) > len(s) {
				return false
			}
			p := info.pos
			pe := p + info.size
			for ; p < pe; p++ {
				if s[bp] != rb.byte[p] {
					return false
				}
				bp++
			}
		}
		rb.reset()
		bp = quickSpan(&rb, bp)
	}
	return true
}

// patchTail fixes a case where a rune may be incorrectly normalized
// if it is followed by illegal continuation bytes. It returns the
// patched buffer and whether there were trailing continuation bytes.
func patchTail(rb *reorderBuffer, buf []byte) ([]byte, bool) {
	info, p := lastRuneStart(&rb.f, buf)
	if p == -1 || info.size == 0 {
		return buf, false
	}
	end := p + int(info.size)
	extra := len(buf) - end
	if extra > 0 {
		// Potentially allocating memory. However, this only
		// happens with ill-formed UTF-8.
		x := make([]byte, 0)
		x = append(x, buf[len(buf)-extra:]...)
		buf = decomposeToLastBoundary(rb, buf[:end])
		if rb.f.composing {
			rb.compose()
		}
		buf = rb.flush(buf)
		return append(buf, x...), true
	}
	return buf, false
}

func appendQuick(rb *reorderBuffer, dst []byte, i int) ([]byte, int) {
	if rb.nsrc == i {
		return dst, i
	}
	end := quickSpan(rb, i)
	return rb.src.appendSlice(dst, i, end), end
}

// Append returns f(append(out, b...)).
// The buffer out must be nil, empty, or equal to f(out).
func (f Form) Append(out []byte, src ...byte) []byte {
	if len(src) == 0 {
		return out
	}
	rb := reorderBuffer{}
	rb.init(f, src)
	return doAppend(&rb, out, 0)
}

func doAppend(rb *reorderBuffer, out []byte, p int) []byte {
	src, n := rb.src, rb.nsrc
	doMerge := len(out) > 0
	if q := src.skipNonStarter(p); q > p {
		// Move leading non-starters to destination.
		out = src.appendSlice(out, p, q)
		buf, endsInError := patchTail(rb, out)
		if endsInError {
			out = buf
			doMerge = false // no need to merge, ends with illegal UTF-8
		} else {
			out = decomposeToLastBoundary(rb, buf) // force decomposition
		}
		p = q
	}
	fd := &rb.f
	if doMerge {
		var info Properties
		if p < n {
			info = fd.info(src, p)
			if p == 0 && !info.BoundaryBefore() {
				out = decomposeToLastBoundary(rb, out)
			}
		}
		if info.size == 0 || info.BoundaryBefore() {
			if fd.composing {
				rb.compose()
			}
			out = rb.flush(out)
			if info.size == 0 {
				// Append incomplete UTF-8 encoding.
				return src.appendSlice(out, p, n)
			}
		}
	}
	if rb.nrune == 0 {
		out, p = appendQuick(rb, out, p)
	}
	for p < n {
		p = decomposeSegment(rb, p)
		if fd.composing {
			rb.compose()
		}
		out = rb.flush(out)
		out, p = appendQuick(rb, out, p)
	}
	return out
}

// AppendString returns f(append(out, []byte(s))).
// The buffer out must be nil, empty, or equal to f(out).
func (f Form) AppendString(out []byte, src string) []byte {
	if len(src) == 0 {
		return out
	}
	rb := reorderBuffer{}
	rb.initString(f, src)
	return doAppend(&rb, out, 0)
}

// QuickSpan returns a boundary n such that b[0:n] == f(b[0:n]).
// It is not guaranteed to return the largest such n.
func (f Form) QuickSpan(b []byte) int {
	rb := reorderBuffer{}
	rb.init(f, b)
	n := quickSpan(&rb, 0)
	return n
}

func quickSpan(rb *reorderBuffer, i int) int {
	var lastCC uint8
	var nc int
	lastSegStart := i
	src, n := rb.src, rb.nsrc
	for i < n {
		if j := src.skipASCII(i, n); i != j {
			i = j
			lastSegStart = i - 1
			lastCC = 0
			nc = 0
			continue
		}
		info := rb.f.info(src, i)
		if info.size == 0 {
			// include incomplete runes
			return n
		}
		cc := info.ccc
		if rb.f.composing {
			if !info.isYesC() {
				break
			}
		} else {
			if !info.isYesD() {
				break
			}
		}
		if cc == 0 {
			lastSegStart = i
			nc = 0
		} else {
			if nc >= maxCombiningChars {
				lastSegStart = i
				lastCC = cc
				nc = 1
			} else {
				if lastCC > cc {
					return lastSegStart
				}
				nc++
			}
		}
		lastCC = cc
		i += int(info.size)
	}
	if i == n {
		return n
	}
	if rb.f.composing {
		return lastSegStart
	}
	return i
}

// QuickSpanString returns a boundary n such that b[0:n] == f(s[0:n]).
// It is not guaranteed to return the largest such n.
func (f Form) QuickSpanString(s string) int {
	rb := reorderBuffer{}
	rb.initString(f, s)
	return quickSpan(&rb, 0)
}

// FirstBoundary returns the position i of the first boundary in b
// or -1 if b contains no boundary.
func (f Form) FirstBoundary(b []byte) int {
	rb := reorderBuffer{}
	rb.init(f, b)
	return firstBoundary(&rb)
}

func firstBoundary(rb *reorderBuffer) int {
	src, nsrc := rb.src, rb.nsrc
	i := src.skipNonStarter(0)
	if i >= nsrc {
		return -1
	}
	fd := &rb.f
	info := fd.info(src, i)
	for n := 0; info.size != 0 && !info.BoundaryBefore(); {
		i += int(info.size)
		if n++; n >= maxCombiningChars {
			return i
		}
		if i >= nsrc {
			if !info.BoundaryAfter() {
				return -1
			}
			return nsrc
		}
		info = fd.info(src, i)
	}
	if info.size == 0 {
		return -1
	}
	return i
}

// FirstBoundaryInString returns the position i of the first boundary in s
// or -1 if s contains no boundary.
func (f Form) FirstBoundaryInString(s string) int {
	rb := reorderBuffer{}
	rb.initString(f, s)
	return firstBoundary(&rb)
}

// LastBoundary returns the position i of the last boundary in b
// or -1 if b contains no boundary.
func (f Form) LastBoundary(b []byte) int {
	return lastBoundary(formTable[f], b)
}

func lastBoundary(fd *formInfo, b []byte) int {
	i := len(b)
	info, p := lastRuneStart(fd, b)
	if p == -1 {
		return -1
	}
	if info.size == 0 { // ends with incomplete rune
		if p == 0 { // starts with incomplete rune
			return -1
		}
		i = p
		info, p = lastRuneStart(fd, b[:i])
		if p == -1 { // incomplete UTF-8 encoding or non-starter bytes without a starter
			return i
		}
	}
	if p+int(info.size) != i { // trailing non-starter bytes: illegal UTF-8
		return i
	}
	if info.BoundaryAfter() {
		return i
	}
	i = p
	for n := 0; i >= 0 && !info.BoundaryBefore(); {
		info, p = lastRuneStart(fd, b[:i])
		if n++; n >= maxCombiningChars {
			return len(b)
		}
		if p+int(info.size) != i {
			if p == -1 { // no boundary found
				return -1
			}
			return i // boundary after an illegal UTF-8 encoding
		}
		i = p
	}
	return i
}

// decomposeSegment scans the first segment in src into rb.
// It returns the number of bytes consumed from src.
// TODO(mpvl): consider inserting U+034f (Combining Grapheme Joiner)
// when we detect a sequence of 30+ non-starter chars.
func decomposeSegment(rb *reorderBuffer, sp int) int {
	// Force one character to be consumed.
	info := rb.f.info(rb.src, sp)
	if info.size == 0 {
		return 0
	}
	for rb.insert(rb.src, sp, info) {
		sp += int(info.size)
		if sp >= rb.nsrc {
			break
		}
		info = rb.f.info(rb.src, sp)
		bound := info.BoundaryBefore()
		if bound || info.size == 0 {
			break
		}
	}
	return sp
}

// lastRuneStart returns the runeInfo and position of the last
// rune in buf or the zero runeInfo and -1 if no rune was found.
func lastRuneStart(fd *formInfo, buf []byte) (Properties, int) {
	p := len(buf) - 1
	for ; p >= 0 && !utf8.RuneStart(buf[p]); p-- {
	}
	if p < 0 {
		return Properties{}, -1
	}
	return fd.info(inputBytes(buf), p), p
}

// decomposeToLastBoundary finds an open segment at the end of the buffer
// and scans it into rb. Returns the buffer minus the last segment.
func decomposeToLastBoundary(rb *reorderBuffer, buf []byte) []byte {
	fd := &rb.f
	info, i := lastRuneStart(fd, buf)
	if int(info.size) != len(buf)-i {
		// illegal trailing continuation bytes
		return buf
	}
	if info.BoundaryAfter() {
		return buf
	}
	var add [maxBackRunes]Properties // stores runeInfo in reverse order
	add[0] = info
	padd := 1
	n := 1
	p := len(buf) - int(info.size)
	for ; p >= 0 && !info.BoundaryBefore(); p -= int(info.size) {
		info, i = lastRuneStart(fd, buf[:p])
		if int(info.size) != p-i {
			break
		}
		// Check that decomposition doesn't result in overflow.
		if info.hasDecomposition() {
			if isHangul(buf) {
				i += int(info.size)
				n++
			} else {
				dcomp := info.Decomposition()
				for i := 0; i < len(dcomp); {
					inf := rb.f.info(inputBytes(dcomp), i)
					i += int(inf.size)
					n++
				}
			}
		} else {
			n++
		}
		if n > maxBackRunes {
			break
		}
		add[padd] = info
		padd++
	}
	pp := p
	for padd--; padd >= 0; padd-- {
		info = add[padd]
		rb.insert(inputBytes(buf), pp, info)
		pp += int(info.size)
	}
	return buf[:p]
}
<|endoftext|>


<commit_before>package main

import (
	"big"
)

type Point [3]int64
type Vector [3]int64
type Triangle [3]Point
type Line [2]Point
type Cube [2]Point

const MaxJ = 10

func NewVector(p1, p2 Point) Vector {
	return Vector{p2[0] - p1[0], p2[1] - p1[1], p2[2] - p1[2]}
}

func VectorProduct(a, b Vector) Vector {
	return Vector{
		a[1]*b[2] - a[2]*b[1],
		a[2]*b[0] - a[0]*b[2],
		a[0]*b[1] - a[1]*b[0],
	}
}

func ScalarProduct(a, b Vector) (s *big.Int) {
	s = big.NewInt(0)
	for i := 0; i < 3; i++ {
		tmp := big.NewInt(0)
		s.Add(s, tmp.Mul(big.NewInt(a[i]), big.NewInt(b[i])))
	}
	return
}

func DotInPlane(p, a, b, c Point, r int64) bool {
	va := NewVector(c, a)
	vb := NewVector(c, b)
	vc := NewVector(c, p)
	v := VectorProduct(va, vb)

	s := ScalarProduct(vc, v)
	s.Mul(s, s)
	s.Mul(s, big.NewInt(4))

	r2 := big.NewInt(r)
	r2.Mul(r2, r2)

	v2 := ScalarProduct(v, v)

	t := big.NewInt(0)
	t.Mul(r2, v2)

	return t.Cmp(s) >= 0
}

func len2(v Vector) (s *big.Int) {
	s = big.NewInt(0)
	for i := 0; i < 3; i++ {
		tmp := big.NewInt(v[i])
		tmp.Mul(tmp, tmp)
		s.Add(s, tmp)
	}
	return
}

func sameSide(p, a, b, c Point, r int64) bool {
	ab := NewVector(a, b)
	ac := NewVector(a, c)
	ap := NewVector(a, p)
	v1 := VectorProduct(ab, ac)
	v2 := VectorProduct(ab, ap)
	s := ScalarProduct(v1, v2)
	if s.Cmp(big.NewInt(0)) >= 0 {
		return true
	}
	h2 := len2(v2)
	h2.Mul(h2, big.NewInt(4))
	r2 := big.NewInt(r)
	r2.Mul(r2, r2)
	r2.Mul(r2, len2(ab))
	return r2.Cmp(h2) >= 0
}

func inplaneDotInTriangle(p, a, b, c Point, r int64) bool {
	return sameSide(p, a, b, c, r) &&
		sameSide(p, b, c, a, r) &&
		sameSide(p, c, a, b, r)
}

func DotInTriangle(p, a, b, c Point, r int64) bool {
	return DotInPlane(p, a, b, c, r) && inplaneDotInTriangle(p, a, b, c, r)
}

func DotOnLine(p Point, line Line, r int64) bool {
	v1 := NewVector(line[0], line[1])
	v2 := NewVector(line[0], p)
	v3 := VectorProduct(v1, v2)
	l3 := len2(v3)
	l3.Mul(l3, big.NewInt(4))
	r2 := big.NewInt(r)
	r2.Mul(r2, r2)
	r2.Mul(r2, len2(v1))
	return r2.Cmp(l3) >= 0
}

func IntersectLines(l1, l2 Line, scale int64) (p Point, ok bool) {
	panic("IntersectLines is not implemented")
}

func to(a, n int64) int64 {
	if a >= 0 {
		return (a + (n-1)/2) / n
	}
	return -((-a + n/2) / n)
}

func toGrid(p Point, scale int64) Point {
	return Point{to(p[0], scale), to(p[1], scale), to(p[2], scale)}
}

func scalePoint(p Point, scale int64) Point {
	return Point{p[0] * scale, p[1] * scale, p[2] * scale}
}

func findJ(p1, p2 Point, scale int64) (j uint) {
	for j = 0; j < 31; j++ {
		var r2 int64
		for z := 0; z < 3; z++ {
			diff := int64(p1[z] - p2[z])
			r2 += diff * diff
		}
		//		fmt.Fprintf(os.Stderr, "r2: %d, j: %d, scale: %d\n", r2, j, scale)
		if r2 < (int64(scale)*int64(scale))<<(2*j) {
			return j + 2
		}
	}
	panic("unreachable")
}

func peq(p1, p2 Point) bool {
	for z := 0; z < 3; z++ {
		if p1[z] != p2[z] {
			return false
		}
	}
	return true
}

type pointSlice []Point

func (ps pointSlice) Len() int {
	return len(ps)
}

func (ps pointSlice) Less(i, j int) (res bool) {
	//	defer func() {
	//		fmt.Fprintf(os.Stderr, "Less(%d,%d): %v, %v\n", i, j, res, ps)
	//	}()
	for z := 0; z < 3; z++ {
		if ps[i][z] < ps[j][z] {
			return true
		}
		if ps[i][z] > ps[j][z] {
			return false
		}
	}
	return false
}

func (ps pointSlice) Swap(i, j int) {
	//	fmt.Fprintf(os.Stderr, "Swap(%d, %d), before: %v\n", i, j, ps)
	ps[i], ps[j] = ps[j], ps[i]
	//	fmt.Fprintf(os.Stderr, "Swap(%d, %d),  after: %v\n", i, j, ps)
}

func uniq(ps []Point) (res []Point) {
	res = ps[:0]
	for i, p := range ps {
		if i > 0 && peq(ps[i-1], p) {
			continue
		}
		res = append(res, ps[i])
	}
	return
}

func scoreDiff(p1, p2 Point) (res int) {
	for i := 0; i < 3; i++ {
		if p1[i] != p2[i] {
			res++
		}
	}
	return
}

func AddDot(a, b, c Point, scale int64, vol VolumeSetter, i0, i1 int64, j0, j1 uint, last1 Point, color uint16) Point {
	m := j0
	if m < j1 {
		m = j1
	}
	i2 := 1<<m - i0*(1<<(m-j0)) - i1*(1<<(m-j1))
	var p Point
	for z := 0; z < 3; z++ {
		p[z] = int64(i0)*(int64(1)<<uint(m-j0))*a[z] +
			int64(i1)*(int64(1)<<uint(m-j1))*b[z] +
			int64(i2)*c[z]
		p[z] >>= m
	}
	//			fmt.Fprintf(os.Stderr, "AllTriangleDots1, 60, p=%v\n", p)
	p = toGrid(p, scale)
	/*	var last2 Point
		if i1 != 0 {
			if i1 < 0 {
				fmt.Fprintf(os.Stderr, "ogogo! i1 < 0, i1: %d\n", i1)
				panic("aaaa!")
			}
			if scoreDiff(last1, p) > 1 {
				//			fmt.Fprintf(os.Stderr, "So, there is a problem; i1: %d, j1: %d\n", i1, j1)
				var delta uint
				for j1+delta <= MaxJ {
					delta++
					last2 = AddDot(a, b, c, scale, vol, i0, i1*(1<<delta)-1, j0, j1+delta, last1)
					//				fmt.Fprintf(os.Stderr, "last1: %v, p: %v, last2: %v\n", last1, p, last2)
					if !peq(p, last2) {
						break
					}
				}
				//			if j1 > MaxJ {
				//				fmt.Fprintf(os.Stderr, "%d = j1 > MaxJ = %d\n", j1, MaxJ)
				//			}
			}
			//		if peq(p, last2) {
			//			fmt.Fprintf(os.Stderr, "opa! last2 == p\n")
			//		}
		}*/
	vol.Set(int(p[0]), int(p[1]), int(p[2]), color)
	//	if scoreDiff(last1, p) > 1 && scoreDiff(last2, p) > 1 {
	//		fmt.Fprintf(os.Stderr, "Returning bad result. last1: %v, p: %v, last2: %v, i1: %d, j1: %d\n", last1, p, last2, i1, j1)
	//	}
	return p
}

func AllTriangleDots(a, b, c Point, scale int64, vol VolumeSetter, color uint16) {
	//	fmt.Fprintf(os.Stderr, "AllTriangleDots1, 0, a=%v, b=%v, c=%v\n", a, b, c)
	j0 := findJ(a, c, scale)
	//	fmt.Fprintf(os.Stderr, "AllTriangleDots1, 10, j0=%d\n", j0)
	j1 := findJ(a, b, scale)
	//	fmt.Fprintf(os.Stderr, "AllTriangleDots1, 20, j1=%d\n", j1)
	m := j0
	if m < j1 {
		m = j1
	}
	//	fmt.Fprintf(os.Stderr, "AllTriangleDots1, 30, m=%d\n", m)
	for i0 := 0; i0 <= 1<<j0; i0++ {
		var last1 Point
		for i1 := 0; i0*(1<<(m-j0))+i1*(1<<(m-j1)) <= 1<<m; i1++ {
			last1 = AddDot(a, b, c, scale, vol, int64(i0), int64(i1), j0, j1, last1, color)

		}
		//		fmt.Fprintf(os.Stderr, "AllTriangleDots1, 90\n")
	}
	//	fmt.Fprintf(os.Stderr, "AllTriangleDots1, 94, res: %v\n", res)
	//	fmt.Fprintf(os.Stderr, "AllTriangleDots1, 95, res: %v\n", res)
	//	fmt.Fprintf(os.Stderr, "AllTriangleDots1, 100, res: %v\n", res)
}

func checkAlphaInd(num, den int64, a, b, p, q *Point, ind int) bool {
	if den == 0 {
		return false
	}
	if den < 0 {
		num = -num
		den = -den
	}
	if num < 0 || num > den {
		// 0 <= \alpha <= 1
		return false
	}
	left := a[ind]*den + num*(b[ind]-a[ind])
	if left < p[ind]*den {
		return false
	}
	if left > q[ind]*den {
		return false
	}
	return true
}

func checkAlpha(num, den int64, a, b, p, q *Point) bool {
	for i := 0; i < 3; i++ {
		if !checkAlphaInd(num, den, a, b, p, q, i) {
			return false
		}
	}
	return true
}

func getAlphaPoint(num, den int64, a, b *Point) (res Point) {
	for i := 0; i < 3; i++ {
		// This is not the best thing to do, because we divide on the calculated value.
		// This can lead to an unpredictable behavior, but we say "fine" for now.
		res[i] = a[i] + (num*(b[i]-a[i]))/den
	}
	return
}

func det3(v0, v1, v2 Vector) int64 {
	return v0[0]*v1[1]*v2[2] + v0[1]*v1[2]*v2[0] + v0[2]*v1[0]*v2[1] -
		v0[0]*v1[2]*v2[1] - v0[1]*v1[0]*v2[2] - v0[2]*v1[1]*v2[0]
}

func MeshVolume(triangles []Triangle, scale int64) (res int64) {
	for _, t := range triangles {
		res += det3(Vector(t[0]), Vector(t[1]), Vector(t[2]))
	}
	return res / 6
}
<commit_msg>Remove commented diagnostics from AllTrianglePoints<commit_after>package main

import (
	"big"
)

type Point [3]int64
type Vector [3]int64
type Triangle [3]Point
type Line [2]Point
type Cube [2]Point

const MaxJ = 10

func NewVector(p1, p2 Point) Vector {
	return Vector{p2[0] - p1[0], p2[1] - p1[1], p2[2] - p1[2]}
}

func VectorProduct(a, b Vector) Vector {
	return Vector{
		a[1]*b[2] - a[2]*b[1],
		a[2]*b[0] - a[0]*b[2],
		a[0]*b[1] - a[1]*b[0],
	}
}

func ScalarProduct(a, b Vector) (s *big.Int) {
	s = big.NewInt(0)
	for i := 0; i < 3; i++ {
		tmp := big.NewInt(0)
		s.Add(s, tmp.Mul(big.NewInt(a[i]), big.NewInt(b[i])))
	}
	return
}

func DotInPlane(p, a, b, c Point, r int64) bool {
	va := NewVector(c, a)
	vb := NewVector(c, b)
	vc := NewVector(c, p)
	v := VectorProduct(va, vb)

	s := ScalarProduct(vc, v)
	s.Mul(s, s)
	s.Mul(s, big.NewInt(4))

	r2 := big.NewInt(r)
	r2.Mul(r2, r2)

	v2 := ScalarProduct(v, v)

	t := big.NewInt(0)
	t.Mul(r2, v2)

	return t.Cmp(s) >= 0
}

func len2(v Vector) (s *big.Int) {
	s = big.NewInt(0)
	for i := 0; i < 3; i++ {
		tmp := big.NewInt(v[i])
		tmp.Mul(tmp, tmp)
		s.Add(s, tmp)
	}
	return
}

func sameSide(p, a, b, c Point, r int64) bool {
	ab := NewVector(a, b)
	ac := NewVector(a, c)
	ap := NewVector(a, p)
	v1 := VectorProduct(ab, ac)
	v2 := VectorProduct(ab, ap)
	s := ScalarProduct(v1, v2)
	if s.Cmp(big.NewInt(0)) >= 0 {
		return true
	}
	h2 := len2(v2)
	h2.Mul(h2, big.NewInt(4))
	r2 := big.NewInt(r)
	r2.Mul(r2, r2)
	r2.Mul(r2, len2(ab))
	return r2.Cmp(h2) >= 0
}

func inplaneDotInTriangle(p, a, b, c Point, r int64) bool {
	return sameSide(p, a, b, c, r) &&
		sameSide(p, b, c, a, r) &&
		sameSide(p, c, a, b, r)
}

func DotInTriangle(p, a, b, c Point, r int64) bool {
	return DotInPlane(p, a, b, c, r) && inplaneDotInTriangle(p, a, b, c, r)
}

func DotOnLine(p Point, line Line, r int64) bool {
	v1 := NewVector(line[0], line[1])
	v2 := NewVector(line[0], p)
	v3 := VectorProduct(v1, v2)
	l3 := len2(v3)
	l3.Mul(l3, big.NewInt(4))
	r2 := big.NewInt(r)
	r2.Mul(r2, r2)
	r2.Mul(r2, len2(v1))
	return r2.Cmp(l3) >= 0
}

func to(a, n int64) int64 {
	if a >= 0 {
		return (a + (n-1)/2) / n
	}
	return -((-a + n/2) / n)
}

func toGrid(p Point, scale int64) Point {
	return Point{to(p[0], scale), to(p[1], scale), to(p[2], scale)}
}

func scalePoint(p Point, scale int64) Point {
	return Point{p[0] * scale, p[1] * scale, p[2] * scale}
}

func findJ(p1, p2 Point, scale int64) (j uint) {
	for j = 0; j < 31; j++ {
		var r2 int64
		for z := 0; z < 3; z++ {
			diff := int64(p1[z] - p2[z])
			r2 += diff * diff
		}
		if r2 < (int64(scale)*int64(scale))<<(2*j) {
			return j + 2
		}
	}
	panic("unreachable")
}

func peq(p1, p2 Point) bool {
	for z := 0; z < 3; z++ {
		if p1[z] != p2[z] {
			return false
		}
	}
	return true
}

type pointSlice []Point

func (ps pointSlice) Len() int {
	return len(ps)
}

func (ps pointSlice) Less(i, j int) (res bool) {
	for z := 0; z < 3; z++ {
		if ps[i][z] < ps[j][z] {
			return true
		}
		if ps[i][z] > ps[j][z] {
			return false
		}
	}
	return false
}

func (ps pointSlice) Swap(i, j int) {
	ps[i], ps[j] = ps[j], ps[i]
}

func uniq(ps []Point) (res []Point) {
	res = ps[:0]
	for i, p := range ps {
		if i > 0 && peq(ps[i-1], p) {
			continue
		}
		res = append(res, ps[i])
	}
	return
}

func scoreDiff(p1, p2 Point) (res int) {
	for i := 0; i < 3; i++ {
		if p1[i] != p2[i] {
			res++
		}
	}
	return
}

func AddDot(a, b, c Point, scale int64, vol VolumeSetter, i0, i1 int64, j0, j1 uint, last1 Point, color uint16) Point {
	m := j0
	if m < j1 {
		m = j1
	}
	i2 := 1<<m - i0*(1<<(m-j0)) - i1*(1<<(m-j1))
	var p Point
	for z := 0; z < 3; z++ {
		p[z] = int64(i0)*(int64(1)<<uint(m-j0))*a[z] +
			int64(i1)*(int64(1)<<uint(m-j1))*b[z] +
			int64(i2)*c[z]
		p[z] >>= m
	}

	p = toGrid(p, scale)
	vol.Set(int(p[0]), int(p[1]), int(p[2]), color)

	return p
}

func AllTriangleDots(a, b, c Point, scale int64, vol VolumeSetter, color uint16) {
	j0 := findJ(a, c, scale)
	j1 := findJ(a, b, scale)

	m := j0
	if m < j1 {
		m = j1
	}

	for i0 := 0; i0 <= 1<<j0; i0++ {
		var last1 Point
		for i1 := 0; i0*(1<<(m-j0))+i1*(1<<(m-j1)) <= 1<<m; i1++ {
			last1 = AddDot(a, b, c, scale, vol, int64(i0), int64(i1), j0, j1, last1, color)

		}
	}
}

func checkAlphaInd(num, den int64, a, b, p, q *Point, ind int) bool {
	if den == 0 {
		return false
	}
	if den < 0 {
		num = -num
		den = -den
	}
	if num < 0 || num > den {
		// 0 <= \alpha <= 1
		return false
	}
	left := a[ind]*den + num*(b[ind]-a[ind])
	if left < p[ind]*den {
		return false
	}
	if left > q[ind]*den {
		return false
	}
	return true
}

func checkAlpha(num, den int64, a, b, p, q *Point) bool {
	for i := 0; i < 3; i++ {
		if !checkAlphaInd(num, den, a, b, p, q, i) {
			return false
		}
	}
	return true
}

func getAlphaPoint(num, den int64, a, b *Point) (res Point) {
	for i := 0; i < 3; i++ {
		// This is not the best thing to do, because we divide on the calculated value.
		// This can lead to an unpredictable behavior, but we say "fine" for now.
		res[i] = a[i] + (num*(b[i]-a[i]))/den
	}
	return
}

func det3(v0, v1, v2 Vector) int64 {
	return v0[0]*v1[1]*v2[2] + v0[1]*v1[2]*v2[0] + v0[2]*v1[0]*v2[1] -
		v0[0]*v1[2]*v2[1] - v0[1]*v1[0]*v2[2] - v0[2]*v1[1]*v2[0]
}

func MeshVolume(triangles []Triangle, scale int64) (res int64) {
	for _, t := range triangles {
		res += det3(Vector(t[0]), Vector(t[1]), Vector(t[2]))
	}
	return res / 6
}
<|endoftext|>


<commit_before><commit_msg>Added ListObjects which will return a list of objects in a container/directory.<commit_after><|endoftext|>


<commit_before>/*
 *
 * Copyright 2014, Google Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *     * Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above
 * copyright notice, this list of conditions and the following disclaimer
 * in the documentation and/or other materials provided with the
 * distribution.
 *     * Neither the name of Google Inc. nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

/*
Package transport defines and implements message oriented communication channel
to complete various transactions (e.g., an RPC).
*/
package transport // import "google.golang.org/grpc/transport"

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"net"
	"sync"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/metadata"
)

// recvMsg represents the received msg from the transport. All transport
// protocol specific info has been removed.
type recvMsg struct {
	data []byte
	// nil: received some data
	// io.EOF: stream is completed. data is nil.
	// other non-nil error: transport failure. data is nil.
	err error
}

func (recvMsg) isItem() bool {
	return true
}

// All items in an out of a recvBuffer should be the same type.
type item interface {
	isItem() bool
}

// recvBuffer is an unbounded channel of item.
type recvBuffer struct {
	c       chan item
	mu      sync.Mutex
	backlog []item
}

func newRecvBuffer() *recvBuffer {
	b := &recvBuffer{
		c: make(chan item, 1),
	}
	return b
}

func (b *recvBuffer) put(r item) {
	b.mu.Lock()
	defer b.mu.Unlock()
	b.backlog = append(b.backlog, r)
	select {
	case b.c <- b.backlog[0]:
		b.backlog = b.backlog[1:]
	default:
	}
}

func (b *recvBuffer) load() {
	b.mu.Lock()
	defer b.mu.Unlock()
	if len(b.backlog) > 0 {
		select {
		case b.c <- b.backlog[0]:
			b.backlog = b.backlog[1:]
		default:
		}
	}
}

// get returns the channel that receives an item in the buffer.
//
// Upon receipt of an item, the caller should call load to send another
// item onto the channel if there is any.
func (b *recvBuffer) get() <-chan item {
	return b.c
}

// recvBufferReader implements io.Reader interface to read the data from
// recvBuffer.
type recvBufferReader struct {
	ctx  context.Context
	recv *recvBuffer
	last *bytes.Reader // Stores the remaining data in the previous calls.
	err  error
}

// Read reads the next len(p) bytes from last. If last is drained, it tries to
// read additional data from recv. It blocks if there no additional data available
// in recv. If Read returns any non-nil error, it will continue to return that error.
func (r *recvBufferReader) Read(p []byte) (n int, err error) {
	if r.err != nil {
		return 0, r.err
	}
	defer func() { r.err = err }()
	if r.last != nil && r.last.Len() > 0 {
		// Read remaining data left in last call.
		return r.last.Read(p)
	}
	select {
	case <-r.ctx.Done():
		return 0, ContextErr(r.ctx.Err())
	case i := <-r.recv.get():
		r.recv.load()
		m := i.(*recvMsg)
		if m.err != nil {
			return 0, m.err
		}
		r.last = bytes.NewReader(m.data)
		return r.last.Read(p)
	}
}

type streamState uint8

const (
	streamActive    streamState = iota
	streamWriteDone             // EndStream sent
	streamReadDone              // EndStream received
	streamDone                  // sendDone and recvDone or RSTStreamFrame is sent or received.
)

// Stream represents an RPC in the transport layer.
type Stream struct {
	id uint32
	// nil for client side Stream.
	st ServerTransport
	// ctx is the associated context of the stream.
	ctx    context.Context
	cancel context.CancelFunc
	// method records the associated RPC method of the stream.
	method string
	buf    *recvBuffer
	dec    io.Reader

	// updateStreams indicates whether the transport's streamsQuota needed
	// to be updated when this stream is closed. It is false when the transport
	// sticks to the initial infinite value of the number of concurrent streams.
	// Ture otherwise.
	updateStreams bool
	fc            *inFlow
	recvQuota     uint32
	// The accumulated inbound quota pending for window update.
	updateQuota uint32
	// The handler to control the window update procedure for both this
	// particular stream and the associated transport.
	windowHandler func(int)

	sendQuotaPool *quotaPool
	// Close headerChan to indicate the end of reception of header metadata.
	headerChan chan struct{}
	// header caches the received header metadata.
	header metadata.MD
	// The key-value map of trailer metadata.
	trailer metadata.MD

	mu sync.RWMutex // guard the following
	// headerOK becomes true from the first header is about to send.
	headerOk bool
	state    streamState
	// true iff headerChan is closed. Used to avoid closing headerChan
	// multiple times.
	headerDone bool
	// the status received from the server.
	statusCode codes.Code
	statusDesc string
}

// Header acquires the key-value pairs of header metadata once it
// is available. It blocks until i) the metadata is ready or ii) there is no
// header metadata or iii) the stream is cancelled/expired.
func (s *Stream) Header() (metadata.MD, error) {
	select {
	case <-s.ctx.Done():
		return nil, ContextErr(s.ctx.Err())
	case <-s.headerChan:
		return s.header.Copy(), nil
	}
}

// Trailer returns the cached trailer metedata. Note that if it is not called
// after the entire stream is done, it could return an empty MD. Client
// side only.
func (s *Stream) Trailer() metadata.MD {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.trailer.Copy()
}

// ServerTransport returns the underlying ServerTransport for the stream.
// The client side stream always returns nil.
func (s *Stream) ServerTransport() ServerTransport {
	return s.st
}

// Context returns the context of the stream.
func (s *Stream) Context() context.Context {
	return s.ctx
}

// Method returns the method for the stream.
func (s *Stream) Method() string {
	return s.method
}

// StatusCode returns statusCode received from the server.
func (s *Stream) StatusCode() codes.Code {
	return s.statusCode
}

// StatusDesc returns statusDesc received from the server.
func (s *Stream) StatusDesc() string {
	return s.statusDesc
}

// ErrIllegalTrailerSet indicates that the trailer has already been set or it
// is too late to do so.
var ErrIllegalTrailerSet = errors.New("transport: trailer has been set")

// SetTrailer sets the trailer metadata which will be sent with the RPC status
// by the server. This can only be called at most once. Server side only.
func (s *Stream) SetTrailer(md metadata.MD) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.trailer != nil {
		return ErrIllegalTrailerSet
	}
	s.trailer = md.Copy()
	return nil
}

func (s *Stream) write(m recvMsg) {
	s.buf.put(&m)
}

// Read reads all the data available for this Stream from the transport and
// passes them into the decoder, which converts them into a gRPC message stream.
// The error is io.EOF when the stream is done or another non-nil error if
// the stream broke.
func (s *Stream) Read(p []byte) (n int, err error) {
	n, err = s.dec.Read(p)
	if err != nil {
		return
	}
	s.windowHandler(n)
	return
}

type key int

// The key to save transport.Stream in the context.
const streamKey = key(0)

// newContextWithStream creates a new context from ctx and attaches stream
// to it.
func newContextWithStream(ctx context.Context, stream *Stream) context.Context {
	return context.WithValue(ctx, streamKey, stream)
}

// StreamFromContext returns the stream saved in ctx.
func StreamFromContext(ctx context.Context) (s *Stream, ok bool) {
	s, ok = ctx.Value(streamKey).(*Stream)
	return
}

// state of transport
type transportState int

const (
	reachable transportState = iota
	unreachable
	closing
)

// NewServerTransport creates a ServerTransport with conn or non-nil error
// if it fails.
func NewServerTransport(protocol string, conn net.Conn, maxStreams uint32) (ServerTransport, error) {
	return newHTTP2Server(conn, maxStreams)
}

// ConnectOptions covers all relevant options for dialing a server.
type ConnectOptions struct {
	// UserAgent is the application user agent.
	UserAgent string
	// Dialer specifies how to dial a network address.
	Dialer func(string, time.Duration) (net.Conn, error)
	// AuthOptions stores the credentials required to setup a client connection and issue RPCs.
	AuthOptions []credentials.Credentials
	// Timeout specifies the timeout for dialing a client connection.
	Timeout time.Duration
}

// NewClientTransport establishes the transport with the required ConnectOptions
// and returns it to the caller.
func NewClientTransport(target string, opts *ConnectOptions) (ClientTransport, error) {
	return newHTTP2Client(target, opts)
}

// Options provides additional hints and information for message
// transmission.
type Options struct {
	// Indicate whether it is the last piece for this stream.
	Last bool
	// The hint to transport impl whether the data could be buffered for
	// batching write. Transport impl can feel free to ignore it.
	Delay bool
}

// CallHdr carries the information of a particular RPC.
type CallHdr struct {
	Host   string // peer host
	Method string // the operation to perform on the specified host
}

// ClientTransport is the common interface for all gRPC client side transport
// implementations.
type ClientTransport interface {
	// Close tears down this transport. Once it returns, the transport
	// should not be accessed any more. The caller must make sure this
	// is called only once.
	Close() error

	// Write sends the data for the given stream. A nil stream indicates
	// the write is to be performed on the transport as a whole.
	Write(s *Stream, data []byte, opts *Options) error

	// NewStream creates a Stream for an RPC.
	NewStream(ctx context.Context, callHdr *CallHdr) (*Stream, error)

	// CloseStream clears the footprint of a stream when the stream is
	// not needed any more. The err indicates the error incurred when
	// CloseStream is called. Must be called when a stream is finished
	// unless the associated transport is closing.
	CloseStream(stream *Stream, err error)

	// Error returns a channel that is closed when some I/O error
	// happens. Typically the caller should have a goroutine to monitor
	// this in order to take action (e.g., close the current transport
	// and create a new one) in error case. It should not return nil
	// once the transport is initiated.
	Error() <-chan struct{}
}

// ServerTransport is the common interface for all gRPC server side transport
// implementations.
type ServerTransport interface {
	// WriteStatus sends the status of a stream to the client.
	WriteStatus(s *Stream, statusCode codes.Code, statusDesc string) error
	// Write sends the data for the given stream.
	Write(s *Stream, data []byte, opts *Options) error
	// WriteHeader sends the header metedata for the given stream.
	WriteHeader(s *Stream, md metadata.MD) error
	// HandleStreams receives incoming streams using the given handler.
	HandleStreams(func(*Stream))
	// Close tears down the transport. Once it is called, the transport
	// should not be accessed any more. All the pending streams and their
	// handlers will be terminated asynchronously.
	Close() error
}

// StreamErrorf creates an StreamError with the specified error code and description.
func StreamErrorf(c codes.Code, format string, a ...interface{}) StreamError {
	return StreamError{
		Code: c,
		Desc: fmt.Sprintf(format, a...),
	}
}

// ConnectionErrorf creates an ConnectionError with the specified error description.
func ConnectionErrorf(format string, a ...interface{}) ConnectionError {
	return ConnectionError{
		Desc: fmt.Sprintf(format, a...),
	}
}

// ConnectionError is an error that results in the termination of the
// entire connection and the retry of all the active streams.
type ConnectionError struct {
	Desc string
}

func (e ConnectionError) Error() string {
	return fmt.Sprintf("connection error: desc = %q", e.Desc)
}

// Define some common ConnectionErrors.
var ErrConnClosing = ConnectionError{Desc: "transport is closing"}

// StreamError is an error that only affects one stream within a connection.
type StreamError struct {
	Code codes.Code
	Desc string
}

func (e StreamError) Error() string {
	return fmt.Sprintf("stream error: code = %d desc = %q", e.Code, e.Desc)
}

// ContextErr converts the error from context package into a StreamError.
func ContextErr(err error) StreamError {
	switch err {
	case context.DeadlineExceeded:
		return StreamErrorf(codes.DeadlineExceeded, "%v", err)
	case context.Canceled:
		return StreamErrorf(codes.Canceled, "%v", err)
	}
	panic(fmt.Sprintf("Unexpected error from context packet: %v", err))
}

// wait blocks until it can receive from ctx.Done, closing, or proceed.
// If it receives from ctx.Done, it returns 0, the StreamError for ctx.Err.
// If it receives from closing, it returns 0, ErrConnClosing.
// If it receives from proceed, it returns the received integer, nil.
func wait(ctx context.Context, closing <-chan struct{}, proceed <-chan int) (int, error) {
	select {
	case <-ctx.Done():
		return 0, ContextErr(ctx.Err())
	case <-closing:
		return 0, ErrConnClosing
	case i := <-proceed:
		return i, nil
	}
}
<commit_msg>fix a comment<commit_after>/*
 *
 * Copyright 2014, Google Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *     * Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above
 * copyright notice, this list of conditions and the following disclaimer
 * in the documentation and/or other materials provided with the
 * distribution.
 *     * Neither the name of Google Inc. nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

/*
Package transport defines and implements message oriented communication channel
to complete various transactions (e.g., an RPC).
*/
package transport // import "google.golang.org/grpc/transport"

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"net"
	"sync"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/metadata"
)

// recvMsg represents the received msg from the transport. All transport
// protocol specific info has been removed.
type recvMsg struct {
	data []byte
	// nil: received some data
	// io.EOF: stream is completed. data is nil.
	// other non-nil error: transport failure. data is nil.
	err error
}

func (recvMsg) isItem() bool {
	return true
}

// All items in an out of a recvBuffer should be the same type.
type item interface {
	isItem() bool
}

// recvBuffer is an unbounded channel of item.
type recvBuffer struct {
	c       chan item
	mu      sync.Mutex
	backlog []item
}

func newRecvBuffer() *recvBuffer {
	b := &recvBuffer{
		c: make(chan item, 1),
	}
	return b
}

func (b *recvBuffer) put(r item) {
	b.mu.Lock()
	defer b.mu.Unlock()
	b.backlog = append(b.backlog, r)
	select {
	case b.c <- b.backlog[0]:
		b.backlog = b.backlog[1:]
	default:
	}
}

func (b *recvBuffer) load() {
	b.mu.Lock()
	defer b.mu.Unlock()
	if len(b.backlog) > 0 {
		select {
		case b.c <- b.backlog[0]:
			b.backlog = b.backlog[1:]
		default:
		}
	}
}

// get returns the channel that receives an item in the buffer.
//
// Upon receipt of an item, the caller should call load to send another
// item onto the channel if there is any.
func (b *recvBuffer) get() <-chan item {
	return b.c
}

// recvBufferReader implements io.Reader interface to read the data from
// recvBuffer.
type recvBufferReader struct {
	ctx  context.Context
	recv *recvBuffer
	last *bytes.Reader // Stores the remaining data in the previous calls.
	err  error
}

// Read reads the next len(p) bytes from last. If last is drained, it tries to
// read additional data from recv. It blocks if there no additional data available
// in recv. If Read returns any non-nil error, it will continue to return that error.
func (r *recvBufferReader) Read(p []byte) (n int, err error) {
	if r.err != nil {
		return 0, r.err
	}
	defer func() { r.err = err }()
	if r.last != nil && r.last.Len() > 0 {
		// Read remaining data left in last call.
		return r.last.Read(p)
	}
	select {
	case <-r.ctx.Done():
		return 0, ContextErr(r.ctx.Err())
	case i := <-r.recv.get():
		r.recv.load()
		m := i.(*recvMsg)
		if m.err != nil {
			return 0, m.err
		}
		r.last = bytes.NewReader(m.data)
		return r.last.Read(p)
	}
}

type streamState uint8

const (
	streamActive    streamState = iota
	streamWriteDone             // EndStream sent
	streamReadDone              // EndStream received
	streamDone                  // sendDone and recvDone or RSTStreamFrame is sent or received.
)

// Stream represents an RPC in the transport layer.
type Stream struct {
	id uint32
	// nil for client side Stream.
	st ServerTransport
	// ctx is the associated context of the stream.
	ctx    context.Context
	cancel context.CancelFunc
	// method records the associated RPC method of the stream.
	method string
	buf    *recvBuffer
	dec    io.Reader

	// updateStreams indicates whether the transport's streamsQuota needed
	// to be updated when this stream is closed. It is false when the transport
	// sticks to the initial infinite value of the number of concurrent streams.
	// Ture otherwise.
	updateStreams bool
	fc            *inFlow
	recvQuota     uint32
	// The accumulated inbound quota pending for window update.
	updateQuota uint32
	// The handler to control the window update procedure for both this
	// particular stream and the associated transport.
	windowHandler func(int)

	sendQuotaPool *quotaPool
	// Close headerChan to indicate the end of reception of header metadata.
	headerChan chan struct{}
	// header caches the received header metadata.
	header metadata.MD
	// The key-value map of trailer metadata.
	trailer metadata.MD

	mu sync.RWMutex // guard the following
	// headerOK becomes true from the first header is about to send.
	headerOk bool
	state    streamState
	// true iff headerChan is closed. Used to avoid closing headerChan
	// multiple times.
	headerDone bool
	// the status received from the server.
	statusCode codes.Code
	statusDesc string
}

// Header acquires the key-value pairs of header metadata once it
// is available. It blocks until i) the metadata is ready or ii) there is no
// header metadata or iii) the stream is cancelled/expired.
func (s *Stream) Header() (metadata.MD, error) {
	select {
	case <-s.ctx.Done():
		return nil, ContextErr(s.ctx.Err())
	case <-s.headerChan:
		return s.header.Copy(), nil
	}
}

// Trailer returns the cached trailer metedata. Note that if it is not called
// after the entire stream is done, it could return an empty MD. Client
// side only.
func (s *Stream) Trailer() metadata.MD {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.trailer.Copy()
}

// ServerTransport returns the underlying ServerTransport for the stream.
// The client side stream always returns nil.
func (s *Stream) ServerTransport() ServerTransport {
	return s.st
}

// Context returns the context of the stream.
func (s *Stream) Context() context.Context {
	return s.ctx
}

// Method returns the method for the stream.
func (s *Stream) Method() string {
	return s.method
}

// StatusCode returns statusCode received from the server.
func (s *Stream) StatusCode() codes.Code {
	return s.statusCode
}

// StatusDesc returns statusDesc received from the server.
func (s *Stream) StatusDesc() string {
	return s.statusDesc
}

// ErrIllegalTrailerSet indicates that the trailer has already been set or it
// is too late to do so.
var ErrIllegalTrailerSet = errors.New("transport: trailer has been set")

// SetTrailer sets the trailer metadata which will be sent with the RPC status
// by the server. This can only be called at most once. Server side only.
func (s *Stream) SetTrailer(md metadata.MD) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.trailer != nil {
		return ErrIllegalTrailerSet
	}
	s.trailer = md.Copy()
	return nil
}

func (s *Stream) write(m recvMsg) {
	s.buf.put(&m)
}

// Read reads all the data available for this Stream from the transport and
// passes them into the decoder, which converts them into a gRPC message stream.
// The error is io.EOF when the stream is done or another non-nil error if
// the stream broke.
func (s *Stream) Read(p []byte) (n int, err error) {
	n, err = s.dec.Read(p)
	if err != nil {
		return
	}
	s.windowHandler(n)
	return
}

type key int

// The key to save transport.Stream in the context.
const streamKey = key(0)

// newContextWithStream creates a new context from ctx and attaches stream
// to it.
func newContextWithStream(ctx context.Context, stream *Stream) context.Context {
	return context.WithValue(ctx, streamKey, stream)
}

// StreamFromContext returns the stream saved in ctx.
func StreamFromContext(ctx context.Context) (s *Stream, ok bool) {
	s, ok = ctx.Value(streamKey).(*Stream)
	return
}

// state of transport
type transportState int

const (
	reachable transportState = iota
	unreachable
	closing
)

// NewServerTransport creates a ServerTransport with conn or non-nil error
// if it fails.
func NewServerTransport(protocol string, conn net.Conn, maxStreams uint32) (ServerTransport, error) {
	return newHTTP2Server(conn, maxStreams)
}

// ConnectOptions covers all relevant options for dialing a server.
type ConnectOptions struct {
	// UserAgent is the application user agent.
	UserAgent string
	// Dialer specifies how to dial a network address.
	Dialer func(string, time.Duration) (net.Conn, error)
	// AuthOptions stores the credentials required to setup a client connection and/or issue RPCs.
	AuthOptions []credentials.Credentials
	// Timeout specifies the timeout for dialing a client connection.
	Timeout time.Duration
}

// NewClientTransport establishes the transport with the required ConnectOptions
// and returns it to the caller.
func NewClientTransport(target string, opts *ConnectOptions) (ClientTransport, error) {
	return newHTTP2Client(target, opts)
}

// Options provides additional hints and information for message
// transmission.
type Options struct {
	// Indicate whether it is the last piece for this stream.
	Last bool
	// The hint to transport impl whether the data could be buffered for
	// batching write. Transport impl can feel free to ignore it.
	Delay bool
}

// CallHdr carries the information of a particular RPC.
type CallHdr struct {
	Host   string // peer host
	Method string // the operation to perform on the specified host
}

// ClientTransport is the common interface for all gRPC client side transport
// implementations.
type ClientTransport interface {
	// Close tears down this transport. Once it returns, the transport
	// should not be accessed any more. The caller must make sure this
	// is called only once.
	Close() error

	// Write sends the data for the given stream. A nil stream indicates
	// the write is to be performed on the transport as a whole.
	Write(s *Stream, data []byte, opts *Options) error

	// NewStream creates a Stream for an RPC.
	NewStream(ctx context.Context, callHdr *CallHdr) (*Stream, error)

	// CloseStream clears the footprint of a stream when the stream is
	// not needed any more. The err indicates the error incurred when
	// CloseStream is called. Must be called when a stream is finished
	// unless the associated transport is closing.
	CloseStream(stream *Stream, err error)

	// Error returns a channel that is closed when some I/O error
	// happens. Typically the caller should have a goroutine to monitor
	// this in order to take action (e.g., close the current transport
	// and create a new one) in error case. It should not return nil
	// once the transport is initiated.
	Error() <-chan struct{}
}

// ServerTransport is the common interface for all gRPC server side transport
// implementations.
type ServerTransport interface {
	// WriteStatus sends the status of a stream to the client.
	WriteStatus(s *Stream, statusCode codes.Code, statusDesc string) error
	// Write sends the data for the given stream.
	Write(s *Stream, data []byte, opts *Options) error
	// WriteHeader sends the header metedata for the given stream.
	WriteHeader(s *Stream, md metadata.MD) error
	// HandleStreams receives incoming streams using the given handler.
	HandleStreams(func(*Stream))
	// Close tears down the transport. Once it is called, the transport
	// should not be accessed any more. All the pending streams and their
	// handlers will be terminated asynchronously.
	Close() error
}

// StreamErrorf creates an StreamError with the specified error code and description.
func StreamErrorf(c codes.Code, format string, a ...interface{}) StreamError {
	return StreamError{
		Code: c,
		Desc: fmt.Sprintf(format, a...),
	}
}

// ConnectionErrorf creates an ConnectionError with the specified error description.
func ConnectionErrorf(format string, a ...interface{}) ConnectionError {
	return ConnectionError{
		Desc: fmt.Sprintf(format, a...),
	}
}

// ConnectionError is an error that results in the termination of the
// entire connection and the retry of all the active streams.
type ConnectionError struct {
	Desc string
}

func (e ConnectionError) Error() string {
	return fmt.Sprintf("connection error: desc = %q", e.Desc)
}

// Define some common ConnectionErrors.
var ErrConnClosing = ConnectionError{Desc: "transport is closing"}

// StreamError is an error that only affects one stream within a connection.
type StreamError struct {
	Code codes.Code
	Desc string
}

func (e StreamError) Error() string {
	return fmt.Sprintf("stream error: code = %d desc = %q", e.Code, e.Desc)
}

// ContextErr converts the error from context package into a StreamError.
func ContextErr(err error) StreamError {
	switch err {
	case context.DeadlineExceeded:
		return StreamErrorf(codes.DeadlineExceeded, "%v", err)
	case context.Canceled:
		return StreamErrorf(codes.Canceled, "%v", err)
	}
	panic(fmt.Sprintf("Unexpected error from context packet: %v", err))
}

// wait blocks until it can receive from ctx.Done, closing, or proceed.
// If it receives from ctx.Done, it returns 0, the StreamError for ctx.Err.
// If it receives from closing, it returns 0, ErrConnClosing.
// If it receives from proceed, it returns the received integer, nil.
func wait(ctx context.Context, closing <-chan struct{}, proceed <-chan int) (int, error) {
	select {
	case <-ctx.Done():
		return 0, ContextErr(ctx.Err())
	case <-closing:
		return 0, ErrConnClosing
	case i := <-proceed:
		return i, nil
	}
}
<|endoftext|>


<commit_before>package registry

import (
	"encoding/json"
	"fmt"
	"path"

	log "github.com/coreos/fleet/Godeps/_workspace/src/github.com/golang/glog"

	"github.com/coreos/fleet/etcd"
	"github.com/coreos/fleet/unit"
)

const (
	unitPrefix = "/unit/"
	// Legacy versions of fleet stored payloads instead of units
	payloadPrefix = "/payload/"
)

func (r *EtcdRegistry) storeOrGetUnit(u unit.Unit) (err error) {
	json, err := marshal(u)
	if err != nil {
		return err
	}

	req := etcd.Create{
		Key:   r.hashedUnitPath(u.Hash()),
		Value: json,
	}
	_, err = r.etcd.Do(&req)
	// unit is already stored
	if err != nil && isNodeExist(err) {
		// TODO(jonboulle): verify more here?
		err = nil
	}
	return
}

// getUnitFromLegacyPayload tries to extract a Unit from a legacy JobPayload of the given name
func (r *EtcdRegistry) getUnitFromLegacyPayload(name string) (*unit.Unit, error) {
	req := etcd.Get{
		Key:       path.Join(r.keyPrefix, payloadPrefix, name),
		Recursive: true,
		Sorted:    true,
	}
	resp, err := r.etcd.Do(&req)
	if err != nil {
		if isKeyNotFound(err) {
			err = nil
		}
		return nil, err
	}

	var ljp LegacyJobPayload
	if err := unmarshal(resp.Node.Value, &ljp); err != nil {
		return nil, fmt.Errorf("error unmarshaling LegacyJobPayload(%s): %v", name, err)
	}
	if ljp.Name != name {
		return nil, fmt.Errorf("payload name in Registry (%s) does not match expected name (%s)", ljp.Name, name)
	}
	// After the unmarshaling, the LegacyPayload should contain a fully hydrated Unit
	return &ljp.Unit, nil
}

// getUnitByHash retrieves from the Registry the Unit associated with the given Hash
func (r *EtcdRegistry) getUnitByHash(hash unit.Hash) *unit.Unit {
	req := etcd.Get{
		Key:       r.hashedUnitPath(hash),
		Recursive: true,
	}
	resp, err := r.etcd.Do(&req)
	if err != nil {
		if isKeyNotFound(err) {
			err = nil
		}
		return nil
	}
	var u unit.Unit
	if err := unmarshal(resp.Node.Value, &u); err != nil {
		log.Errorf("error unmarshaling Unit(%s): %v", hash, err)
		return nil
	}
	return &u
}

func (r *EtcdRegistry) hashedUnitPath(hash unit.Hash) string {
	return path.Join(r.keyPrefix, unitPrefix, hash.String())
}

// LegacyJobPayload deals with the legacy concept of a "JobPayload" (deprecated by Units).
// The associated marshaling/unmarshaling methods deal with Payloads encoded in this legacy format.
type LegacyJobPayload struct {
	Name string
	Unit unit.Unit
}

func (ljp *LegacyJobPayload) UnmarshalJSON(data []byte) error {
	var ljpm legacyJobPayloadModel
	err := json.Unmarshal(data, &ljpm)
	if err != nil {
		return fmt.Errorf("unable to JSON-deserialize object: %s", err)
	}

	var u *unit.Unit
	if len(ljpm.Unit.Raw) > 0 {
		u, err = unit.NewUnit(ljpm.Unit.Raw)
	} else {
		u, err = unit.NewUnitFromLegacyContents(ljpm.Unit.Contents)
	}
	if err != nil {
		return err
	}

	ljp.Unit = *u
	ljp.Name = ljpm.Name

	return nil
}

// legacyJobPayloadModel is an abstraction to deal with serialized LegacyJobPayloads
type legacyJobPayloadModel struct {
	Name string
	Unit unitFileModel
}

// unitFileModel is an abstraction to deal with serialized LegacyJobPayloads
type unitFileModel struct {
	Contents map[string]map[string]string
	Raw      string
}
<commit_msg>registry: use unitModel for Unit serialization<commit_after>package registry

import (
	"encoding/json"
	"fmt"
	"path"

	log "github.com/coreos/fleet/Godeps/_workspace/src/github.com/golang/glog"

	"github.com/coreos/fleet/etcd"
	"github.com/coreos/fleet/unit"
)

const (
	unitPrefix = "/unit/"
	// Legacy versions of fleet stored payloads instead of units
	payloadPrefix = "/payload/"
)

func (r *EtcdRegistry) storeOrGetUnit(u unit.Unit) (err error) {
	um := unitModel{
		Raw: u.String(),
	}

	json, err := marshal(um)
	if err != nil {
		return err
	}

	req := etcd.Create{
		Key:   r.hashedUnitPath(u.Hash()),
		Value: json,
	}
	_, err = r.etcd.Do(&req)
	// unit is already stored
	if err != nil && isNodeExist(err) {
		// TODO(jonboulle): verify more here?
		err = nil
	}
	return
}

// getUnitFromLegacyPayload tries to extract a Unit from a legacy JobPayload of the given name
func (r *EtcdRegistry) getUnitFromLegacyPayload(name string) (*unit.Unit, error) {
	req := etcd.Get{
		Key:       path.Join(r.keyPrefix, payloadPrefix, name),
		Recursive: true,
		Sorted:    true,
	}
	resp, err := r.etcd.Do(&req)
	if err != nil {
		if isKeyNotFound(err) {
			err = nil
		}
		return nil, err
	}

	var ljp LegacyJobPayload
	if err := unmarshal(resp.Node.Value, &ljp); err != nil {
		return nil, fmt.Errorf("error unmarshaling LegacyJobPayload(%s): %v", name, err)
	}
	if ljp.Name != name {
		return nil, fmt.Errorf("payload name in Registry (%s) does not match expected name (%s)", ljp.Name, name)
	}
	// After the unmarshaling, the LegacyPayload should contain a fully hydrated Unit
	return &ljp.Unit, nil
}

// getUnitByHash retrieves from the Registry the Unit associated with the given Hash
func (r *EtcdRegistry) getUnitByHash(hash unit.Hash) *unit.Unit {
	req := etcd.Get{
		Key:       r.hashedUnitPath(hash),
		Recursive: true,
	}
	resp, err := r.etcd.Do(&req)
	if err != nil {
		if isKeyNotFound(err) {
			err = nil
		}
		return nil
	}
	var um unitModel
	if err := unmarshal(resp.Node.Value, &um); err != nil {
		log.Errorf("error unmarshaling Unit(%s): %v", hash, err)
		return nil
	}

	u, err := unit.NewUnit(um.Raw)
	if err != nil {
		log.Errorf("error parsing Unit(%s): %v", hash, err)
		return nil
	}

	return u
}

func (r *EtcdRegistry) hashedUnitPath(hash unit.Hash) string {
	return path.Join(r.keyPrefix, unitPrefix, hash.String())
}

// LegacyJobPayload deals with the legacy concept of a "JobPayload" (deprecated by Units).
// The associated marshaling/unmarshaling methods deal with Payloads encoded in this legacy format.
type LegacyJobPayload struct {
	Name string
	Unit unit.Unit
}

func (ljp *LegacyJobPayload) UnmarshalJSON(data []byte) error {
	var ljpm legacyJobPayloadModel
	err := json.Unmarshal(data, &ljpm)
	if err != nil {
		return fmt.Errorf("unable to JSON-deserialize object: %s", err)
	}

	var u *unit.Unit
	if len(ljpm.Unit.Raw) > 0 {
		u, err = unit.NewUnit(ljpm.Unit.Raw)
	} else {
		u, err = unit.NewUnitFromLegacyContents(ljpm.Unit.Contents)
	}
	if err != nil {
		return err
	}

	ljp.Unit = *u
	ljp.Name = ljpm.Name

	return nil
}

// legacyJobPayloadModel is an abstraction to deal with serialized LegacyJobPayloads
type legacyJobPayloadModel struct {
	Name string
	Unit struct {
		Contents map[string]map[string]string
		Raw      string
	}
}

type unitModel struct {
	Raw string
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"image"
	"image/color"
	"image/gif"
	"image/png"
	"io"
	"os"
	"path/filepath"

	"golang.org/x/image/draw"
)

var (
	width       = flag.Int("width", 506, "target min width")
	height      = flag.Int("height", 0, "target min height")
	repeat      = flag.Int("repeat", 3, "repeat count")
	transparent = flag.Bool("transparent", false, "transparent background")
	duplicate   = flag.Bool("duplicate", false, "use duplication instead of repeating animation")
)

func handleGif(infile io.Reader, outfile io.Writer) error {
	source, err := gif.DecodeAll(infile)
	if err != nil {
		return fmt.Errorf("failed to decode gif: %v", err)
	}

	size := image.Pt(source.Config.Width, source.Config.Height)
	if size.X < *width {
		size.X = *width
	}
	if size.Y < *height {
		size.Y = *height
	}

	var target gif.GIF
	target.Config = source.Config
	target.Config.Width = size.X
	target.Config.Height = size.Y

	target.LoopCount = source.LoopCount
	target.BackgroundIndex = source.BackgroundIndex

	offset := image.Pt(
		size.X/2-source.Config.Width/2,
		size.Y/2-source.Config.Height/2,
	)

	for i, m := range source.Image {
		d := image.NewPaletted(image.Rectangle{image.ZP, size}, m.Palette)
		if !*transparent {
			for k := range d.Pix {
				d.Pix[k] = m.Pix[0]
			}
		}
		draw.Draw(d, m.Bounds().Add(offset), m, image.ZP, draw.Src)

		if *duplicate {
			target.Image = append(target.Image, d)
			target.Delay = append(target.Delay, source.Delay[i]/2)
			if len(source.Disposal) > 0 {
				target.Disposal = append(target.Disposal, source.Disposal[i])
			}
			if i != len(source.Image)-1 {
				target.Image = append(target.Image, d)
				target.Delay = append(target.Delay, source.Delay[i]/2)
				if len(source.Disposal) > 0 {
					target.Disposal = append(target.Disposal, source.Disposal[i])
				}
			}
		} else {
			target.Image = append(target.Image, d)
			target.Delay = append(target.Delay, source.Delay[i])
			if len(source.Disposal) > 0 {
				target.Disposal = append(target.Disposal, source.Disposal[i])
			}
		}
	}

	if !*duplicate {
		n := len(source.Image)
		for k := 1; k < *repeat; k++ {
			for i := 0; i < n; i++ {
				target.Image = append(target.Image, target.Image[i])
				target.Delay = append(target.Delay, target.Delay[i])
				if len(target.Disposal) > 0 {
					target.Disposal = append(target.Disposal, target.Disposal[i])
				}
			}
		}
	}

	err = gif.EncodeAll(outfile, &target)
	if err != nil {
		return fmt.Errorf("failed to encode to gif: %v", err)
	}

	return nil
}

func handlePng(infile io.Reader, outfile io.Writer) error {
	source, err := png.Decode(infile)
	if err != nil {
		return fmt.Errorf("failed to decode png: %v", err)
	}

	size := source.Bounds().Size()
	if size.X < *width {
		size.X = *width
	}
	if size.Y < *height {
		size.Y = *height
	}

	offset := image.Pt(
		size.X/2-source.Bounds().Dx()/2,
		size.Y/2-source.Bounds().Dy()/2,
	)

	target := image.NewRGBA(image.Rectangle{image.ZP, size})
	if !*transparent {
		draw.Draw(target, target.Bounds(), &image.Uniform{color.White}, image.ZP, draw.Src)
	}
	draw.Draw(target, source.Bounds().Add(offset), source, image.ZP, draw.Src)

	err = png.Encode(outfile, target)
	if err != nil {
		return fmt.Errorf("failed to encode to png: %v", err)
	}

	return nil
}
func main() {
	flag.Parse()

	if flag.Arg(0) == "" || flag.Arg(1) == "" {
		flag.Usage()
		os.Exit(1)
	}

	infile, err := os.Open(flag.Arg(0))
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}
	defer infile.Close()

	outfile, err := os.Create(flag.Arg(1))
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}
	defer outfile.Close()

	switch filepath.Ext(flag.Arg(0)) {
	case ".gif":
		err := handleGif(infile, outfile)
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed twitterifying gif: %v\n", err)
			os.Exit(1)
		}
	case ".png":
		err := handlePng(infile, outfile)
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed twitterifying png: %v\n", err)
			os.Exit(1)
		}
	}
}
<commit_msg>Add duration overriding.<commit_after>package main

import (
	"flag"
	"fmt"
	"image"
	"image/color"
	"image/gif"
	"image/png"
	"io"
	"os"
	"path/filepath"

	"golang.org/x/image/draw"
)

var (
	width       = flag.Int("width", 506, "target min width")
	height      = flag.Int("height", 0, "target min height")
	repeat      = flag.Int("repeat", 3, "repeat count")
	transparent = flag.Bool("transparent", false, "transparent background")
	duplicate   = flag.Bool("duplicate", false, "use duplication instead of repeating animation")
	duration    = flag.Int("duration", 0, "override frame duration")
)

func handleGif(infile io.Reader, outfile io.Writer) error {
	// TODO: fix handling of different disposals

	source, err := gif.DecodeAll(infile)
	if err != nil {
		return fmt.Errorf("failed to decode gif: %v", err)
	}

	size := image.Pt(source.Config.Width, source.Config.Height)
	if size.X < *width {
		size.X = *width
	}
	if size.Y < *height {
		size.Y = *height
	}

	var target gif.GIF
	target.Config = source.Config
	target.Config.Width = size.X
	target.Config.Height = size.Y

	target.LoopCount = source.LoopCount
	target.BackgroundIndex = source.BackgroundIndex

	offset := image.Pt(
		size.X/2-source.Config.Width/2,
		size.Y/2-source.Config.Height/2,
	)

	for i, m := range source.Image {
		d := image.NewPaletted(image.Rectangle{image.ZP, size}, m.Palette)
		if !*transparent {
			for k := range d.Pix {
				d.Pix[k] = m.Pix[0]
			}
		}
		draw.Draw(d, m.Bounds().Add(offset), m, image.ZP, draw.Src)

		delay := source.Delay[i]
		if *duration > 0 {
			delay = *duration
		}

		if *duplicate {
			target.Image = append(target.Image, d)
			target.Delay = append(target.Delay, delay/2)
			if len(source.Disposal) > 0 {
				target.Disposal = append(target.Disposal, source.Disposal[i])
			}
			if i != len(source.Image)-1 {
				target.Image = append(target.Image, d)
				target.Delay = append(target.Delay, delay/2)
				if len(source.Disposal) > 0 {
					target.Disposal = append(target.Disposal, source.Disposal[i])
				}
			}
		} else {
			target.Image = append(target.Image, d)
			target.Delay = append(target.Delay, delay)
			if len(source.Disposal) > 0 {
				target.Disposal = append(target.Disposal, source.Disposal[i])
			}
		}
	}

	if !*duplicate {
		n := len(source.Image)
		for k := 1; k < *repeat; k++ {
			for i := 0; i < n; i++ {
				target.Image = append(target.Image, target.Image[i])
				target.Delay = append(target.Delay, target.Delay[i])
				if len(target.Disposal) > 0 {
					target.Disposal = append(target.Disposal, target.Disposal[i])
				}
			}
		}
	}

	err = gif.EncodeAll(outfile, &target)
	if err != nil {
		return fmt.Errorf("failed to encode to gif: %v", err)
	}

	return nil
}

func handlePng(infile io.Reader, outfile io.Writer) error {
	source, err := png.Decode(infile)
	if err != nil {
		return fmt.Errorf("failed to decode png: %v", err)
	}

	size := source.Bounds().Size()
	if size.X < *width {
		size.X = *width
	}
	if size.Y < *height {
		size.Y = *height
	}

	offset := image.Pt(
		size.X/2-source.Bounds().Dx()/2,
		size.Y/2-source.Bounds().Dy()/2,
	)

	target := image.NewRGBA(image.Rectangle{image.ZP, size})
	if !*transparent {
		draw.Draw(target, target.Bounds(), &image.Uniform{color.White}, image.ZP, draw.Src)
	}
	draw.Draw(target, source.Bounds().Add(offset), source, image.ZP, draw.Src)

	err = png.Encode(outfile, target)
	if err != nil {
		return fmt.Errorf("failed to encode to png: %v", err)
	}

	return nil
}
func main() {
	flag.Parse()

	if flag.Arg(0) == "" || flag.Arg(1) == "" {
		flag.Usage()
		os.Exit(1)
	}

	infile, err := os.Open(flag.Arg(0))
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}
	defer infile.Close()

	outfile, err := os.Create(flag.Arg(1))
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}
	defer outfile.Close()

	switch filepath.Ext(flag.Arg(0)) {
	case ".gif":
		err := handleGif(infile, outfile)
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed twitterifying gif: %v\n", err)
			os.Exit(1)
		}
	case ".png":
		err := handlePng(infile, outfile)
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed twitterifying png: %v\n", err)
			os.Exit(1)
		}
	}
}
<|endoftext|>