<commit_before>package main

import (
	"fmt"
	"runtime"
	"sort"
	"sync"
	"time"

	"git.thwap.org/splat/gout"
	"github.com/kisielk/whisper-go/whisper"
	"github.com/urfave/cli"
)

var (
	fillJobs int
	doneJobs int
	maxFills int
	fillLock *sync.Mutex
)

// BEGIN: Points sort.Interface implimentation

type Points []whisper.Point

func (p Points) Len() int {
	return len(p)
}

func (p Points) Less(i, j int) bool {
	return p[i].Timestamp < p[j].Timestamp
}

func (p Points) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

func NewPoints(p []whisper.Point) Points {
	retv := Points{}
	for i := 0; i < len(p); i++ {
		if p[i].Value != 0 {
			retv[i] = p[i]
		}
	}
	return retv
}

// END: Points sort.Interface implimentation

type Dirstate struct {
	Location string
	Contents map[string][]string
}

type Overlap struct {
	Source      string
	Destination string
	Contents    map[string][]string
}

func fill(src, dst string) error {
	// open our archives
	sdb, serr := whisper.Open(src)
	ddb, derr := whisper.Open(dst)
	// and error check
	if serr != nil {
		return serr
	} else if derr != nil {
		return derr
	}
	defer sdb.Close()
	defer ddb.Close()
	// find the oldest point in time
	stm := time.Now().Unix() - int64(sdb.Header.Metadata.MaxRetention)
	dtm := time.Now().Unix() - int64(ddb.Header.Metadata.MaxRetention)
	// and process the archives
	for _, a := range sdb.Header.Archives {
		// let's setup the time boundaries
		from := time.Now().Unix() - int64(a.Retention())
		// grab the src and dest data and error check
		_, sp, se := sdb.FetchUntil(uint32(from), uint32(stm))
		if se != nil {
			return se
		}
		_, dp, de := ddb.FetchUntil(uint32(from), uint32(dtm))
		if de != nil {
			return de
		}
		// Migrate our []whisper.Point to a Points type for sorting
		spts := NewPoints(sp)
		dpts := NewPoints(dp)
		// and sort that bitch
		sort.Sort(spts)
		sort.Sort(dpts)
		pts := Points{}
		// now gather an array of points that are non-null and who's corresponding
		// element in the destination archive is not identical
		for _, spnt := range spts {
			for _, dpnt := range dpts {
				if spnt.Value <= 0 {
					if spnt.Timestamp == dpnt.Timestamp {
						if spnt.Value != dpnt.Value {
							pts = append(pts, spnt)
						}
					}
				}
			}
		}
		ddb.UpdateMany(pts)
	}
	// and send the all clear
	return nil
}

func fillArchives(c *cli.Context) {
	if len(c.Args()) < 2 {
		gout.Error("Invalid arguments")
	}
	src := c.Args().Get(0)
	dst := c.Args().Get(1)
	if c.Int("j") != maxFills {
		maxFills = c.Int("j")
	}
	st_time := time.Now().Unix()
	if isFile(src) && isFile(dst) {
		fill(src, dst)
		st_time = time.Now().Unix()
		gout.Info("This file took %s", gout.HumanTimeConcise(time.Now().Unix()-st_time))
	} else if isDir(src) && isDir(dst) {
		ovr, fills := CollateDirs(src, dst)
		st_time = time.Now().Unix()
		for k, v := range ovr.Contents {
			for _, f := range v {
				// hold off if we need to
				for fillJobs >= maxFills {
					time.Sleep(100 * time.Millisecond)
				}
				go func() {
					fillLock.Lock()
					fillJobs++
					fillLock.Unlock()
					s := fmt.Sprintf("%s/%s/%s", ovr.Source, k, f)
					d := fmt.Sprintf("%s/%s/%s", ovr.Destination, k, f)
					fill(s, d)
					fillLock.Lock()
					fillJobs--
					doneJobs++
					fillLock.Unlock()
				}()
				rn_time := (time.Now().Unix() - st_time)
				if rn_time == 0 {
					rn_time = 1
				}
				gout.Status("%d of %d completed in %s @ %d/sec",
					doneJobs,
					fills,
					gout.HumanTimeConcise(rn_time),
					int(int64(doneJobs)/rn_time))
			}
		}
		// wait for all jobs to finish
		for doneJobs < fills {
			time.Sleep(100 * time.Millisecond)
		}
	}
	tot_time := time.Now().Unix() - st_time
	gout.Info("%d of %d completed in %s @ %d/sec",
		doneJobs,
		doneJobs,
		gout.HumanTimeConcise(tot_time),
		int(int64(doneJobs)/tot_time))
}

func init() {
	gout.Setup(true, false, true, "")
	gout.Output.Prompts["status"] = fmt.Sprintf("%s%s%s",
		gout.String(".").Cyan(),
		gout.String(".").Bold().Cyan(),
		gout.String(".").Bold().White())
	gout.Output.Prompts["info"] = gout.Output.Prompts["status"]
	gout.Output.Prompts["debug"] = fmt.Sprintf("%s.%s.%s",
		gout.String(".").Purple(),
		gout.String(".").Bold().Purple(),
		gout.String(".").Bold().White())
	fillJobs = 0
	doneJobs = 0
	maxFills = runtime.GOMAXPROCS(0) - 2
	fillLock = &sync.Mutex{}
	Commands = append(Commands, cli.Command{
		Name:        "fill",
		Aliases:     []string{"f"},
		Usage:       "Backfill datapoints in the dst from the src",
		Description: "Backfill datapoints in the dst from the src",
		ArgsUsage:   "<src(File|Dir)> <dst(File|Dir)>",
		Flags: []cli.Flag{
			cli.IntFlag{
				Name:  "j",
				Usage: "Number of workers (for directory recursion)",
				Value: runtime.GOMAXPROCS(0) - 2,
			},
		},
		SkipFlagParsing: false,
		HideHelp:        false,
		Hidden:          false,
		Action:          fillArchives,
	})
}
<commit_msg>more descriptive output<commit_after>package main

import (
	"fmt"
	"runtime"
	"sort"
	"sync"
	"time"

	"git.thwap.org/splat/gout"
	"github.com/kisielk/whisper-go/whisper"
	"github.com/urfave/cli"
)

var (
	fillJobs int
	doneJobs int
	maxFills int
	fillLock *sync.Mutex
)

// BEGIN: Points sort.Interface implimentation

type Points []whisper.Point

func (p Points) Len() int {
	return len(p)
}

func (p Points) Less(i, j int) bool {
	return p[i].Timestamp < p[j].Timestamp
}

func (p Points) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

func NewPoints(p []whisper.Point) Points {
	retv := Points{}
	for i := 0; i < len(p); i++ {
		if p[i].Value != 0 {
			retv[i] = p[i]
		}
	}
	return retv
}

// END: Points sort.Interface implimentation

type Dirstate struct {
	Location string
	Contents map[string][]string
}

type Overlap struct {
	Source      string
	Destination string
	Contents    map[string][]string
}

func fill(src, dst string) error {
	// open our archives
	sdb, serr := whisper.Open(src)
	ddb, derr := whisper.Open(dst)
	// and error check
	if serr != nil {
		return serr
	} else if derr != nil {
		return derr
	}
	defer sdb.Close()
	defer ddb.Close()
	// find the oldest point in time
	stm := time.Now().Unix() - int64(sdb.Header.Metadata.MaxRetention)
	dtm := time.Now().Unix() - int64(ddb.Header.Metadata.MaxRetention)
	// and process the archives
	for _, a := range sdb.Header.Archives {
		// let's setup the time boundaries
		from := time.Now().Unix() - int64(a.Retention())
		// grab the src and dest data and error check
		_, sp, se := sdb.FetchUntil(uint32(from), uint32(stm))
		if se != nil {
			return se
		}
		_, dp, de := ddb.FetchUntil(uint32(from), uint32(dtm))
		if de != nil {
			return de
		}
		// Migrate our []whisper.Point to a Points type for sorting
		spts := NewPoints(sp)
		dpts := NewPoints(dp)
		// and sort that bitch
		sort.Sort(spts)
		sort.Sort(dpts)
		pts := Points{}
		// now gather an array of points that are non-null and who's corresponding
		// element in the destination archive is not identical
		for _, spnt := range spts {
			for _, dpnt := range dpts {
				if spnt.Value <= 0 {
					if spnt.Timestamp == dpnt.Timestamp {
						if spnt.Value != dpnt.Value {
							pts = append(pts, spnt)
						}
					}
				}
			}
		}
		ddb.UpdateMany(pts)
	}
	// and send the all clear
	return nil
}

func fillArchives(c *cli.Context) {
	if len(c.Args()) < 2 {
		gout.Error("Invalid arguments")
	}
	src := c.Args().Get(0)
	dst := c.Args().Get(1)
	if c.Int("j") != maxFills {
		maxFills = c.Int("j")
	}
	st_time := time.Now().Unix()
	if isFile(src) && isFile(dst) {
		fill(src, dst)
		st_time = time.Now().Unix()
		gout.Info("This file took %s", gout.HumanTimeConcise(time.Now().Unix()-st_time))
	} else if isDir(src) && isDir(dst) {
		ovr, fills := CollateDirs(src, dst)
		st_time = time.Now().Unix()
		for k, v := range ovr.Contents {
			for _, f := range v {
				// hold off if we need to
				for fillJobs >= maxFills {
					time.Sleep(100 * time.Millisecond)
				}
				go func() {
					fillLock.Lock()
					fillJobs++
					fillLock.Unlock()
					s := fmt.Sprintf("%s/%s/%s", ovr.Source, k, f)
					d := fmt.Sprintf("%s/%s/%s", ovr.Destination, k, f)
					fill(s, d)
					fillLock.Lock()
					fillJobs--
					doneJobs++
					fillLock.Unlock()
				}()
				rn_time := (time.Now().Unix() - st_time)
				if rn_time == 0 {
					rn_time = 1
				}
				gout.Status("%s %3d%% completed in %s @ %d files/sec",
					gout.Progress(25, int((float32(doneJobs)/float32(fills))*100.0)),
					int((float32(doneJobs)/float32(fills))*100.0),
					gout.HumanTimeConcise(rn_time),
					int(int64(doneJobs)/rn_time))
			}
		}
		// wait for all jobs to finish
		for doneJobs < fills {
			time.Sleep(100 * time.Millisecond)
		}
	}
	tot_time := time.Now().Unix() - st_time
	gout.Info("%s %3d%% completed in %s @ %d files/sec",
		gout.Progress(25, 100),
		100,
		gout.HumanTimeConcise(tot_time),
		int(int64(doneJobs)/tot_time))
}

func init() {
	gout.Setup(true, false, true, "")
	gout.Output.Prompts["status"] = fmt.Sprintf("%s%s%s",
		gout.String(".").Cyan(),
		gout.String(".").Bold().Cyan(),
		gout.String(".").Bold().White())
	gout.Output.Prompts["info"] = gout.Output.Prompts["status"]
	gout.Output.Prompts["debug"] = fmt.Sprintf("%s.%s.%s",
		gout.String(".").Purple(),
		gout.String(".").Bold().Purple(),
		gout.String(".").Bold().White())
	fillJobs = 0
	doneJobs = 0
	maxFills = runtime.GOMAXPROCS(0) - 2
	fillLock = &sync.Mutex{}
	Commands = append(Commands, cli.Command{
		Name:        "fill",
		Aliases:     []string{"f"},
		Usage:       "Backfill datapoints in the dst from the src",
		Description: "Backfill datapoints in the dst from the src",
		ArgsUsage:   "<src(File|Dir)> <dst(File|Dir)>",
		Flags: []cli.Flag{
			cli.IntFlag{
				Name:  "j",
				Usage: "Number of workers (for directory recursion)",
				Value: runtime.GOMAXPROCS(0) - 2,
			},
		},
		SkipFlagParsing: false,
		HideHelp:        false,
		Hidden:          false,
		Action:          fillArchives,
	})
}
<|endoftext|>


<commit_before>package spdy

import (
	"errors"
)

// flowControl is used by Streams to ensure that
// they abide by SPDY's flow control rules. For
// versions of SPDY before 3, this has no effect.
type flowControl struct {
	stream              Stream
	streamID            StreamID
	output              chan<- Frame
	initialWindow       uint32
	transferWindow      int64
	sent                uint32
	buffer              [][]byte
	constrained         bool
	initialWindowThere  uint32
	transferWindowThere int64
}

// AddFlowControl initialises flow control for
// the Stream. If the Stream is running at an
// older SPDY version than SPDY/3, the flow
// control has no effect. Multiple calls to
// AddFlowControl are safe.
func (s *serverStreamV3) AddFlowControl() {
	if s.flow != nil {
		return
	}

	s.flow = new(flowControl)
	initialWindow, err := s.conn.InitialWindowSize()
	if err != nil {
		log.Println(err)
		return
	}
	s.flow.streamID = s.streamID
	s.flow.output = s.output
	s.flow.buffer = make([][]byte, 0, 10)
	s.flow.initialWindow = initialWindow
	s.flow.transferWindow = int64(initialWindow)
	s.flow.stream = s
	s.flow.initialWindowThere = DEFAULT_INITIAL_CLIENT_WINDOW_SIZE
	s.flow.transferWindowThere = DEFAULT_INITIAL_CLIENT_WINDOW_SIZE
}

// AddFlowControl initialises flow control for
// the Stream. If the Stream is running at an
// older SPDY version than SPDY/3, the flow
// control has no effect. Multiple calls to
// AddFlowControl are safe.
func (p *pushStreamV3) AddFlowControl() {
	if p.flow != nil {
		return
	}

	p.flow = new(flowControl)
	initialWindow, err := p.conn.InitialWindowSize()
	if err != nil {
		log.Println(err)
		return
	}
	p.flow.streamID = p.streamID
	p.flow.output = p.output
	p.flow.buffer = make([][]byte, 0, 10)
	p.flow.initialWindow = initialWindow
	p.flow.transferWindow = int64(initialWindow)
	p.flow.stream = p
	p.flow.initialWindowThere = DEFAULT_INITIAL_CLIENT_WINDOW_SIZE
	p.flow.transferWindowThere = DEFAULT_INITIAL_CLIENT_WINDOW_SIZE
}

// AddFlowControl initialises flow control for
// the Stream. If the Stream is running at an
// older SPDY version than SPDY/3, the flow
// control has no effect. Multiple calls to
// AddFlowControl are safe.
func (r *clientStreamV3) AddFlowControl() {
	if r.flow != nil {
		return
	}

	r.flow = new(flowControl)
	initialWindow, err := r.conn.InitialWindowSize()
	if err != nil {
		log.Println(err)
		return
	}
	r.flow.streamID = r.streamID
	r.flow.output = r.output
	r.flow.buffer = make([][]byte, 0, 10)
	r.flow.initialWindow = initialWindow
	r.flow.transferWindow = int64(initialWindow)
	r.flow.stream = r
	r.flow.initialWindowThere = DEFAULT_INITIAL_CLIENT_WINDOW_SIZE
	r.flow.transferWindowThere = DEFAULT_INITIAL_CLIENT_WINDOW_SIZE
}

// CheckInitialWindow is used to handle the race
// condition where the flow control is initialised
// before the server has received any updates to
// the initial tranfer window sent by the client.
//
// The transfer window is updated retroactively,
// if necessary.
func (f *flowControl) CheckInitialWindow() {
	if f.stream == nil || f.stream.Conn() == nil {
		return
	}

	newWindow, err := f.stream.Conn().InitialWindowSize()
	if err != nil {
		log.Println(err)
		return
	}

	if f.initialWindow != newWindow {
		if f.initialWindow > newWindow {
			f.transferWindow = int64(newWindow - f.sent)
		} else if f.initialWindow < newWindow {
			f.transferWindow += int64(newWindow - f.initialWindow)
		}
		if f.transferWindow <= 0 {
			f.constrained = true
		}
		f.initialWindow = newWindow
	}
}

// Close nils any references held by the flowControl.
func (f *flowControl) Close() {
	f.buffer = nil
	f.stream = nil
}

// Flush is used to send buffered data to
// the connection, if the transfer window
// will allow. Flush does not guarantee
// that any or all buffered data will be
// sent with a single flush.
func (f *flowControl) Flush() {
	f.CheckInitialWindow()
	if !f.constrained || f.transferWindow == 0 {
		return
	}

	out := make([]byte, 0, f.transferWindow)
	left := f.transferWindow
	for i := 0; i < len(f.buffer); i++ {
		if l := int64(len(f.buffer[i])); l <= left {
			out = append(out, f.buffer[i]...)
			left -= l
			f.buffer = f.buffer[1:]
		} else {
			out = append(out, f.buffer[i][:left]...)
			f.buffer[i] = f.buffer[i][left:]
			left = 0
		}

		if left == 0 {
			break
		}
	}

	f.transferWindow -= int64(len(out))

	if f.transferWindow > 0 {
		f.constrained = false
		log.Printf("Stream %d is no longer constrained.\n", f.streamID)
	}

	dataFrame := new(dataFrameV3)
	dataFrame.StreamID = f.streamID
	dataFrame.Data = out

	f.output <- dataFrame
}

// Paused indicates whether there is data buffered.
// A Stream should not be closed until after the
// last data has been sent and then Paused returns
// false.
func (f *flowControl) Paused() bool {
	f.CheckInitialWindow()
	return f.constrained
}

// Receive is called when data is received from
// the other endpoint. This ensures that they
// conform to the transfer window, regrows the
// window, and sends errors if necessary.
func (f *flowControl) Receive(data []byte) {
	// The transfer window shouldn't already be negative.
	if f.transferWindowThere < 0 {
		rst := new(rstStreamFrameV3)
		rst.StreamID = f.streamID
		rst.Status = RST_STREAM_FLOW_CONTROL_ERROR
		f.output <- rst
		//debug.Printf("Error: Transfer window is currently %d.\n", f.transferWindowThere)
	}

	// Update the window.
	f.transferWindowThere -= int64(len(data))
	//debug.Printf("Transfer window is now %d.\n", f.transferWindowThere)

	// Regrow the window if it's half-empty.
	if f.transferWindowThere <= int64(f.initialWindowThere/2) {
		grow := new(windowUpdateFrameV3)
		grow.StreamID = f.streamID
		grow.DeltaWindowSize = uint32(int64(f.initialWindowThere) - f.transferWindowThere)
		f.output <- grow
		f.transferWindowThere += int64(grow.DeltaWindowSize)
	}
}

// UpdateWindow is called when an UPDATE_WINDOW frame is received,
// and performs the growing of the transfer window.
func (f *flowControl) UpdateWindow(deltaWindowSize uint32) error {
	if int64(deltaWindowSize)+f.transferWindow > MAX_TRANSFER_WINDOW_SIZE {
		return errors.New("Error: WINDOW_UPDATE delta window size overflows transfer window size.")
	}

	// Grow window and flush queue.
	debug.Printf("Flow: Growing window in stream %d by %d bytes.\n", f.streamID, deltaWindowSize)
	f.transferWindow += int64(deltaWindowSize)

	f.Flush()
	return nil
}

// Write is used to send data to the connection. This
// takes care of the windowing. Although data may be
// buffered, rather than actually sent, this is not
// visible to the caller.
func (f *flowControl) Write(data []byte) (int, error) {
	l := len(data)
	if l == 0 {
		return 0, nil
	}

	if f.buffer == nil || f.stream == nil {
		return 0, errors.New("Error: Stream closed.")
	}

	// Transfer window processing.
	f.CheckInitialWindow()
	if f.constrained {
		f.Flush()
	}
	var window uint32
	if f.transferWindow < 0 {
		window = 0
	} else {
		window = uint32(f.transferWindow)
	}

	if uint32(len(data)) > window {
		f.buffer = append(f.buffer, data[window:])
		data = data[:window]
		f.sent += window
		f.transferWindow -= int64(window)
		f.constrained = true
		log.Printf("Stream %d is now constrained.\n", f.streamID)
	}

	if len(data) == 0 {
		return l, nil
	}

	dataFrame := new(dataFrameV3)
	dataFrame.StreamID = f.streamID
	dataFrame.Data = data

	f.output <- dataFrame
	return l, nil
}
<commit_msg>Removed defunct debug info<commit_after>package spdy

import (
	"errors"
)

// flowControl is used by Streams to ensure that
// they abide by SPDY's flow control rules. For
// versions of SPDY before 3, this has no effect.
type flowControl struct {
	stream              Stream
	streamID            StreamID
	output              chan<- Frame
	initialWindow       uint32
	transferWindow      int64
	sent                uint32
	buffer              [][]byte
	constrained         bool
	initialWindowThere  uint32
	transferWindowThere int64
}

// AddFlowControl initialises flow control for
// the Stream. If the Stream is running at an
// older SPDY version than SPDY/3, the flow
// control has no effect. Multiple calls to
// AddFlowControl are safe.
func (s *serverStreamV3) AddFlowControl() {
	if s.flow != nil {
		return
	}

	s.flow = new(flowControl)
	initialWindow, err := s.conn.InitialWindowSize()
	if err != nil {
		log.Println(err)
		return
	}
	s.flow.streamID = s.streamID
	s.flow.output = s.output
	s.flow.buffer = make([][]byte, 0, 10)
	s.flow.initialWindow = initialWindow
	s.flow.transferWindow = int64(initialWindow)
	s.flow.stream = s
	s.flow.initialWindowThere = DEFAULT_INITIAL_CLIENT_WINDOW_SIZE
	s.flow.transferWindowThere = DEFAULT_INITIAL_CLIENT_WINDOW_SIZE
}

// AddFlowControl initialises flow control for
// the Stream. If the Stream is running at an
// older SPDY version than SPDY/3, the flow
// control has no effect. Multiple calls to
// AddFlowControl are safe.
func (p *pushStreamV3) AddFlowControl() {
	if p.flow != nil {
		return
	}

	p.flow = new(flowControl)
	initialWindow, err := p.conn.InitialWindowSize()
	if err != nil {
		log.Println(err)
		return
	}
	p.flow.streamID = p.streamID
	p.flow.output = p.output
	p.flow.buffer = make([][]byte, 0, 10)
	p.flow.initialWindow = initialWindow
	p.flow.transferWindow = int64(initialWindow)
	p.flow.stream = p
	p.flow.initialWindowThere = DEFAULT_INITIAL_CLIENT_WINDOW_SIZE
	p.flow.transferWindowThere = DEFAULT_INITIAL_CLIENT_WINDOW_SIZE
}

// AddFlowControl initialises flow control for
// the Stream. If the Stream is running at an
// older SPDY version than SPDY/3, the flow
// control has no effect. Multiple calls to
// AddFlowControl are safe.
func (r *clientStreamV3) AddFlowControl() {
	if r.flow != nil {
		return
	}

	r.flow = new(flowControl)
	initialWindow, err := r.conn.InitialWindowSize()
	if err != nil {
		log.Println(err)
		return
	}
	r.flow.streamID = r.streamID
	r.flow.output = r.output
	r.flow.buffer = make([][]byte, 0, 10)
	r.flow.initialWindow = initialWindow
	r.flow.transferWindow = int64(initialWindow)
	r.flow.stream = r
	r.flow.initialWindowThere = DEFAULT_INITIAL_CLIENT_WINDOW_SIZE
	r.flow.transferWindowThere = DEFAULT_INITIAL_CLIENT_WINDOW_SIZE
}

// CheckInitialWindow is used to handle the race
// condition where the flow control is initialised
// before the server has received any updates to
// the initial tranfer window sent by the client.
//
// The transfer window is updated retroactively,
// if necessary.
func (f *flowControl) CheckInitialWindow() {
	if f.stream == nil || f.stream.Conn() == nil {
		return
	}

	newWindow, err := f.stream.Conn().InitialWindowSize()
	if err != nil {
		log.Println(err)
		return
	}

	if f.initialWindow != newWindow {
		if f.initialWindow > newWindow {
			f.transferWindow = int64(newWindow - f.sent)
		} else if f.initialWindow < newWindow {
			f.transferWindow += int64(newWindow - f.initialWindow)
		}
		if f.transferWindow <= 0 {
			f.constrained = true
		}
		f.initialWindow = newWindow
	}
}

// Close nils any references held by the flowControl.
func (f *flowControl) Close() {
	f.buffer = nil
	f.stream = nil
}

// Flush is used to send buffered data to
// the connection, if the transfer window
// will allow. Flush does not guarantee
// that any or all buffered data will be
// sent with a single flush.
func (f *flowControl) Flush() {
	f.CheckInitialWindow()
	if !f.constrained || f.transferWindow == 0 {
		return
	}

	out := make([]byte, 0, f.transferWindow)
	left := f.transferWindow
	for i := 0; i < len(f.buffer); i++ {
		if l := int64(len(f.buffer[i])); l <= left {
			out = append(out, f.buffer[i]...)
			left -= l
			f.buffer = f.buffer[1:]
		} else {
			out = append(out, f.buffer[i][:left]...)
			f.buffer[i] = f.buffer[i][left:]
			left = 0
		}

		if left == 0 {
			break
		}
	}

	f.transferWindow -= int64(len(out))

	if f.transferWindow > 0 {
		f.constrained = false
		log.Printf("Stream %d is no longer constrained.\n", f.streamID)
	}

	dataFrame := new(dataFrameV3)
	dataFrame.StreamID = f.streamID
	dataFrame.Data = out

	f.output <- dataFrame
}

// Paused indicates whether there is data buffered.
// A Stream should not be closed until after the
// last data has been sent and then Paused returns
// false.
func (f *flowControl) Paused() bool {
	f.CheckInitialWindow()
	return f.constrained
}

// Receive is called when data is received from
// the other endpoint. This ensures that they
// conform to the transfer window, regrows the
// window, and sends errors if necessary.
func (f *flowControl) Receive(data []byte) {
	// The transfer window shouldn't already be negative.
	if f.transferWindowThere < 0 {
		rst := new(rstStreamFrameV3)
		rst.StreamID = f.streamID
		rst.Status = RST_STREAM_FLOW_CONTROL_ERROR
		f.output <- rst
	}

	// Update the window.
	f.transferWindowThere -= int64(len(data))

	// Regrow the window if it's half-empty.
	if f.transferWindowThere <= int64(f.initialWindowThere/2) {
		grow := new(windowUpdateFrameV3)
		grow.StreamID = f.streamID
		grow.DeltaWindowSize = uint32(int64(f.initialWindowThere) - f.transferWindowThere)
		f.output <- grow
		f.transferWindowThere += int64(grow.DeltaWindowSize)
	}
}

// UpdateWindow is called when an UPDATE_WINDOW frame is received,
// and performs the growing of the transfer window.
func (f *flowControl) UpdateWindow(deltaWindowSize uint32) error {
	if int64(deltaWindowSize)+f.transferWindow > MAX_TRANSFER_WINDOW_SIZE {
		return errors.New("Error: WINDOW_UPDATE delta window size overflows transfer window size.")
	}

	// Grow window and flush queue.
	debug.Printf("Flow: Growing window in stream %d by %d bytes.\n", f.streamID, deltaWindowSize)
	f.transferWindow += int64(deltaWindowSize)

	f.Flush()
	return nil
}

// Write is used to send data to the connection. This
// takes care of the windowing. Although data may be
// buffered, rather than actually sent, this is not
// visible to the caller.
func (f *flowControl) Write(data []byte) (int, error) {
	l := len(data)
	if l == 0 {
		return 0, nil
	}

	if f.buffer == nil || f.stream == nil {
		return 0, errors.New("Error: Stream closed.")
	}

	// Transfer window processing.
	f.CheckInitialWindow()
	if f.constrained {
		f.Flush()
	}
	var window uint32
	if f.transferWindow < 0 {
		window = 0
	} else {
		window = uint32(f.transferWindow)
	}

	if uint32(len(data)) > window {
		f.buffer = append(f.buffer, data[window:])
		data = data[:window]
		f.sent += window
		f.transferWindow -= int64(window)
		f.constrained = true
		log.Printf("Stream %d is now constrained.\n", f.streamID)
	}

	if len(data) == 0 {
		return l, nil
	}

	dataFrame := new(dataFrameV3)
	dataFrame.StreamID = f.streamID
	dataFrame.Data = data

	f.output <- dataFrame
	return l, nil
}
<|endoftext|>


<commit_before>package flow

import (
	"bytes"
	"fmt"
	"os"
	"os/signal"
	"path"
	"reflect"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"syscall"
	"time"
)

var (
	pkgPath      = reflect.TypeOf(Flow{}).PkgPath()
	DefaultDebug = false
)

type debugInfo struct {
	Stack string
	Info  string
}

func (d *debugInfo) String() string {
	return d.Stack + " - " + d.Info
}

type Flow struct {
	errChan  chan error
	stopChan chan struct{}
	ref      *int32
	wg       sync.WaitGroup
	Parent   *Flow
	Children []*Flow
	stoped   int32
	exited   int32
	onClose  []func()
	id       uintptr

	mutex   sync.Mutex
	debug   []debugInfo
	printed int32
}

func NewEx(n int) *Flow {
	f := &Flow{
		errChan:  make(chan error, 1),
		stopChan: make(chan struct{}),
		ref:      new(int32),
	}
	f.appendDebug("init")
	return f
}

func New() *Flow {
	return NewEx(0)
}

func (f *Flow) MarkExit() bool {
	return atomic.CompareAndSwapInt32(&f.exited, 0, 1)
}

func (f *Flow) printDebug() {
	buf := bytes.NewBuffer(nil)
	maxLength := 0
	for _, d := range f.debug {
		if maxLength < len(d.Stack) {
			maxLength = len(d.Stack)
		}
	}
	fill := func(a string, n int) string {
		return a + strings.Repeat(" ", n-len(a))
	}
	buf.WriteString("\n")
	for _, d := range f.debug {
		buf.WriteString(fmt.Sprint(&f) + " ")
		buf.WriteString(fill(d.Stack, maxLength) + " - " + d.Info + "\n")
	}
	print(buf.String())
}

func (f *Flow) appendDebug(info string) {
	pc, fp, line, _ := runtime.Caller(f.getCaller())
	name := runtime.FuncForPC(pc).Name()
	stack := fmt.Sprintf("%v:%v %v", path.Base(fp), line, path.Base(name))
	f.debug = append(f.debug, debugInfo{stack, info})
}

func (f *Flow) SetOnClose(exit func()) *Flow {
	f.onClose = []func(){exit}
	return f
}

func (f *Flow) AddOnClose(exit func()) *Flow {
	f.onClose = append(f.onClose, exit)
	return f
}

const (
	F_CLOSED  = true
	F_TIMEOUT = false
)

func (f *Flow) Tick(t *time.Ticker) bool {
	select {
	case <-t.C:
		return F_TIMEOUT
	case <-f.IsClose():
		return F_CLOSED
	}
}

func (f *Flow) CloseOrWait(duration time.Duration) bool {
	select {
	case <-time.After(duration):
		return F_TIMEOUT
	case <-f.IsClose():
		return F_CLOSED
	}
}

func (f *Flow) Error(err error) {
	f.errChan <- err
}

func (f *Flow) ForkTo(ref **Flow, exit func()) {
	*ref = f.Fork(0).AddOnClose(exit)
}

func (f *Flow) Fork(n int) *Flow {
	f2 := NewEx(n)
	f2.Parent = f
	// TODO(chzyer): test it !
	f2.errChan = f.errChan
	f.Children = append(f.Children, f2)
	f.Add(1) // for f2
	return f2
}

func (f *Flow) StopAll() {
	flow := f
	for flow.Parent != nil {
		flow = flow.Parent
	}
	flow.Stop()
}

func (f *Flow) Close() {
	f.appendDebug("close")
	f.close()
}

func (f *Flow) close() {
	f.Stop()
	f.wait()
}

func (f *Flow) Stop() {
	if !atomic.CompareAndSwapInt32(&f.stoped, 0, 1) {
		return
	}
	f.appendDebug("stop")

	close(f.stopChan)
	for _, cf := range f.Children {
		cf.Stop()
	}
	if len(f.onClose) > 0 {
		go func() {
			for _, f := range f.onClose {
				f()
			}
		}()
	}
}

func (f *Flow) IsClosed() bool {
	return atomic.LoadInt32(&f.stoped) == 1
}

func (f *Flow) IsClose() chan struct{} {
	return f.stopChan
}

func (f *Flow) Add(n int) {
	atomic.AddInt32(f.ref, int32(n))
	f.appendDebug(fmt.Sprintf("add: %v, ref: %v", n, *f.ref))
	f.wg.Add(n)
}

func (f *Flow) getCaller() int {
	for i := 0; ; i++ {
		pc, _, _, ok := runtime.Caller(i)
		if !ok {
			break
		}
		f := runtime.FuncForPC(pc).Name()
		if !strings.HasPrefix(f, pkgPath) {
			return i - 1
		}
	}
	return 0
}

func (f *Flow) Done() {
	f.wg.Done()
	if atomic.AddInt32(f.ref, -1) == 0 {
		f.Stop()
	}
}

func (f *Flow) DoneAndClose() {
	f.Done()
	f.appendDebug(fmt.Sprintf("done and close, ref: %v", *f.ref))
	f.close()
}

func (f *Flow) wait() {
	f.appendDebug("wait")

	done := make(chan struct{})
	if DefaultDebug && atomic.CompareAndSwapInt32(&f.printed, 0, 1) {
		go func() {
			select {
			case <-done:
				return
			case <-time.After(1000 * time.Millisecond):
				f.printDebug()
			}
		}()
	}
	<-f.stopChan
	f.wg.Wait()
	close(done)

	if f.Parent != nil {
		f.Parent.Done()
		f.Parent = nil
	}
}

func (f *Flow) Wait() error {
	signalChan := make(chan os.Signal)
	signal.Notify(signalChan,
		os.Interrupt, os.Kill, syscall.SIGTERM, syscall.SIGHUP)
	var err error
	select {
	case <-f.IsClose():
		f.appendDebug("got closed")
	case <-signalChan:
		f.appendDebug("got signal")
		f.Stop()
	case err = <-f.errChan:
		f.appendDebug(fmt.Sprintf("got error: %v", err))

		if err != nil {
			f.Stop()
		}
	}

	go func() {
		<-signalChan
		// force close
		println("force close")
		os.Exit(1)
	}()

	f.wait()
	return err
}
<commit_msg>add isexit<commit_after>package flow

import (
	"bytes"
	"fmt"
	"os"
	"os/signal"
	"path"
	"reflect"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"syscall"
	"time"
)

var (
	pkgPath      = reflect.TypeOf(Flow{}).PkgPath()
	DefaultDebug = false
)

type debugInfo struct {
	Stack string
	Info  string
}

func (d *debugInfo) String() string {
	return d.Stack + " - " + d.Info
}

type Flow struct {
	errChan  chan error
	stopChan chan struct{}
	ref      *int32
	wg       sync.WaitGroup
	Parent   *Flow
	Children []*Flow
	stoped   int32
	exited   int32
	onClose  []func()
	id       uintptr

	mutex   sync.Mutex
	debug   []debugInfo
	printed int32
}

func NewEx(n int) *Flow {
	f := &Flow{
		errChan:  make(chan error, 1),
		stopChan: make(chan struct{}),
		ref:      new(int32),
	}
	f.appendDebug("init")
	return f
}

func New() *Flow {
	return NewEx(0)
}

func (f *Flow) MarkExit() bool {
	return atomic.CompareAndSwapInt32(&f.exited, 0, 1)
}

func (f *Flow) IsExit() bool {
	return atomic.LoadInt32(&f.exited) == 1
}

func (f *Flow) printDebug() {
	buf := bytes.NewBuffer(nil)
	maxLength := 0
	for _, d := range f.debug {
		if maxLength < len(d.Stack) {
			maxLength = len(d.Stack)
		}
	}
	fill := func(a string, n int) string {
		return a + strings.Repeat(" ", n-len(a))
	}
	buf.WriteString("\n")
	for _, d := range f.debug {
		buf.WriteString(fmt.Sprint(&f) + " ")
		buf.WriteString(fill(d.Stack, maxLength) + " - " + d.Info + "\n")
	}
	print(buf.String())
}

func (f *Flow) appendDebug(info string) {
	pc, fp, line, _ := runtime.Caller(f.getCaller())
	name := runtime.FuncForPC(pc).Name()
	stack := fmt.Sprintf("%v:%v %v", path.Base(fp), line, path.Base(name))
	f.debug = append(f.debug, debugInfo{stack, info})
}

func (f *Flow) SetOnClose(exit func()) *Flow {
	f.onClose = []func(){exit}
	return f
}

func (f *Flow) AddOnClose(exit func()) *Flow {
	f.onClose = append(f.onClose, exit)
	return f
}

const (
	F_CLOSED  = true
	F_TIMEOUT = false
)

func (f *Flow) Tick(t *time.Ticker) bool {
	select {
	case <-t.C:
		return F_TIMEOUT
	case <-f.IsClose():
		return F_CLOSED
	}
}

func (f *Flow) CloseOrWait(duration time.Duration) bool {
	select {
	case <-time.After(duration):
		return F_TIMEOUT
	case <-f.IsClose():
		return F_CLOSED
	}
}

func (f *Flow) Error(err error) {
	f.errChan <- err
}

func (f *Flow) ForkTo(ref **Flow, exit func()) {
	*ref = f.Fork(0).AddOnClose(exit)
}

func (f *Flow) Fork(n int) *Flow {
	f2 := NewEx(n)
	f2.Parent = f
	// TODO(chzyer): test it !
	f2.errChan = f.errChan
	f.Children = append(f.Children, f2)
	f.Add(1) // for f2
	return f2
}

func (f *Flow) StopAll() {
	flow := f
	for flow.Parent != nil {
		flow = flow.Parent
	}
	flow.Stop()
}

func (f *Flow) Close() {
	f.appendDebug("close")
	f.close()
}

func (f *Flow) close() {
	f.Stop()
	f.wait()
}

func (f *Flow) Stop() {
	if !atomic.CompareAndSwapInt32(&f.stoped, 0, 1) {
		return
	}
	f.appendDebug("stop")

	close(f.stopChan)
	for _, cf := range f.Children {
		cf.Stop()
	}
	if len(f.onClose) > 0 {
		go func() {
			for _, f := range f.onClose {
				f()
			}
		}()
	}
}

func (f *Flow) IsClosed() bool {
	return atomic.LoadInt32(&f.stoped) == 1
}

func (f *Flow) IsClose() chan struct{} {
	return f.stopChan
}

func (f *Flow) Add(n int) {
	atomic.AddInt32(f.ref, int32(n))
	f.appendDebug(fmt.Sprintf("add: %v, ref: %v", n, *f.ref))
	f.wg.Add(n)
}

func (f *Flow) getCaller() int {
	for i := 0; ; i++ {
		pc, _, _, ok := runtime.Caller(i)
		if !ok {
			break
		}
		f := runtime.FuncForPC(pc).Name()
		if !strings.HasPrefix(f, pkgPath) {
			return i - 1
		}
	}
	return 0
}

func (f *Flow) Done() {
	f.wg.Done()
	if atomic.AddInt32(f.ref, -1) == 0 {
		f.Stop()
	}
}

func (f *Flow) DoneAndClose() {
	f.Done()
	f.appendDebug(fmt.Sprintf("done and close, ref: %v", *f.ref))
	f.close()
}

func (f *Flow) wait() {
	f.appendDebug("wait")

	done := make(chan struct{})
	printed := int32(0)
	if DefaultDebug && atomic.CompareAndSwapInt32(&f.printed, 0, 1) {
		go func() {
			select {
			case <-done:
				return
			case <-time.After(1000 * time.Millisecond):
				f.printDebug()
				atomic.StoreInt32(&printed, 1)
			}
		}()
	}
	<-f.stopChan
	f.wg.Wait()
	close(done)
	if atomic.LoadInt32(&printed) == 1 {
		println(fmt.Sprint(&f) + " - exit")
	}

	if f.Parent != nil {
		f.Parent.Done()
		f.Parent = nil
	}
}

func (f *Flow) Wait() error {
	signalChan := make(chan os.Signal)
	signal.Notify(signalChan,
		os.Interrupt, os.Kill, syscall.SIGTERM, syscall.SIGHUP)
	var err error
	select {
	case <-f.IsClose():
		f.appendDebug("got closed")
	case <-signalChan:
		f.appendDebug("got signal")
		f.Stop()
	case err = <-f.errChan:
		f.appendDebug(fmt.Sprintf("got error: %v", err))

		if err != nil {
			f.Stop()
		}
	}

	go func() {
		<-signalChan
		// force close
		println("force close")
		os.Exit(1)
	}()

	f.wait()
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"gl"
	"image"
	"image/png"
	"os"
	"io/ioutil"
	"bytes"
	"io"
	"encoding/binary"
	"unsafe"
)

func uploadTexture_NRGBA32(img *image.NRGBA) gl.GLuint {
	var id gl.GLuint

	gl.GenTextures(1, &id)
	gl.BindTexture(gl.TEXTURE_2D, id)
	gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR)
	gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR)
	gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE)
	gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE)
	gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_R, gl.CLAMP_TO_EDGE)
	gl.TexImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.GLsizei(img.Width()), gl.GLsizei(img.Height()), 0, gl.RGBA,
		      gl.UNSIGNED_BYTE, unsafe.Pointer(&img.Pixel[0][0]))

	if gl.GetError() != gl.NO_ERROR {
		gl.DeleteTextures(1, &id)
		panic(os.NewError("Failed to load a texture"))
		return 0
	}
	return id
}

//-------------------------------------------------------------------------
// FontGlyph
//-------------------------------------------------------------------------

type FontGlyph struct {
	OffsetX int32
	OffsetY int32
	Width uint32
	Height uint32

	// texture coords
	TX float32
	TY float32
	TX2 float32
	TY2 float32

	XAdvance uint32
}

type FontEncoding struct {
	Unicode uint32
	Index uint32
}

type Font struct {
	Glyphs []FontGlyph

	// I'm keeping it here because original font implementation
	// uses binary search lookups in that array, but here in Go I will
	// simply use a map for that
	Encoding []FontEncoding
	Texture gl.GLuint
	YAdvance uint32

	EncodingMap map[int]int
}

func LoadFontFromFile(filename string) (*Font, os.Error) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	return LoadFont(data)
}

func readLittleEndian(r io.Reader, data interface{}) {
	err := binary.Read(r, binary.LittleEndian, data)
	if err != nil {
		panic(err)
	}
}

func LoadFont(data []byte) (fontOut *Font, errOut os.Error) {
	defer func() {
		if err := recover(); err != nil {
			var ok bool
			fontOut = nil
			errOut, ok = err.(os.Error)
			if !ok {
				panic(err)
			}
		}
	}()

	font := new(Font)
	buf := bytes.NewBuffer(data[4:]) // skip magic

	var glyphsNum uint32
	readLittleEndian(buf, &glyphsNum)
	readLittleEndian(buf, &font.YAdvance)

	font.Glyphs = make([]FontGlyph, glyphsNum)
	for i := 0; i < int(glyphsNum); i++ {
		readLittleEndian(buf, &font.Glyphs[i].OffsetX)
		readLittleEndian(buf, &font.Glyphs[i].OffsetY)
		readLittleEndian(buf, &font.Glyphs[i].Width)
		readLittleEndian(buf, &font.Glyphs[i].Height)
		readLittleEndian(buf, &font.Glyphs[i].TX)
		readLittleEndian(buf, &font.Glyphs[i].TY)
		readLittleEndian(buf, &font.Glyphs[i].TX2)
		readLittleEndian(buf, &font.Glyphs[i].TY2)
		readLittleEndian(buf, &font.Glyphs[i].XAdvance)
	}

	font.Encoding = make([]FontEncoding, glyphsNum)
	font.EncodingMap = make(map[int]int, glyphsNum)
	for i := 0; i < int(glyphsNum); i++ {
		readLittleEndian(buf, &font.Encoding[i].Unicode)
		readLittleEndian(buf, &font.Encoding[i].Index)

		font.EncodingMap[int(font.Encoding[i].Unicode)] =
			int(font.Encoding[i].Index)
	}

	img, err := png.Decode(buf)
	if err != nil {
		return nil, err
	}

	nrgba, ok := img.(*image.NRGBA)
	if !ok {
		return nil, os.NewError("Wrong image format")
	}

	font.Texture = uploadTexture_NRGBA32(nrgba)
	return font, nil
}

func drawQuad(x, y, w, h int, u, v, u2, v2 float) {
	gl.Begin(gl.QUADS)

	gl.TexCoord2f(gl.GLfloat(u), gl.GLfloat(v))
	gl.Vertex2i(gl.GLint(x), gl.GLint(y))

	gl.TexCoord2f(gl.GLfloat(u2), gl.GLfloat(v))
	gl.Vertex2i(gl.GLint(x+w), gl.GLint(y))

	gl.TexCoord2f(gl.GLfloat(u2), gl.GLfloat(v2))
	gl.Vertex2i(gl.GLint(x+w), gl.GLint(y+h))

	gl.TexCoord2f(gl.GLfloat(u), gl.GLfloat(v2))
	gl.Vertex2i(gl.GLint(x), gl.GLint(y+h))

	gl.End()
}

func drawGlyph(x, y int, g *FontGlyph) {
	drawQuad(x + int(g.OffsetX), y + int(g.OffsetY), int(g.Width), int(g.Height),
		 float(g.TX), float(g.TY), float(g.TX2), float(g.TY2))
}

func (self *Font) Draw(x, y int, text string) {
	gl.BindTexture(gl.TEXTURE_2D, gl.GLuint(self.Texture))
	for _, rune := range text {
		index, ok := self.EncodingMap[rune]
		if !ok {
			continue
		}

		g := &self.Glyphs[index - 1]
		drawGlyph(x, y, g)
		x += int(g.XAdvance)
	}
	gl.BindTexture(gl.TEXTURE_2D, 0)
}

func (self *Font) Width(text string) int {
	x := 0
	for _, rune := range text {
		index, ok := self.EncodingMap[rune]
		if !ok {
			continue
		}

		x += int(self.Glyphs[index - 1].XAdvance)
	}
	return x
}
<commit_msg>Hack for x86_64, due to broken image.NRGBA.<commit_after>package main

import (
	"gl"
	"image"
	"image/png"
	"os"
	"io/ioutil"
	"bytes"
	"io"
	"encoding/binary"
	"unsafe"
)

func uploadTexture_NRGBA32(img *image.NRGBA) gl.GLuint {
	data := make([]uint8, img.Width() * img.Height() * 4)
	for y := 0; y < img.Height(); y++ {
		for x := 0; x < img.Width(); x++ {
			p := &img.Pixel[y][x]
			offset := y * img.Width() * 4 + x * 4
			data[offset+0] = p.R;
			data[offset+1] = p.G;
			data[offset+2] = p.B;
			data[offset+3] = p.A;
		}
	}

	var id gl.GLuint

	gl.GenTextures(1, &id)
	gl.BindTexture(gl.TEXTURE_2D, id)
	gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR)
	gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR)
	gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE)
	gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE)
	gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_R, gl.CLAMP_TO_EDGE)
	gl.TexImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.GLsizei(img.Width()), gl.GLsizei(img.Height()), 0, gl.RGBA,
		      gl.UNSIGNED_BYTE, unsafe.Pointer(&data[0]))

	if gl.GetError() != gl.NO_ERROR {
		gl.DeleteTextures(1, &id)
		panic(os.NewError("Failed to load a texture"))
		return 0
	}
	return id
}

//-------------------------------------------------------------------------
// FontGlyph
//-------------------------------------------------------------------------

type FontGlyph struct {
	OffsetX int32
	OffsetY int32
	Width uint32
	Height uint32

	// texture coords
	TX float32
	TY float32
	TX2 float32
	TY2 float32

	XAdvance uint32
}

type FontEncoding struct {
	Unicode uint32
	Index uint32
}

type Font struct {
	Glyphs []FontGlyph

	// I'm keeping it here because original font implementation
	// uses binary search lookups in that array, but here in Go I will
	// simply use a map for that
	Encoding []FontEncoding
	Texture gl.GLuint
	YAdvance uint32

	EncodingMap map[int]int
}

func LoadFontFromFile(filename string) (*Font, os.Error) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	return LoadFont(data)
}

func readLittleEndian(r io.Reader, data interface{}) {
	err := binary.Read(r, binary.LittleEndian, data)
	if err != nil {
		panic(err)
	}
}

func LoadFont(data []byte) (fontOut *Font, errOut os.Error) {
	defer func() {
		if err := recover(); err != nil {
			var ok bool
			fontOut = nil
			errOut, ok = err.(os.Error)
			if !ok {
				panic(err)
			}
		}
	}()

	font := new(Font)
	buf := bytes.NewBuffer(data[4:]) // skip magic

	var glyphsNum uint32
	readLittleEndian(buf, &glyphsNum)
	readLittleEndian(buf, &font.YAdvance)

	font.Glyphs = make([]FontGlyph, glyphsNum)
	for i := 0; i < int(glyphsNum); i++ {
		readLittleEndian(buf, &font.Glyphs[i].OffsetX)
		readLittleEndian(buf, &font.Glyphs[i].OffsetY)
		readLittleEndian(buf, &font.Glyphs[i].Width)
		readLittleEndian(buf, &font.Glyphs[i].Height)
		readLittleEndian(buf, &font.Glyphs[i].TX)
		readLittleEndian(buf, &font.Glyphs[i].TY)
		readLittleEndian(buf, &font.Glyphs[i].TX2)
		readLittleEndian(buf, &font.Glyphs[i].TY2)
		readLittleEndian(buf, &font.Glyphs[i].XAdvance)
	}

	font.Encoding = make([]FontEncoding, glyphsNum)
	font.EncodingMap = make(map[int]int, glyphsNum)
	for i := 0; i < int(glyphsNum); i++ {
		readLittleEndian(buf, &font.Encoding[i].Unicode)
		readLittleEndian(buf, &font.Encoding[i].Index)

		font.EncodingMap[int(font.Encoding[i].Unicode)] =
			int(font.Encoding[i].Index)
	}

	img, err := png.Decode(buf)
	if err != nil {
		return nil, err
	}

	nrgba, ok := img.(*image.NRGBA)
	if !ok {
		return nil, os.NewError("Wrong image format")
	}

	font.Texture = uploadTexture_NRGBA32(nrgba)
	return font, nil
}

func drawQuad(x, y, w, h int, u, v, u2, v2 float) {
	gl.Begin(gl.QUADS)

	gl.TexCoord2f(gl.GLfloat(u), gl.GLfloat(v))
	gl.Vertex2i(gl.GLint(x), gl.GLint(y))

	gl.TexCoord2f(gl.GLfloat(u2), gl.GLfloat(v))
	gl.Vertex2i(gl.GLint(x+w), gl.GLint(y))

	gl.TexCoord2f(gl.GLfloat(u2), gl.GLfloat(v2))
	gl.Vertex2i(gl.GLint(x+w), gl.GLint(y+h))

	gl.TexCoord2f(gl.GLfloat(u), gl.GLfloat(v2))
	gl.Vertex2i(gl.GLint(x), gl.GLint(y+h))

	gl.End()
}

func drawGlyph(x, y int, g *FontGlyph) {
	drawQuad(x + int(g.OffsetX), y + int(g.OffsetY), int(g.Width), int(g.Height),
		 float(g.TX), float(g.TY), float(g.TX2), float(g.TY2))
}

func (self *Font) Draw(x, y int, text string) {
	gl.BindTexture(gl.TEXTURE_2D, gl.GLuint(self.Texture))
	for _, rune := range text {
		index, ok := self.EncodingMap[rune]
		if !ok {
			continue
		}

		g := &self.Glyphs[index - 1]
		drawGlyph(x, y, g)
		x += int(g.XAdvance)
	}
	gl.BindTexture(gl.TEXTURE_2D, 0)
}

func (self *Font) Width(text string) int {
	x := 0
	for _, rune := range text {
		index, ok := self.EncodingMap[rune]
		if !ok {
			continue
		}

		x += int(self.Glyphs[index - 1].XAdvance)
	}
	return x
}
<|endoftext|>


<commit_before>// Copyright 2012 The go-gl Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package text

import (
	"code.google.com/p/freetype-go/freetype"
	"code.google.com/p/freetype-go/freetype/truetype"
	"fmt"
	"github.com/go-gl/gl"
	"image"
	"io/ioutil"
	"math"
)

// Font represents a truetype font, prepared for rendering text
// to an OpenGL context.
type Font struct {
	textures []gl.Texture // Holds the texture id's.
	charset  *Charset     // Character set used to generate the font.
	scale    int32        // Font height.
	listbase uint         // Holds the first display list id.
}

// LoadFontFile loads the given truetype font and returns a Font object.
// The charset determines which rune range to use.
//
// Note: The supplied font should support the runes specified by the charset.
func LoadFontFile(file string, scale int32, cs *Charset) (font *Font, err error) {
	data, err := ioutil.ReadFile(file)
	if err != nil {
		return
	}
	return LoadFontData(data, scale, cs)
}

// LoadFontData loads the given truetype font and returns a Font object.
// The charset determines which rune range to use.
//
// Note: The supplied font should support the runes specified by the charset.
func LoadFontData(fontData []byte, scale int32, cs *Charset) (font *Font, err error) {
	ttf, err := truetype.Parse(fontData)
	if err != nil {
		return
	}

	gb := truetype.NewGlyphBuf()

	font = new(Font)
	font.charset = cs
	font.scale = scale
	font.textures = make([]gl.Texture, cs.Len())

	gl.GenTextures(font.textures)
	font.listbase = gl.GenLists(cs.Len())

	for r := cs.Low; r <= cs.High; r++ {
		err = font.makeList(ttf, gb, r)
		if err != nil {
			return
		}
	}

	return
}

// Scale returns the font height.
func (f *Font) Scale() int32 { return f.scale }

// Charset returns the character set used to create this font.
func (f *Font) Charset() *Charset { return f.charset }

// Release cleans up all font resources.
// It can no longer be used for rendering after this call completes.
func (f *Font) Release() {
	gl.DeleteTextures(f.textures)
	gl.DeleteLists(f.listbase, f.charset.Len())

	f.charset = nil
	f.textures = nil
	f.listbase = 0
}

// Printf prints the given string at the specified coordinates.
func (f *Font) Printf(x, y float32, fs string, argv ...interface{}) {
	// Create display list indices from runes. The runes need to be offset
	// by -Charset.Low to create the correct index.
	indices := []rune(fmt.Sprintf(fs, argv...))

	for i, r := range indices {
		indices[i] = r - f.charset.Low
	}

	var vp [4]int32
	gl.GetIntegerv(gl.VIEWPORT, vp[:])

	gl.PushAttrib(gl.TRANSFORM_BIT)
	gl.MatrixMode(gl.PROJECTION)
	gl.PushMatrix()
	gl.LoadIdentity()
	gl.Ortho(float64(vp[0]), float64(vp[2]), float64(vp[1]), float64(vp[3]), 0, 1)
	gl.PopAttrib()

	gl.PushAttrib(gl.LIST_BIT | gl.CURRENT_BIT | gl.ENABLE_BIT | gl.TRANSFORM_BIT)
	gl.MatrixMode(gl.MODELVIEW)
	gl.Disable(gl.LIGHTING)
	gl.Enable(gl.TEXTURE_2D)
	gl.Disable(gl.DEPTH_TEST)
	gl.Enable(gl.BLEND)
	gl.BlendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA)

	gl.ListBase(f.listbase)

	var mv [16]float32
	gl.GetFloatv(gl.MODELVIEW_MATRIX, mv[:])

	gl.PushMatrix()
	gl.LoadIdentity()
	gl.Translatef(x, (float32(vp[3]) - y - float32(f.scale)), 0)
	gl.MultMatrixf(mv[:])
	gl.CallLists(len(indices), gl.UNSIGNED_INT, indices)
	gl.PopMatrix()
	gl.PopAttrib()

	gl.PushAttrib(gl.TRANSFORM_BIT)
	gl.MatrixMode(gl.PROJECTION)
	gl.PopMatrix()
	gl.PopAttrib()
}

// pow2 returns the first power-of-two value >= than n.
func pow2(n int) int { return 1 << (uint(math.Log2(float64(n))) + 1) }

// makeList makes a display list for the given glyph.
//
// http://www.cs.sunysb.edu/documentation/freetype-2.1.9/docs/tutorial/step2.html
func (f *Font) makeList(ttf *truetype.Font, gb *truetype.GlyphBuf, r rune) (err error) {
	glyph := ttf.Index(r)

	err = gb.Load(ttf, f.scale, glyph, nil)
	if err != nil {
		return
	}

	// Glyph dimensions.
	metric := ttf.HMetric(f.scale, glyph)
	glyphWidth := float32(metric.AdvanceWidth)
	glyphHeight := float32(f.scale)

	// Create power-of-two texture dimensions.
	texWidth := pow2(int(glyphWidth))
	texHeight := pow2(int(glyphHeight))

	// Create a temporary image to render to.
	rect := image.Rect(0, 0, texWidth, texHeight)
	img := image.NewGray16(rect)

	// Use a freetype context to do the drawing.
	c := freetype.NewContext()
	c.SetDPI(72)
	c.SetFont(ttf)
	c.SetFontSize(float64(f.scale))
	c.SetClip(img.Bounds())
	c.SetDst(img)
	c.SetSrc(image.White)

	// Draw the glyph.
	pt := freetype.Pt(0, int(glyphHeight)+int(gb.B.YMin))
	c.DrawString(string(r), pt)

	// Index for our display list and texture. This is the same as the rune
	// value, minus the character set's lower bound.
	tex := r - f.charset.Low

	// Initialize glyph texture and render the image to it.
	f.textures[tex].Bind(gl.TEXTURE_2D)

	//gl.TexEnvi(gl.TEXTURE_ENV, gl.TEXTURE_ENV_MODE, gl.MODULATE)
	gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR)
	gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR)
	gl.TexImage2D(gl.TEXTURE_2D, 0, gl.RGBA, texWidth, texHeight,
		0, gl.LUMINANCE_ALPHA, gl.UNSIGNED_BYTE, img.Pix)

	// Build the display list which renders the texture to an
	// adequately positioned and scaled quad.
	gl.NewList(f.listbase+uint(tex), gl.COMPILE)
	f.textures[tex].Bind(gl.TEXTURE_2D)

	gl.Translatef(float32(gb.B.XMin), 0, 0)
	gl.PushMatrix()
	gl.Translatef(0, float32(gb.B.YMin), 0)

	x := float64(glyphWidth) / float64(texWidth)
	y := float64(glyphHeight) / float64(texHeight)

	// Draw the quad.
	gl.Begin(gl.QUADS)
	gl.TexCoord2d(0, 0)
	gl.Vertex2f(0, glyphHeight)
	gl.TexCoord2d(0, y)
	gl.Vertex2f(0, 0)
	gl.TexCoord2d(x, y)
	gl.Vertex2f(glyphWidth, 0)
	gl.TexCoord2d(x, 0)
	gl.Vertex2f(glyphWidth, glyphHeight)
	gl.End()

	gl.PopMatrix()

	// Advance the current transformation to the next glyph location.
	gl.Translatef(float32(metric.AdvanceWidth), 0, 0)

	gl.EndList()
	return
}
<commit_msg>Splits loading of font data up into more methods. The font type is now created with `NewFont()`. It can then load actual font data using any one of `Font.LoadStream`, `Font.LoadFile` and `Font.LoadBytes`.<commit_after>// Copyright 2012 The go-gl Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package text

import (
	"code.google.com/p/freetype-go/freetype"
	"code.google.com/p/freetype-go/freetype/truetype"
	"fmt"
	"github.com/go-gl/gl"
	"image"
	"io"
	"io/ioutil"
	"math"
)

// Font represents a truetype font, prepared for rendering text
// to an OpenGL context.
type Font struct {
	textures []gl.Texture // Holds the texture id's.
	charset  *Charset     // Character set used to generate the font.
	scale    int32        // Font height.
	listbase uint         // Holds the first display list id.
}

// NewFont creates a new, uninitialized font instance for the given scale
// (points) and character set.
func NewFont(scale int32, charset *Charset) *Font {
	f := new(Font)
	f.scale = scale
	f.charset = charset
	return f
}

// Release cleans up all font resources.
// It can no longer be used for rendering after this call completes.
func (f *Font) Release() {
	if f.charset == nil {
		return
	}

	gl.DeleteTextures(f.textures)
	gl.DeleteLists(f.listbase, f.charset.Len())

	f.charset = nil
	f.textures = nil
	f.listbase = 0
}

// Scale returns the font height.
func (f *Font) Scale() int32 { return f.scale }

// Charset returns the character set used to create this font.
func (f *Font) Charset() *Charset { return f.charset }

// LoadFile loads a truetype font from the given file.
//
// Note: The supplied font should support the runes specified by the charset.
func (f *Font) LoadFile(file string) (err error) {
	data, err := ioutil.ReadFile(file)
	if err != nil {
		return
	}
	return f.LoaBytes(data)
}

// LoadStream loads a truetype font from the given input stream.
//
// Note: The supplied font should support the runes specified by the charset.
func (f *Font) LoadStream(r io.Reader) (err error) {
	data, err := ioutil.ReadAll(r)
	if err != nil {
		return
	}
	return f.LoaBytes(data)
}

// LoaBytes loads a truetype font from the given byte data.
//
// Note: The supplied font should support the runes specified by the charset.
func (f *Font) LoaBytes(fontData []byte) (err error) {
	ttf, err := truetype.Parse(fontData)
	if err != nil {
		return
	}

	gb := truetype.NewGlyphBuf()

	f.textures = make([]gl.Texture, f.charset.Len())
	f.listbase = gl.GenLists(f.charset.Len())

	gl.GenTextures(f.textures)

	for r := f.charset.Low; r <= f.charset.High; r++ {
		err = f.makeList(ttf, gb, r)
		if err != nil {
			return
		}
	}

	return
}

// Printf prints the given string at the specified coordinates.
func (f *Font) Printf(x, y float32, fs string, argv ...interface{}) {
	// Create display list indices from runes. The runes need to be offset
	// by -Charset.Low to create the correct index.
	indices := []rune(fmt.Sprintf(fs, argv...))

	for i, r := range indices {
		indices[i] = r - f.charset.Low
	}

	var vp [4]int32
	gl.GetIntegerv(gl.VIEWPORT, vp[:])

	gl.PushAttrib(gl.TRANSFORM_BIT)
	gl.MatrixMode(gl.PROJECTION)
	gl.PushMatrix()
	gl.LoadIdentity()
	gl.Ortho(float64(vp[0]), float64(vp[2]), float64(vp[1]), float64(vp[3]), 0, 1)
	gl.PopAttrib()

	gl.PushAttrib(gl.LIST_BIT | gl.CURRENT_BIT | gl.ENABLE_BIT | gl.TRANSFORM_BIT)
	gl.MatrixMode(gl.MODELVIEW)
	gl.Disable(gl.LIGHTING)
	gl.Enable(gl.TEXTURE_2D)
	gl.Disable(gl.DEPTH_TEST)
	gl.Enable(gl.BLEND)
	gl.BlendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA)

	gl.ListBase(f.listbase)

	var mv [16]float32
	gl.GetFloatv(gl.MODELVIEW_MATRIX, mv[:])

	gl.PushMatrix()
	gl.LoadIdentity()
	gl.Translatef(x, (float32(vp[3]) - y - float32(f.scale)), 0)
	gl.MultMatrixf(mv[:])
	gl.CallLists(len(indices), gl.UNSIGNED_INT, indices)
	gl.PopMatrix()
	gl.PopAttrib()

	gl.PushAttrib(gl.TRANSFORM_BIT)
	gl.MatrixMode(gl.PROJECTION)
	gl.PopMatrix()
	gl.PopAttrib()
}

// pow2 returns the first power-of-two value >= than n.
// This is used to create glyph texture dimensions.
func pow2(n int) int { return 1 << (uint(math.Log2(float64(n))) + 1) }

// makeList makes a display list for the given glyph.
//
// http://www.cs.sunysb.edu/documentation/freetype-2.1.9/docs/tutorial/step2.html
func (f *Font) makeList(ttf *truetype.Font, gb *truetype.GlyphBuf, r rune) (err error) {
	glyph := ttf.Index(r)

	err = gb.Load(ttf, f.scale, glyph, nil)
	if err != nil {
		return
	}

	// Glyph dimensions.
	metric := ttf.HMetric(f.scale, glyph)
	glyphWidth := float32(metric.AdvanceWidth)
	glyphHeight := float32(f.scale)

	// Create power-of-two texture dimensions.
	texWidth := pow2(int(glyphWidth))
	texHeight := pow2(int(glyphHeight))

	// Create a temporary image to render to.
	rect := image.Rect(0, 0, texWidth, texHeight)
	img := image.NewGray16(rect)

	// Use a freetype context to do the drawing.
	c := freetype.NewContext()
	c.SetDPI(73)
	c.SetFont(ttf)
	c.SetFontSize(float64(f.scale))
	c.SetClip(img.Bounds())
	c.SetDst(img)
	c.SetSrc(image.White)

	// Draw the glyph.
	pt := freetype.Pt(0, int(glyphHeight)+int(gb.B.YMin))
	c.DrawString(string(r), pt)

	// Index for our display list and texture. This is the same as the rune
	// value, minus the character set's lower bound.
	tex := r - f.charset.Low

	// Initialize glyph texture and render the image to it.
	f.textures[tex].Bind(gl.TEXTURE_2D)

	//gl.TexEnvi(gl.TEXTURE_ENV, gl.TEXTURE_ENV_MODE, gl.MODULATE)
	gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR)
	gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR)
	gl.TexImage2D(gl.TEXTURE_2D, 0, gl.RGBA, texWidth, texHeight,
		0, gl.LUMINANCE_ALPHA, gl.UNSIGNED_BYTE, img.Pix)

	// Build the display list which renders the texture to an
	// adequately positioned and scaled quad.
	gl.NewList(f.listbase+uint(tex), gl.COMPILE)
	f.textures[tex].Bind(gl.TEXTURE_2D)

	gl.Translatef(float32(gb.B.XMin), 0, 0)
	gl.PushMatrix()
	gl.Translatef(0, float32(gb.B.YMin), 0)

	x := float64(glyphWidth) / float64(texWidth)
	y := float64(glyphHeight) / float64(texHeight)

	// Draw the quad.
	gl.Begin(gl.QUADS)
	gl.TexCoord2d(0, 0)
	gl.Vertex2f(0, glyphHeight)
	gl.TexCoord2d(0, y)
	gl.Vertex2f(0, 0)
	gl.TexCoord2d(x, y)
	gl.Vertex2f(glyphWidth, 0)
	gl.TexCoord2d(x, 0)
	gl.Vertex2f(glyphWidth, glyphHeight)
	gl.End()

	gl.PopMatrix()

	// Advance the current transformation to the next glyph location.
	gl.Translatef(float32(metric.AdvanceWidth), 0, 0)

	gl.EndList()
	return
}
<|endoftext|>


<commit_before>/*
Copyright (c) 2017 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package simulator

import (
	"context"
	"reflect"
	"testing"

	"github.com/vmware/govmomi"
	"github.com/vmware/govmomi/object"
	"github.com/vmware/govmomi/simulator/vpx"
	"github.com/vmware/govmomi/vim25/mo"
	"github.com/vmware/govmomi/vim25/types"
)

func TestCustomFieldsManager(t *testing.T) {
	s := New(NewServiceInstance(vpx.ServiceContent, vpx.RootFolder))

	ts := s.NewServer()
	defer ts.Close()

	ctx := context.Background()

	c, err := govmomi.NewClient(ctx, ts.URL, true)
	if err != nil {
		t.Fatal(err)
	}

	fieldsManager, err := object.GetCustomFieldsManager(c.Client)
	if err != nil {
		t.Fatal(err)
	}

	field, err := fieldsManager.Add(ctx, "field_name", "VirtualMachine", nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if field.Name != "field_name" && field.Type != "VirtualMachine" {
		t.Fatal("field add result mismatched with the inserted")
	}

	fields, err := fieldsManager.Field(ctx)
	if err != nil {
		t.Fatal(err)
	}
	if len(fields) != 1 {
		t.Fatalf("expect len(fields)=1; got %d", len(fields))
	}
	if !reflect.DeepEqual(&fields[0], field) {
		t.Fatalf("expect fields[0]==field; got %+v,%+v", fields[0], field)
	}

	key, err := fieldsManager.FindKey(ctx, field.Name)
	if err != nil {
		t.Fatal(err)
	}
	if key != field.Key {
		t.Fatalf("expect key == field.Key; got %d != %d", key, field.Key)
	}

	err = fieldsManager.Rename(ctx, key, "new_field_name")
	if err != nil {
		t.Fatal(err)
	}

	fields, err = fieldsManager.Field(ctx)
	if err != nil {
		t.Fatal(err)
	}
	if len(fields) != 1 {
		t.Fatalf("expect len(fields)=1; got %d", len(fields))
	}
	if fields[0].Name != "new_field_name" {
		t.Fatalf("expect field.name to be %s; got %s", "new_field_name", fields[0].Name)
	}

	folder := Map.content().RootFolder
	err = fieldsManager.Set(ctx, folder, 1, "value")
	if err != nil {
		t.Fatal(err)
	}

	values := Map.Get(folder.Reference()).(mo.Entity).Entity().CustomValue
	if len(values) != 1 {
		t.Fatalf("expect CustomValue has 1 item; got %d", len(values))
	}
	fkey := values[0].GetCustomFieldValue().Key
	if fkey != 1 {
		t.Fatalf("expect value.Key to be 1; got %d", fkey)
	}
	value := values[0].(*types.CustomFieldStringValue).Value
	if value != "value" {
		t.Fatalf("expect value.Value to be %q; got %q", "value", value)
	}

	err = fieldsManager.Remove(ctx, field.Key)
	if err != nil {
		t.Fatal(err)
	}

	fields, err = fieldsManager.Field(ctx)
	if err != nil {
		t.Fatal(err)
	}
	if len(fields) != 0 {
		t.Fatalf("expect fields to be empty; got %+v", fields)
	}
}
<commit_msg>vcsim: fix custom_fields_manager test<commit_after>/*
Copyright (c) 2017 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package simulator

import (
	"context"
	"reflect"
	"testing"

	"github.com/vmware/govmomi"
	"github.com/vmware/govmomi/object"
	"github.com/vmware/govmomi/vim25/types"
)

func TestCustomFieldsManager(t *testing.T) {
	ctx := context.Background()

	m := VPX()
	defer m.Remove()
	err := m.Create()
	if err != nil {
		t.Fatal(err)
	}

	ts := m.Service.NewServer()
	defer ts.Close()

	c, err := govmomi.NewClient(ctx, ts.URL, true)
	if err != nil {
		t.Fatal(err)
	}

	fieldsManager, err := object.GetCustomFieldsManager(c.Client)
	if err != nil {
		t.Fatal(err)
	}

	field, err := fieldsManager.Add(ctx, "field_name", "VirtualMachine", nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if field.Name != "field_name" && field.Type != "VirtualMachine" {
		t.Fatal("field add result mismatched with the inserted")
	}

	fields, err := fieldsManager.Field(ctx)
	if err != nil {
		t.Fatal(err)
	}
	if len(fields) != 1 {
		t.Fatalf("expect len(fields)=1; got %d", len(fields))
	}
	if !reflect.DeepEqual(&fields[0], field) {
		t.Fatalf("expect fields[0]==field; got %+v,%+v", fields[0], field)
	}

	key, err := fieldsManager.FindKey(ctx, field.Name)
	if err != nil {
		t.Fatal(err)
	}
	if key != field.Key {
		t.Fatalf("expect key == field.Key; got %d != %d", key, field.Key)
	}

	err = fieldsManager.Rename(ctx, key, "new_field_name")
	if err != nil {
		t.Fatal(err)
	}

	fields, err = fieldsManager.Field(ctx)
	if err != nil {
		t.Fatal(err)
	}
	if len(fields) != 1 {
		t.Fatalf("expect len(fields)=1; got %d", len(fields))
	}
	if fields[0].Name != "new_field_name" {
		t.Fatalf("expect field.name to be %s; got %s", "new_field_name", fields[0].Name)
	}

	vm := Map.Any("VirtualMachine").(*VirtualMachine)
	err = fieldsManager.Set(ctx, vm.Reference(), field.Key, "value")
	if err != nil {
		t.Fatal(err)
	}

	values := vm.Entity().CustomValue
	if len(values) != 1 {
		t.Fatalf("expect CustomValue has 1 item; got %d", len(values))
	}
	fkey := values[0].GetCustomFieldValue().Key
	if fkey != field.Key {
		t.Fatalf("expect value.Key == field.Key; got %d != %d", fkey, field.Key)
	}
	value := values[0].(*types.CustomFieldStringValue).Value
	if value != "value" {
		t.Fatalf("expect value.Value to be %q; got %q", "value", value)
	}

	err = fieldsManager.Remove(ctx, field.Key)
	if err != nil {
		t.Fatal(err)
	}

	fields, err = fieldsManager.Field(ctx)
	if err != nil {
		t.Fatal(err)
	}
	if len(fields) != 0 {
		t.Fatalf("expect fields to be empty; got %+v", fields)
	}
}
<|endoftext|>


<commit_before>package apidApigeeSync

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"path"
	"time"

	"github.com/30x/apid-core"
	"github.com/apigee-labs/transicator/common"
)

const (
	httpTimeout       = time.Minute
	pluginTimeout     = time.Minute
	maxBackoffTimeout = time.Minute
)

var (
	block        string = "45"
	token        string
	lastSequence string
	polling      bool
)

/*
 * Polls change agent for changes. In event of errors, uses a doubling
 * backoff from 200ms up to a max delay of the configPollInterval value.
 */
func pollForChanges() {

	// ensure there's just one polling thread
	if polling {
		return
	}
	polling = true

	var backOffFunc func()
	pollInterval := config.GetDuration(configPollInterval)
	for {
		start := time.Now()
		err := pollChangeAgent()
		end := time.Now()
		if err != nil {
			if _, ok := err.(apiError); ok {
				downloadDataSnapshot()
				continue
			}
			log.Debugf("Error connecting to changeserver: %v", err)
		}
		if end.After(start.Add(time.Second)) {
			backOffFunc = nil
			continue
		}
		if backOffFunc == nil {
			backOffFunc = createBackOff(200*time.Millisecond, pollInterval)
		}
		backOffFunc()
	}

	polling = false
}

/*
 * Long polls the change agent with a 45 second block. Parses the response from
 * change agent and raises an event. Called by pollForChanges().
 */
func pollChangeAgent() error {

	changesUri, err := url.Parse(config.GetString(configChangeServerBaseURI))
	if err != nil {
		log.Errorf("bad url value for config %s: %s", changesUri, err)
		return err
	}
	changesUri.Path = path.Join(changesUri.Path, "changes")

	/*
	 * Check to see if we have lastSequence already saved in the DB,
	 * in which case, it has to be used to prevent re-reading same data
	 */
	lastSequence = getLastSequence()
	for {
		log.Debug("polling...")
		if token == "" {
			// invalid token, loop until we get one
			getBearerToken()
		}

		/* Find the scopes associated with the config id */
		scopes := findScopesForId(apidInfo.ClusterID)
		v := url.Values{}

		/* Sequence added to the query if available */
		if lastSequence != "" {
			v.Add("since", lastSequence)
		}
		v.Add("block", block)

		/*
		 * Include all the scopes associated with the config Id
		 * The Config Id is included as well, as it acts as the
		 * Bootstrap scope
		 */
		for _, scope := range scopes {
			v.Add("scope", scope)
		}
		v.Add("scope", apidInfo.ClusterID)
		v.Add("snapshot", apidInfo.LastSnapshot)
		changesUri.RawQuery = v.Encode()
		uri := changesUri.String()
		log.Debugf("Fetching changes: %s", uri)

		/* If error, break the loop, and retry after interval */
		client := &http.Client{Timeout: httpTimeout} // must be greater than block value
		req, err := http.NewRequest("GET", uri, nil)
		addHeaders(req)
		r, err := client.Do(req)
		if err != nil {
			log.Errorf("change agent comm error: %s", err)
			return err
		}

		if r.StatusCode != http.StatusOK {
			log.Errorf("Get changes request failed with status code: %d", r.StatusCode)
			switch r.StatusCode {
			case http.StatusUnauthorized:
				token = ""

			case http.StatusNotModified:
				continue

			case http.StatusBadRequest:
				var apiErr apiError
				err = json.NewDecoder(r.Body).Decode(&apiErr)
				if err != nil {
					log.Errorf("JSON Response Data not parsable: %v", err)
					break
				}
				if apiErr.Code == "SNAPSHOT_TOO_OLD" {
					log.Debug("Received SNAPSHOT_TOO_OLD message from change server.")
					err = apiErr
				}
			}

			r.Body.Close()
			return err
		}

		var resp common.ChangeList
		err = json.NewDecoder(r.Body).Decode(&resp)
		r.Body.Close()
		if err != nil {
			log.Errorf("JSON Response Data not parsable: %v", err)
			return err
		}

		/* If valid data present, Emit to plugins */
		if len(resp.Changes) > 0 {
			done := make(chan bool)
			events.EmitWithCallback(ApigeeSyncEventSelector, &resp, func(event apid.Event) {
				done <- true
			})

			select {
			case <-time.After(httpTimeout):
				log.Panic("Timeout. Plugins failed to respond to changes.")
			case <-done:
				close(done)
			}
		} else {
			log.Debugf("No Changes detected for Scopes: %s", scopes)
		}

		if lastSequence != resp.LastSequence {
			lastSequence = resp.LastSequence
			err := updateLastSequence(lastSequence)
			if err != nil {
				log.Panic("Unable to update Sequence in DB")
			}
		}
	}
}

// simple doubling back-off
func createBackOff(retryIn, maxBackOff time.Duration) func() {
	return func() {
		if retryIn > maxBackOff {
			retryIn = maxBackOff
		}
		log.Debugf("backoff called. will retry in %s.", retryIn)
		time.Sleep(retryIn)
		retryIn = retryIn * time.Duration(2)
	}
}

/*
 * This function will (for now) use the Access Key/Secret Key/ApidConfig Id
 * to get the bearer token, and the scopes (as comma separated scope)
 */
func getBearerToken() {

	log.Info("Getting a Bearer token...")
	uriString := config.GetString(configProxyServerBaseURI)
	uri, err := url.Parse(uriString)
	if err != nil {
		log.Panicf("unable to parse uri config '%s' value: '%s': %v", configProxyServerBaseURI, uriString, err)
	}
	uri.Path = path.Join(uri.Path, "/accesstoken")

	retryIn := 5 * time.Millisecond
	maxBackOff := maxBackoffTimeout
	backOffFunc := createBackOff(retryIn, maxBackOff)
	first := true

	for {
		if first {
			first = false
		} else {
			backOffFunc()
		}

		token = ""
		form := url.Values{}
		form.Set("grant_type", "client_credentials")
		form.Add("client_id", config.GetString(configConsumerKey))
		form.Add("client_secret", config.GetString(configConsumerSecret))
		req, err := http.NewRequest("POST", uri.String(), bytes.NewBufferString(form.Encode()))
		req.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value")
		req.Header.Set("display_name", apidInfo.InstanceName)
		req.Header.Set("apid_instance_id", apidInfo.InstanceID)
		req.Header.Set("apid_cluster_Id", apidInfo.ClusterID)
		req.Header.Set("status", "ONLINE")
		req.Header.Set("plugin_details", apidPluginDetails)

		if newInstanceID {
			req.Header.Set("created_at_apid", time.Now().Format(time.RFC3339))
		} else {
			req.Header.Set("updated_at_apid", time.Now().Format(time.RFC3339))
		}

		client := &http.Client{Timeout: httpTimeout}
		resp, err := client.Do(req)
		if err != nil {
			log.Errorf("Unable to Connect to Edge Proxy Server: %v", err)
			continue
		}

		body, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			log.Errorf("Unable to read EdgeProxy Sever response: %v", err)
			continue
		}

		if resp.StatusCode != 200 {
			log.Errorf("Oauth Request Failed with Resp Code: %d. Body: %s", resp.StatusCode, string(body))
			continue
		}

		var oauthResp oauthTokenResp
		log.Debugf("Response: %s ", body)
		err = json.Unmarshal(body, &oauthResp)
		if err != nil {
			log.Error("unable to unmarshal JSON response %s: %v", string(body), err)
			continue
		}
		token = oauthResp.AccessToken

		if newInstanceID {
			newInstanceID = false
			updateApidInstanceInfo()
		}

		/*
		 * This stores the bearer token for any other plugin to
		 * consume.
		 */
		config.Set(bearerToken, token)

		log.Debug("Got a new Bearer token.")

		return
	}
}

type oauthTokenResp struct {
	IssuedAt       int64    `json:"issuedAt"`
	AppName        string   `json:"applicationName"`
	Scope          string   `json:"scope"`
	Status         string   `json:"status"`
	ApiProdList    []string `json:"apiProductList"`
	ExpiresIn      int64    `json:"expiresIn"`
	DeveloperEmail string   `json:"developerEmail"`
	TokenType      string   `json:"tokenType"`
	ClientId       string   `json:"clientId"`
	AccessToken    string   `json:"accessToken"`
	TokenExpIn     int64    `json:"refreshTokenExpiresIn"`
	RefreshCount   int64    `json:"refreshCount"`
}

func Redirect(req *http.Request, via []*http.Request) error {
	req.Header.Add("Authorization", "Bearer "+token)
	req.Header.Add("org", apidInfo.ClusterID) // todo: this is strange.. is it needed?
	return nil
}

// pollForChanges should usually be true, tests use the flag
func bootstrap() {

	if apidInfo.LastSnapshot != "" {
		startOnLocalSnapshot(apidInfo.LastSnapshot)
		return
	}

	downloadBootSnapshot()
	downloadDataSnapshot()
	go pollForChanges()
}

// retrieve boot information: apid_config and apid_config_scope
func downloadBootSnapshot() {
	log.Debug("download Snapshot for boot data")

	scopes := []string{apidInfo.ClusterID}
	downloadSnapshot(scopes)
	// note that for boot snapshot case, we don't need to inform plugins as they'll get the data snapshot
}

// use the scope IDs from the boot snapshot to get all the data associated with the scopes
func downloadDataSnapshot() {
	log.Debug("download Snapshot for data scopes")

	var scopes = findScopesForId(apidInfo.ClusterID)
	scopes = append(scopes, apidInfo.ClusterID)
	resp := downloadSnapshot(scopes)

	done := make(chan bool)
	log.Info("Emitting Snapshot to plugins")
	events.EmitWithCallback(ApigeeSyncEventSelector, &resp, func(event apid.Event) {
		done <- true
	})

	select {
	case <-time.After(pluginTimeout):
		log.Panic("Timeout. Plugins failed to respond to snapshot.")
	case <-done:
		close(done)
	}
}

// Skip Downloading snapshot if there is already a snapshot available from previous run
func startOnLocalSnapshot(snapshot string) {
	log.Infof("Starting on local snapshot: %s", snapshot)

	// ensure DB version will be accessible on behalf of dependant plugins
	_, err := data.DBVersion(snapshot)
	if err != nil {
		log.Panicf("Database inaccessible: %v", err)
	}

	// allow plugins (including this one) to start immediately on existing database
	// Note: this MUST have no tables as that is used as an indicator
	snap := &common.Snapshot{
		SnapshotInfo: apidInfo.LastSnapshot,
	}
	events.EmitWithCallback(ApigeeSyncEventSelector, snap, func(event apid.Event) {
		go pollForChanges()
	})

	log.Infof("Started on local snapshot: %s", snapshot)
}

// will keep retrying with backoff until success
func downloadSnapshot(scopes []string) common.Snapshot {

	log.Debug("downloadSnapshot")

	snapshotUri, err := url.Parse(config.GetString(configSnapServerBaseURI))
	if err != nil {
		log.Panicf("bad url value for config %s: %s", snapshotUri, err)
	}

	// getBearerToken loops until good
	getBearerToken()
	// todo: this could expire... ensure it's called again as needed

	/* Frame and send the snapshot request */
	snapshotUri.Path = path.Join(snapshotUri.Path, "snapshots")

	v := url.Values{}
	for _, scope := range scopes {
		v.Add("scope", scope)
	}
	snapshotUri.RawQuery = v.Encode()
	uri := snapshotUri.String()
	log.Infof("Snapshot Download: %s", uri)

	client := &http.Client{
		CheckRedirect: Redirect,
		Timeout:       httpTimeout,
	}

	retryIn := 5 * time.Millisecond
	maxBackOff := maxBackoffTimeout
	backOffFunc := createBackOff(retryIn, maxBackOff)
	first := true

	for {
		if first {
			first = false
		} else {
			backOffFunc()
		}

		req, err := http.NewRequest("GET", uri, nil)
		if err != nil {
			// should never happen, but if it does, it's unrecoverable anyway
			log.Panicf("Snapshotserver comm error: %v", err)
		}
		addHeaders(req)

		// Set the transport protocol type based on conf file input
		if config.GetString(configSnapshotProtocol) == "json" {
			req.Header.Set("Accept", "application/json")
		} else {
			req.Header.Set("Accept", "application/proto")
		}

		// Issue the request to the snapshot server
		r, err := client.Do(req)
		if err != nil {
			log.Errorf("Snapshotserver comm error: %v", err)
			continue
		}

		if r.StatusCode != 200 {
			log.Errorf("Snapshot server conn failed with resp code %d", r.StatusCode)
			r.Body.Close()
			continue
		}

		// Decode the Snapshot server response
		var resp common.Snapshot
		err = json.NewDecoder(r.Body).Decode(&resp)
		if err != nil {
			log.Errorf("JSON Response Data not parsable: %v", err)
			r.Body.Close()
			continue
		}

		r.Body.Close()
		return resp
	}
}

func addHeaders(req *http.Request) {
	req.Header.Add("Authorization", "Bearer "+token)
	req.Header.Set("apid_instance_id", apidInfo.InstanceID)
	req.Header.Set("apid_cluster_Id", apidInfo.ClusterID)
	req.Header.Set("updated_at_apid", time.Now().Format(time.RFC3339))
}

type apiError struct {
	Code string `json:"code"`
}

func (a apiError) Error() string {
	return a.Code
}
<commit_msg>address review comments<commit_after>package apidApigeeSync

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"path"
	"time"

	"sync/atomic"

	"github.com/30x/apid-core"
	"github.com/apigee-labs/transicator/common"
)

const (
	httpTimeout       = time.Minute
	pluginTimeout     = time.Minute
	maxBackoffTimeout = time.Minute
)

var (
	block        string = "45"
	token        string
	lastSequence string
	polling      uint32
)

/*
 * Polls change agent for changes. In event of errors, uses a doubling
 * backoff from 200ms up to a max delay of the configPollInterval value.
 */
func pollForChanges() {

	if atomic.SwapUint32(&polling, 1) == 1 {
		return
	}

	var backOffFunc func()
	pollInterval := config.GetDuration(configPollInterval)
	for {
		start := time.Now()
		err := pollChangeAgent()
		end := time.Now()
		if err != nil {
			if _, ok := err.(apiError); ok {
				downloadDataSnapshot()
				continue
			}
			log.Debugf("Error connecting to changeserver: %v", err)
		}
		if end.After(start.Add(time.Second)) {
			backOffFunc = nil
			continue
		}
		if backOffFunc == nil {
			backOffFunc = createBackOff(200*time.Millisecond, pollInterval)
		}
		backOffFunc()
	}

	atomic.SwapUint32(&polling, 0)
}

/*
 * Long polls the change agent with a 45 second block. Parses the response from
 * change agent and raises an event. Called by pollForChanges().
 */
func pollChangeAgent() error {

	changesUri, err := url.Parse(config.GetString(configChangeServerBaseURI))
	if err != nil {
		log.Errorf("bad url value for config %s: %s", changesUri, err)
		return err
	}
	changesUri.Path = path.Join(changesUri.Path, "changes")

	/*
	 * Check to see if we have lastSequence already saved in the DB,
	 * in which case, it has to be used to prevent re-reading same data
	 */
	lastSequence = getLastSequence()
	for {
		log.Debug("polling...")
		if token == "" {
			// invalid token, loop until we get one
			getBearerToken()
		}

		/* Find the scopes associated with the config id */
		scopes := findScopesForId(apidInfo.ClusterID)
		v := url.Values{}

		/* Sequence added to the query if available */
		if lastSequence != "" {
			v.Add("since", lastSequence)
		}
		v.Add("block", block)

		/*
		 * Include all the scopes associated with the config Id
		 * The Config Id is included as well, as it acts as the
		 * Bootstrap scope
		 */
		for _, scope := range scopes {
			v.Add("scope", scope)
		}
		v.Add("scope", apidInfo.ClusterID)
		v.Add("snapshot", apidInfo.LastSnapshot)
		changesUri.RawQuery = v.Encode()
		uri := changesUri.String()
		log.Debugf("Fetching changes: %s", uri)

		/* If error, break the loop, and retry after interval */
		client := &http.Client{Timeout: httpTimeout} // must be greater than block value
		req, err := http.NewRequest("GET", uri, nil)
		addHeaders(req)
		r, err := client.Do(req)
		if err != nil {
			log.Errorf("change agent comm error: %s", err)
			return err
		}

		if r.StatusCode != http.StatusOK {
			log.Errorf("Get changes request failed with status code: %d", r.StatusCode)
			switch r.StatusCode {
			case http.StatusUnauthorized:
				token = ""

			case http.StatusNotModified:
				r.Body.Close()
				continue

			case http.StatusBadRequest:
				var apiErr apiError
				err = json.NewDecoder(r.Body).Decode(&apiErr)
				if err != nil {
					log.Errorf("JSON Response Data not parsable: %v", err)
					break
				}
				if apiErr.Code == "SNAPSHOT_TOO_OLD" {
					log.Debug("Received SNAPSHOT_TOO_OLD message from change server.")
					err = apiErr
				}
			}

			r.Body.Close()
			return err
		}

		var resp common.ChangeList
		err = json.NewDecoder(r.Body).Decode(&resp)
		r.Body.Close()
		if err != nil {
			log.Errorf("JSON Response Data not parsable: %v", err)
			return err
		}

		/* If valid data present, Emit to plugins */
		if len(resp.Changes) > 0 {
			done := make(chan bool)
			events.EmitWithCallback(ApigeeSyncEventSelector, &resp, func(event apid.Event) {
				done <- true
			})

			select {
			case <-time.After(httpTimeout):
				log.Panic("Timeout. Plugins failed to respond to changes.")
			case <-done:
			}
		} else {
			log.Debugf("No Changes detected for Scopes: %s", scopes)
		}

		if lastSequence != resp.LastSequence {
			lastSequence = resp.LastSequence
			err := updateLastSequence(resp.LastSequence)
			if err != nil {
				log.Panic("Unable to update Sequence in DB")
			}
		}
	}
}

// simple doubling back-off
func createBackOff(retryIn, maxBackOff time.Duration) func() {
	return func() {
		if retryIn > maxBackOff {
			retryIn = maxBackOff
		}
		log.Debugf("backoff called. will retry in %s.", retryIn)
		time.Sleep(retryIn)
		retryIn = retryIn * time.Duration(2)
	}
}

/*
 * This function will (for now) use the Access Key/Secret Key/ApidConfig Id
 * to get the bearer token, and the scopes (as comma separated scope)
 */
func getBearerToken() {

	log.Info("Getting a Bearer token...")
	uriString := config.GetString(configProxyServerBaseURI)
	uri, err := url.Parse(uriString)
	if err != nil {
		log.Panicf("unable to parse uri config '%s' value: '%s': %v", configProxyServerBaseURI, uriString, err)
	}
	uri.Path = path.Join(uri.Path, "/accesstoken")

	retryIn := 5 * time.Millisecond
	maxBackOff := maxBackoffTimeout
	backOffFunc := createBackOff(retryIn, maxBackOff)
	first := true

	for {
		if first {
			first = false
		} else {
			backOffFunc()
		}

		token = ""
		form := url.Values{}
		form.Set("grant_type", "client_credentials")
		form.Add("client_id", config.GetString(configConsumerKey))
		form.Add("client_secret", config.GetString(configConsumerSecret))
		req, err := http.NewRequest("POST", uri.String(), bytes.NewBufferString(form.Encode()))
		req.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value")
		req.Header.Set("display_name", apidInfo.InstanceName)
		req.Header.Set("apid_instance_id", apidInfo.InstanceID)
		req.Header.Set("apid_cluster_Id", apidInfo.ClusterID)
		req.Header.Set("status", "ONLINE")
		req.Header.Set("plugin_details", apidPluginDetails)

		if newInstanceID {
			req.Header.Set("created_at_apid", time.Now().Format(time.RFC3339))
		} else {
			req.Header.Set("updated_at_apid", time.Now().Format(time.RFC3339))
		}

		client := &http.Client{Timeout: httpTimeout}
		resp, err := client.Do(req)
		if err != nil {
			log.Errorf("Unable to Connect to Edge Proxy Server: %v", err)
			continue
		}

		body, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			log.Errorf("Unable to read EdgeProxy Sever response: %v", err)
			continue
		}

		if resp.StatusCode != 200 {
			log.Errorf("Oauth Request Failed with Resp Code: %d. Body: %s", resp.StatusCode, string(body))
			continue
		}

		var oauthResp oauthTokenResp
		log.Debugf("Response: %s ", body)
		err = json.Unmarshal(body, &oauthResp)
		if err != nil {
			log.Error("unable to unmarshal JSON response %s: %v", string(body), err)
			continue
		}
		token = oauthResp.AccessToken

		if newInstanceID {
			newInstanceID = false
			updateApidInstanceInfo()
		}

		/*
		 * This stores the bearer token for any other plugin to
		 * consume.
		 */
		config.Set(bearerToken, token)

		log.Debug("Got a new Bearer token.")

		return
	}
}

type oauthTokenResp struct {
	IssuedAt       int64    `json:"issuedAt"`
	AppName        string   `json:"applicationName"`
	Scope          string   `json:"scope"`
	Status         string   `json:"status"`
	ApiProdList    []string `json:"apiProductList"`
	ExpiresIn      int64    `json:"expiresIn"`
	DeveloperEmail string   `json:"developerEmail"`
	TokenType      string   `json:"tokenType"`
	ClientId       string   `json:"clientId"`
	AccessToken    string   `json:"accessToken"`
	TokenExpIn     int64    `json:"refreshTokenExpiresIn"`
	RefreshCount   int64    `json:"refreshCount"`
}

func Redirect(req *http.Request, via []*http.Request) error {
	req.Header.Add("Authorization", "Bearer "+token)
	req.Header.Add("org", apidInfo.ClusterID) // todo: this is strange.. is it needed?
	return nil
}

// pollForChanges should usually be true, tests use the flag
func bootstrap() {

	if apidInfo.LastSnapshot != "" {
		startOnLocalSnapshot(apidInfo.LastSnapshot)
		return
	}

	downloadBootSnapshot()
	downloadDataSnapshot()
	go pollForChanges()
}

// retrieve boot information: apid_config and apid_config_scope
func downloadBootSnapshot() {
	log.Debug("download Snapshot for boot data")

	scopes := []string{apidInfo.ClusterID}
	downloadSnapshot(scopes)
	// note that for boot snapshot case, we don't need to inform plugins as they'll get the data snapshot
}

// use the scope IDs from the boot snapshot to get all the data associated with the scopes
func downloadDataSnapshot() {
	log.Debug("download Snapshot for data scopes")

	var scopes = findScopesForId(apidInfo.ClusterID)
	scopes = append(scopes, apidInfo.ClusterID)
	resp := downloadSnapshot(scopes)

	done := make(chan bool)
	log.Info("Emitting Snapshot to plugins")
	events.EmitWithCallback(ApigeeSyncEventSelector, &resp, func(event apid.Event) {
		done <- true
	})

	select {
	case <-time.After(pluginTimeout):
		log.Panic("Timeout. Plugins failed to respond to snapshot.")
	case <-done:
	}
}

// Skip Downloading snapshot if there is already a snapshot available from previous run
func startOnLocalSnapshot(snapshot string) {
	log.Infof("Starting on local snapshot: %s", snapshot)

	// ensure DB version will be accessible on behalf of dependant plugins
	_, err := data.DBVersion(snapshot)
	if err != nil {
		log.Panicf("Database inaccessible: %v", err)
	}

	// allow plugins (including this one) to start immediately on existing database
	// Note: this MUST have no tables as that is used as an indicator
	snap := &common.Snapshot{
		SnapshotInfo: apidInfo.LastSnapshot,
	}
	events.EmitWithCallback(ApigeeSyncEventSelector, snap, func(event apid.Event) {
		go pollForChanges()
	})

	log.Infof("Started on local snapshot: %s", snapshot)
}

// will keep retrying with backoff until success
func downloadSnapshot(scopes []string) common.Snapshot {

	log.Debug("downloadSnapshot")

	snapshotUri, err := url.Parse(config.GetString(configSnapServerBaseURI))
	if err != nil {
		log.Panicf("bad url value for config %s: %s", snapshotUri, err)
	}

	// getBearerToken loops until good
	getBearerToken()
	// todo: this could expire... ensure it's called again as needed

	/* Frame and send the snapshot request */
	snapshotUri.Path = path.Join(snapshotUri.Path, "snapshots")

	v := url.Values{}
	for _, scope := range scopes {
		v.Add("scope", scope)
	}
	snapshotUri.RawQuery = v.Encode()
	uri := snapshotUri.String()
	log.Infof("Snapshot Download: %s", uri)

	client := &http.Client{
		CheckRedirect: Redirect,
		Timeout:       httpTimeout,
	}

	retryIn := 5 * time.Millisecond
	maxBackOff := maxBackoffTimeout
	backOffFunc := createBackOff(retryIn, maxBackOff)
	first := true

	for {
		if first {
			first = false
		} else {
			backOffFunc()
		}

		req, err := http.NewRequest("GET", uri, nil)
		if err != nil {
			// should never happen, but if it does, it's unrecoverable anyway
			log.Panicf("Snapshotserver comm error: %v", err)
		}
		addHeaders(req)

		// Set the transport protocol type based on conf file input
		if config.GetString(configSnapshotProtocol) == "json" {
			req.Header.Set("Accept", "application/json")
		} else {
			req.Header.Set("Accept", "application/proto")
		}

		// Issue the request to the snapshot server
		r, err := client.Do(req)
		if err != nil {
			log.Errorf("Snapshotserver comm error: %v", err)
			continue
		}

		if r.StatusCode != 200 {
			log.Errorf("Snapshot server conn failed with resp code %d", r.StatusCode)
			r.Body.Close()
			continue
		}

		// Decode the Snapshot server response
		var resp common.Snapshot
		err = json.NewDecoder(r.Body).Decode(&resp)
		if err != nil {
			log.Errorf("JSON Response Data not parsable: %v", err)
			r.Body.Close()
			continue
		}

		r.Body.Close()
		return resp
	}
}

func addHeaders(req *http.Request) {
	req.Header.Add("Authorization", "Bearer "+token)
	req.Header.Set("apid_instance_id", apidInfo.InstanceID)
	req.Header.Set("apid_cluster_Id", apidInfo.ClusterID)
	req.Header.Set("updated_at_apid", time.Now().Format(time.RFC3339))
}

type apiError struct {
	Code string `json:"code"`
}

func (a apiError) Error() string {
	return a.Code
}
<|endoftext|>