<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package math

/*
	The algorithm is based in part on "Optimal Partitioning of
	Newton's Method for Calculating Roots", by Gunter Meinardus
	and G. D. Taylor, Mathematics of Computation © 1980 American
	Mathematical Society.
	(http://www.jstor.org/stable/2006387?seq=9, accessed 11-Feb-2010)
*/

// Cbrt returns the cube root of its argument.
//
// Special cases are:
//	Cbrt(±0) = ±0
//	Cbrt(±Inf) = ±Inf
//	Cbrt(NaN) = NaN
func Cbrt(x float64) float64 {
	const (
		A1 = 1.662848358e-01
		A2 = 1.096040958e+00
		A3 = 4.105032829e-01
		A4 = 5.649335816e-01
		B1 = 2.639607233e-01
		B2 = 8.699282849e-01
		B3 = 1.629083358e-01
		B4 = 2.824667908e-01
		C1 = 4.190115298e-01
		C2 = 6.904625373e-01
		C3 = 6.46502159e-02
		C4 = 1.412333954e-01
	)
	// TODO(rsc): Remove manual inlining of IsNaN, IsInf
	// when compiler does it for us
	// special cases
	switch {
	case x == 0 || x != x || x < -MaxFloat64 || x > MaxFloat64: // x == 0 || IsNaN(x) || IsInf(x, 0):
		return x
	}
	sign := false
	if x < 0 {
		x = -x
		sign = true
	}
	// Reduce argument
	f, e := Frexp(x)
	m := e % 3
	if m > 0 {
		m -= 3
		e -= m // e is multiple of 3
	}
	f = Ldexp(f, m) // 0.125 <= f < 1.0

	// Estimate cube root
	switch m {
	case 0: // 0.5 <= f < 1.0
		f = A1*f + A2 - A3/(A4+f)
	case -1: // 0.25 <= f < 0.5
		f = B1*f + B2 - B3/(B4+f)
	default: // 0.125 <= f < 0.25
		f = C1*f + C2 - C3/(C4+f)
	}
	y := Ldexp(f, e/3) // e/3 = exponent of cube root

	// Iterate
	s := y * y * y
	t := s + x
	y *= (t + x) / (s + t)
	// Reiterate
	s = (y*y*y - x) / x
	y -= y * (((14.0/81.0)*s-(2.0/9.0))*s + (1.0 / 3.0)) * s
	if sign {
		y = -y
	}
	return y
}
<commit_msg>math: faster Cbrt<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package math

/*
	The algorithm is based in part on "Optimal Partitioning of
	Newton's Method for Calculating Roots", by Gunter Meinardus
	and G. D. Taylor, Mathematics of Computation © 1980 American
	Mathematical Society.
	(http://www.jstor.org/stable/2006387?seq=9, accessed 11-Feb-2010)
*/

// Cbrt returns the cube root of its argument.
//
// Special cases are:
//	Cbrt(±0) = ±0
//	Cbrt(±Inf) = ±Inf
//	Cbrt(NaN) = NaN
func Cbrt(x float64) float64 {
	const (
		A1 = 1.662848358e-01
		A2 = 1.096040958e+00
		A3 = 4.105032829e-01
		A4 = 5.649335816e-01
		B1 = 2.639607233e-01
		B2 = 8.699282849e-01
		B3 = 1.629083358e-01
		B4 = 2.824667908e-01
		C1 = 4.190115298e-01
		C2 = 6.904625373e-01
		C3 = 6.46502159e-02
		C4 = 1.412333954e-01
	)
	// TODO(rsc): Remove manual inlining of IsNaN, IsInf
	// when compiler does it for us
	// special cases
	switch {
	case x == 0 || x != x || x < -MaxFloat64 || x > MaxFloat64: // x == 0 || IsNaN(x) || IsInf(x, 0):
		return x
	}
	sign := false
	if x < 0 {
		x = -x
		sign = true
	}
	// Reduce argument and estimate cube root
	f, e := Frexp(x) // 0.5 <= f < 1.0
	m := e % 3
	if m > 0 {
		m -= 3
		e -= m // e is multiple of 3
	}
	switch m {
	case 0: // 0.5 <= f < 1.0
		f = A1*f + A2 - A3/(A4+f)
	case -1:
		f *= 0.5 // 0.25 <= f < 0.5
		f = B1*f + B2 - B3/(B4+f)
	default: // m == -2
		f *= 0.25 // 0.125 <= f < 0.25
		f = C1*f + C2 - C3/(C4+f)
	}
	y := Ldexp(f, e/3) // e/3 = exponent of cube root

	// Iterate
	s := y * y * y
	t := s + x
	y *= (t + x) / (s + t)
	// Reiterate
	s = (y*y*y - x) / x
	y -= y * (((14.0/81.0)*s-(2.0/9.0))*s + (1.0 / 3.0)) * s
	if sign {
		y = -y
	}
	return y
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The path package implements utility routines for manipulating
// slash-separated filename paths.
package path

import (
	"io/ioutil"
	"os"
	"strings"
)

// Clean returns the shortest path name equivalent to path
// by purely lexical processing.  It applies the following rules
// iteratively until no further processing can be done:
//
//	1. Replace multiple slashes with a single slash.
//	2. Eliminate each . path name element (the current directory).
//	3. Eliminate each inner .. path name element (the parent directory)
//	   along with the non-.. element that precedes it.
//	4. Eliminate .. elements that begin a rooted path:
//	   that is, replace "/.." by "/" at the beginning of a path.
//
// If the result of this process is an empty string, Clean
// returns the string ".".
//
// See also Rob Pike, ``Lexical File Names in Plan 9 or
// Getting Dot-Dot right,''
// http://plan9.bell-labs.com/sys/doc/lexnames.html
func Clean(path string) string {
	if path == "" {
		return "."
	}

	rooted := path[0] == '/'
	n := len(path)

	// Invariants:
	//	reading from path; r is index of next byte to process.
	//	writing to buf; w is index of next byte to write.
	//	dotdot is index in buf where .. must stop, either because
	//		it is the leading slash or it is a leading ../../.. prefix.
	buf := []byte(path)
	r, w, dotdot := 0, 0, 0
	if rooted {
		r, w, dotdot = 1, 1, 1
	}

	for r < n {
		switch {
		case path[r] == '/':
			// empty path element
			r++
		case path[r] == '.' && (r+1 == n || path[r+1] == '/'):
			// . element
			r++
		case path[r] == '.' && path[r+1] == '.' && (r+2 == n || path[r+2] == '/'):
			// .. element: remove to last /
			r += 2
			switch {
			case w > dotdot:
				// can backtrack
				w--
				for w > dotdot && buf[w] != '/' {
					w--
				}
			case !rooted:
				// cannot backtrack, but not rooted, so append .. element.
				if w > 0 {
					buf[w] = '/'
					w++
				}
				buf[w] = '.'
				w++
				buf[w] = '.'
				w++
				dotdot = w
			}
		default:
			// real path element.
			// add slash if needed
			if rooted && w != 1 || !rooted && w != 0 {
				buf[w] = '/'
				w++
			}
			// copy element
			for ; r < n && path[r] != '/'; r++ {
				buf[w] = path[r]
				w++
			}
		}
	}

	// Turn empty string into "."
	if w == 0 {
		buf[w] = '.'
		w++
	}

	return string(buf[0:w])
}

// Split splits path immediately following the final slash,
// separating it into a directory and file name component.
// If there is no slash in path, Split returns an empty dir and
// file set to path.
func Split(path string) (dir, file string) {
	for i := len(path) - 1; i >= 0; i-- {
		if path[i] == '/' {
			return path[0 : i+1], path[i+1:]
		}
	}
	return "", path
}

// Join joins any number of path elements into a single path, adding a
// separating slash if necessary.  All empty strings are ignored.
func Join(elem ...string) string {
	for i, e := range elem {
		if e != "" {
			return Clean(strings.Join(elem[i:], "/"))
		}
	}
	return ""
}

// Ext returns the file name extension used by path.
// The extension is the suffix beginning at the final dot
// in the final slash-separated element of path;
// it is empty if there is no dot.
func Ext(path string) string {
	for i := len(path) - 1; i >= 0 && path[i] != '/'; i-- {
		if path[i] == '.' {
			return path[i:]
		}
	}
	return ""
}

// Visitor methods are invoked for corresponding file tree entries
// visited by Walk. The parameter path is the full path of d relative
// to root.
type Visitor interface {
	VisitDir(path string, f *os.FileInfo) bool
	VisitFile(path string, f *os.FileInfo)
}

func walk(path string, f *os.FileInfo, v Visitor, errors chan<- os.Error) {
	if !f.IsDirectory() {
		v.VisitFile(path, f)
		return
	}

	if !v.VisitDir(path, f) {
		return // skip directory entries
	}

	list, err := ioutil.ReadDir(path)
	if err != nil {
		if errors != nil {
			errors <- err
		}
	}

	for _, e := range list {
		walk(Join(path, e.Name), e, v, errors)
	}
}

// Walk walks the file tree rooted at root, calling v.VisitDir or
// v.VisitFile for each directory or file in the tree, including root.
// If v.VisitDir returns false, Walk skips the directory's entries;
// otherwise it invokes itself for each directory entry in sorted order.
// An error reading a directory does not abort the Walk.
// If errors != nil, Walk sends each directory read error
// to the channel.  Otherwise Walk discards the error.
func Walk(root string, v Visitor, errors chan<- os.Error) {
	f, err := os.Lstat(root)
	if err != nil {
		if errors != nil {
			errors <- err
		}
		return // can't progress
	}
	walk(root, f, v, errors)
}

// Base returns the last path element of the slash-separated name.
// Trailing slashes are removed before extracting the last element.  If the name is
// empty, "." is returned.  If it consists entirely of slashes, "/" is returned.
func Base(name string) string {
	if name == "" {
		return "."
	}
	// Strip trailing slashes.
	for len(name) > 0 && name[len(name)-1] == '/' {
		name = name[0 : len(name)-1]
	}
	// Find the last element
	if i := strings.LastIndex(name, "/"); i >= 0 {
		name = name[i+1:]
	}
	// If empty now, it had only slashes.
	if name == "" {
		return "/"
	}
	return name
}
<commit_msg>path: fix typo in path.Visitor<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The path package implements utility routines for manipulating
// slash-separated filename paths.
package path

import (
	"io/ioutil"
	"os"
	"strings"
)

// Clean returns the shortest path name equivalent to path
// by purely lexical processing.  It applies the following rules
// iteratively until no further processing can be done:
//
//	1. Replace multiple slashes with a single slash.
//	2. Eliminate each . path name element (the current directory).
//	3. Eliminate each inner .. path name element (the parent directory)
//	   along with the non-.. element that precedes it.
//	4. Eliminate .. elements that begin a rooted path:
//	   that is, replace "/.." by "/" at the beginning of a path.
//
// If the result of this process is an empty string, Clean
// returns the string ".".
//
// See also Rob Pike, ``Lexical File Names in Plan 9 or
// Getting Dot-Dot right,''
// http://plan9.bell-labs.com/sys/doc/lexnames.html
func Clean(path string) string {
	if path == "" {
		return "."
	}

	rooted := path[0] == '/'
	n := len(path)

	// Invariants:
	//	reading from path; r is index of next byte to process.
	//	writing to buf; w is index of next byte to write.
	//	dotdot is index in buf where .. must stop, either because
	//		it is the leading slash or it is a leading ../../.. prefix.
	buf := []byte(path)
	r, w, dotdot := 0, 0, 0
	if rooted {
		r, w, dotdot = 1, 1, 1
	}

	for r < n {
		switch {
		case path[r] == '/':
			// empty path element
			r++
		case path[r] == '.' && (r+1 == n || path[r+1] == '/'):
			// . element
			r++
		case path[r] == '.' && path[r+1] == '.' && (r+2 == n || path[r+2] == '/'):
			// .. element: remove to last /
			r += 2
			switch {
			case w > dotdot:
				// can backtrack
				w--
				for w > dotdot && buf[w] != '/' {
					w--
				}
			case !rooted:
				// cannot backtrack, but not rooted, so append .. element.
				if w > 0 {
					buf[w] = '/'
					w++
				}
				buf[w] = '.'
				w++
				buf[w] = '.'
				w++
				dotdot = w
			}
		default:
			// real path element.
			// add slash if needed
			if rooted && w != 1 || !rooted && w != 0 {
				buf[w] = '/'
				w++
			}
			// copy element
			for ; r < n && path[r] != '/'; r++ {
				buf[w] = path[r]
				w++
			}
		}
	}

	// Turn empty string into "."
	if w == 0 {
		buf[w] = '.'
		w++
	}

	return string(buf[0:w])
}

// Split splits path immediately following the final slash,
// separating it into a directory and file name component.
// If there is no slash in path, Split returns an empty dir and
// file set to path.
func Split(path string) (dir, file string) {
	for i := len(path) - 1; i >= 0; i-- {
		if path[i] == '/' {
			return path[0 : i+1], path[i+1:]
		}
	}
	return "", path
}

// Join joins any number of path elements into a single path, adding a
// separating slash if necessary.  All empty strings are ignored.
func Join(elem ...string) string {
	for i, e := range elem {
		if e != "" {
			return Clean(strings.Join(elem[i:], "/"))
		}
	}
	return ""
}

// Ext returns the file name extension used by path.
// The extension is the suffix beginning at the final dot
// in the final slash-separated element of path;
// it is empty if there is no dot.
func Ext(path string) string {
	for i := len(path) - 1; i >= 0 && path[i] != '/'; i-- {
		if path[i] == '.' {
			return path[i:]
		}
	}
	return ""
}

// Visitor methods are invoked for corresponding file tree entries
// visited by Walk. The parameter path is the full path of f relative
// to root.
type Visitor interface {
	VisitDir(path string, f *os.FileInfo) bool
	VisitFile(path string, f *os.FileInfo)
}

func walk(path string, f *os.FileInfo, v Visitor, errors chan<- os.Error) {
	if !f.IsDirectory() {
		v.VisitFile(path, f)
		return
	}

	if !v.VisitDir(path, f) {
		return // skip directory entries
	}

	list, err := ioutil.ReadDir(path)
	if err != nil {
		if errors != nil {
			errors <- err
		}
	}

	for _, e := range list {
		walk(Join(path, e.Name), e, v, errors)
	}
}

// Walk walks the file tree rooted at root, calling v.VisitDir or
// v.VisitFile for each directory or file in the tree, including root.
// If v.VisitDir returns false, Walk skips the directory's entries;
// otherwise it invokes itself for each directory entry in sorted order.
// An error reading a directory does not abort the Walk.
// If errors != nil, Walk sends each directory read error
// to the channel.  Otherwise Walk discards the error.
func Walk(root string, v Visitor, errors chan<- os.Error) {
	f, err := os.Lstat(root)
	if err != nil {
		if errors != nil {
			errors <- err
		}
		return // can't progress
	}
	walk(root, f, v, errors)
}

// Base returns the last path element of the slash-separated name.
// Trailing slashes are removed before extracting the last element.  If the name is
// empty, "." is returned.  If it consists entirely of slashes, "/" is returned.
func Base(name string) string {
	if name == "" {
		return "."
	}
	// Strip trailing slashes.
	for len(name) > 0 && name[len(name)-1] == '/' {
		name = name[0 : len(name)-1]
	}
	// Find the last element
	if i := strings.LastIndex(name, "/"); i >= 0 {
		name = name[i+1:]
	}
	// If empty now, it had only slashes.
	if name == "" {
		return "/"
	}
	return name
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package time

import (
	"os"
	"sync"
)

// A Ticker holds a synchronous channel that delivers `ticks' of a clock
// at intervals.
type Ticker struct {
	C        <-chan int64 // The channel on which the ticks are delivered.
	c        chan<- int64 // The same channel, but the end we use.
	ns       int64
	shutdown chan bool // Buffered channel used to signal shutdown.
	nextTick int64
	next     *Ticker
}

// Stop turns off a ticker.  After Stop, no more ticks will be sent.
func (t *Ticker) Stop() {
	// Make it non-blocking so multiple Stops don't block.
	_ = t.shutdown <- true
}

// Tick is a convenience wrapper for NewTicker providing access to the ticking
// channel only.  Useful for clients that have no need to shut down the ticker.
func Tick(ns int64) <-chan int64 {
	if ns <= 0 {
		return nil
	}
	return NewTicker(ns).C
}

type alarmer struct {
	wakeUp   chan bool // wakeup signals sent/received here
	wakeMeAt chan int64
	wakeTime int64
}

// Set alarm to go off at time ns, if not already set earlier.
func (a *alarmer) set(ns int64) {
	switch {
	case a.wakeTime > ns:
		// Next tick we expect is too late; shut down the late runner
		// and (after fallthrough) start a new wakeLoop.
		a.wakeMeAt <- -1
		fallthrough
	case a.wakeMeAt == nil:
		// There's no wakeLoop, start one.
		a.wakeMeAt = make(chan int64, 10)
		go wakeLoop(a.wakeMeAt, a.wakeUp)
		fallthrough
	case a.wakeTime == 0:
		// Nobody else is waiting; it's just us.
		a.wakeTime = ns
		a.wakeMeAt <- ns
	default:
		// There's already someone scheduled.
	}
}

// Channel to notify tickerLoop of new Tickers being created.
var newTicker chan *Ticker

func startTickerLoop() {
	newTicker = make(chan *Ticker)
	go tickerLoop()
}

// wakeLoop delivers ticks at scheduled times, sleeping until the right moment.
// If another, earlier Ticker is created while it sleeps, tickerLoop() will start a new
// wakeLoop but they will share the wakeUp channel and signal that this one
// is done by giving it a negative time request.
func wakeLoop(wakeMeAt chan int64, wakeUp chan bool) {
	for {
		wakeAt := <-wakeMeAt
		if wakeAt < 0 { // tickerLoop has started another wakeLoop
			return
		}
		now := Nanoseconds()
		if wakeAt > now {
			Sleep(wakeAt - now)
			now = Nanoseconds()
		}
		wakeUp <- true
	}
}

// A single tickerLoop serves all ticks to Tickers.  It waits for two events:
// either the creation of a new Ticker or a tick from the alarm,
// signalling a time to wake up one or more Tickers.
func tickerLoop() {
	// Represents the next alarm to be delivered.
	var alarm alarmer
	// All wakeLoops deliver wakeups to this channel.
	alarm.wakeUp = make(chan bool, 10)
	var now, prevTime, wakeTime int64
	var tickers *Ticker
	for {
		select {
		case t := <-newTicker:
			// Add Ticker to list
			t.next = tickers
			tickers = t
			// Arrange for a new alarm if this one precedes the existing one.
			alarm.set(t.nextTick)
		case <-alarm.wakeUp:
			now = Nanoseconds()
			// Ignore an old time due to a dying wakeLoop
			if now < prevTime {
				continue
			}
			wakeTime = now + 1e15 // very long in the future
			var prev *Ticker = nil
			// Scan list of tickers, delivering updates to those
			// that need it and determining the next wake time.
			// TODO(r): list should be sorted in time order.
			for t := tickers; t != nil; t = t.next {
				if _, ok := <-t.shutdown; ok {
					// Ticker is done; remove it from list.
					if prev == nil {
						tickers = t.next
					} else {
						prev.next = t.next
					}
					continue
				}
				if t.nextTick <= now {
					if len(t.c) == 0 {
						// Only send if there's room.  We must not block.
						// The channel is allocated with a one-element
						// buffer, which is sufficient: if he hasn't picked
						// up the last tick, no point in sending more.
						t.c <- now
					}
					t.nextTick += t.ns
					if t.nextTick <= now {
						// Still behind; advance in one big step.
						t.nextTick += (now - t.nextTick + t.ns) / t.ns * t.ns
					}
				}
				if t.nextTick < wakeTime {
					wakeTime = t.nextTick
				}
				prev = t
			}
			if tickers != nil {
				// Please send wakeup at earliest required time.
				// If there are no tickers, don't bother.
				alarm.wakeMeAt <- wakeTime
			} else {
				alarm.wakeTime = 0
			}
		}
		prevTime = now
	}
}

var onceStartTickerLoop sync.Once

// NewTicker returns a new Ticker containing a channel that will
// send the time, in nanoseconds, every ns nanoseconds.  It adjusts the
// intervals to make up for pauses in delivery of the ticks. The value of
// ns must be greater than zero; if not, NewTicker will panic.
func NewTicker(ns int64) *Ticker {
	if ns <= 0 {
		panic(os.ErrorString("non-positive interval for NewTicker"))
	}
	c := make(chan int64, 1) //  See comment on send in tickerLoop
	t := &Ticker{
		C:        c,
		c:        c,
		ns:       ns,
		shutdown: make(chan bool, 1),
		nextTick: Nanoseconds() + ns,
	}
	onceStartTickerLoop.Do(startTickerLoop)
	// must be run in background so global Tickers can be created
	go func() { newTicker <- t }()
	return t
}
<commit_msg>time: fix tick accuracy when using multiple Tickers<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package time

import (
	"os"
	"sync"
)

// A Ticker holds a synchronous channel that delivers `ticks' of a clock
// at intervals.
type Ticker struct {
	C        <-chan int64 // The channel on which the ticks are delivered.
	c        chan<- int64 // The same channel, but the end we use.
	ns       int64
	shutdown chan bool // Buffered channel used to signal shutdown.
	nextTick int64
	next     *Ticker
}

// Stop turns off a ticker.  After Stop, no more ticks will be sent.
func (t *Ticker) Stop() {
	// Make it non-blocking so multiple Stops don't block.
	_ = t.shutdown <- true
}

// Tick is a convenience wrapper for NewTicker providing access to the ticking
// channel only.  Useful for clients that have no need to shut down the ticker.
func Tick(ns int64) <-chan int64 {
	if ns <= 0 {
		return nil
	}
	return NewTicker(ns).C
}

type alarmer struct {
	wakeUp   chan bool // wakeup signals sent/received here
	wakeMeAt chan int64
	wakeTime int64
}

// Set alarm to go off at time ns, if not already set earlier.
func (a *alarmer) set(ns int64) {
	switch {
	case a.wakeTime > ns:
		// Next tick we expect is too late; shut down the late runner
		// and (after fallthrough) start a new wakeLoop.
		close(a.wakeMeAt)
		fallthrough
	case a.wakeMeAt == nil:
		// There's no wakeLoop, start one.
		a.wakeMeAt = make(chan int64)
		a.wakeUp = make(chan bool, 1)
		go wakeLoop(a.wakeMeAt, a.wakeUp)
		fallthrough
	case a.wakeTime == 0:
		// Nobody else is waiting; it's just us.
		a.wakeTime = ns
		a.wakeMeAt <- ns
	default:
		// There's already someone scheduled.
	}
}

// Channel to notify tickerLoop of new Tickers being created.
var newTicker chan *Ticker

func startTickerLoop() {
	newTicker = make(chan *Ticker)
	go tickerLoop()
}

// wakeLoop delivers ticks at scheduled times, sleeping until the right moment.
// If another, earlier Ticker is created while it sleeps, tickerLoop() will start a new
// wakeLoop and signal that this one is done by closing the wakeMeAt channel.
func wakeLoop(wakeMeAt chan int64, wakeUp chan bool) {
	for wakeAt := range wakeMeAt {
		Sleep(wakeAt - Nanoseconds())
		wakeUp <- true
	}
}

// A single tickerLoop serves all ticks to Tickers.  It waits for two events:
// either the creation of a new Ticker or a tick from the alarm,
// signalling a time to wake up one or more Tickers.
func tickerLoop() {
	// Represents the next alarm to be delivered.
	var alarm alarmer
	var now, wakeTime int64
	var tickers *Ticker
	for {
		select {
		case t := <-newTicker:
			// Add Ticker to list
			t.next = tickers
			tickers = t
			// Arrange for a new alarm if this one precedes the existing one.
			alarm.set(t.nextTick)
		case <-alarm.wakeUp:
			now = Nanoseconds()
			wakeTime = now + 1e15 // very long in the future
			var prev *Ticker = nil
			// Scan list of tickers, delivering updates to those
			// that need it and determining the next wake time.
			// TODO(r): list should be sorted in time order.
			for t := tickers; t != nil; t = t.next {
				if _, ok := <-t.shutdown; ok {
					// Ticker is done; remove it from list.
					if prev == nil {
						tickers = t.next
					} else {
						prev.next = t.next
					}
					continue
				}
				if t.nextTick <= now {
					if len(t.c) == 0 {
						// Only send if there's room.  We must not block.
						// The channel is allocated with a one-element
						// buffer, which is sufficient: if he hasn't picked
						// up the last tick, no point in sending more.
						t.c <- now
					}
					t.nextTick += t.ns
					if t.nextTick <= now {
						// Still behind; advance in one big step.
						t.nextTick += (now - t.nextTick + t.ns) / t.ns * t.ns
					}
				}
				if t.nextTick < wakeTime {
					wakeTime = t.nextTick
				}
				prev = t
			}
			if tickers != nil {
				// Please send wakeup at earliest required time.
				// If there are no tickers, don't bother.
				alarm.wakeTime = wakeTime
				alarm.wakeMeAt <- wakeTime
			} else {
				alarm.wakeTime = 0
			}
		}
	}
}

var onceStartTickerLoop sync.Once

// NewTicker returns a new Ticker containing a channel that will
// send the time, in nanoseconds, every ns nanoseconds.  It adjusts the
// intervals to make up for pauses in delivery of the ticks. The value of
// ns must be greater than zero; if not, NewTicker will panic.
func NewTicker(ns int64) *Ticker {
	if ns <= 0 {
		panic(os.ErrorString("non-positive interval for NewTicker"))
	}
	c := make(chan int64, 1) //  See comment on send in tickerLoop
	t := &Ticker{
		C:        c,
		c:        c,
		ns:       ns,
		shutdown: make(chan bool, 1),
		nextTick: Nanoseconds() + ns,
	}
	onceStartTickerLoop.Do(startTickerLoop)
	// must be run in background so global Tickers can be created
	go func() { newTicker <- t }()
	return t
}
<|endoftext|>


<commit_before>// Application that captures SKPs from CT's webpage archives.
package main

import (
	"encoding/csv"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"sync"
	"time"

	"github.com/skia-dev/glog"

	"go.skia.org/infra/ct/go/util"
	"go.skia.org/infra/ct/go/worker_scripts/worker_common"
	"go.skia.org/infra/go/common"
	skutil "go.skia.org/infra/go/util"
)

const (
	// The number of goroutines that will run in parallel to capture SKPs.
	WORKER_POOL_SIZE = 10
)

var (
	workerNum          = flag.Int("worker_num", 1, "The number of this CT worker. It will be in the {1..100} range.")
	pagesetType        = flag.String("pageset_type", util.PAGESET_TYPE_MOBILE_10k, "The type of pagesets to create SKPs from. Eg: 10k, Mobile10k, All.")
	chromiumBuild      = flag.String("chromium_build", "", "The chromium build that will be used to create the SKPs.")
	runID              = flag.String("run_id", "", "The unique run id (typically requester + timestamp).")
	targetPlatform     = flag.String("target_platform", util.PLATFORM_LINUX, "The platform the benchmark will run on (Android / Linux).")
	chromeCleanerTimer = flag.Duration("cleaner_timer", 30*time.Minute, "How often all chrome processes will be killed on this slave.")
)

func main() {
	defer common.LogPanic()
	worker_common.Init()
	if !*worker_common.Local {
		defer util.CleanTmpDir()
	}
	defer util.TimeTrack(time.Now(), "Capturing SKPs")
	defer glog.Flush()

	// Validate required arguments.
	if *chromiumBuild == "" {
		glog.Error("Must specify --chromium_build")
		return
	}
	if *runID == "" {
		glog.Error("Must specify --run_id")
		return
	}
	if *targetPlatform == util.PLATFORM_ANDROID {
		glog.Error("Android is not yet supported for capturing SKPs.")
		return
	}

	// Reset the local chromium checkout.
	if err := util.ResetCheckout(util.ChromiumSrcDir); err != nil {
		glog.Errorf("Could not reset %s: %s", util.ChromiumSrcDir, err)
		return
	}
	// Sync the local chromium checkout.
	if err := util.SyncDir(util.ChromiumSrcDir); err != nil {
		glog.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err)
		return
	}

	// Create the task file so that the master knows this worker is still busy.
	skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_CAPTURING_SKPS))
	defer util.DeleteTaskFile(util.ACTIVITY_CAPTURING_SKPS)

	// Instantiate GsUtil object.
	gs, err := util.NewGsUtil(nil)
	if err != nil {
		glog.Error(err)
		return
	}

	// Download the specified chromium build.
	if err := gs.DownloadChromiumBuild(*chromiumBuild); err != nil {
		glog.Error(err)
		return
	}
	// Delete the chromium build to save space when we are done.
	defer skutil.RemoveAll(filepath.Join(util.ChromiumBuildsDir, *chromiumBuild))
	chromiumBinary := filepath.Join(util.ChromiumBuildsDir, *chromiumBuild, util.BINARY_CHROME)
	if *targetPlatform == util.PLATFORM_ANDROID {
		// Install the APK on the Android device.
		if err := util.InstallChromeAPK(*chromiumBuild); err != nil {
			glog.Errorf("Could not install the chromium APK: %s", err)
			return
		}
	}

	// Download pagesets if they do not exist locally.
	if err := gs.DownloadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil {
		glog.Error(err)
		return
	}
	pathToPagesets := filepath.Join(util.PagesetsDir, *pagesetType)

	// Download archives if they do not exist locally.
	if err := gs.DownloadWorkerArtifacts(util.WEB_ARCHIVES_DIR_NAME, *pagesetType, *workerNum); err != nil {
		glog.Error(err)
		return
	}

	// Create the dir that SKPs will be stored in.
	pathToSkps := filepath.Join(util.SkpsDir, *pagesetType, *chromiumBuild)
	// Delete and remake the local SKPs directory.
	skutil.RemoveAll(pathToSkps)
	skutil.MkdirAll(pathToSkps, 0700)

	// Establish output paths.
	localOutputDir := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, *runID)
	skutil.RemoveAll(localOutputDir)
	skutil.MkdirAll(localOutputDir, 0700)
	defer skutil.RemoveAll(localOutputDir)

	// Construct path to the ct_run_benchmark python script.
	_, currentFile, _, _ := runtime.Caller(0)
	pathToPyFiles := filepath.Join(
		filepath.Dir((filepath.Dir(filepath.Dir(filepath.Dir(currentFile))))),
		"py")

	timeoutSecs := util.PagesetTypeToInfo[*pagesetType].CaptureSKPsTimeoutSecs
	fileInfos, err := ioutil.ReadDir(pathToPagesets)
	if err != nil {
		glog.Errorf("Unable to read the pagesets dir %s: %s", pathToPagesets, err)
		return
	}

	// Create channel that contains all pageset file names. This channel will
	// be consumed by the worker pool.
	pagesetRequests := util.GetClosedChannelOfPagesets(fileInfos)

	var wg sync.WaitGroup
	// Use a RWMutex for the chromeProcessesCleaner goroutine to communicate to
	// the workers (acting as "readers") when it wants to be the "writer" and
	// kill all zombie chrome processes.
	var mutex sync.RWMutex

	// Loop through workers in the worker pool.
	for i := 0; i < WORKER_POOL_SIZE; i++ {
		// Increment the WaitGroup counter.
		wg.Add(1)

		// Create and run a goroutine closure that captures SKPs.
		go func() {
			// Decrement the WaitGroup counter when the goroutine completes.
			defer wg.Done()

			for pagesetName := range pagesetRequests {

				mutex.RLock()

				// Read the pageset.
				pagesetPath := filepath.Join(pathToPagesets, pagesetName)
				decodedPageset, err := util.ReadPageset(pagesetPath)
				if err != nil {
					glog.Errorf("Could not read %s: %s", pagesetPath, err)
					continue
				}

				glog.Infof("===== Processing %s =====", pagesetPath)

				skutil.LogErr(os.Chdir(pathToPyFiles))
				args := []string{
					filepath.Join(util.TelemetryBinariesDir, util.BINARY_RUN_BENCHMARK),
					util.BenchmarksToTelemetryName[util.BENCHMARK_SKPICTURE_PRINTER],
					"--also-run-disabled-tests",
					"--page-repeat=1", // Only need one run for SKPs.
					"--skp-outdir=" + pathToSkps,
					"--extra-browser-args=" + util.DEFAULT_BROWSER_ARGS,
					"--user-agent=" + decodedPageset.UserAgent,
					"--urls-list=" + decodedPageset.UrlsList,
					"--archive-data-file=" + decodedPageset.ArchiveDataFile,
				}
				// Figure out which browser should be used.
				if *targetPlatform == util.PLATFORM_ANDROID {
					args = append(args, "--browser=android-chromium")
				} else {
					args = append(args, "--browser=exact", "--browser-executable="+chromiumBinary)
				}
				// Set the PYTHONPATH to the pagesets and the telemetry dirs.
				env := []string{
					fmt.Sprintf("PYTHONPATH=%s:%s:%s:%s:$PYTHONPATH", pathToPagesets, util.TelemetryBinariesDir, util.TelemetrySrcDir, util.CatapultSrcDir),
					"DISPLAY=:0",
				}
				skutil.LogErr(
					util.ExecuteCmd("python", args, env, time.Duration(timeoutSecs)*time.Second, nil, nil))

				mutex.RUnlock()

			}
		}()
	}

	if !*worker_common.Local {
		// Start the cleaner.
		go util.ChromeProcessesCleaner(&mutex, *chromeCleanerTimer)
	}

	// Wait for all spawned goroutines to complete.
	wg.Wait()

	// Move, validate and upload all SKP files.
	// List all directories in pathToSkps and copy out the skps.
	skpFileInfos, err := ioutil.ReadDir(pathToSkps)
	if err != nil {
		glog.Errorf("Unable to read %s: %s", pathToSkps, err)
		return
	}
	for _, fileInfo := range skpFileInfos {
		if !fileInfo.IsDir() {
			// We are only interested in directories.
			continue
		}
		skpName := fileInfo.Name()
		// Find the largest layer in this directory.
		layerInfos, err := ioutil.ReadDir(filepath.Join(pathToSkps, skpName))
		if err != nil {
			glog.Errorf("Unable to read %s: %s", filepath.Join(pathToSkps, skpName), err)
		}
		if len(layerInfos) > 0 {
			largestLayerInfo := layerInfos[0]
			for _, layerInfo := range layerInfos {
				fmt.Println(layerInfo.Size())
				if layerInfo.Size() > largestLayerInfo.Size() {
					largestLayerInfo = layerInfo
				}
			}
			// Only save SKPs greater than 6000 bytes. Less than that are probably
			// malformed.
			if largestLayerInfo.Size() > 6000 {
				layerPath := filepath.Join(pathToSkps, skpName, largestLayerInfo.Name())
				skutil.Rename(layerPath, filepath.Join(pathToSkps, skpName+".skp"))
			} else {
				glog.Warningf("Skipping %s because size was less than 5000 bytes", skpName)
			}
		}
		// We extracted what we needed from the directory, now delete it.
		skutil.RemoveAll(filepath.Join(pathToSkps, skpName))
	}

	glog.Info("Calling remove_invalid_skps.py")
	// Sync Skia tree.
	skutil.LogErr(util.SyncDir(util.SkiaTreeDir))
	// Build tools.
	skutil.LogErr(util.BuildSkiaTools())
	// Run remove_invalid_skps.py
	pathToRemoveSKPs := filepath.Join(pathToPyFiles, "remove_invalid_skps.py")
	pathToSKPInfo := filepath.Join(util.SkiaTreeDir, "out", "Release", "skpinfo")
	args := []string{
		pathToRemoveSKPs,
		"--skp_dir=" + pathToSkps,
		"--path_to_skpinfo=" + pathToSKPInfo,
	}
	skutil.LogErr(util.ExecuteCmd("python", args, []string{}, util.REMOVE_INVALID_SKPS_TIMEOUT,
		nil, nil))

	// Write timestamp to the SKPs dir.
	skutil.LogErr(util.CreateTimestampFile(pathToSkps))

	// Upload SKPs dir to Google Storage.
	if err := gs.UploadWorkerArtifacts(util.SKPS_DIR_NAME, filepath.Join(*pagesetType, *chromiumBuild), *workerNum); err != nil {
		glog.Error(err)
		return
	}
}

func getRowsFromCSV(csvPath string) ([]string, []string, error) {
	csvFile, err := os.Open(csvPath)
	defer skutil.Close(csvFile)
	if err != nil {
		return nil, nil, fmt.Errorf("Could not open %s: %s", csvPath, err)
	}
	reader := csv.NewReader(csvFile)
	reader.FieldsPerRecord = -1
	rawCSVdata, err := reader.ReadAll()
	if err != nil {
		return nil, nil, fmt.Errorf("Could not read %s: %s", csvPath, err)
	}
	if len(rawCSVdata) != 2 {
		return nil, nil, fmt.Errorf("No data in %s", csvPath)
	}
	return rawCSVdata[0], rawCSVdata[1], nil
}

func writeRowsToCSV(csvPath string, headers, values []string) error {
	csvFile, err := os.OpenFile(csvPath, os.O_WRONLY, 666)
	defer skutil.Close(csvFile)
	if err != nil {
		return fmt.Errorf("Could not open %s: %s", csvPath, err)
	}
	writer := csv.NewWriter(csvFile)
	defer writer.Flush()
	for _, row := range [][]string{headers, values} {
		if err := writer.Write(row); err != nil {
			return fmt.Errorf("Could not write to %s: %s", csvPath, err)
		}
	}
	return nil
}
<commit_msg>[CT] Temporarily use "skip-reencoding-on-skp-capture" flag when capturing SKPs<commit_after>// Application that captures SKPs from CT's webpage archives.
package main

import (
	"encoding/csv"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"sync"
	"time"

	"github.com/skia-dev/glog"

	"go.skia.org/infra/ct/go/util"
	"go.skia.org/infra/ct/go/worker_scripts/worker_common"
	"go.skia.org/infra/go/common"
	skutil "go.skia.org/infra/go/util"
)

const (
	// The number of goroutines that will run in parallel to capture SKPs.
	WORKER_POOL_SIZE = 10
)

var (
	workerNum          = flag.Int("worker_num", 1, "The number of this CT worker. It will be in the {1..100} range.")
	pagesetType        = flag.String("pageset_type", util.PAGESET_TYPE_MOBILE_10k, "The type of pagesets to create SKPs from. Eg: 10k, Mobile10k, All.")
	chromiumBuild      = flag.String("chromium_build", "", "The chromium build that will be used to create the SKPs.")
	runID              = flag.String("run_id", "", "The unique run id (typically requester + timestamp).")
	targetPlatform     = flag.String("target_platform", util.PLATFORM_LINUX, "The platform the benchmark will run on (Android / Linux).")
	chromeCleanerTimer = flag.Duration("cleaner_timer", 30*time.Minute, "How often all chrome processes will be killed on this slave.")
)

func main() {
	defer common.LogPanic()
	worker_common.Init()
	if !*worker_common.Local {
		defer util.CleanTmpDir()
	}
	defer util.TimeTrack(time.Now(), "Capturing SKPs")
	defer glog.Flush()

	// Validate required arguments.
	if *chromiumBuild == "" {
		glog.Error("Must specify --chromium_build")
		return
	}
	if *runID == "" {
		glog.Error("Must specify --run_id")
		return
	}
	if *targetPlatform == util.PLATFORM_ANDROID {
		glog.Error("Android is not yet supported for capturing SKPs.")
		return
	}

	// Reset the local chromium checkout.
	if err := util.ResetCheckout(util.ChromiumSrcDir); err != nil {
		glog.Errorf("Could not reset %s: %s", util.ChromiumSrcDir, err)
		return
	}
	// Sync the local chromium checkout.
	if err := util.SyncDir(util.ChromiumSrcDir); err != nil {
		glog.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err)
		return
	}

	// Create the task file so that the master knows this worker is still busy.
	skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_CAPTURING_SKPS))
	defer util.DeleteTaskFile(util.ACTIVITY_CAPTURING_SKPS)

	// Instantiate GsUtil object.
	gs, err := util.NewGsUtil(nil)
	if err != nil {
		glog.Error(err)
		return
	}

	// Download the specified chromium build.
	if err := gs.DownloadChromiumBuild(*chromiumBuild); err != nil {
		glog.Error(err)
		return
	}
	// Delete the chromium build to save space when we are done.
	defer skutil.RemoveAll(filepath.Join(util.ChromiumBuildsDir, *chromiumBuild))
	chromiumBinary := filepath.Join(util.ChromiumBuildsDir, *chromiumBuild, util.BINARY_CHROME)
	if *targetPlatform == util.PLATFORM_ANDROID {
		// Install the APK on the Android device.
		if err := util.InstallChromeAPK(*chromiumBuild); err != nil {
			glog.Errorf("Could not install the chromium APK: %s", err)
			return
		}
	}

	// Download pagesets if they do not exist locally.
	if err := gs.DownloadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil {
		glog.Error(err)
		return
	}
	pathToPagesets := filepath.Join(util.PagesetsDir, *pagesetType)

	// Download archives if they do not exist locally.
	if err := gs.DownloadWorkerArtifacts(util.WEB_ARCHIVES_DIR_NAME, *pagesetType, *workerNum); err != nil {
		glog.Error(err)
		return
	}

	// Create the dir that SKPs will be stored in.
	pathToSkps := filepath.Join(util.SkpsDir, *pagesetType, *chromiumBuild)
	// Delete and remake the local SKPs directory.
	skutil.RemoveAll(pathToSkps)
	skutil.MkdirAll(pathToSkps, 0700)

	// Establish output paths.
	localOutputDir := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, *runID)
	skutil.RemoveAll(localOutputDir)
	skutil.MkdirAll(localOutputDir, 0700)
	defer skutil.RemoveAll(localOutputDir)

	// Construct path to the ct_run_benchmark python script.
	_, currentFile, _, _ := runtime.Caller(0)
	pathToPyFiles := filepath.Join(
		filepath.Dir((filepath.Dir(filepath.Dir(filepath.Dir(currentFile))))),
		"py")

	timeoutSecs := util.PagesetTypeToInfo[*pagesetType].CaptureSKPsTimeoutSecs
	fileInfos, err := ioutil.ReadDir(pathToPagesets)
	if err != nil {
		glog.Errorf("Unable to read the pagesets dir %s: %s", pathToPagesets, err)
		return
	}

	// Create channel that contains all pageset file names. This channel will
	// be consumed by the worker pool.
	pagesetRequests := util.GetClosedChannelOfPagesets(fileInfos)

	var wg sync.WaitGroup
	// Use a RWMutex for the chromeProcessesCleaner goroutine to communicate to
	// the workers (acting as "readers") when it wants to be the "writer" and
	// kill all zombie chrome processes.
	var mutex sync.RWMutex

	// Loop through workers in the worker pool.
	for i := 0; i < WORKER_POOL_SIZE; i++ {
		// Increment the WaitGroup counter.
		wg.Add(1)

		// Create and run a goroutine closure that captures SKPs.
		go func() {
			// Decrement the WaitGroup counter when the goroutine completes.
			defer wg.Done()

			for pagesetName := range pagesetRequests {

				mutex.RLock()

				// Read the pageset.
				pagesetPath := filepath.Join(pathToPagesets, pagesetName)
				decodedPageset, err := util.ReadPageset(pagesetPath)
				if err != nil {
					glog.Errorf("Could not read %s: %s", pagesetPath, err)
					continue
				}

				glog.Infof("===== Processing %s =====", pagesetPath)

				skutil.LogErr(os.Chdir(pathToPyFiles))
				args := []string{
					filepath.Join(util.TelemetryBinariesDir, util.BINARY_RUN_BENCHMARK),
					util.BenchmarksToTelemetryName[util.BENCHMARK_SKPICTURE_PRINTER],
					"--also-run-disabled-tests",
					"--page-repeat=1", // Only need one run for SKPs.
					"--skp-outdir=" + pathToSkps,
					"--extra-browser-args=--skip-reencoding-on-skp-capture " + util.DEFAULT_BROWSER_ARGS,
					"--user-agent=" + decodedPageset.UserAgent,
					"--urls-list=" + decodedPageset.UrlsList,
					"--archive-data-file=" + decodedPageset.ArchiveDataFile,
				}
				// Figure out which browser should be used.
				if *targetPlatform == util.PLATFORM_ANDROID {
					args = append(args, "--browser=android-chromium")
				} else {
					args = append(args, "--browser=exact", "--browser-executable="+chromiumBinary)
				}
				// Set the PYTHONPATH to the pagesets and the telemetry dirs.
				env := []string{
					fmt.Sprintf("PYTHONPATH=%s:%s:%s:%s:$PYTHONPATH", pathToPagesets, util.TelemetryBinariesDir, util.TelemetrySrcDir, util.CatapultSrcDir),
					"DISPLAY=:0",
				}
				skutil.LogErr(
					util.ExecuteCmd("python", args, env, time.Duration(timeoutSecs)*time.Second, nil, nil))

				mutex.RUnlock()

			}
		}()
	}

	if !*worker_common.Local {
		// Start the cleaner.
		go util.ChromeProcessesCleaner(&mutex, *chromeCleanerTimer)
	}

	// Wait for all spawned goroutines to complete.
	wg.Wait()

	// Move, validate and upload all SKP files.
	// List all directories in pathToSkps and copy out the skps.
	skpFileInfos, err := ioutil.ReadDir(pathToSkps)
	if err != nil {
		glog.Errorf("Unable to read %s: %s", pathToSkps, err)
		return
	}
	for _, fileInfo := range skpFileInfos {
		if !fileInfo.IsDir() {
			// We are only interested in directories.
			continue
		}
		skpName := fileInfo.Name()
		// Find the largest layer in this directory.
		layerInfos, err := ioutil.ReadDir(filepath.Join(pathToSkps, skpName))
		if err != nil {
			glog.Errorf("Unable to read %s: %s", filepath.Join(pathToSkps, skpName), err)
		}
		if len(layerInfos) > 0 {
			largestLayerInfo := layerInfos[0]
			for _, layerInfo := range layerInfos {
				fmt.Println(layerInfo.Size())
				if layerInfo.Size() > largestLayerInfo.Size() {
					largestLayerInfo = layerInfo
				}
			}
			// Only save SKPs greater than 6000 bytes. Less than that are probably
			// malformed.
			if largestLayerInfo.Size() > 6000 {
				layerPath := filepath.Join(pathToSkps, skpName, largestLayerInfo.Name())
				skutil.Rename(layerPath, filepath.Join(pathToSkps, skpName+".skp"))
			} else {
				glog.Warningf("Skipping %s because size was less than 5000 bytes", skpName)
			}
		}
		// We extracted what we needed from the directory, now delete it.
		skutil.RemoveAll(filepath.Join(pathToSkps, skpName))
	}

	glog.Info("Calling remove_invalid_skps.py")
	// Sync Skia tree.
	skutil.LogErr(util.SyncDir(util.SkiaTreeDir))
	// Build tools.
	skutil.LogErr(util.BuildSkiaTools())
	// Run remove_invalid_skps.py
	pathToRemoveSKPs := filepath.Join(pathToPyFiles, "remove_invalid_skps.py")
	pathToSKPInfo := filepath.Join(util.SkiaTreeDir, "out", "Release", "skpinfo")
	args := []string{
		pathToRemoveSKPs,
		"--skp_dir=" + pathToSkps,
		"--path_to_skpinfo=" + pathToSKPInfo,
	}
	skutil.LogErr(util.ExecuteCmd("python", args, []string{}, util.REMOVE_INVALID_SKPS_TIMEOUT,
		nil, nil))

	// Write timestamp to the SKPs dir.
	skutil.LogErr(util.CreateTimestampFile(pathToSkps))

	// Upload SKPs dir to Google Storage.
	if err := gs.UploadWorkerArtifacts(util.SKPS_DIR_NAME, filepath.Join(*pagesetType, *chromiumBuild), *workerNum); err != nil {
		glog.Error(err)
		return
	}
}

func getRowsFromCSV(csvPath string) ([]string, []string, error) {
	csvFile, err := os.Open(csvPath)
	defer skutil.Close(csvFile)
	if err != nil {
		return nil, nil, fmt.Errorf("Could not open %s: %s", csvPath, err)
	}
	reader := csv.NewReader(csvFile)
	reader.FieldsPerRecord = -1
	rawCSVdata, err := reader.ReadAll()
	if err != nil {
		return nil, nil, fmt.Errorf("Could not read %s: %s", csvPath, err)
	}
	if len(rawCSVdata) != 2 {
		return nil, nil, fmt.Errorf("No data in %s", csvPath)
	}
	return rawCSVdata[0], rawCSVdata[1], nil
}

func writeRowsToCSV(csvPath string, headers, values []string) error {
	csvFile, err := os.OpenFile(csvPath, os.O_WRONLY, 666)
	defer skutil.Close(csvFile)
	if err != nil {
		return fmt.Errorf("Could not open %s: %s", csvPath, err)
	}
	writer := csv.NewWriter(csvFile)
	defer writer.Flush()
	for _, row := range [][]string{headers, values} {
		if err := writer.Write(row); err != nil {
			return fmt.Errorf("Could not write to %s: %s", csvPath, err)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package daemon // import "github.com/docker/docker/daemon"

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/daemon/names"
)

var (
	validCheckpointNameChars   = names.RestrictedNameChars
	validCheckpointNamePattern = names.RestrictedNamePattern
)

// getCheckpointDir verifies checkpoint directory for create,remove, list options and checks if checkpoint already exists
func getCheckpointDir(checkDir, checkpointID, ctrName, ctrID, ctrCheckpointDir string, create bool) (string, error) {
	var checkpointDir string
	var err2 error
	if checkDir != "" {
		checkpointDir = checkDir
	} else {
		checkpointDir = ctrCheckpointDir
	}
	checkpointAbsDir := filepath.Join(checkpointDir, checkpointID)
	stat, err := os.Stat(checkpointAbsDir)
	if create {
		switch {
		case err == nil && stat.IsDir():
			err2 = fmt.Errorf("checkpoint with name %s already exists for container %s", checkpointID, ctrName)
		case err != nil && os.IsNotExist(err):
			err2 = os.MkdirAll(checkpointAbsDir, 0700)
		case err != nil:
			err2 = err
		case err == nil:
			err2 = fmt.Errorf("%s exists and is not a directory", checkpointAbsDir)
		}
	} else {
		switch {
		case err != nil:
			err2 = fmt.Errorf("checkpoint %s does not exist for container %s", checkpointID, ctrName)
		case err == nil && stat.IsDir():
			err2 = nil
		case err == nil:
			err2 = fmt.Errorf("%s exists and is not a directory", checkpointAbsDir)
		}
	}
	return checkpointAbsDir, err2
}

// CheckpointCreate checkpoints the process running in a container with CRIU
func (daemon *Daemon) CheckpointCreate(name string, config types.CheckpointCreateOptions) error {
	container, err := daemon.GetContainer(name)
	if err != nil {
		return err
	}

	if !container.IsRunning() {
		return fmt.Errorf("Container %s not running", name)
	}

	if container.Config.Tty {
		return fmt.Errorf("checkpoint not support on containers with tty")
	}

	if !validCheckpointNamePattern.MatchString(config.CheckpointID) {
		return fmt.Errorf("Invalid checkpoint ID (%s), only %s are allowed", config.CheckpointID, validCheckpointNameChars)
	}

	checkpointDir, err := getCheckpointDir(config.CheckpointDir, config.CheckpointID, name, container.ID, container.CheckpointDir(), true)
	if err != nil {
		return fmt.Errorf("cannot checkpoint container %s: %s", name, err)
	}

	err = daemon.containerd.CreateCheckpoint(context.Background(), container.ID, checkpointDir, config.Exit)
	if err != nil {
		os.RemoveAll(checkpointDir)
		return fmt.Errorf("Cannot checkpoint container %s: %s", name, err)
	}

	daemon.LogContainerEvent(container, "checkpoint")

	return nil
}

// CheckpointDelete deletes the specified checkpoint
func (daemon *Daemon) CheckpointDelete(name string, config types.CheckpointDeleteOptions) error {
	container, err := daemon.GetContainer(name)
	if err != nil {
		return err
	}
	checkpointDir, err := getCheckpointDir(config.CheckpointDir, config.CheckpointID, name, container.ID, container.CheckpointDir(), false)
	if err == nil {
		return os.RemoveAll(checkpointDir)
	}
	return err
}

// CheckpointList lists all checkpoints of the specified container
func (daemon *Daemon) CheckpointList(name string, config types.CheckpointListOptions) ([]types.Checkpoint, error) {
	var out []types.Checkpoint

	container, err := daemon.GetContainer(name)
	if err != nil {
		return nil, err
	}

	checkpointDir, err := getCheckpointDir(config.CheckpointDir, "", name, container.ID, container.CheckpointDir(), false)
	if err != nil {
		return nil, err
	}

	if err := os.MkdirAll(checkpointDir, 0755); err != nil {
		return nil, err
	}

	dirs, err := ioutil.ReadDir(checkpointDir)
	if err != nil {
		return nil, err
	}

	for _, d := range dirs {
		if !d.IsDir() {
			continue
		}
		path := filepath.Join(checkpointDir, d.Name(), "config.json")
		data, err := ioutil.ReadFile(path)
		if err != nil {
			return nil, err
		}
		var cpt types.Checkpoint
		if err := json.Unmarshal(data, &cpt); err != nil {
			return nil, err
		}
		out = append(out, cpt)
	}

	return out, nil
}
<commit_msg>Fix CheckpointList<commit_after>package daemon // import "github.com/docker/docker/daemon"

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/daemon/names"
)

var (
	validCheckpointNameChars   = names.RestrictedNameChars
	validCheckpointNamePattern = names.RestrictedNamePattern
)

// getCheckpointDir verifies checkpoint directory for create,remove, list options and checks if checkpoint already exists
func getCheckpointDir(checkDir, checkpointID, ctrName, ctrID, ctrCheckpointDir string, create bool) (string, error) {
	var checkpointDir string
	var err2 error
	if checkDir != "" {
		checkpointDir = checkDir
	} else {
		checkpointDir = ctrCheckpointDir
	}
	checkpointAbsDir := filepath.Join(checkpointDir, checkpointID)
	stat, err := os.Stat(checkpointAbsDir)
	if create {
		switch {
		case err == nil && stat.IsDir():
			err2 = fmt.Errorf("checkpoint with name %s already exists for container %s", checkpointID, ctrName)
		case err != nil && os.IsNotExist(err):
			err2 = os.MkdirAll(checkpointAbsDir, 0700)
		case err != nil:
			err2 = err
		case err == nil:
			err2 = fmt.Errorf("%s exists and is not a directory", checkpointAbsDir)
		}
	} else {
		switch {
		case err != nil:
			err2 = fmt.Errorf("checkpoint %s does not exist for container %s", checkpointID, ctrName)
		case err == nil && stat.IsDir():
			err2 = nil
		case err == nil:
			err2 = fmt.Errorf("%s exists and is not a directory", checkpointAbsDir)
		}
	}
	return checkpointAbsDir, err2
}

// CheckpointCreate checkpoints the process running in a container with CRIU
func (daemon *Daemon) CheckpointCreate(name string, config types.CheckpointCreateOptions) error {
	container, err := daemon.GetContainer(name)
	if err != nil {
		return err
	}

	if !container.IsRunning() {
		return fmt.Errorf("Container %s not running", name)
	}

	if container.Config.Tty {
		return fmt.Errorf("checkpoint not support on containers with tty")
	}

	if !validCheckpointNamePattern.MatchString(config.CheckpointID) {
		return fmt.Errorf("Invalid checkpoint ID (%s), only %s are allowed", config.CheckpointID, validCheckpointNameChars)
	}

	checkpointDir, err := getCheckpointDir(config.CheckpointDir, config.CheckpointID, name, container.ID, container.CheckpointDir(), true)
	if err != nil {
		return fmt.Errorf("cannot checkpoint container %s: %s", name, err)
	}

	err = daemon.containerd.CreateCheckpoint(context.Background(), container.ID, checkpointDir, config.Exit)
	if err != nil {
		os.RemoveAll(checkpointDir)
		return fmt.Errorf("Cannot checkpoint container %s: %s", name, err)
	}

	daemon.LogContainerEvent(container, "checkpoint")

	return nil
}

// CheckpointDelete deletes the specified checkpoint
func (daemon *Daemon) CheckpointDelete(name string, config types.CheckpointDeleteOptions) error {
	container, err := daemon.GetContainer(name)
	if err != nil {
		return err
	}
	checkpointDir, err := getCheckpointDir(config.CheckpointDir, config.CheckpointID, name, container.ID, container.CheckpointDir(), false)
	if err == nil {
		return os.RemoveAll(checkpointDir)
	}
	return err
}

// CheckpointList lists all checkpoints of the specified container
func (daemon *Daemon) CheckpointList(name string, config types.CheckpointListOptions) ([]types.Checkpoint, error) {
	var out []types.Checkpoint

	container, err := daemon.GetContainer(name)
	if err != nil {
		return nil, err
	}

	checkpointDir, err := getCheckpointDir(config.CheckpointDir, "", name, container.ID, container.CheckpointDir(), false)
	if err != nil {
		return nil, err
	}

	if err := os.MkdirAll(checkpointDir, 0755); err != nil {
		return nil, err
	}

	dirs, err := ioutil.ReadDir(checkpointDir)
	if err != nil {
		return nil, err
	}

	for _, d := range dirs {
		if !d.IsDir() {
			continue
		}
		cpt := types.Checkpoint{Name: d.Name()}
		out = append(out, cpt)
	}

	return out, nil
}
<|endoftext|>


<commit_before>package gpnsconfig

import (
	"flag"
	"github.com/msbranco/goconfig"
	"log"
)

type APPLICATION_MODE int

const (
	SERVER_MODE   APPLICATION_MODE = iota
	REGISTER_MODE APPLICATION_MODE = iota
	SEND_MODE     APPLICATION_MODE = iota
)

func ParseConfig() APPLICATION_MODE {
	var aws_config_file string
	var base_config_file string
	var register bool
	var send bool
	var input_file string
	var output_file string

	flag.StringVar(&base_config_file, "baseConfig", "./config/base.conf", "The path to the base configuration file")
	flag.StringVar(&aws_config_file, "awsConfig", "./config/aws.conf", "The path to the aws configuration file")

	flag.BoolVar(&register, "register", false, "Set flag to run in client mode and register a set of devices. If true inputFile and outputFile must be set.")
	flag.BoolVar(&send, "send", false, "Set flag to run in client mode and send push notifications to a set of arns. If true inputFile and outputFile must be set.")

	flag.StringVar(&input_file, "inputFile", "", "The path to the Device IDs or Arns file")
	flag.StringVar(&output_file, "outputFile", "", "The path to the Device IDs or Arns file")
	flag.Parse()

	log.Printf("Using base configuration file: %s", base_config_file)
	baseConfig, err := goconfig.ReadConfigFile(base_config_file)
	checkError("Unable to parse base config", err)

	log.Printf("Using aws configuration file: %s", aws_config_file)
	awsConfig, err := goconfig.ReadConfigFile(aws_config_file)
	checkError("Unable to parse AWS config", err)

	parseBaseConfig(baseConfig)
	parseAwsConfig(awsConfig)

	if register {
		log.Printf("Running in client mode, registering devices listed in %s, and printing arns in %s", input_file, output_file)
		return REGISTER_MODE
	} else if send {
		log.Printf("Running in client mode, sending pusn notes to ARNs listed in %s, and printing results in %s", input_file, output_file)
		return SEND_MODE
	} else {
		log.Printf("Running in server mode")
		return SERVER_MODE
	}

}

func checkError(message string, err error) {
	if err != nil {
		log.Fatalf("%s: %v", message, err)
	}
}
<commit_msg>Adding flag for the message file<commit_after>package gpnsconfig

import (
	"flag"
	"github.com/msbranco/goconfig"
	"log"
)

type APPLICATION_MODE int

const (
	SERVER_MODE   APPLICATION_MODE = iota
	REGISTER_MODE APPLICATION_MODE = iota
	SEND_MODE     APPLICATION_MODE = iota
)

func ParseConfig() APPLICATION_MODE {
	var aws_config_file string
	var base_config_file string
	var register bool
	var send bool
	var input_file string
	var output_file string
	var message_file string

	flag.StringVar(&base_config_file, "baseConfig", "./config/base.conf", "The path to the base configuration file")
	flag.StringVar(&aws_config_file, "awsConfig", "./config/aws.conf", "The path to the aws configuration file")

	flag.BoolVar(&register, "register", false, "Set flag to run in client mode and register a set of devices. If true inputFile and outputFile must be set.")
	flag.BoolVar(&send, "send", false, "Set flag to run in client mode and send push notifications to a set of arns. If true inputFile and outputFile must be set.")

	flag.StringVar(&input_file, "inputFile", "", "The path to the Device IDs or Arns file")
	flag.StringVar(&output_file, "outputFile", "", "The path to the Device IDs or Arns file")
	flag.StringVar(&message_file, "messageFile", "", "The path to the file containing the notificaito message to be sent out")

	flag.Parse()

	log.Printf("Using base configuration file: %s", base_config_file)
	baseConfig, err := goconfig.ReadConfigFile(base_config_file)
	checkError("Unable to parse base config", err)

	log.Printf("Using aws configuration file: %s", aws_config_file)
	awsConfig, err := goconfig.ReadConfigFile(aws_config_file)
	checkError("Unable to parse AWS config", err)

	parseBaseConfig(baseConfig)
	parseAwsConfig(awsConfig)

	if register {
		log.Printf("Running in client mode, registering devices listed in %s, and printing arns in %s", input_file, output_file)
		return REGISTER_MODE
	} else if send {
		log.Printf("Running in client mode, sending pusn notes to ARNs listed in %s, and printing results in %s", input_file, output_file)
		return SEND_MODE
	} else {
		log.Printf("Running in server mode")
		return SERVER_MODE
	}

}

func checkError(message string, err error) {
	if err != nil {
		log.Fatalf("%s: %v", message, err)
	}
}
<|endoftext|>


<commit_before>package transactionpool

import (
	"testing"
	"time"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

// TestFindSets checks that the findSets functions is properly parsing and
// combining transactions into their minimal sets.
func TestFindSets(t *testing.T) {
	// Graph a graph which is a chain. Graph will be invalid, but we don't need
	// the consensus set, so no worries.
	graph1Size := 5
	edges := make([]types.TransactionGraphEdge, 0, graph1Size)
	for i := 0; i < graph1Size; i++ {
		edges = append(edges, types.TransactionGraphEdge{
			Dest:   i + 1,
			Fee:    types.NewCurrency64(5),
			Source: i,
			Value:  types.NewCurrency64(100),
		})
	}
	graph1, err := types.TransactionGraph(types.SiacoinOutputID{}, edges)
	if err != nil {
		t.Fatal(err)
	}

	// Split the graph using findSets. Result should be a single set with 5
	// transactions.
	sets := findSets(graph1)
	if len(sets) != 1 {
		t.Fatal("there should be only one set")
	}
	if len(sets[0]) != graph1Size {
		t.Error("findSets is not grouping the transactions correctly")
	}

	// Create a second graph to check it can handle two graphs.
	graph2Size := 6
	edges = make([]types.TransactionGraphEdge, 0, graph2Size)
	for i := 0; i < graph2Size; i++ {
		edges = append(edges, types.TransactionGraphEdge{
			Dest:   i + 1,
			Fee:    types.NewCurrency64(5),
			Source: i,
			Value:  types.NewCurrency64(100),
		})
	}
	graph2, err := types.TransactionGraph(types.SiacoinOutputID{1}, edges)
	if err != nil {
		t.Fatal(err)
	}
	sets = findSets(append(graph1, graph2...))
	if len(sets) != 2 {
		t.Fatal("there should be two sets")
	}
	lens := make(map[int]struct{})
	lens[len(sets[0])] = struct{}{}
	lens[len(sets[1])] = struct{}{}
	_, exists1 := lens[graph1Size]
	_, exists2 := lens[graph2Size]
	if !exists1 || !exists2 {
		t.Fatal("there should be five transactions in each set")
	}

	// Create a diamond graph to make sure it can handle diamond graph.
	edges = make([]types.TransactionGraphEdge, 0, 5)
	sources := []int{0, 0, 1, 2, 3}
	dests := []int{1, 2, 3, 3, 4}
	for i := 0; i < 5; i++ {
		edges = append(edges, types.TransactionGraphEdge{
			Dest:   dests[i],
			Fee:    types.NewCurrency64(5),
			Source: sources[i],
			Value:  types.NewCurrency64(100),
		})
	}
	graph3, err := types.TransactionGraph(types.SiacoinOutputID{2}, edges)
	graph3Size := len(graph3)
	if err != nil {
		t.Fatal(err)
	}
	sets = findSets(append(graph1, append(graph2, graph3...)...))
	if len(sets) != 3 {
		t.Fatal("there should be two sets")
	}
	lens = make(map[int]struct{})
	lens[len(sets[0])] = struct{}{}
	lens[len(sets[1])] = struct{}{}
	lens[len(sets[2])] = struct{}{}
	_, exists1 = lens[graph1Size]
	_, exists2 = lens[graph2Size]
	_, exists3 := lens[graph3Size]
	if !exists1 || !exists2 || !exists3 {
		t.Fatal("sets have wrong counts", exists1, exists2, exists3)
	}

	// Sporadically weave the transactions and make sure the set finder still
	// parses the sets correctly (sets can assumed to be ordered, but not all in
	// a row).
	var sporadic []types.Transaction
	for len(graph1) > 0 || len(graph2) > 0 || len(graph3) > 0 {
		if len(graph1) > 0 {
			sporadic = append(sporadic, graph1[0])
			graph1 = graph1[1:]
		}
		if len(graph2) > 0 {
			sporadic = append(sporadic, graph2[0])
			graph2 = graph2[1:]
		}
		if len(graph3) > 0 {
			sporadic = append(sporadic, graph3[0])
			graph3 = graph3[1:]
		}
	}
	if len(sporadic) != graph1Size+graph2Size+graph3Size {
		t.Error("sporadic block creation failed")
	}
	// Result of findSets should match previous result.
	sets = findSets(sporadic)
	if len(sets) != 3 {
		t.Fatal("there should be two sets")
	}
	lens = make(map[int]struct{})
	lens[len(sets[0])] = struct{}{}
	lens[len(sets[1])] = struct{}{}
	lens[len(sets[2])] = struct{}{}
	_, exists1 = lens[graph1Size]
	_, exists2 = lens[graph2Size]
	_, exists3 = lens[graph3Size]
	if !exists1 || !exists2 || !exists3 {
		t.Fatal("sets have wrong counts", exists1, exists2, exists3)
	}
}

// TestArbDataOnly tries submitting a transaction with only arbitrary data to
// the transaction pool. Then a block is mined, putting the transaction on the
// blockchain. The arb data transaction should no longer be in the transaction
// pool.
func TestArbDataOnly(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	tpt, err := createTpoolTester(t.Name())
	if err != nil {
		t.Fatal(err)
	}
	defer tpt.Close()
	txn := types.Transaction{
		ArbitraryData: [][]byte{
			append(modules.PrefixNonSia[:], []byte("arb-data")...),
		},
	}
	err = tpt.tpool.AcceptTransactionSet([]types.Transaction{txn})
	if err != nil {
		t.Fatal(err)
	}
	if len(tpt.tpool.TransactionList()) != 1 {
		t.Error("expecting to see a transaction in the transaction pool")
	}
	_, err = tpt.miner.AddBlock()
	if err != nil {
		t.Fatal(err)
	}
	if len(tpt.tpool.TransactionList()) != 0 {
		t.Error("transaction was not cleared from the transaction pool")
	}
}

// TestValidRevertedTransaction verifies that if a transaction appears in a
// block's reverted transactions, it is added correctly to the pool.
func TestValidRevertedTransaction(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}

	tpt, err := createTpoolTester(t.Name())
	if err != nil {
		t.Fatal(err)
	}
	defer tpt.Close()

	tpt2, err := blankTpoolTester(t.Name() + "-tpt2")
	if err != nil {
		t.Fatal(err)
	}
	defer tpt2.Close()

	// connect the testers and wait for them to have the same current block
	err = tpt2.gateway.Connect(tpt.gateway.Address())
	if err != nil {
		t.Fatal(err)
	}
	success := false
	for start := time.Now(); time.Since(start) < time.Minute; time.Sleep(time.Millisecond * 100) {
		if tpt.cs.CurrentBlock().ID() == tpt2.cs.CurrentBlock().ID() {
			success = true
			break
		}
	}
	if !success {
		t.Fatal("testers did not have the same block height after one minute")
	}

	// disconnect the testers
	err = tpt2.gateway.Disconnect(tpt.gateway.Address())
	if err != nil {
		t.Fatal(err)
	}

	// make some transactions on tpt
	var txnSets [][]types.Transaction
	for i := 0; i < 5; i++ {
		txns, err := tpt.wallet.SendSiacoins(types.SiacoinPrecision.Mul64(1000), types.UnlockHash{})
		if err != nil {
			t.Fatal(err)
		}
		txnSets = append(txnSets, txns)
	}
	// mine some blocks to cause a re-org
	for i := 0; i < 3; i++ {
		_, err = tpt.miner.AddBlock()
		if err != nil {
			t.Fatal(err)
		}
	}
	// put tpt2 at a higher height
	for i := 0; i < 10; i++ {
		_, err = tpt2.miner.AddBlock()
		if err != nil {
			t.Fatal(err)
		}
	}

	// connect the testers and wait for them to have the same current block
	err = tpt.gateway.Connect(tpt2.gateway.Address())
	if err != nil {
		t.Fatal(err)
	}
	success = false
	for start := time.Now(); time.Since(start) < time.Minute; time.Sleep(time.Millisecond * 100) {
		if tpt.cs.CurrentBlock().ID() == tpt2.cs.CurrentBlock().ID() {
			success = true
			break
		}
	}
	if !success {
		t.Fatal("testers did not have the same block height after one minute")
	}

	// verify the transaction pool still has the reorged txns
	for _, txnSet := range txnSets {
		for _, txn := range txnSet {
			_, _, exists := tpt.tpool.Transaction(txn.ID())
			if !exists {
				t.Error("Transaction was not re-added to the transaction pool after being re-orged out of the blockchain:", txn.ID())
			}
		}
	}

	// Try to get the transactoins into a block.
	_, err = tpt.miner.AddBlock()
	if err != nil {
		t.Fatal(err)
	}
	if len(tpt.tpool.TransactionList()) != 0 {
		t.Error("Does not seem that the transactions were added to the transaction pool.")
	}
}

// TestTransactionPoolPruning verifies that the transaction pool correctly
// prunes transactions older than maxTxnAge.
func TestTransactionPoolPruning(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}

	tpt, err := createTpoolTester(t.Name())
	if err != nil {
		t.Fatal(err)
	}
	defer tpt.Close()
	tpt2, err := blankTpoolTester(t.Name() + "-tpt2")
	if err != nil {
		t.Fatal(err)
	}
	defer tpt2.Close()

	// connect the testers and wait for them to have the same current block
	err = tpt2.gateway.Connect(tpt.gateway.Address())
	if err != nil {
		t.Fatal(err)
	}
	success := false
	for start := time.Now(); time.Since(start) < time.Minute; time.Sleep(time.Millisecond * 100) {
		if tpt.cs.CurrentBlock().ID() == tpt2.cs.CurrentBlock().ID() {
			success = true
			break
		}
	}
	if !success {
		t.Fatal("testers did not have the same block height after one minute")
	}

	// disconnect tpt, create an unconfirmed transaction on tpt, mine maxTxnAge
	// blocks on tpt2 and reconnect. The unconfirmed transactions should be
	// removed from tpt's pool.
	err = tpt.gateway.Disconnect(tpt2.gateway.Address())
	if err != nil {
		t.Fatal(err)
	}
	txns, err := tpt.wallet.SendSiacoins(types.SiacoinPrecision.Mul64(1000), types.UnlockHash{})
	if err != nil {
		t.Fatal(err)
	}
	for i := types.BlockHeight(0); i < maxTxnAge+1; i++ {
		_, err = tpt2.miner.AddBlock()
		if err != nil {
			t.Fatal(err)
		}
	}

	// reconnect the testers
	err = tpt.gateway.Connect(tpt2.gateway.Address())
	if err != nil {
		t.Fatal(err)
	}
	success = false
	for start := time.Now(); time.Since(start) < time.Minute; time.Sleep(time.Millisecond * 100) {
		if tpt.cs.CurrentBlock().ID() == tpt2.cs.CurrentBlock().ID() {
			success = true
			break
		}
	}
	if !success {
		t.Fatal("testers did not have the same block height after one minute")
	}

	for _, txn := range txns {
		_, _, exists := tpt.tpool.Transaction(txn.ID())
		if exists {
			t.Fatal("transaction pool had a transaction that should have been pruned")
		}
	}
	if len(tpt.tpool.TransactionList()) != 0 {
		t.Fatal("should have no unconfirmed transactions")
	}
	if len(tpt.tpool.knownObjects) != 0 {
		t.Fatal("should have no known objects")
	}
	if len(tpt.tpool.transactionSetDiffs) != 0 {
		t.Fatal("should have no transaction set diffs")
	}
	if tpt.tpool.transactionListSize != 0 {
		t.Fatal("transactionListSize should be zero")
	}
}

// TestUpdateBlockHeight verifies that the transactionpool updates its internal
// block height correctly.
func TestUpdateBlockHeight(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}

	tpt, err := blankTpoolTester(t.Name())
	if err != nil {
		t.Fatal(err)
	}
	defer tpt.Close()

	targetHeight := 20
	for i := 0; i < targetHeight; i++ {
		_, err = tpt.miner.AddBlock()
		if err != nil {
			t.Fatal(err)
		}
	}
	if tpt.tpool.blockHeight != types.BlockHeight(targetHeight) {
		t.Fatalf("transaction pool had the wrong block height, got %v wanted %v\n", tpt.tpool.blockHeight, targetHeight)
	}
}
<commit_msg>switching to sorting method for checking test results<commit_after>package transactionpool

import (
	"sort"
	"testing"
	"time"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

// TestFindSets checks that the findSets functions is properly parsing and
// combining transactions into their minimal sets.
func TestFindSets(t *testing.T) {
	// Graph a graph which is a chain. Graph will be invalid, but we don't need
	// the consensus set, so no worries.
	graph1Size := 5
	edges := make([]types.TransactionGraphEdge, 0, graph1Size)
	for i := 0; i < graph1Size; i++ {
		edges = append(edges, types.TransactionGraphEdge{
			Dest:   i + 1,
			Fee:    types.NewCurrency64(5),
			Source: i,
			Value:  types.NewCurrency64(100),
		})
	}
	graph1, err := types.TransactionGraph(types.SiacoinOutputID{}, edges)
	if err != nil {
		t.Fatal(err)
	}

	// Split the graph using findSets. Result should be a single set with 5
	// transactions.
	sets := findSets(graph1)
	if len(sets) != 1 {
		t.Fatal("there should be only one set")
	}
	if len(sets[0]) != graph1Size {
		t.Error("findSets is not grouping the transactions correctly")
	}

	// Create a second graph to check it can handle two graphs.
	graph2Size := 6
	edges = make([]types.TransactionGraphEdge, 0, graph2Size)
	for i := 0; i < graph2Size; i++ {
		edges = append(edges, types.TransactionGraphEdge{
			Dest:   i + 1,
			Fee:    types.NewCurrency64(5),
			Source: i,
			Value:  types.NewCurrency64(100),
		})
	}
	graph2, err := types.TransactionGraph(types.SiacoinOutputID{1}, edges)
	if err != nil {
		t.Fatal(err)
	}
	sets = findSets(append(graph1, graph2...))
	if len(sets) != 2 {
		t.Fatal("there should be two sets")
	}
	lens := []int{len(sets[0]), len(sets[1])}
	sort.Ints(lens)
	expected := []int{graph1Size, graph2Size}
	sort.Ints(expected)
	if lens[0] != expected[0] || lens[1] != expected[1] {
		t.Error("Resulting sets do not have the right lengths")
	}

	// Create a diamond graph to make sure it can handle diamond graph.
	edges = make([]types.TransactionGraphEdge, 0, 5)
	sources := []int{0, 0, 1, 2, 3}
	dests := []int{1, 2, 3, 3, 4}
	for i := 0; i < 5; i++ {
		edges = append(edges, types.TransactionGraphEdge{
			Dest:   dests[i],
			Fee:    types.NewCurrency64(5),
			Source: sources[i],
			Value:  types.NewCurrency64(100),
		})
	}
	graph3, err := types.TransactionGraph(types.SiacoinOutputID{2}, edges)
	graph3Size := len(graph3)
	if err != nil {
		t.Fatal(err)
	}
	sets = findSets(append(graph1, append(graph2, graph3...)...))
	if len(sets) != 3 {
		t.Fatal("there should be two sets")
	}
	lens = []int{len(sets[0]), len(sets[1]), len(sets[2])}
	sort.Ints(lens)
	expected = []int{graph1Size, graph2Size, graph3Size}
	sort.Ints(expected)
	if lens[0] != expected[0] || lens[1] != expected[1] || lens[2] != expected[2] {
		t.Error("Resulting sets do not have the right lengths")
	}

	// Sporadically weave the transactions and make sure the set finder still
	// parses the sets correctly (sets can assumed to be ordered, but not all in
	// a row).
	var sporadic []types.Transaction
	for len(graph1) > 0 || len(graph2) > 0 || len(graph3) > 0 {
		if len(graph1) > 0 {
			sporadic = append(sporadic, graph1[0])
			graph1 = graph1[1:]
		}
		if len(graph2) > 0 {
			sporadic = append(sporadic, graph2[0])
			graph2 = graph2[1:]
		}
		if len(graph3) > 0 {
			sporadic = append(sporadic, graph3[0])
			graph3 = graph3[1:]
		}
	}
	if len(sporadic) != graph1Size+graph2Size+graph3Size {
		t.Error("sporadic block creation failed")
	}
	// Result of findSets should match previous result.
	sets = findSets(sporadic)
	if len(sets) != 3 {
		t.Fatal("there should be two sets")
	}
	lens = []int{len(sets[0]), len(sets[1]), len(sets[2])}
	sort.Ints(lens)
	expected = []int{graph1Size, graph2Size, graph3Size}
	sort.Ints(expected)
	if lens[0] != expected[0] || lens[1] != expected[1] || lens[2] != expected[2] {
		t.Error("Resulting sets do not have the right lengths")
	}
}

// TestArbDataOnly tries submitting a transaction with only arbitrary data to
// the transaction pool. Then a block is mined, putting the transaction on the
// blockchain. The arb data transaction should no longer be in the transaction
// pool.
func TestArbDataOnly(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	tpt, err := createTpoolTester(t.Name())
	if err != nil {
		t.Fatal(err)
	}
	defer tpt.Close()
	txn := types.Transaction{
		ArbitraryData: [][]byte{
			append(modules.PrefixNonSia[:], []byte("arb-data")...),
		},
	}
	err = tpt.tpool.AcceptTransactionSet([]types.Transaction{txn})
	if err != nil {
		t.Fatal(err)
	}
	if len(tpt.tpool.TransactionList()) != 1 {
		t.Error("expecting to see a transaction in the transaction pool")
	}
	_, err = tpt.miner.AddBlock()
	if err != nil {
		t.Fatal(err)
	}
	if len(tpt.tpool.TransactionList()) != 0 {
		t.Error("transaction was not cleared from the transaction pool")
	}
}

// TestValidRevertedTransaction verifies that if a transaction appears in a
// block's reverted transactions, it is added correctly to the pool.
func TestValidRevertedTransaction(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}

	tpt, err := createTpoolTester(t.Name())
	if err != nil {
		t.Fatal(err)
	}
	defer tpt.Close()

	tpt2, err := blankTpoolTester(t.Name() + "-tpt2")
	if err != nil {
		t.Fatal(err)
	}
	defer tpt2.Close()

	// connect the testers and wait for them to have the same current block
	err = tpt2.gateway.Connect(tpt.gateway.Address())
	if err != nil {
		t.Fatal(err)
	}
	success := false
	for start := time.Now(); time.Since(start) < time.Minute; time.Sleep(time.Millisecond * 100) {
		if tpt.cs.CurrentBlock().ID() == tpt2.cs.CurrentBlock().ID() {
			success = true
			break
		}
	}
	if !success {
		t.Fatal("testers did not have the same block height after one minute")
	}

	// disconnect the testers
	err = tpt2.gateway.Disconnect(tpt.gateway.Address())
	if err != nil {
		t.Fatal(err)
	}

	// make some transactions on tpt
	var txnSets [][]types.Transaction
	for i := 0; i < 5; i++ {
		txns, err := tpt.wallet.SendSiacoins(types.SiacoinPrecision.Mul64(1000), types.UnlockHash{})
		if err != nil {
			t.Fatal(err)
		}
		txnSets = append(txnSets, txns)
	}
	// mine some blocks to cause a re-org
	for i := 0; i < 3; i++ {
		_, err = tpt.miner.AddBlock()
		if err != nil {
			t.Fatal(err)
		}
	}
	// put tpt2 at a higher height
	for i := 0; i < 10; i++ {
		_, err = tpt2.miner.AddBlock()
		if err != nil {
			t.Fatal(err)
		}
	}

	// connect the testers and wait for them to have the same current block
	err = tpt.gateway.Connect(tpt2.gateway.Address())
	if err != nil {
		t.Fatal(err)
	}
	success = false
	for start := time.Now(); time.Since(start) < time.Minute; time.Sleep(time.Millisecond * 100) {
		if tpt.cs.CurrentBlock().ID() == tpt2.cs.CurrentBlock().ID() {
			success = true
			break
		}
	}
	if !success {
		t.Fatal("testers did not have the same block height after one minute")
	}

	// verify the transaction pool still has the reorged txns
	for _, txnSet := range txnSets {
		for _, txn := range txnSet {
			_, _, exists := tpt.tpool.Transaction(txn.ID())
			if !exists {
				t.Error("Transaction was not re-added to the transaction pool after being re-orged out of the blockchain:", txn.ID())
			}
		}
	}

	// Try to get the transactoins into a block.
	_, err = tpt.miner.AddBlock()
	if err != nil {
		t.Fatal(err)
	}
	if len(tpt.tpool.TransactionList()) != 0 {
		t.Error("Does not seem that the transactions were added to the transaction pool.")
	}
}

// TestTransactionPoolPruning verifies that the transaction pool correctly
// prunes transactions older than maxTxnAge.
func TestTransactionPoolPruning(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}

	tpt, err := createTpoolTester(t.Name())
	if err != nil {
		t.Fatal(err)
	}
	defer tpt.Close()
	tpt2, err := blankTpoolTester(t.Name() + "-tpt2")
	if err != nil {
		t.Fatal(err)
	}
	defer tpt2.Close()

	// connect the testers and wait for them to have the same current block
	err = tpt2.gateway.Connect(tpt.gateway.Address())
	if err != nil {
		t.Fatal(err)
	}
	success := false
	for start := time.Now(); time.Since(start) < time.Minute; time.Sleep(time.Millisecond * 100) {
		if tpt.cs.CurrentBlock().ID() == tpt2.cs.CurrentBlock().ID() {
			success = true
			break
		}
	}
	if !success {
		t.Fatal("testers did not have the same block height after one minute")
	}

	// disconnect tpt, create an unconfirmed transaction on tpt, mine maxTxnAge
	// blocks on tpt2 and reconnect. The unconfirmed transactions should be
	// removed from tpt's pool.
	err = tpt.gateway.Disconnect(tpt2.gateway.Address())
	if err != nil {
		t.Fatal(err)
	}
	txns, err := tpt.wallet.SendSiacoins(types.SiacoinPrecision.Mul64(1000), types.UnlockHash{})
	if err != nil {
		t.Fatal(err)
	}
	for i := types.BlockHeight(0); i < maxTxnAge+1; i++ {
		_, err = tpt2.miner.AddBlock()
		if err != nil {
			t.Fatal(err)
		}
	}

	// reconnect the testers
	err = tpt.gateway.Connect(tpt2.gateway.Address())
	if err != nil {
		t.Fatal(err)
	}
	success = false
	for start := time.Now(); time.Since(start) < time.Minute; time.Sleep(time.Millisecond * 100) {
		if tpt.cs.CurrentBlock().ID() == tpt2.cs.CurrentBlock().ID() {
			success = true
			break
		}
	}
	if !success {
		t.Fatal("testers did not have the same block height after one minute")
	}

	for _, txn := range txns {
		_, _, exists := tpt.tpool.Transaction(txn.ID())
		if exists {
			t.Fatal("transaction pool had a transaction that should have been pruned")
		}
	}
	if len(tpt.tpool.TransactionList()) != 0 {
		t.Fatal("should have no unconfirmed transactions")
	}
	if len(tpt.tpool.knownObjects) != 0 {
		t.Fatal("should have no known objects")
	}
	if len(tpt.tpool.transactionSetDiffs) != 0 {
		t.Fatal("should have no transaction set diffs")
	}
	if tpt.tpool.transactionListSize != 0 {
		t.Fatal("transactionListSize should be zero")
	}
}

// TestUpdateBlockHeight verifies that the transactionpool updates its internal
// block height correctly.
func TestUpdateBlockHeight(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}

	tpt, err := blankTpoolTester(t.Name())
	if err != nil {
		t.Fatal(err)
	}
	defer tpt.Close()

	targetHeight := 20
	for i := 0; i < targetHeight; i++ {
		_, err = tpt.miner.AddBlock()
		if err != nil {
			t.Fatal(err)
		}
	}
	if tpt.tpool.blockHeight != types.BlockHeight(targetHeight) {
		t.Fatalf("transaction pool had the wrong block height, got %v wanted %v\n", tpt.tpool.blockHeight, targetHeight)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net"
	"os"
	"runtime"
	"strings"

	"github.com/sirupsen/logrus"

	"k8s.io/client-go/kubernetes"
	restclient "k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"

	ovncluster "github.com/openvswitch/ovn-kubernetes/go-controller/pkg/cluster"
	"github.com/openvswitch/ovn-kubernetes/go-controller/pkg/factory"
	"github.com/openvswitch/ovn-kubernetes/go-controller/pkg/ovn"
	util "github.com/openvswitch/ovn-kubernetes/go-controller/pkg/util"
)

func main() {
	// auth flags
	kubeconfig := flag.String("kubeconfig", "", "absolute path to the kubeconfig file")
	server := flag.String("apiserver", "https://localhost:8443", "URL to the Kubernetes apiserver")
	rootCAFile := flag.String("ca-cert", "", "CA cert for the Kubernetes api server")
	token := flag.String("token", "", "Bearer token to use for establishing ovn infrastructure")
	clusterSubnet := flag.String("cluster-subnet", "11.11.0.0/16", "Cluster wide IP subnet to use")
	clusterServicesSubnet := flag.String("service-cluster-ip-range", "",
		"A CIDR notation IP range from which k8s assigns service cluster "+
			"IPs. This should be the same as the one provided for "+
			"kube-apiserver \"-service-cluster-ip-range\" option.")

	// IP address and port of the northbound API server
	ovnNorth := flag.String("ovn-north-db", "", "IP address and port of the OVN northbound API (eg, ssl://1.2.3.4:6641).  Leave empty to use a local unix socket.")

	// SSL-related flags for securing the northbound API server
	ovnNorthServerPrivKey := flag.String("ovn-north-server-privkey", "", "Private key that the OVN northbound API should use for securing the API.  Leave empty to use local unix socket.")
	ovnNorthServerCert := flag.String("ovn-north-server-cert", "", "Server certificate that the OVN northbound API should use for securing the API.  Leave empty to use local unix socket.")
	ovnNorthServerCACert := flag.String("ovn-north-server-cacert", "", "CA certificate that the OVN northbound API should use for securing the API.  Leave empty to use local unix socket.")

	// SSL-related flags for clients connecting to the northbound API
	ovnNorthClientPrivKey := flag.String("ovn-north-client-privkey", "", "Private key that the client should use for talking to the OVN database.  Leave empty to use local unix socket.")
	ovnNorthClientCert := flag.String("ovn-north-client-cert", "", "Client certificate that the client should use for talking to the OVN database.  Leave empty to use local unix socket.")
	ovnNorthClientCACert := flag.String("ovn-north-client-cacert", "", "CA certificate that the client should use for talking to the OVN database.  Leave empty to use local unix socket.")

	// IP address and port of the southbound database server
	ovnSouth := flag.String("ovn-south-db", "", "IP address and port of the OVN southbound database (eg, ssl://1.2.3.4:6642).")

	// SSL-related flags for securing the southbound database server
	ovnSouthServerPrivKey := flag.String("ovn-south-server-privkey", "", "Private key that the OVN southbound database should use for securing the API.")
	ovnSouthServerCert := flag.String("ovn-south-server-cert", "", "Server certificate that the OVN southbound database should use for securing the API.")
	ovnSouthServerCACert := flag.String("ovn-south-server-cacert", "", "CA certificate that the OVN southbound database should use for securing the API.")

	// SSL-related flags for clients connecting to the southbound database
	ovnSouthClientPrivKey := flag.String("ovn-south-client-privkey", "", "Private key that the client should use for talking to the OVN database.")
	ovnSouthClientCert := flag.String("ovn-south-client-cert", "", "Client certificate that the client should use for talking to the OVN database.")
	ovnSouthClientCACert := flag.String("ovn-south-client-cacert", "", "CA certificate that the client should use for talking to the OVN database.")

	// mode flags
	netController := flag.Bool("net-controller", false, "Flag to start the central controller that watches pods/services/policies")
	master := flag.String("init-master", "", "initialize master, requires the hostname as argument")
	node := flag.String("init-node", "", "initialize node, requires the name that node is registered with in kubernetes cluster")

	// log flags
	verbose := flag.Int("loglevel", 4,
		"loglevels 5=debug, 4=info, 3=warn, 2=error, 1=fatal")
	logFile := flag.String("logfile", "",
		"logfile name (with path) for ovnkube to write to.")

	// gateway flags
	gatewayInit := flag.Bool("init-gateways", false,
		"initialize a gateway in the minion. Only useful with \"init-node\"")
	gatewayIntf := flag.String("gateway-interface", "",
		"The interface in minions that will be the gateway interface. "+
			"If none specified, then the node's interface on which the "+
			"default gateway is configured will be used as the gateway "+
			"interface. Only useful with \"init-gateways\"")
	gatewayNextHop := flag.String("gateway-nexthop", "",
		"The external default gateway which is used as a next hop by "+
			"OVN gateway.  This is many times just the default gateway "+
			"of the node in question. If not specified, the default gateway"+
			"configured in the node is used. Only useful with "+
			"\"init-gateways\"")
	gatewaySpareIntf := flag.Bool("gateway-spare-interface", false,
		"If true, assumes that \"gateway-interface\" provided can be "+
			"exclusively used for the OVN gateway.  When true, only OVN"+
			"related traffic can flow through this interface")

	// Enable nodeport
	nodePortEnable := flag.Bool("nodeport", false,
		"Setup nodeport based ingress on gateways.")

	flag.Parse()

	// Process log flags
	logrus.SetLevel(logrus.Level(*verbose))
	logrus.SetOutput(os.Stderr)
	if *logFile != "" {
		file, err := os.OpenFile(*logFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY,
			0660)
		if err != nil {
			logrus.Errorf("failed to open logfile %s (%v). Ignoring..",
				*logFile, err)
		} else {
			defer file.Close()
			logrus.SetOutput(file)
		}
	}

	// Process auth flags
	var config *restclient.Config
	var err error
	if *kubeconfig != "" {
		// uses the current context in kubeconfig
		config, err = clientcmd.BuildConfigFromFlags("", *kubeconfig)
	} else if *server != "" && *token != "" && ((*rootCAFile != "") || !strings.HasPrefix(*server, "https")) {
		config, err = util.CreateConfig(*server, *token, *rootCAFile)
	} else {
		err = fmt.Errorf("Provide kubeconfig file or give server/token/tls credentials")
	}
	if err != nil {
		panic(err.Error())
	}

	// creates the clientset
	clientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		panic(err.Error())
	}

	// create factory and start the controllers asked for
	stopChan := make(chan struct{})
	factory, err := factory.NewWatchFactory(clientset, stopChan)
	if err != nil {
		panic(err.Error)
	}

	if *master != "" || *node != "" {
		clusterController := ovncluster.NewClusterController(clientset, factory)
		clusterController.KubeServer = *server
		clusterController.CACert = *rootCAFile
		clusterController.Token = *token
		clusterController.HostSubnetLength = 8
		clusterController.GatewayInit = *gatewayInit
		clusterController.GatewayIntf = *gatewayIntf
		clusterController.GatewayNextHop = *gatewayNextHop
		clusterController.GatewaySpareIntf = *gatewaySpareIntf
		_, clusterController.ClusterIPNet, err = net.ParseCIDR(*clusterSubnet)
		if err != nil {
			panic(err.Error)
		}

		if *clusterServicesSubnet != "" {
			var servicesSubnet *net.IPNet
			_, servicesSubnet, err = net.ParseCIDR(
				*clusterServicesSubnet)
			if err != nil {
				panic(err.Error)
			}
			clusterController.ClusterServicesSubnet = servicesSubnet.String()
		}

		clusterController.NorthDBClientAuth, err = ovncluster.NewOvnDBAuth(*ovnNorth, *ovnNorthClientPrivKey, *ovnNorthClientCert, *ovnNorthClientCACert, false)
		if err != nil {
			panic(err.Error())
		}
		clusterController.SouthDBClientAuth, err = ovncluster.NewOvnDBAuth(*ovnSouth, *ovnSouthClientPrivKey, *ovnSouthClientCert, *ovnSouthClientCACert, false)
		if err != nil {
			panic(err.Error())
		}

		if *node != "" {
			if *token == "" {
				panic("Cannot initialize node without service account 'token'. Please provide one with --token argument")
			}

			err := clusterController.StartClusterNode(*node)
			if err != nil {
				logrus.Errorf(err.Error())
				panic(err.Error())
			}
		}
		clusterController.NodePortEnable = *nodePortEnable

		if *master != "" {
			if runtime.GOOS == "windows" {
				panic("Windows is not supported as master node")
			}
			clusterController.NorthDBServerAuth, err = ovncluster.NewOvnDBAuth(*ovnNorth, *ovnNorthServerPrivKey, *ovnNorthServerCert, *ovnNorthServerCACert, true)
			if err != nil {
				panic(err.Error())
			}
			clusterController.SouthDBServerAuth, err = ovncluster.NewOvnDBAuth(*ovnSouth, *ovnSouthServerPrivKey, *ovnSouthServerCert, *ovnSouthServerCACert, true)
			if err != nil {
				panic(err.Error())
			}

			// run the cluster controller to init the master
			err := clusterController.StartClusterMaster(*master)
			if err != nil {
				logrus.Errorf(err.Error())
				panic(err.Error())
			}
		}
	}
	if *netController {
		ovnController := ovn.NewOvnController(clientset, factory)
		ovnController.NodePortEnable = *nodePortEnable
		ovnController.Run()
	}
	if *master != "" || *netController {
		// run forever
		select {}
	}
}
<commit_msg>ovnkube: convert CLI args to urfave/cli<commit_after>package main

import (
	"fmt"
	"net"
	"os"
	"runtime"
	"strings"

	"github.com/sirupsen/logrus"
	"github.com/urfave/cli"

	"k8s.io/client-go/kubernetes"
	restclient "k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"

	ovncluster "github.com/openvswitch/ovn-kubernetes/go-controller/pkg/cluster"
	"github.com/openvswitch/ovn-kubernetes/go-controller/pkg/factory"
	"github.com/openvswitch/ovn-kubernetes/go-controller/pkg/ovn"
	util "github.com/openvswitch/ovn-kubernetes/go-controller/pkg/util"
)

func main() {
	c := cli.NewApp()
	c.Name = "ovnkube"
	c.Usage = "run ovnkube to start master, node, and gateway services"
	c.Version = "0.0.1"
	c.Flags = []cli.Flag{
		// Kubernetes-related options
		cli.StringFlag{
			Name:  "kubeconfig",
			Usage: "absolute path to the kubeconfig file",
		},
		cli.StringFlag{
			Name:  "apiserver",
			Value: "https://localhost:8443",
			Usage: "URL to the Kubernetes apiserver",
		},
		cli.StringFlag{
			Name:  "ca-cert",
			Usage: "CA cert for the Kubernetes api server",
		},
		cli.StringFlag{
			Name:  "token",
			Usage: "Bearer token to use for establishing ovn infrastructure",
		},
		cli.StringFlag{
			Name:  "cluster-subnet",
			Value: "11.11.0.0/16",
			Usage: "Cluster wide IP subnet to use",
		},
		cli.StringFlag{
			Name: "service-cluster-ip-range",
			Usage: "A CIDR notation IP range from which k8s assigns " +
				"service cluster IPs. This should be the same as the one " +
				"provided for kube-apiserver \"-service-cluster-ip-range\" " +
				"option.",
		},

		// OVN northbound database options
		cli.StringFlag{
			Name:  "ovn-north-db",
			Usage: "IP address and port of the OVN northbound API (eg, ssl://1.2.3.4:6641).  Leave empty to use a local unix socket.",
		},
		cli.StringFlag{
			Name:  "ovn-north-server-privkey",
			Usage: "Private key that the OVN northbound API should use for securing the API.  Leave empty to use local unix socket.",
		},
		cli.StringFlag{
			Name:  "ovn-north-server-cert",
			Usage: "Server certificate that the OVN northbound API should use for securing the API.  Leave empty to use local unix socket.",
		},
		cli.StringFlag{
			Name:  "ovn-north-server-cacert",
			Usage: "CA certificate that the OVN northbound API should use for securing the API.  Leave empty to use local unix socket.",
		},
		cli.StringFlag{
			Name:  "ovn-north-client-privkey",
			Usage: "Private key that the client should use for talking to the OVN database.  Leave empty to use local unix socket.",
		},
		cli.StringFlag{
			Name:  "ovn-north-client-cert",
			Usage: "Client certificate that the client should use for talking to the OVN database.  Leave empty to use local unix socket.",
		},
		cli.StringFlag{
			Name:  "ovn-north-client-cacert",
			Usage: "CA certificate that the client should use for talking to the OVN database.  Leave empty to use local unix socket.",
		},

		// OVN southbound database options
		cli.StringFlag{
			Name:  "ovn-south-db",
			Usage: "IP address and port of the OVN southbound API (eg, ssl://1.2.3.4:6642).  Leave empty to use a local unix socket.",
		},
		cli.StringFlag{
			Name:  "ovn-south-server-privkey",
			Usage: "Private key that the OVN southbound API should use for securing the API.  Leave empty to use local unix socket.",
		},
		cli.StringFlag{
			Name:  "ovn-south-server-cert",
			Usage: "Server certificate that the OVN southbound API should use for securing the API.  Leave empty to use local unix socket.",
		},
		cli.StringFlag{
			Name:  "ovn-south-server-cacert",
			Usage: "CA certificate that the OVN southbound API should use for securing the API.  Leave empty to use local unix socket.",
		},
		cli.StringFlag{
			Name:  "ovn-south-client-privkey",
			Usage: "Private key that the client should use for talking to the OVN database.  Leave empty to use local unix socket.",
		},
		cli.StringFlag{
			Name:  "ovn-south-client-cert",
			Usage: "Client certificate that the client should use for talking to the OVN database.  Leave empty to use local unix socket.",
		},
		cli.StringFlag{
			Name:  "ovn-south-client-cacert",
			Usage: "CA certificate that the client should use for talking to the OVN database.  Leave empty to use local unix socket.",
		},

		// Mode flags
		cli.BoolFlag{
			Name:  "net-controller",
			Usage: "Flag to start the central controller that watches pods/services/policies",
		},
		cli.StringFlag{
			Name:  "init-master",
			Usage: "initialize master, requires the hostname as argument",
		},
		cli.StringFlag{
			Name:  "init-node",
			Usage: "initialize node, requires the name that node is registered with in kubernetes cluster",
		},

		// Log flags
		cli.IntFlag{
			Name:  "loglevel",
			Usage: "loglevels 5=debug, 4=info, 3=warn, 2=error, 1=fatal",
			Value: 4,
		},
		cli.StringFlag{
			Name:  "logfile",
			Usage: "logfile name (with path) for ovnkube to write to.",
		},

		// Gateway flags
		cli.BoolFlag{
			Name:  "init-gateways",
			Usage: "initialize a gateway in the minion. Only useful with \"init-node\"",
		},
		cli.StringFlag{
			Name: "gateway-interface",
			Usage: "The interface in minions that will be the gateway interface. " +
				"If none specified, then the node's interface on which the " +
				"default gateway is configured will be used as the gateway " +
				"interface. Only useful with \"init-gateways\"",
		},
		cli.StringFlag{
			Name: "gateway-nexthop",
			Usage: "The external default gateway which is used as a next hop by " +
				"OVN gateway.  This is many times just the default gateway " +
				"of the node in question. If not specified, the default gateway" +
				"configured in the node is used. Only useful with " +
				"\"init-gateways\"",
		},
		cli.BoolFlag{
			Name: "gateway-spare-interface",
			Usage: "If true, assumes that \"gateway-interface\" provided can be " +
				"exclusively used for the OVN gateway.  When true, only OVN" +
				"related traffic can flow through this interface",
		},

		cli.BoolFlag{
			Name:  "nodeport",
			Usage: "Setup nodeport based ingress on gateways.",
		},
	}
	c.Action = func(c *cli.Context) error {
		return runOvnKube(c)
	}

	if err := c.Run(os.Args); err != nil {
		logrus.Fatal(err)
	}
}

func runOvnKube(ctx *cli.Context) error {
	// Process log flags
	logrus.SetLevel(logrus.Level(ctx.Int("loglevel")))
	logrus.SetOutput(os.Stderr)
	logFile := ctx.String("logfile")
	if logFile != "" {
		file, err := os.OpenFile(logFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY,
			0660)
		if err != nil {
			logrus.Errorf("failed to open logfile %s (%v). Ignoring..",
				logFile, err)
		} else {
			defer file.Close()
			logrus.SetOutput(file)
		}
	}

	// Process auth flags
	var config *restclient.Config
	var err error

	kubeconfig := ctx.String("kubeconfig")
	server := ctx.String("apiserver")
	rootCAFile := ctx.String("ca-cert")
	token := ctx.String("token")
	if kubeconfig != "" {
		// uses the current context in kubeconfig
		config, err = clientcmd.BuildConfigFromFlags("", kubeconfig)
	} else if server != "" && token != "" && ((rootCAFile != "") || !strings.HasPrefix(server, "https")) {
		config, err = util.CreateConfig(server, token, rootCAFile)
	} else {
		err = fmt.Errorf("Provide kubeconfig file or give server/token/tls credentials")
	}
	if err != nil {
		panic(err.Error())
	}

	// creates the clientset
	clientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		panic(err.Error())
	}

	// create factory and start the controllers asked for
	stopChan := make(chan struct{})
	factory, err := factory.NewWatchFactory(clientset, stopChan)
	if err != nil {
		panic(err.Error)
	}

	netController := ctx.Bool("net-controller")
	master := ctx.String("init-master")
	node := ctx.String("init-node")
	nodePortEnable := ctx.Bool("nodeport")

	if master != "" || node != "" {
		clusterController := ovncluster.NewClusterController(clientset, factory)
		clusterController.KubeServer = server
		clusterController.CACert = rootCAFile
		clusterController.Token = token
		clusterController.HostSubnetLength = 8
		clusterController.GatewayInit = ctx.Bool("init-gateways")
		clusterController.GatewayIntf = ctx.String("gateway-interface")
		clusterController.GatewayNextHop = ctx.String("gateway-nexthop")
		clusterController.GatewaySpareIntf = ctx.Bool("gateway-spare-interface")
		_, clusterController.ClusterIPNet, err = net.ParseCIDR(ctx.String("cluster-subnet"))
		if err != nil {
			panic(err.Error)
		}

		clusterServicesSubnet := ctx.String("service-cluster-ip-range")
		if clusterServicesSubnet != "" {
			var servicesSubnet *net.IPNet
			_, servicesSubnet, err = net.ParseCIDR(
				clusterServicesSubnet)
			if err != nil {
				panic(err.Error)
			}
			clusterController.ClusterServicesSubnet = servicesSubnet.String()
		}

		ovnNorth := ctx.String("ovn-north-db")
		ovnNorthClientPrivKey := ctx.String("ovn-north-client-privkey")
		ovnNorthClientCert := ctx.String("ovn-north-client-cert")
		ovnNorthClientCACert := ctx.String("ovn-north-client-cacert")
		clusterController.NorthDBClientAuth, err = ovncluster.NewOvnDBAuth(ovnNorth, ovnNorthClientPrivKey, ovnNorthClientCert, ovnNorthClientCACert, false)
		if err != nil {
			panic(err.Error())
		}

		ovnSouth := ctx.String("ovn-south-db")
		ovnSouthClientPrivKey := ctx.String("ovn-south-client-privkey")
		ovnSouthClientCert := ctx.String("ovn-south-client-cert")
		ovnSouthClientCACert := ctx.String("ovn-south-client-cacert")
		clusterController.SouthDBClientAuth, err = ovncluster.NewOvnDBAuth(ovnSouth, ovnSouthClientPrivKey, ovnSouthClientCert, ovnSouthClientCACert, false)
		if err != nil {
			panic(err.Error())
		}

		if node != "" {
			if token == "" {
				panic("Cannot initialize node without service account 'token'. Please provide one with --token argument")
			}

			err := clusterController.StartClusterNode(node)
			if err != nil {
				logrus.Errorf(err.Error())
				panic(err.Error())
			}
		}
		clusterController.NodePortEnable = nodePortEnable

		if master != "" {
			if runtime.GOOS == "windows" {
				panic("Windows is not supported as master node")
			}
			ovnNorthServerPrivKey := ctx.String("ovn-north-server-privkey")
			ovnNorthServerCert := ctx.String("ovn-north-server-cert")
			ovnNorthServerCACert := ctx.String("ovn-north-server-cacert")
			clusterController.NorthDBServerAuth, err = ovncluster.NewOvnDBAuth(ovnNorth, ovnNorthServerPrivKey, ovnNorthServerCert, ovnNorthServerCACert, true)
			if err != nil {
				panic(err.Error())
			}

			ovnSouthServerPrivKey := ctx.String("ovn-south-server-privkey")
			ovnSouthServerCert := ctx.String("ovn-south-server-cert")
			ovnSouthServerCACert := ctx.String("ovn-south-server-cacert")
			clusterController.SouthDBServerAuth, err = ovncluster.NewOvnDBAuth(ovnSouth, ovnSouthServerPrivKey, ovnSouthServerCert, ovnSouthServerCACert, true)
			if err != nil {
				panic(err.Error())
			}

			// run the cluster controller to init the master
			err := clusterController.StartClusterMaster(master)
			if err != nil {
				logrus.Errorf(err.Error())
				panic(err.Error())
			}
		}
	}
	if netController {
		ovnController := ovn.NewOvnController(clientset, factory)
		ovnController.NodePortEnable = nodePortEnable
		ovnController.Run()
	}
	if master != "" || netController {
		// run forever
		select {}
	}

	return nil
}
<|endoftext|>