<commit_before>// Copyright 2016 Stratumn SAS. All rights reserved.
// Use of this source code is governed by the license
// that can be found in the LICENSE file.

// Package batchfossilizer implements a fossilizer that fossilize batches of data using a Merkle tree.
// The evidence will contain the Merkle root, the Merkle path, and a timestamp.
package batchfossilizer

import (
	"encoding/gob"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/stratumn/go/fossilizer"
	"github.com/stratumn/goprivate/merkle"
	"github.com/stratumn/goprivate/types"
)

const (
	// Name is the name set in the fossilizer's information.
	Name = "batch"

	// Description is the description set in the fossilizer's information.
	Description = "Stratumn Batch Fossilizer"

	// DefaultInterval is the default interval between batches.
	DefaultInterval = time.Minute

	// DefaultMaxLeaves if the default maximum number of leaves of a Merkle tree.
	DefaultMaxLeaves = 32 * 1024

	// DefaultArchive is whether to archive completed batches by default.
	DefaultArchive = true

	// DefaultStopBatch is whether to do a batch on stop by default.
	DefaultStopBatch = true

	// DefaultFSync is whether to fsync after saving a hash to disk by default.
	DefaultFSync = false

	// PendingExt is the pending hashes filename extension.
	PendingExt = "pending"

	// DirPerm is the directory's permissions.
	DirPerm = 0644

	// FilePerm is the files's permissions.
	FilePerm = 0644
)

// Config contains configuration options for the fossilizer.
type Config struct {
	// A version string that will set in the store's information.
	Version string

	// A git commit sha that will set in the store's information.
	Commit string

	// Interval between batches.
	Interval time.Duration

	// Maximum number of leaves of a Merkle tree.
	MaxLeaves int

	// Where to store pending hashes.
	// If empty, pending hashes are not saved and will be lost if stopped abruptly.
	Path string

	// Whether to archive completed batches.
	Archive bool

	// Whether to do a batch on stop.
	StopBatch bool

	// Whether to fsync after saving a hash to disk.
	FSync bool
}

// Info is the info returned by GetInfo.
type Info struct {
	Name        string `json:"name"`
	Description string `json:"description"`
	Version     string `json:"version"`
	Commit      string `json:"commit"`
}

// Evidence is the evidence sent to the result channel.
type Evidence struct {
	Time int64          `json:"time"`
	Root *types.Bytes32 `json:"merkleRoot"`
	Path merkle.Path    `json:"merklePath"`
}

// EvidenceWrapper wraps evidence with a namespace.
type EvidenceWrapper struct {
	Evidence *Evidence `json:"batch"`
}

type batch struct {
	leaves []types.Bytes32
	meta   [][]byte
	path   string
}

type chunk struct {
	Data []byte
	Meta []byte
}

// Fossilizer is the type that implements github.com/stratumn/go/fossilizer.Adapter.
type Fossilizer struct {
	config      *Config
	resultChans []chan *fossilizer.Result
	leaves      []types.Bytes32
	meta        [][]byte
	file        *os.File
	encoder     *gob.Encoder
	mutex       sync.Mutex
	waitGroup   sync.WaitGroup
	closeChan   chan error
}

// New creates an instance of a Fossilizer.
func New(config *Config) (*Fossilizer, error) {
	maxLeaves := config.MaxLeaves
	if maxLeaves == 0 {
		maxLeaves = DefaultMaxLeaves
	}

	a := &Fossilizer{
		config:    config,
		leaves:    make([]types.Bytes32, 0, maxLeaves),
		meta:      make([][]byte, 0, maxLeaves),
		closeChan: make(chan error),
	}

	if a.config.Path != "" {
		if err := os.MkdirAll(a.config.Path, DirPerm); err != nil {
			return nil, err
		}

		if err := a.recover(); err != nil {
			return nil, err
		}
	}

	return a, nil
}

// GetInfo implements github.com/stratumn/go/fossilizer.Adapter.GetInfo.
func (a *Fossilizer) GetInfo() (interface{}, error) {
	return &Info{
		Name:        Name,
		Description: Description,
		Version:     a.config.Version,
		Commit:      a.config.Commit,
	}, nil
}

// AddResultChan implements github.com/stratumn/go/fossilizer.Adapter.AddResultChan.
func (a *Fossilizer) AddResultChan(resultChan chan *fossilizer.Result) {
	a.resultChans = append(a.resultChans, resultChan)
}

// Fossilize implements github.com/stratumn/go/fossilizer.Adapter.Fossilize.
func (a *Fossilizer) Fossilize(data []byte, meta []byte) error {
	a.mutex.Lock()
	defer a.mutex.Unlock()

	if a.closeChan == nil {
		return errors.New("fossilizer is stopped")
	}

	if a.config.Path != "" {
		if a.file == nil {
			if err := a.open(); err != nil {
				return err
			}
		}

		if err := a.write(data, meta); err != nil {
			return err
		}
	}

	var leaf types.Bytes32
	copy(leaf[:], data)
	a.leaves = append(a.leaves, leaf)
	a.meta = append(a.meta, meta)

	maxLeaves := a.config.MaxLeaves
	if maxLeaves == 0 {
		maxLeaves = DefaultMaxLeaves
	}
	if len(a.leaves) >= maxLeaves {
		if err := a.makeBatch(); err != nil {
			a.closeChan <- err
			return err
		}
	}

	return nil
}

// Start starts the fossilizer.
func (a *Fossilizer) Start() error {
	interval := a.config.Interval
	if interval == 0 {
		interval = DefaultInterval
	}

	for {
		select {
		case <-time.After(interval):
			a.mutex.Lock()
			if len(a.leaves) > 0 {
				if err := a.makeBatch(); err != nil {
					return err
				}
			}
			a.mutex.Unlock()
		case err := <-a.closeChan:
			return err
		}
	}
}

// Stop stops the fossilizer.
func (a *Fossilizer) Stop() error {
	a.mutex.Lock()
	defer a.mutex.Unlock()

	close(a.closeChan)
	a.closeChan = nil

	if a.config.StopBatch && len(a.leaves) > 0 {
		if err := a.makeBatch(); err != nil {
			return err
		}
	}

	a.waitGroup.Wait()

	if a.file != nil {
		return a.file.Close()
	}

	return nil
}

func (a *Fossilizer) batch(b batch) {
	defer a.waitGroup.Done()

	tree, err := merkle.NewStaticTree(b.leaves)

	if err != nil {
		a.closeChan <- err
		return
	}

	var (
		meta = b.meta
		ts   = time.Now().UTC().Unix()
		root = tree.Root()
	)

	for i := 0; i < tree.LeavesLen(); i++ {
		leaf := tree.Leaf(i)
		r := &fossilizer.Result{
			Evidence: &EvidenceWrapper{
				&Evidence{
					Time: ts,
					Root: root,
					Path: tree.Path(i),
				},
			},
			Data: leaf[:],
			Meta: meta[i],
		}

		for _, c := range a.resultChans {
			c <- r
		}
	}

	if b.path != "" {
		if a.config.Archive {
			path := filepath.Join(a.config.Path, root.String())
			if err := os.Rename(b.path, path); err != nil {
				log.Printf("Error: %s", err)
			}
		} else {
			if err := os.Remove(b.path); err != nil {
				log.Printf("Error: %s", err)
			}
		}
	}
}

func (a *Fossilizer) makeBatch() error {
	var path string

	if a.file != nil {
		path = a.file.Name()
		if err := a.file.Close(); err != nil {
			return err
		}
		a.file = nil
	}

	a.waitGroup.Add(1)
	go a.batch(batch{a.leaves, a.meta, path})

	maxLeaves := a.config.MaxLeaves
	if maxLeaves == 0 {
		maxLeaves = DefaultMaxLeaves
	}

	a.leaves, a.meta = make([]types.Bytes32, 0, maxLeaves), make([][]byte, 0, maxLeaves)

	return nil
}

func (a *Fossilizer) recover() error {
	matches, err := filepath.Glob(filepath.Join(a.config.Path, "*."+PendingExt))
	if err != nil {
		return err
	}

	for _, path := range matches {
		file, err := os.OpenFile(path, os.O_RDONLY|os.O_EXCL, FilePerm)
		if err != nil {
			return err
		}

		dec := gob.NewDecoder(file)

		for {
			var c chunk

			if err := dec.Decode(&c); err != nil {
				if err == io.EOF {
					break
				}
				file.Close()
				return err
			}

			if err := a.Fossilize(c.Data, c.Meta); err != nil {
				file.Close()
				return err
			}
		}

		a.waitGroup.Wait()
		file.Close()

		if err := os.Remove(path); err != nil {
			return err
		}

		log.Printf("Recovered pending hashes file %q", filepath.Base(path))
	}

	return nil
}

func (a *Fossilizer) open() error {
	filename := fmt.Sprintf("%d.%s", time.Now().UTC().UnixNano(), PendingExt)
	path := filepath.Join(a.config.Path, filename)
	file, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY|os.O_EXCL|os.O_CREATE, FilePerm)
	if err != nil {
		return err
	}

	a.file, a.encoder = file, gob.NewEncoder(file)

	return nil
}

func (a *Fossilizer) write(data []byte, meta []byte) error {
	if err := a.encoder.Encode(chunk{data, meta}); err != nil {
		return err
	}

	if a.config.FSync {
		if err := a.file.Sync(); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>batchfossilizer: Polish<commit_after>// Copyright 2016 Stratumn SAS. All rights reserved.
// Use of this source code is governed by the license
// that can be found in the LICENSE file.

// Package batchfossilizer implements a fossilizer that fossilize batches of data using a Merkle tree.
// The evidence will contain the Merkle root, the Merkle path, and a timestamp.
package batchfossilizer

import (
	"encoding/gob"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/stratumn/go/fossilizer"
	"github.com/stratumn/goprivate/merkle"
	"github.com/stratumn/goprivate/types"
)

const (
	// Name is the name set in the fossilizer's information.
	Name = "batch"

	// Description is the description set in the fossilizer's information.
	Description = "Stratumn Batch Fossilizer"

	// DefaultInterval is the default interval between batches.
	DefaultInterval = time.Minute

	// DefaultMaxLeaves if the default maximum number of leaves of a Merkle tree.
	DefaultMaxLeaves = 32 * 1024

	// DefaultArchive is whether to archive completed batches by default.
	DefaultArchive = true

	// DefaultStopBatch is whether to do a batch on stop by default.
	DefaultStopBatch = true

	// DefaultFSync is whether to fsync after saving a hash to disk by default.
	DefaultFSync = false

	// PendingExt is the pending hashes filename extension.
	PendingExt = "pending"

	// DirPerm is the directory's permissions.
	DirPerm = 0644

	// FilePerm is the files's permissions.
	FilePerm = 0644
)

// Config contains configuration options for the fossilizer.
type Config struct {
	// A version string that will set in the store's information.
	Version string

	// A git commit sha that will set in the store's information.
	Commit string

	// Interval between batches.
	Interval time.Duration

	// Maximum number of leaves of a Merkle tree.
	MaxLeaves int

	// Where to store pending hashes.
	// If empty, pending hashes are not saved and will be lost if stopped abruptly.
	Path string

	// Whether to archive completed batches.
	Archive bool

	// Whether to do a batch on stop.
	StopBatch bool

	// Whether to fsync after saving a hash to disk.
	FSync bool
}

// Info is the info returned by GetInfo.
type Info struct {
	Name        string `json:"name"`
	Description string `json:"description"`
	Version     string `json:"version"`
	Commit      string `json:"commit"`
}

// Evidence is the evidence sent to the result channel.
type Evidence struct {
	Time int64          `json:"time"`
	Root *types.Bytes32 `json:"merkleRoot"`
	Path merkle.Path    `json:"merklePath"`
}

// EvidenceWrapper wraps evidence with a namespace.
type EvidenceWrapper struct {
	Evidence *Evidence `json:"batch"`
}

type batch struct {
	leaves []types.Bytes32
	meta   [][]byte
	path   string
}

type chunk struct {
	Data []byte
	Meta []byte
}

// Fossilizer is the type that implements github.com/stratumn/go/fossilizer.Adapter.
type Fossilizer struct {
	config      *Config
	resultChans []chan *fossilizer.Result
	leaves      []types.Bytes32
	meta        [][]byte
	file        *os.File
	encoder     *gob.Encoder
	mutex       sync.Mutex
	waitGroup   sync.WaitGroup
	closeChan   chan error
}

// New creates an instance of a Fossilizer.
func New(config *Config) (*Fossilizer, error) {
	maxLeaves := config.MaxLeaves
	if maxLeaves == 0 {
		maxLeaves = DefaultMaxLeaves
	}

	a := &Fossilizer{
		config:    config,
		leaves:    make([]types.Bytes32, 0, maxLeaves),
		meta:      make([][]byte, 0, maxLeaves),
		closeChan: make(chan error),
	}

	if a.config.Path != "" {
		if err := os.MkdirAll(a.config.Path, DirPerm); err != nil {
			return nil, err
		}

		if err := a.recover(); err != nil {
			return nil, err
		}
	}

	return a, nil
}

// GetInfo implements github.com/stratumn/go/fossilizer.Adapter.GetInfo.
func (a *Fossilizer) GetInfo() (interface{}, error) {
	return &Info{
		Name:        Name,
		Description: Description,
		Version:     a.config.Version,
		Commit:      a.config.Commit,
	}, nil
}

// AddResultChan implements github.com/stratumn/go/fossilizer.Adapter.AddResultChan.
func (a *Fossilizer) AddResultChan(resultChan chan *fossilizer.Result) {
	a.resultChans = append(a.resultChans, resultChan)
}

// Fossilize implements github.com/stratumn/go/fossilizer.Adapter.Fossilize.
func (a *Fossilizer) Fossilize(data []byte, meta []byte) error {
	a.mutex.Lock()
	defer a.mutex.Unlock()

	if a.closeChan == nil {
		return errors.New("fossilizer is stopped")
	}

	if a.config.Path != "" {
		if a.file == nil {
			if err := a.open(); err != nil {
				return err
			}
		}

		if err := a.write(data, meta); err != nil {
			return err
		}
	}

	var leaf types.Bytes32
	copy(leaf[:], data)
	a.leaves = append(a.leaves, leaf)
	a.meta = append(a.meta, meta)

	maxLeaves := a.config.MaxLeaves
	if maxLeaves == 0 {
		maxLeaves = DefaultMaxLeaves
	}
	if len(a.leaves) >= maxLeaves {
		if err := a.requestBatch(); err != nil {
			a.closeChan <- err
			return err
		}
		log.Printf("Requested new batch because the maximum number of leaves (%d) was reached", maxLeaves)
	}

	return nil
}

// Start starts the fossilizer.
func (a *Fossilizer) Start() error {
	interval := a.config.Interval
	if interval == 0 {
		interval = DefaultInterval
	}

	for {
		select {
		case <-time.After(interval):
			a.mutex.Lock()
			if len(a.leaves) > 0 {
				if err := a.requestBatch(); err != nil {
					return err
				}
				log.Printf("Requested new batch because the %s interval was reached", interval)
			} else {
				log.Printf("No batch is needed after the %s interval because there are no pending hashes", interval)
			}
			a.mutex.Unlock()
		case err := <-a.closeChan:
			return err
		}
	}
}

// Stop stops the fossilizer.
func (a *Fossilizer) Stop() error {
	a.mutex.Lock()
	defer a.mutex.Unlock()

	close(a.closeChan)
	a.closeChan = nil

	if a.config.StopBatch {
		if len(a.leaves) > 0 {
			if err := a.requestBatch(); err != nil {
				return err
			}
			log.Print("Requested final batch for pending hashes")
		} else {
			log.Print("No final batch is needed because there are no pending hashes")
		}
	}

	a.waitGroup.Wait()

	if a.file != nil {
		return a.file.Close()
	}

	return nil
}

func (a *Fossilizer) batch(b batch) {
	defer a.waitGroup.Done()

	tree, err := merkle.NewStaticTree(b.leaves)

	if err != nil {
		a.closeChan <- err
		return
	}

	var (
		meta = b.meta
		ts   = time.Now().UTC().Unix()
		root = tree.Root()
	)

	log.Printf("Created tree with Merkle root %q", root)

	for i := 0; i < tree.LeavesLen(); i++ {
		leaf := tree.Leaf(i)
		r := &fossilizer.Result{
			Evidence: &EvidenceWrapper{
				&Evidence{
					Time: ts,
					Root: root,
					Path: tree.Path(i),
				},
			},
			Data: leaf[:],
			Meta: meta[i],
		}

		for _, c := range a.resultChans {
			c <- r
		}
	}

	log.Printf("Sent evidence for batch with Merkle root %q", root)

	if b.path != "" {
		if a.config.Archive {
			path := filepath.Join(a.config.Path, root.String())
			if err := os.Rename(b.path, path); err != nil {
				log.Printf("Error: %s", err)
			}
			log.Printf("Renamed pending hashes file %q to %q", filepath.Base(b.path), filepath.Base(path))
		} else {
			if err := os.Remove(b.path); err != nil {
				log.Printf("Error: %s", err)
			}
			log.Printf("Removed pending hashes file %q", filepath.Base(b.path))
		}
	}

	log.Printf("Finished batch with Merkle root %q", root)
}

func (a *Fossilizer) requestBatch() error {
	var path string

	if a.file != nil {
		path = a.file.Name()
		if err := a.file.Close(); err != nil {
			return err
		}
		a.file = nil
	}

	a.waitGroup.Add(1)
	go a.batch(batch{a.leaves, a.meta, path})

	maxLeaves := a.config.MaxLeaves
	if maxLeaves == 0 {
		maxLeaves = DefaultMaxLeaves
	}

	a.leaves, a.meta = make([]types.Bytes32, 0, maxLeaves), make([][]byte, 0, maxLeaves)

	return nil
}

func (a *Fossilizer) recover() error {
	matches, err := filepath.Glob(filepath.Join(a.config.Path, "*."+PendingExt))
	if err != nil {
		return err
	}

	for _, path := range matches {
		file, err := os.OpenFile(path, os.O_RDONLY|os.O_EXCL, FilePerm)
		if err != nil {
			return err
		}

		dec := gob.NewDecoder(file)

		for {
			var c chunk

			if err := dec.Decode(&c); err != nil {
				if err == io.EOF {
					break
				}
				file.Close()
				return err
			}

			if err := a.Fossilize(c.Data, c.Meta); err != nil {
				file.Close()
				return err
			}
		}

		a.waitGroup.Wait()
		file.Close()

		if err := os.Remove(path); err != nil {
			return err
		}

		log.Printf("Recovered pending hashes file %q", filepath.Base(path))
	}

	return nil
}

func (a *Fossilizer) open() error {
	filename := fmt.Sprintf("%d.%s", time.Now().UTC().UnixNano(), PendingExt)
	path := filepath.Join(a.config.Path, filename)
	file, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY|os.O_EXCL|os.O_CREATE, FilePerm)
	if err != nil {
		return err
	}
	log.Print("Opened pending hashes file %s", filepath.Base(path))

	a.file, a.encoder = file, gob.NewEncoder(file)

	return nil
}

func (a *Fossilizer) write(data []byte, meta []byte) error {
	if err := a.encoder.Encode(chunk{data, meta}); err != nil {
		return err
	}

	if a.config.FSync {
		if err := a.file.Sync(); err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package bubbles

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"sync"
	"time"
)

const (
	// DefaultMaxDocumentsPerBatch is the number of documents a batch needs to
	// have before it is send. This is per connection.
	DefaultMaxDocumentsPerBatch = 10

	// DefaultFlushTimeout is the maximum time we batch something before we try
	// to send it to a server.
	DefaultFlushTimeout = 10 * time.Second

	// DefaultServerTimeout is the time we give ES to respond. This is also the
	// maximum time Stop() will take.
	DefaultServerTimeout = 10 * time.Second

	// DefaultConnCount is the number of connections per hosts.
	DefaultConnCount = 2

	serverErrorWait = 1 * time.Second
)

// Bubbles is the main struct to control a queue of Actions going to the
// ElasticSearch servers.
type Bubbles struct {
	q                chan Action
	retryQ           chan Action
	error            chan ActionError
	quit             chan struct{}
	wg               sync.WaitGroup
	maxDocumentCount int
	connCount        int
	flushTimeout     time.Duration
	serverTimeout    time.Duration
}

type bulkRes struct {
	Took   int                        `json:"took"`
	Items  []map[string]bulkResStatus `json:"items"`
	Errors bool                       `json:"errors"`
}

type bulkResStatus struct {
	Index   string `json:"_index"`
	Type    string `json:"_type"`
	ID      string `json:"_id"`
	Version int    `json:"_version"`
	Status  int    `json:"status"`
	Error   string `json:"error"`
}

// ActionError wraps an Action we won't retry. It implements the error interface.
type ActionError struct {
	Action Action
	Msg    string
	Server string
}

func (e ActionError) Error() string {
	return fmt.Sprintf("%s: %s", e.Server, e.Msg)
}

// Opt is any option to New()
type Opt func(*Bubbles)

// OptConnCount is an option to New() to specify the number of connections per
// host. The default is DefaultConnCount.
func OptConnCount(n int) Opt {
	return func(b *Bubbles) {
		b.connCount = n
	}
}

// OptFlush is an option to New() to specify the flush timeout of a batch. The
// default is DefaultFlushTimeout.
func OptFlush(d time.Duration) Opt {
	return func(b *Bubbles) {
		b.flushTimeout = d
	}
}

// OptServerTimeout is an option to New() to specify the timeout of a single
// batch POST to ES. This value is also the maximum time Stop() will take. All
// actions in a bulk which is timed out will be retried. The default is
// DefaultServerTimeout.
func OptServerTimeout(d time.Duration) Opt {
	return func(b *Bubbles) {
		b.serverTimeout = d
	}
}

// OptMaxDocs is an option to New() to specify maximum number of documents in a
// single batch. The default is DefaultMaxDocumentsPerBatch.
func OptMaxDocs(n int) Opt {
	return func(b *Bubbles) {
		b.maxDocumentCount = n
	}
}

// New makes a new ES bulk inserter. It needs a list with 'ip:port' addresses,
// options are added via the Opt* functions. Be sure to read the Errors()
// channel.
func New(addrs []string, opts ...Opt) *Bubbles {
	b := Bubbles{
		q:                make(chan Action),
		error:            make(chan ActionError, 10),
		quit:             make(chan struct{}),
		maxDocumentCount: DefaultMaxDocumentsPerBatch,
		connCount:        DefaultConnCount,
		flushTimeout:     DefaultFlushTimeout,
		serverTimeout:    DefaultServerTimeout,
	}
	for _, o := range opts {
		o(&b)
	}
	b.retryQ = make(chan Action, len(addrs)*b.connCount)

	// Start a go routine per connection per host
	for _, a := range addrs {
		for i := 0; i < b.connCount; i++ {
			b.wg.Add(1)
			go func(a string) {
				client(&b, a)
				b.wg.Done()
			}(a)
		}
	}
	return &b
}

// Errors returns a channel with all actions we won't retry.
func (b *Bubbles) Errors() <-chan ActionError {
	return b.error
}

// Enqueue returns the queue to add Actions in a routine. It will block if all bulk
// processors are busy.
func (b *Bubbles) Enqueue() chan<- Action {
	return b.q
}

// Stop shuts down all ES clients. It'll return all Action entries which were
// not yet processed, or were up for a retry. It can take OptServerTimeout to
// complete.
func (b *Bubbles) Stop() []Action {
	close(b.quit)
	// There is no explicit timeout, we rely on b.serverTimeout to shut down
	// everything.
	b.wg.Wait()

	close(b.error)

	// Collect and return elements which are in flight.
	close(b.retryQ)
	close(b.q)
	pending := make([]Action, 0, len(b.q)+len(b.retryQ))
	for a := range b.q {
		pending = append(pending, a)
	}
	for a := range b.retryQ {
		pending = append(pending, a)
	}
	return pending
}

// client talks to ES. This runs in a go routine in a loop and deals with a
// single ES address.
func client(b *Bubbles, addr string) {
	url := fmt.Sprintf("http://%s/_bulk", addr) // TODO: https?
	// log.Printf("starting client to %s\n", addr)
	// defer log.Printf("stopping client to %s\n", addr)

	cl := http.Client{
		Timeout:       b.serverTimeout,
		CheckRedirect: noRedirect,
	}

	for {
		select {
		case <-b.quit:
			return
		default:
		}
		if err := runBatch(b, cl, url); err != nil {
			// runBatch only returns an error on server error.
			log.Printf("server error: %s", err)
			select {
			case <-b.quit:
			case <-time.After(serverErrorWait):
				// TODO: backoff period
			}
		}
	}

}

// runBatch gathers and deals with a batch of actions. It'll return a non-nil
// error if the whole server gave an error.
func runBatch(b *Bubbles, cl http.Client, url string) error {
	actions := make([]Action, 0, b.maxDocumentCount)
	// First use all retry actions.
retry:
	for len(actions) < b.maxDocumentCount {
		select {
		case a := <-b.retryQ:
			actions = append(actions, a)
		default:
			// no more retry actions queued
			break retry
		}
	}

	var t <-chan time.Time
gather:
	for len(actions) < b.maxDocumentCount {
		if t == nil && len(actions) > 0 {
			// Set timeout on the first element we read
			t = time.After(b.flushTimeout)
		}
		select {
		case <-b.quit:
			for _, a := range actions {
				b.retryQ <- a
			}
			return nil
		case <-t:
			// this case is not enabled until we've got an action
			break gather
		case a := <-b.retryQ:
			actions = append(actions, a)
		case a := <-b.q:
			actions = append(actions, a)
		}
	}
	if len(actions) == 0 {
		// no actions. Weird.
		return nil
	}

	res, err := postActions(cl, url, actions)
	if err != nil {
		// A server error. Retry these actions later.
		for _, a := range actions {
			b.retryQ <- a
		}
		return err
	}

	// Server has accepted the request an sich, but there can be errors in the
	// individual actions.
	if !res.Errors {
		// Simple case, no errors present.
		return nil
	}
	// Figure out which actions have errors.
	for i, e := range res.Items {
		a := actions[i] // TODO: sanity check
		el, ok := e[string(a.Type)]
		if !ok {
			// TODO: this
			fmt.Printf("Non matching action!\n")
			continue
		}

		c := el.Status
		switch {
		case c >= 200 && c < 300:
			// Document accepted by ES.
		case c >= 400 && c < 500:
			// Some error. Nothing we can do with it.
			b.error <- ActionError{
				Action: a,
				Msg:    fmt.Sprintf("error %d: %s", c, el.Error),
				Server: url,
			}
		case c >= 500 && c < 600:
			// Server error. Retry it.
			b.retryQ <- a
		default:
			// No idea.
			fmt.Printf("unexpected status: %d. Ignoring document.\n", c)
		}
	}
	return nil
}

func postActions(cl http.Client, url string, actions []Action) (*bulkRes, error) {
	// TODO: bytestring as argument
	// TODO: don't chunk.
	buf := bytes.Buffer{}
	for _, a := range actions {
		buf.Write(a.Buf())
	}

	resp, err := cl.Post(url, "application/json", &buf)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("status %d: %s", resp.StatusCode, string(body))
	}

	var bulk bulkRes
	if err := json.Unmarshal(body, &bulk); err != nil {
		return nil, err
	}
	return &bulk, nil
}

func noRedirect(req *http.Request, via []*http.Request) error {
	return errors.New("no redirect")
}
<commit_msg>Actions can't be empty. Otherwise, we'll have to deal with it.<commit_after>package bubbles

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"sync"
	"time"
)

const (
	// DefaultMaxDocumentsPerBatch is the number of documents a batch needs to
	// have before it is send. This is per connection.
	DefaultMaxDocumentsPerBatch = 10

	// DefaultFlushTimeout is the maximum time we batch something before we try
	// to send it to a server.
	DefaultFlushTimeout = 10 * time.Second

	// DefaultServerTimeout is the time we give ES to respond. This is also the
	// maximum time Stop() will take.
	DefaultServerTimeout = 10 * time.Second

	// DefaultConnCount is the number of connections per hosts.
	DefaultConnCount = 2

	serverErrorWait = 1 * time.Second
)

// Bubbles is the main struct to control a queue of Actions going to the
// ElasticSearch servers.
type Bubbles struct {
	q                chan Action
	retryQ           chan Action
	error            chan ActionError
	quit             chan struct{}
	wg               sync.WaitGroup
	maxDocumentCount int
	connCount        int
	flushTimeout     time.Duration
	serverTimeout    time.Duration
}

type bulkRes struct {
	Took   int                        `json:"took"`
	Items  []map[string]bulkResStatus `json:"items"`
	Errors bool                       `json:"errors"`
}

type bulkResStatus struct {
	Index   string `json:"_index"`
	Type    string `json:"_type"`
	ID      string `json:"_id"`
	Version int    `json:"_version"`
	Status  int    `json:"status"`
	Error   string `json:"error"`
}

// ActionError wraps an Action we won't retry. It implements the error interface.
type ActionError struct {
	Action Action
	Msg    string
	Server string
}

func (e ActionError) Error() string {
	return fmt.Sprintf("%s: %s", e.Server, e.Msg)
}

// Opt is any option to New()
type Opt func(*Bubbles)

// OptConnCount is an option to New() to specify the number of connections per
// host. The default is DefaultConnCount.
func OptConnCount(n int) Opt {
	return func(b *Bubbles) {
		b.connCount = n
	}
}

// OptFlush is an option to New() to specify the flush timeout of a batch. The
// default is DefaultFlushTimeout.
func OptFlush(d time.Duration) Opt {
	return func(b *Bubbles) {
		b.flushTimeout = d
	}
}

// OptServerTimeout is an option to New() to specify the timeout of a single
// batch POST to ES. This value is also the maximum time Stop() will take. All
// actions in a bulk which is timed out will be retried. The default is
// DefaultServerTimeout.
func OptServerTimeout(d time.Duration) Opt {
	return func(b *Bubbles) {
		b.serverTimeout = d
	}
}

// OptMaxDocs is an option to New() to specify maximum number of documents in a
// single batch. The default is DefaultMaxDocumentsPerBatch.
func OptMaxDocs(n int) Opt {
	return func(b *Bubbles) {
		b.maxDocumentCount = n
	}
}

// New makes a new ES bulk inserter. It needs a list with 'ip:port' addresses,
// options are added via the Opt* functions. Be sure to read the Errors()
// channel.
func New(addrs []string, opts ...Opt) *Bubbles {
	b := Bubbles{
		q:                make(chan Action),
		error:            make(chan ActionError, 10),
		quit:             make(chan struct{}),
		maxDocumentCount: DefaultMaxDocumentsPerBatch,
		connCount:        DefaultConnCount,
		flushTimeout:     DefaultFlushTimeout,
		serverTimeout:    DefaultServerTimeout,
	}
	for _, o := range opts {
		o(&b)
	}
	b.retryQ = make(chan Action, len(addrs)*b.connCount)

	// Start a go routine per connection per host
	for _, a := range addrs {
		for i := 0; i < b.connCount; i++ {
			b.wg.Add(1)
			go func(a string) {
				client(&b, a)
				b.wg.Done()
			}(a)
		}
	}
	return &b
}

// Errors returns a channel with all actions we won't retry.
func (b *Bubbles) Errors() <-chan ActionError {
	return b.error
}

// Enqueue returns the queue to add Actions in a routine. It will block if all bulk
// processors are busy.
func (b *Bubbles) Enqueue() chan<- Action {
	return b.q
}

// Stop shuts down all ES clients. It'll return all Action entries which were
// not yet processed, or were up for a retry. It can take OptServerTimeout to
// complete.
func (b *Bubbles) Stop() []Action {
	close(b.quit)
	// There is no explicit timeout, we rely on b.serverTimeout to shut down
	// everything.
	b.wg.Wait()

	close(b.error)

	// Collect and return elements which are in flight.
	close(b.retryQ)
	close(b.q)
	pending := make([]Action, 0, len(b.q)+len(b.retryQ))
	for a := range b.q {
		pending = append(pending, a)
	}
	for a := range b.retryQ {
		pending = append(pending, a)
	}
	return pending
}

// client talks to ES. This runs in a go routine in a loop and deals with a
// single ES address.
func client(b *Bubbles, addr string) {
	url := fmt.Sprintf("http://%s/_bulk", addr) // TODO: https?
	// log.Printf("starting client to %s\n", addr)
	// defer log.Printf("stopping client to %s\n", addr)

	cl := http.Client{
		Timeout:       b.serverTimeout,
		CheckRedirect: noRedirect,
	}

	for {
		select {
		case <-b.quit:
			return
		default:
		}
		if err := runBatch(b, cl, url); err != nil {
			// runBatch only returns an error on server error.
			log.Printf("server error: %s", err)
			select {
			case <-b.quit:
			case <-time.After(serverErrorWait):
				// TODO: backoff period
			}
		}
	}

}

// runBatch gathers and deals with a batch of actions. It'll return a non-nil
// error if the whole server gave an error.
func runBatch(b *Bubbles, cl http.Client, url string) error {
	actions := make([]Action, 0, b.maxDocumentCount)
	// First use all retry actions.
retry:
	for len(actions) < b.maxDocumentCount {
		select {
		case a := <-b.retryQ:
			actions = append(actions, a)
		default:
			// no more retry actions queued
			break retry
		}
	}

	var t <-chan time.Time
gather:
	for len(actions) < b.maxDocumentCount {
		if t == nil && len(actions) > 0 {
			// Set timeout on the first element we read
			t = time.After(b.flushTimeout)
		}
		select {
		case <-b.quit:
			for _, a := range actions {
				b.retryQ <- a
			}
			return nil
		case <-t:
			// this case is not enabled until we've got an action
			break gather
		case a := <-b.retryQ:
			actions = append(actions, a)
		case a := <-b.q:
			actions = append(actions, a)
		}
	}

	res, err := postActions(cl, url, actions)
	if err != nil {
		// A server error. Retry these actions later.
		for _, a := range actions {
			b.retryQ <- a
		}
		return err
	}

	// Server has accepted the request an sich, but there can be errors in the
	// individual actions.
	if !res.Errors {
		// Simple case, no errors present.
		return nil
	}
	// Figure out which actions have errors.
	for i, e := range res.Items {
		a := actions[i] // TODO: sanity check
		el, ok := e[string(a.Type)]
		if !ok {
			// TODO: this
			fmt.Printf("Non matching action!\n")
			continue
		}

		c := el.Status
		switch {
		case c >= 200 && c < 300:
			// Document accepted by ES.
		case c >= 400 && c < 500:
			// Some error. Nothing we can do with it.
			b.error <- ActionError{
				Action: a,
				Msg:    fmt.Sprintf("error %d: %s", c, el.Error),
				Server: url,
			}
		case c >= 500 && c < 600:
			// Server error. Retry it.
			b.retryQ <- a
		default:
			// No idea.
			fmt.Printf("unexpected status: %d. Ignoring document.\n", c)
		}
	}
	return nil
}

func postActions(cl http.Client, url string, actions []Action) (*bulkRes, error) {
	// TODO: bytestring as argument
	// TODO: don't chunk.
	buf := bytes.Buffer{}
	for _, a := range actions {
		buf.Write(a.Buf())
	}

	resp, err := cl.Post(url, "application/json", &buf)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("status %d: %s", resp.StatusCode, string(body))
	}

	var bulk bulkRes
	if err := json.Unmarshal(body, &bulk); err != nil {
		return nil, err
	}
	return &bulk, nil
}

func noRedirect(req *http.Request, via []*http.Request) error {
	return errors.New("no redirect")
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controllers

import (
	"sync"

	k8smetrics "k8s.io/component-base/metrics"
	"k8s.io/component-base/metrics/legacyregistry"
)

var (
	once sync.Once
)

// ControllerMetrics includes all the metrics of the proxy server.
type ControllerMetrics struct {
	controllerInstanceCount *k8smetrics.GaugeVec
}

// NewControllerMetrics create a new ControllerMetrics, configured with default metric names.
func NewControllerMetrics() *ControllerMetrics {
	controllerInstanceCount := k8smetrics.NewGaugeVec(
		&k8smetrics.GaugeOpts{
			Name:           "managed_controller_instances",
			Help:           "Indicates where instances of a controller are currently running",
			StabilityLevel: k8smetrics.ALPHA,
		},
		[]string{"name", "manager"},
	)
	controllerMetrics := &ControllerMetrics{
		controllerInstanceCount: controllerInstanceCount,
	}
	once.Do(func() {
		controllerMetrics.Register()
	})
	return controllerMetrics
}

func (a *ControllerMetrics) Register() {
	legacyregistry.MustRegister(a.controllerInstanceCount)
}

// ControllerStarted sets the controllerInstanceCount to 1.
// These values use set instead of inc/dec to avoid accidentally double counting
// a controller that starts but fails to properly signal when it crashes.
func (a *ControllerMetrics) ControllerStarted(name string, manager string) {
	a.controllerInstanceCount.With(k8smetrics.Labels{"name": name, "manager": manager}).Set(float64(1))
}

// ControllerStopped sets the controllerInstanceCount to 0.
func (a *ControllerMetrics) ControllerStopped(name string, manager string) {
	a.controllerInstanceCount.With(k8smetrics.Labels{"name": name, "manager": manager}).Set(float64(0))
}
<commit_msg>updated name of metric<commit_after>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controllers

import (
	"sync"

	k8smetrics "k8s.io/component-base/metrics"
	"k8s.io/component-base/metrics/legacyregistry"
)

var (
	once sync.Once
)

// ControllerMetrics includes all the metrics of the proxy server.
type ControllerMetrics struct {
	controllerInstanceCount *k8smetrics.GaugeVec
}

// NewControllerMetrics create a new ControllerMetrics, configured with default metric names.
func NewControllerMetrics() *ControllerMetrics {
	controllerInstanceCount := k8smetrics.NewGaugeVec(
		&k8smetrics.GaugeOpts{
			Name:           "managed_controller_started",
			Help:           "Indicates where instances of a controller are currently running",
			StabilityLevel: k8smetrics.ALPHA,
		},
		[]string{"name", "manager"},
	)
	controllerMetrics := &ControllerMetrics{
		controllerInstanceCount: controllerInstanceCount,
	}
	once.Do(func() {
		controllerMetrics.Register()
	})
	return controllerMetrics
}

func (a *ControllerMetrics) Register() {
	legacyregistry.MustRegister(a.controllerInstanceCount)
}

// ControllerStarted sets the controllerInstanceCount to 1.
// These values use set instead of inc/dec to avoid accidentally double counting
// a controller that starts but fails to properly signal when it crashes.
func (a *ControllerMetrics) ControllerStarted(name string, manager string) {
	a.controllerInstanceCount.With(k8smetrics.Labels{"name": name, "manager": manager}).Set(float64(1))
}

// ControllerStopped sets the controllerInstanceCount to 0.
func (a *ControllerMetrics) ControllerStopped(name string, manager string) {
	a.controllerInstanceCount.With(k8smetrics.Labels{"name": name, "manager": manager}).Set(float64(0))
}
<|endoftext|>


<commit_before>// +build windows,amd64

package winapi

import (
	"fmt"
	"time"

	"github.com/google/cabbie/search"
	"github.com/google/cabbie/session"
	"github.com/google/cabbie/updatehistory"
	"github.com/scjalliance/comshim"

	so "github.com/iamacarpet/go-win64api/shared"
)

var updateResultStatus []string = []string{
	"Completed", // Was "Pending", swap to "Completed" to match Update UI in OS
	"Completed", // Was "In Progress", swap to "Completed" to match Update UI in OS
	"Completed",
	"Completed With Errors",
	"Failed",
	"Aborted",
}

func UpdatesPending() (*so.WindowsUpdate, error) {
	retData := &so.WindowsUpdate{}

	comshim.Add(1)
	defer comshim.Done()

	reqUpdates, _, err := listUpdates(false)
	if err != nil {
		return nil, fmt.Errorf("Error getting Windows Update info: %s", err.Error())
	}
	retData.NumUpdates = len(reqUpdates)

	for _, u := range reqUpdates {
		retData.UpdateHistory = append(retData.UpdateHistory, &so.WindowsUpdateHistory{
			EventDate:  time.Now(),
			Status:     "Pending",
			UpdateName: u,
		})
	}

	history, err := updateHistory()
	if err != nil {
		return nil, fmt.Errorf("Error getting update history: %s", err.Error())
	}

	for _, e := range history.Entries {
		retData.UpdateHistory = append(retData.UpdateHistory, &so.WindowsUpdateHistory{
			EventDate:  e.Date,
			Status:     updateResultStatus[int(e.ResultCode)],
			UpdateName: e.Title,
		})
	}

	if retData.NumUpdates > 0 {
		retData.UpdatesReq = true
	}

	return retData, nil
}

func listUpdates(hidden bool) ([]string, []string, error) {
	// Set search criteria
	c := search.BasicSearch + " OR Type='Driver' OR " + search.BasicSearch + " AND Type='Software'"
	if hidden {
		c += " and IsHidden=1"
	} else {
		c += " and IsHidden=0"
	}

	// Start Windows update session
	s, err := session.New()
	if err != nil {
		return nil, nil, fmt.Errorf("failed to create new Windows Update session: %v", err)
	}
	defer s.Close()

	q, err := search.NewSearcher(s, c, []string{}, 1)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to create a new searcher object: %v", err)
	}
	defer q.Close()

	uc, err := q.QueryUpdates()
	if err != nil {
		return nil, nil, fmt.Errorf("error encountered when attempting to query for updates: %v", err)
	}
	defer uc.Close()

	var reqUpdates, optUpdates []string
	for _, u := range uc.Updates {
		// Add to optional updates list if the update does not match the required categories.
		if !u.InCategories([]string{"Critical Updates", "Definition Updates", "Security Updates"}) {
			optUpdates = append(optUpdates, u.Title)
			continue
		}
		// Skip virus updates as they always exist.
		if !u.InCategories([]string{"Definition Updates"}) {
			reqUpdates = append(reqUpdates, u.Title)
		}
	}
	return reqUpdates, optUpdates, nil
}

func updateHistory() (*updatehistory.History, error) {
	// Start Windows update session
	s, err := session.New()
	if err != nil {
		return nil, err
	}
	defer s.Close()

	// Create Update searcher interface
	searcher, err := search.NewSearcher(s, "", []string{}, 1)
	if err != nil {
		return nil, err
	}
	defer searcher.Close()

	return updatehistory.Get(searcher)
}
<commit_msg>Windows Update: Pending -> In Progress<commit_after>// +build windows,amd64

package winapi

import (
	"fmt"
	"time"

	"github.com/google/cabbie/search"
	"github.com/google/cabbie/session"
	"github.com/google/cabbie/updatehistory"
	"github.com/scjalliance/comshim"

	so "github.com/iamacarpet/go-win64api/shared"
)

var updateResultStatus []string = []string{
	"Completed", // Was "Pending", swap to "Completed" to match Update UI in OS
	"Completed", // Was "In Progress", swap to "Completed" to match Update UI in OS
	"Completed",
	"Completed With Errors",
	"Failed",
	"Aborted",
}

func UpdatesPending() (*so.WindowsUpdate, error) {
	retData := &so.WindowsUpdate{}

	comshim.Add(1)
	defer comshim.Done()

	reqUpdates, _, err := listUpdates(false)
	if err != nil {
		return nil, fmt.Errorf("Error getting Windows Update info: %s", err.Error())
	}
	retData.NumUpdates = len(reqUpdates)

	for _, u := range reqUpdates {
		retData.UpdateHistory = append(retData.UpdateHistory, &so.WindowsUpdateHistory{
			EventDate:  time.Now(),
			Status:     "In Progress",
			UpdateName: u,
		})
	}

	history, err := updateHistory()
	if err != nil {
		return nil, fmt.Errorf("Error getting update history: %s", err.Error())
	}

	for _, e := range history.Entries {
		retData.UpdateHistory = append(retData.UpdateHistory, &so.WindowsUpdateHistory{
			EventDate:  e.Date,
			Status:     updateResultStatus[int(e.ResultCode)],
			UpdateName: e.Title,
		})
	}

	if retData.NumUpdates > 0 {
		retData.UpdatesReq = true
	}

	return retData, nil
}

func listUpdates(hidden bool) ([]string, []string, error) {
	// Set search criteria
	c := search.BasicSearch + " OR Type='Driver' OR " + search.BasicSearch + " AND Type='Software'"
	if hidden {
		c += " and IsHidden=1"
	} else {
		c += " and IsHidden=0"
	}

	// Start Windows update session
	s, err := session.New()
	if err != nil {
		return nil, nil, fmt.Errorf("failed to create new Windows Update session: %v", err)
	}
	defer s.Close()

	q, err := search.NewSearcher(s, c, []string{}, 1)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to create a new searcher object: %v", err)
	}
	defer q.Close()

	uc, err := q.QueryUpdates()
	if err != nil {
		return nil, nil, fmt.Errorf("error encountered when attempting to query for updates: %v", err)
	}
	defer uc.Close()

	var reqUpdates, optUpdates []string
	for _, u := range uc.Updates {
		// Add to optional updates list if the update does not match the required categories.
		if !u.InCategories([]string{"Critical Updates", "Definition Updates", "Security Updates"}) {
			optUpdates = append(optUpdates, u.Title)
			continue
		}
		// Skip virus updates as they always exist.
		if !u.InCategories([]string{"Definition Updates"}) {
			reqUpdates = append(reqUpdates, u.Title)
		}
	}
	return reqUpdates, optUpdates, nil
}

func updateHistory() (*updatehistory.History, error) {
	// Start Windows update session
	s, err := session.New()
	if err != nil {
		return nil, err
	}
	defer s.Close()

	// Create Update searcher interface
	searcher, err := search.NewSearcher(s, "", []string{}, 1)
	if err != nil {
		return nil, err
	}
	defer searcher.Close()

	return updatehistory.Get(searcher)
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package nodejs_profile

import (
	"flag"
	"fmt"
	"runtime"

	"v.io/jiri"
	"v.io/jiri/profiles"
	"v.io/jiri/profiles/profilesmanager"
	"v.io/jiri/profiles/profilesutil"
)

type versionSpec struct {
	nodeVersion string
}

func Register(installer, profile string) {
	m := &Manager{
		profileInstaller: installer,
		profileName:      profile,
		qualifiedName:    profiles.QualifiedProfileName(installer, profile),
		versionInfo: profiles.NewVersionInfo(profile, map[string]interface{}{
			"10.24": &versionSpec{"node-v0.10.24"},
		}, "10.24"),
	}
	profilesmanager.Register(m)
}

type Manager struct {
	profileInstaller, profileName, qualifiedName string
	root, nodeRoot                               jiri.RelPath
	nodeSrcDir, nodeInstDir                      jiri.RelPath
	versionInfo                                  *profiles.VersionInfo
	spec                                         versionSpec
}

func (m Manager) Name() string {
	return m.profileName
}

func (m Manager) Installer() string {
	return m.profileInstaller
}
func (m Manager) String() string {
	return fmt.Sprintf("%s[%s]", m.qualifiedName, m.versionInfo.Default())
}

func (m Manager) VersionInfo() *profiles.VersionInfo {
	return m.versionInfo
}

func (m Manager) Info() string {
	return `
The nodejs profile provides support for node. It installs and builds particular,
tested, versions of node.`
}

func (m *Manager) AddFlags(flags *flag.FlagSet, action profiles.Action) {}

func (m *Manager) initForTarget(root jiri.RelPath, target profiles.Target) error {
	if err := m.versionInfo.Lookup(target.Version(), &m.spec); err != nil {
		return err
	}
	m.root = root
	m.nodeRoot = m.root.Join("cout", m.spec.nodeVersion)
	m.nodeInstDir = m.nodeRoot.Join(target.TargetSpecificDirname())
	m.nodeSrcDir = jiri.NewRelPath("third_party", "csrc", m.spec.nodeVersion)
	return nil
}

func (m *Manager) Install(jirix *jiri.X, pdb *profiles.DB, root jiri.RelPath, target profiles.Target) error {
	if err := m.initForTarget(root, target); err != nil {
		return err
	}
	if target.CrossCompiling() {
		return fmt.Errorf("the %q profile does not support cross compilation to %v", m.qualifiedName, target)
	}
	if err := m.installNode(jirix, target); err != nil {
		return err
	}
	target.InstallationDir = string(m.nodeInstDir)
	pdb.InstallProfile(m.profileInstaller, m.profileName, string(m.nodeRoot))
	return pdb.AddProfileTarget(m.profileInstaller, m.profileName, target)
}

func (m *Manager) Uninstall(jirix *jiri.X, pdb *profiles.DB, root jiri.RelPath, target profiles.Target) error {
	if err := m.initForTarget(root, target); err != nil {
		return err
	}
	if err := jirix.NewSeq().RemoveAll(m.nodeInstDir.Abs(jirix)).Done(); err != nil {
		return err
	}
	pdb.RemoveProfileTarget(m.profileInstaller, m.profileName, target)
	return nil
}

func (m *Manager) installNode(jirix *jiri.X, target profiles.Target) error {
	switch target.OS() {
	case "darwin":
	case "linux":
		if err := profilesutil.InstallPackages(jirix, []string{"g++"}); err != nil {
			return err
		}
	default:
		return fmt.Errorf("%q is not supported", target.OS)
	}
	// Build and install NodeJS.
	installNodeFn := func() error {
		return jirix.NewSeq().Pushd(m.nodeSrcDir.Abs(jirix)).
			Run("./configure", fmt.Sprintf("--prefix=%v", m.nodeInstDir.Abs(jirix))).
			Run("make", "clean").
			Run("make", fmt.Sprintf("-j%d", runtime.NumCPU())).
			Last("make", "install")
	}
	return profilesutil.AtomicAction(jirix, installNodeFn, m.nodeInstDir.Abs(jirix), "Build and install node.js")
}
<commit_msg>Update nodejs profile.<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package nodejs_profile

import (
	"flag"
	"fmt"
	"path/filepath"
	"runtime"

	"v.io/jiri"
	"v.io/jiri/profiles"
	"v.io/jiri/profiles/profilesmanager"
	"v.io/jiri/profiles/profilesutil"
	"v.io/x/lib/envvar"
)

type versionSpec struct {
	nodeVersion string
}

func Register(installer, profile string) {
	m := &Manager{
		profileInstaller: installer,
		profileName:      profile,
		qualifiedName:    profiles.QualifiedProfileName(installer, profile),
		versionInfo: profiles.NewVersionInfo(profile, map[string]interface{}{
			"0.10.24": &versionSpec{"node-v0.10.24"},
			"4.4.1":   &versionSpec{"node-v4.4.1"},
		}, "4.4.1"),
	}
	profilesmanager.Register(m)
}

type Manager struct {
	profileInstaller, profileName, qualifiedName string
	root, nodeRoot                               jiri.RelPath
	nodeSrcDir, nodeInstDir                      jiri.RelPath
	versionInfo                                  *profiles.VersionInfo
	spec                                         versionSpec
}

func (m Manager) Name() string {
	return m.profileName
}

func (m Manager) Installer() string {
	return m.profileInstaller
}
func (m Manager) String() string {
	return fmt.Sprintf("%s[%s]", m.qualifiedName, m.versionInfo.Default())
}

func (m Manager) VersionInfo() *profiles.VersionInfo {
	return m.versionInfo
}

func (m Manager) Info() string {
	return `
The nodejs profile provides support for node. It installs and builds particular,
tested, versions of node.`
}

func (m *Manager) AddFlags(flags *flag.FlagSet, action profiles.Action) {}

func (m *Manager) initForTarget(root jiri.RelPath, target profiles.Target) error {
	if err := m.versionInfo.Lookup(target.Version(), &m.spec); err != nil {
		return err
	}
	m.root = root
	m.nodeRoot = m.root.Join("nodejs", m.spec.nodeVersion)
	m.nodeInstDir = m.nodeRoot.Join(target.TargetSpecificDirname())
	return nil
}

func (m *Manager) Install(jirix *jiri.X, pdb *profiles.DB, root jiri.RelPath, target profiles.Target) error {
	if err := m.initForTarget(root, target); err != nil {
		return err
	}
	if target.Version() == "0.10.24" {
		// Install from source only for version 0.10.24.
		m.nodeRoot = m.root.Join("cout", m.spec.nodeVersion)
		m.nodeInstDir = m.nodeRoot.Join(target.TargetSpecificDirname())
		m.nodeSrcDir = jiri.NewRelPath("third_party", "csrc", m.spec.nodeVersion)
		if err := m.installNodeFromSource(jirix, target); err != nil {
			return err
		}
	} else {
		// Install from binary tarballs.
		if err := m.installNodeBinaries(jirix, target, m.nodeInstDir.Abs(jirix)); err != nil {
			return err
		}
	}

	target.Env.Vars = envvar.MergeSlices(target.Env.Vars, []string{
		"NODE_BIN=" + m.nodeInstDir.Join("bin").Symbolic(),
		"PATH=" + m.nodeInstDir.Join("bin").Symbolic(),
	})
	target.InstallationDir = string(m.nodeInstDir)
	pdb.InstallProfile(m.profileInstaller, m.profileName, string(m.nodeRoot))
	return pdb.AddProfileTarget(m.profileInstaller, m.profileName, target)
}

func (m *Manager) Uninstall(jirix *jiri.X, pdb *profiles.DB, root jiri.RelPath, target profiles.Target) error {
	if err := m.initForTarget(root, target); err != nil {
		return err
	}
	if err := jirix.NewSeq().RemoveAll(m.nodeInstDir.Abs(jirix)).Done(); err != nil {
		return err
	}
	pdb.RemoveProfileTarget(m.profileInstaller, m.profileName, target)
	return nil
}

func (m *Manager) installNodeBinaries(jirix *jiri.X, target profiles.Target, outDir string) error {
	if target.Arch() != "amd64" {
		return fmt.Errorf("%q is not supported", target.Arch())
	}

	var arch string
	switch target.Arch() {
	case "amd64":
		arch = "x64"
	case "386":
		arch = "x86"
	default:
		return fmt.Errorf("arch %q is not supported", target.Arch())
	}

	switch target.OS() {
	case "darwin":
		if target.Arch() != "amd64" {
			return fmt.Errorf("arch %q is not supported on darwin", target.Arch())
		}
	case "linux":
	default:
		return fmt.Errorf("os %q is not supported", target.OS())
	}

	tarball := fmt.Sprintf("node-v%s-%s-%s.tar.gz", target.Version(), target.OS(), arch)
	url := fmt.Sprintf("https://nodejs.org/dist/v%s/%s", target.Version(), tarball)
	dirname := fmt.Sprintf("node-v%s-%s-%s", target.Version(), target.OS(), arch)

	tmpDir, err := jirix.NewSeq().TempDir("", "")
	if err != nil {
		return err
	}
	defer jirix.NewSeq().RemoveAll(tmpDir)

	fn := func() error {
		return jirix.NewSeq().
			Pushd(tmpDir).
			Call(func() error {
				return profilesutil.Fetch(jirix, tarball, url)
			}, "fetch nodejs tarball").
			Call(func() error {
				return profilesutil.Untar(jirix, tarball, tmpDir)
			}, "untar nodejs tarball").
			MkdirAll(filepath.Dir(outDir), profilesutil.DefaultDirPerm).
			Rename(filepath.Join(tmpDir, dirname), outDir).
			Done()
	}
	return profilesutil.AtomicAction(jirix, fn, outDir, "Install NodeJS")
}

func (m *Manager) installNodeFromSource(jirix *jiri.X, target profiles.Target) error {
	switch target.OS() {
	case "darwin":
	case "linux":
		if err := profilesutil.InstallPackages(jirix, []string{"g++"}); err != nil {
			return err
		}
	default:
		return fmt.Errorf("%q is not supported", target.OS)
	}
	// Build and install NodeJS.
	installNodeFn := func() error {
		return jirix.NewSeq().Pushd(m.nodeSrcDir.Abs(jirix)).
			Run("./configure", fmt.Sprintf("--prefix=%v", m.nodeInstDir.Abs(jirix))).
			Run("make", "clean").
			Run("make", fmt.Sprintf("-j%d", runtime.NumCPU())).
			Last("make", "install")
	}
	return profilesutil.AtomicAction(jirix, installNodeFn, m.nodeInstDir.Abs(jirix), "Build and install node.js")
}
<|endoftext|>


<commit_before>package gateway

import (
	"context"
	"fmt"
	"sort"
	"strings"
	"sync"

	"github.com/moby/buildkit/cache"
	"github.com/moby/buildkit/executor"
	"github.com/moby/buildkit/frontend/gateway/client"
	"github.com/moby/buildkit/session"
	"github.com/moby/buildkit/solver"
	"github.com/moby/buildkit/solver/llbsolver/mounts"
	opspb "github.com/moby/buildkit/solver/pb"
	"github.com/moby/buildkit/util/stack"
	utilsystem "github.com/moby/buildkit/util/system"
	"github.com/moby/buildkit/worker"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"golang.org/x/sync/errgroup"
)

type NewContainerRequest struct {
	ContainerID string
	NetMode     opspb.NetMode
	Mounts      []Mount
}

// Mount used for the gateway.Container is nearly identical to the client.Mount
// except is has a RefProxy instead of Ref to allow for a common abstraction
// between gateway clients.
type Mount struct {
	Dest      string
	Selector  string
	Readonly  bool
	MountType opspb.MountType
	RefProxy  solver.ResultProxy
	CacheOpt  *opspb.CacheOpt
	SecretOpt *opspb.SecretOpt
	SSHOpt    *opspb.SSHOpt
}

func toProtoMount(m Mount) *opspb.Mount {
	return &opspb.Mount{
		Selector:  m.Selector,
		Dest:      m.Dest,
		Readonly:  m.Readonly,
		MountType: m.MountType,
		CacheOpt:  m.CacheOpt,
		SecretOpt: m.SecretOpt,
		SSHOpt:    m.SSHOpt,
	}
}

func NewContainer(ctx context.Context, e executor.Executor, sm *session.Manager, g session.Group, req NewContainerRequest) (client.Container, error) {
	ctx, cancel := context.WithCancel(ctx)
	eg, ctx := errgroup.WithContext(ctx)
	ctr := &gatewayContainer{
		id:       req.ContainerID,
		netMode:  req.NetMode,
		executor: e,
		errGroup: eg,
		ctx:      ctx,
		cancel:   cancel,
	}

	makeMutable := func(worker worker.Worker, ref cache.ImmutableRef) (cache.MutableRef, error) {
		mRef, err := worker.CacheManager().New(ctx, ref)
		if err != nil {
			return nil, stack.Enable(err)
		}
		ctr.cleanup = append(ctr.cleanup, func() error {
			return stack.Enable(mRef.Release(context.TODO()))
		})
		return mRef, nil
	}

	var mm *mounts.MountManager
	mnts := req.Mounts

	for i, m := range mnts {
		if m.Dest == opspb.RootMount && m.RefProxy != nil {
			res, err := m.RefProxy.Result(ctx)
			if err != nil {
				return nil, stack.Enable(err)
			}
			workerRef, ok := res.Sys().(*worker.WorkerRef)
			if !ok {
				return nil, errors.Errorf("invalid reference for exec %T", res.Sys())
			}

			name := fmt.Sprintf("container %s", req.ContainerID)
			mm = mounts.NewMountManager(name, workerRef.Worker.CacheManager(), sm, workerRef.Worker.MetadataStore())

			ctr.rootFS = workerRef.ImmutableRef
			if !m.Readonly {
				ctr.rootFS, err = makeMutable(workerRef.Worker, workerRef.ImmutableRef)
				if err != nil {
					return nil, stack.Enable(err)
				}
			}

			// delete root mount from list, handled here
			mnts = append(mnts[:i], mnts[i+1:]...)
			break
		}
	}

	if ctr.rootFS == nil {
		return nil, errors.Errorf("root mount required")
	}

	for _, m := range mnts {
		var ref cache.ImmutableRef
		var mountable cache.Mountable
		if m.RefProxy != nil {
			res, err := m.RefProxy.Result(ctx)
			if err != nil {
				return nil, stack.Enable(err)
			}
			workerRef, ok := res.Sys().(*worker.WorkerRef)
			if !ok {
				return nil, errors.Errorf("invalid reference for exec %T", res.Sys())
			}
			ref = workerRef.ImmutableRef
			mountable = ref

			if !m.Readonly {
				mountable, err = makeMutable(workerRef.Worker, ref)
				if err != nil {
					return nil, stack.Enable(err)
				}
			}
		}
		switch m.MountType {
		case opspb.MountType_BIND:
			// nothing to do here
		case opspb.MountType_CACHE:
			mRef, err := mm.MountableCache(ctx, toProtoMount(m), ref)
			if err != nil {
				return nil, err
			}
			mountable = mRef
			ctr.cleanup = append(ctr.cleanup, func() error {
				return stack.Enable(mRef.Release(context.TODO()))
			})
		case opspb.MountType_TMPFS:
			mountable = mm.MountableTmpFS()
		case opspb.MountType_SECRET:
			var err error
			mountable, err = mm.MountableSecret(ctx, toProtoMount(m), g)
			if err != nil {
				return nil, err
			}
			if mountable == nil {
				continue
			}
		case opspb.MountType_SSH:
			var err error
			mountable, err = mm.MountableSSH(ctx, toProtoMount(m), g)
			if err != nil {
				return nil, err
			}
			if mountable == nil {
				continue
			}
		default:
			return nil, errors.Errorf("mount type %s not implemented", m.MountType)
		}

		// validate that there is a mount
		if mountable == nil {
			return nil, errors.Errorf("mount %s has no input", m.Dest)
		}

		execMount := executor.Mount{
			Src:      mountable,
			Selector: m.Selector,
			Dest:     m.Dest,
			Readonly: m.Readonly,
		}

		ctr.mounts = append(ctr.mounts, execMount)
	}

	// sort mounts so parents are mounted first
	sort.Slice(ctr.mounts, func(i, j int) bool {
		return ctr.mounts[i].Dest < ctr.mounts[j].Dest
	})

	return ctr, nil
}

type gatewayContainer struct {
	id           string
	netMode      opspb.NetMode
	securityMode opspb.SecurityMode
	rootFS       cache.Mountable
	mounts       []executor.Mount
	executor     executor.Executor
	started      bool
	errGroup     *errgroup.Group
	mu           sync.Mutex
	cleanup      []func() error
	ctx          context.Context
	cancel       func()
}

func (gwCtr *gatewayContainer) Start(ctx context.Context, req client.StartRequest) (client.ContainerProcess, error) {
	resize := make(chan executor.WinSize)
	procInfo := executor.ProcessInfo{
		Meta: executor.Meta{
			Args:         req.Args,
			Env:          req.Env,
			User:         req.User,
			Cwd:          req.Cwd,
			Tty:          req.Tty,
			NetMode:      gwCtr.netMode,
			SecurityMode: req.SecurityMode,
		},
		Stdin:  req.Stdin,
		Stdout: req.Stdout,
		Stderr: req.Stderr,
		Resize: resize,
	}
	procInfo.Meta.Env = addDefaultEnvvar(procInfo.Meta.Env, "PATH", utilsystem.DefaultPathEnv)
	if req.Tty {
		procInfo.Meta.Env = addDefaultEnvvar(procInfo.Meta.Env, "TERM", "xterm")
	}

	// mark that we have started on the first call to execProcess for this
	// container, so that future calls will call Exec rather than Run
	gwCtr.mu.Lock()
	started := gwCtr.started
	gwCtr.started = true
	gwCtr.mu.Unlock()

	eg, ctx := errgroup.WithContext(gwCtr.ctx)
	gwProc := &gatewayContainerProcess{
		resize:   resize,
		errGroup: eg,
		groupCtx: ctx,
	}

	if !started {
		startedCh := make(chan struct{})
		gwProc.errGroup.Go(func() error {
			logrus.Debugf("Starting new container for %s with args: %q", gwCtr.id, procInfo.Meta.Args)
			err := gwCtr.executor.Run(ctx, gwCtr.id, gwCtr.rootFS, gwCtr.mounts, procInfo, startedCh)
			return stack.Enable(err)
		})
		select {
		case <-ctx.Done():
		case <-startedCh:
		}
	} else {
		gwProc.errGroup.Go(func() error {
			logrus.Debugf("Execing into container %s with args: %q", gwCtr.id, procInfo.Meta.Args)
			err := gwCtr.executor.Exec(ctx, gwCtr.id, procInfo)
			return stack.Enable(err)
		})
	}

	gwCtr.errGroup.Go(gwProc.errGroup.Wait)

	return gwProc, nil
}

func (gwCtr *gatewayContainer) Release(ctx context.Context) error {
	gwCtr.cancel()
	err1 := gwCtr.errGroup.Wait()

	var err2 error
	for i := len(gwCtr.cleanup) - 1; i >= 0; i-- { // call in LIFO order
		err := gwCtr.cleanup[i]()
		if err2 == nil {
			err2 = err
		}
	}

	if err1 != nil {
		return stack.Enable(err1)
	}
	return stack.Enable(err2)
}

type gatewayContainerProcess struct {
	errGroup *errgroup.Group
	groupCtx context.Context
	resize   chan<- executor.WinSize
	mu       sync.Mutex
}

func (gwProc *gatewayContainerProcess) Wait() error {
	err := stack.Enable(gwProc.errGroup.Wait())
	gwProc.mu.Lock()
	defer gwProc.mu.Unlock()
	close(gwProc.resize)
	return err
}

func (gwProc *gatewayContainerProcess) Resize(ctx context.Context, size client.WinSize) error {
	gwProc.mu.Lock()
	defer gwProc.mu.Unlock()

	//  is the container done or should we proceed with sending event?
	select {
	case <-gwProc.groupCtx.Done():
		return nil
	case <-ctx.Done():
		return nil
	default:
	}

	// now we select on contexts again in case p.resize blocks b/c
	// container no longer reading from it.  In that case when
	// the errgroup finishes we want to unblock on the write
	// and exit
	select {
	case <-gwProc.groupCtx.Done():
	case <-ctx.Done():
	case gwProc.resize <- executor.WinSize{Cols: size.Cols, Rows: size.Rows}:
	}
	return nil
}

func addDefaultEnvvar(env []string, k, v string) []string {
	for _, e := range env {
		if strings.HasPrefix(e, k+"=") {
			return env
		}
	}
	return append(env, k+"="+v)
}
<commit_msg>fix linting error<commit_after>package gateway

import (
	"context"
	"fmt"
	"sort"
	"strings"
	"sync"

	"github.com/moby/buildkit/cache"
	"github.com/moby/buildkit/executor"
	"github.com/moby/buildkit/frontend/gateway/client"
	"github.com/moby/buildkit/session"
	"github.com/moby/buildkit/solver"
	"github.com/moby/buildkit/solver/llbsolver/mounts"
	opspb "github.com/moby/buildkit/solver/pb"
	"github.com/moby/buildkit/util/stack"
	utilsystem "github.com/moby/buildkit/util/system"
	"github.com/moby/buildkit/worker"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"golang.org/x/sync/errgroup"
)

type NewContainerRequest struct {
	ContainerID string
	NetMode     opspb.NetMode
	Mounts      []Mount
}

// Mount used for the gateway.Container is nearly identical to the client.Mount
// except is has a RefProxy instead of Ref to allow for a common abstraction
// between gateway clients.
type Mount struct {
	Dest      string
	Selector  string
	Readonly  bool
	MountType opspb.MountType
	RefProxy  solver.ResultProxy
	CacheOpt  *opspb.CacheOpt
	SecretOpt *opspb.SecretOpt
	SSHOpt    *opspb.SSHOpt
}

func toProtoMount(m Mount) *opspb.Mount {
	return &opspb.Mount{
		Selector:  m.Selector,
		Dest:      m.Dest,
		Readonly:  m.Readonly,
		MountType: m.MountType,
		CacheOpt:  m.CacheOpt,
		SecretOpt: m.SecretOpt,
		SSHOpt:    m.SSHOpt,
	}
}

func NewContainer(ctx context.Context, e executor.Executor, sm *session.Manager, g session.Group, req NewContainerRequest) (client.Container, error) {
	ctx, cancel := context.WithCancel(ctx)
	eg, ctx := errgroup.WithContext(ctx)
	ctr := &gatewayContainer{
		id:       req.ContainerID,
		netMode:  req.NetMode,
		executor: e,
		errGroup: eg,
		ctx:      ctx,
		cancel:   cancel,
	}

	makeMutable := func(worker worker.Worker, ref cache.ImmutableRef) (cache.MutableRef, error) {
		mRef, err := worker.CacheManager().New(ctx, ref)
		if err != nil {
			return nil, stack.Enable(err)
		}
		ctr.cleanup = append(ctr.cleanup, func() error {
			return stack.Enable(mRef.Release(context.TODO()))
		})
		return mRef, nil
	}

	var mm *mounts.MountManager
	mnts := req.Mounts

	for i, m := range mnts {
		if m.Dest == opspb.RootMount && m.RefProxy != nil {
			res, err := m.RefProxy.Result(ctx)
			if err != nil {
				return nil, stack.Enable(err)
			}
			workerRef, ok := res.Sys().(*worker.WorkerRef)
			if !ok {
				return nil, errors.Errorf("invalid reference for exec %T", res.Sys())
			}

			name := fmt.Sprintf("container %s", req.ContainerID)
			mm = mounts.NewMountManager(name, workerRef.Worker.CacheManager(), sm, workerRef.Worker.MetadataStore())

			ctr.rootFS = workerRef.ImmutableRef
			if !m.Readonly {
				ctr.rootFS, err = makeMutable(workerRef.Worker, workerRef.ImmutableRef)
				if err != nil {
					return nil, stack.Enable(err)
				}
			}

			// delete root mount from list, handled here
			mnts = append(mnts[:i], mnts[i+1:]...)
			break
		}
	}

	if ctr.rootFS == nil {
		return nil, errors.Errorf("root mount required")
	}

	for _, m := range mnts {
		var ref cache.ImmutableRef
		var mountable cache.Mountable
		if m.RefProxy != nil {
			res, err := m.RefProxy.Result(ctx)
			if err != nil {
				return nil, stack.Enable(err)
			}
			workerRef, ok := res.Sys().(*worker.WorkerRef)
			if !ok {
				return nil, errors.Errorf("invalid reference for exec %T", res.Sys())
			}
			ref = workerRef.ImmutableRef
			mountable = ref

			if !m.Readonly {
				mountable, err = makeMutable(workerRef.Worker, ref)
				if err != nil {
					return nil, stack.Enable(err)
				}
			}
		}
		switch m.MountType {
		case opspb.MountType_BIND:
			// nothing to do here
		case opspb.MountType_CACHE:
			mRef, err := mm.MountableCache(ctx, toProtoMount(m), ref)
			if err != nil {
				return nil, err
			}
			mountable = mRef
			ctr.cleanup = append(ctr.cleanup, func() error {
				return stack.Enable(mRef.Release(context.TODO()))
			})
		case opspb.MountType_TMPFS:
			mountable = mm.MountableTmpFS()
		case opspb.MountType_SECRET:
			var err error
			mountable, err = mm.MountableSecret(ctx, toProtoMount(m), g)
			if err != nil {
				return nil, err
			}
			if mountable == nil {
				continue
			}
		case opspb.MountType_SSH:
			var err error
			mountable, err = mm.MountableSSH(ctx, toProtoMount(m), g)
			if err != nil {
				return nil, err
			}
			if mountable == nil {
				continue
			}
		default:
			return nil, errors.Errorf("mount type %s not implemented", m.MountType)
		}

		// validate that there is a mount
		if mountable == nil {
			return nil, errors.Errorf("mount %s has no input", m.Dest)
		}

		execMount := executor.Mount{
			Src:      mountable,
			Selector: m.Selector,
			Dest:     m.Dest,
			Readonly: m.Readonly,
		}

		ctr.mounts = append(ctr.mounts, execMount)
	}

	// sort mounts so parents are mounted first
	sort.Slice(ctr.mounts, func(i, j int) bool {
		return ctr.mounts[i].Dest < ctr.mounts[j].Dest
	})

	return ctr, nil
}

type gatewayContainer struct {
	id       string
	netMode  opspb.NetMode
	rootFS   cache.Mountable
	mounts   []executor.Mount
	executor executor.Executor
	started  bool
	errGroup *errgroup.Group
	mu       sync.Mutex
	cleanup  []func() error
	ctx      context.Context
	cancel   func()
}

func (gwCtr *gatewayContainer) Start(ctx context.Context, req client.StartRequest) (client.ContainerProcess, error) {
	resize := make(chan executor.WinSize)
	procInfo := executor.ProcessInfo{
		Meta: executor.Meta{
			Args:         req.Args,
			Env:          req.Env,
			User:         req.User,
			Cwd:          req.Cwd,
			Tty:          req.Tty,
			NetMode:      gwCtr.netMode,
			SecurityMode: req.SecurityMode,
		},
		Stdin:  req.Stdin,
		Stdout: req.Stdout,
		Stderr: req.Stderr,
		Resize: resize,
	}
	procInfo.Meta.Env = addDefaultEnvvar(procInfo.Meta.Env, "PATH", utilsystem.DefaultPathEnv)
	if req.Tty {
		procInfo.Meta.Env = addDefaultEnvvar(procInfo.Meta.Env, "TERM", "xterm")
	}

	// mark that we have started on the first call to execProcess for this
	// container, so that future calls will call Exec rather than Run
	gwCtr.mu.Lock()
	started := gwCtr.started
	gwCtr.started = true
	gwCtr.mu.Unlock()

	eg, ctx := errgroup.WithContext(gwCtr.ctx)
	gwProc := &gatewayContainerProcess{
		resize:   resize,
		errGroup: eg,
		groupCtx: ctx,
	}

	if !started {
		startedCh := make(chan struct{})
		gwProc.errGroup.Go(func() error {
			logrus.Debugf("Starting new container for %s with args: %q", gwCtr.id, procInfo.Meta.Args)
			err := gwCtr.executor.Run(ctx, gwCtr.id, gwCtr.rootFS, gwCtr.mounts, procInfo, startedCh)
			return stack.Enable(err)
		})
		select {
		case <-ctx.Done():
		case <-startedCh:
		}
	} else {
		gwProc.errGroup.Go(func() error {
			logrus.Debugf("Execing into container %s with args: %q", gwCtr.id, procInfo.Meta.Args)
			err := gwCtr.executor.Exec(ctx, gwCtr.id, procInfo)
			return stack.Enable(err)
		})
	}

	gwCtr.errGroup.Go(gwProc.errGroup.Wait)

	return gwProc, nil
}

func (gwCtr *gatewayContainer) Release(ctx context.Context) error {
	gwCtr.cancel()
	err1 := gwCtr.errGroup.Wait()

	var err2 error
	for i := len(gwCtr.cleanup) - 1; i >= 0; i-- { // call in LIFO order
		err := gwCtr.cleanup[i]()
		if err2 == nil {
			err2 = err
		}
	}

	if err1 != nil {
		return stack.Enable(err1)
	}
	return stack.Enable(err2)
}

type gatewayContainerProcess struct {
	errGroup *errgroup.Group
	groupCtx context.Context
	resize   chan<- executor.WinSize
	mu       sync.Mutex
}

func (gwProc *gatewayContainerProcess) Wait() error {
	err := stack.Enable(gwProc.errGroup.Wait())
	gwProc.mu.Lock()
	defer gwProc.mu.Unlock()
	close(gwProc.resize)
	return err
}

func (gwProc *gatewayContainerProcess) Resize(ctx context.Context, size client.WinSize) error {
	gwProc.mu.Lock()
	defer gwProc.mu.Unlock()

	//  is the container done or should we proceed with sending event?
	select {
	case <-gwProc.groupCtx.Done():
		return nil
	case <-ctx.Done():
		return nil
	default:
	}

	// now we select on contexts again in case p.resize blocks b/c
	// container no longer reading from it.  In that case when
	// the errgroup finishes we want to unblock on the write
	// and exit
	select {
	case <-gwProc.groupCtx.Done():
	case <-ctx.Done():
	case gwProc.resize <- executor.WinSize{Cols: size.Cols, Rows: size.Rows}:
	}
	return nil
}

func addDefaultEnvvar(env []string, k, v string) []string {
	for _, e := range env {
		if strings.HasPrefix(e, k+"=") {
			return env
		}
	}
	return append(env, k+"="+v)
}
<|endoftext|>


<commit_before>package libvirt

import (
	"encoding/xml"
	"fmt"
	"io"
	"log"
	"net/http"
	"strconv"
	//"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
	//libvirt "gopkg.in/alexzorin/libvirt-go.v2"
	libvirt "github.com/dmacvicar/libvirt-go"
)

func volumeCommonSchema() map[string]*schema.Schema {
	return map[string]*schema.Schema{
		"name": &schema.Schema{
			Type:     schema.TypeString,
			Required: true,
		},
		"pool": &schema.Schema{
			Type:     schema.TypeString,
			Optional: true,
			Default:  "default",
		},
		"source": &schema.Schema{
			Type:     schema.TypeString,
			Optional: true,
		},
		"size": &schema.Schema{
			Type:     schema.TypeInt,
			Optional: true,
			Computed: true,
		},
		"base_volume": &schema.Schema{
			Type:     schema.TypeString,
			Optional: true,
		},
	}
}

func resourceLibvirtVolume() *schema.Resource {
	return &schema.Resource{
		Create: resourceLibvirtVolumeCreate,
		Read:   resourceLibvirtVolumeRead,
		Update: resourceLibvirtVolumeUpdate,
		Delete: resourceLibvirtVolumeDelete,
		Schema: volumeCommonSchema(),
	}
}

func remoteImageSize(url string) (int, error) {
	response, err := http.Head(url)
	if err != nil {
		return 0, err
	}
	length, err := strconv.Atoi(response.Header.Get("Content-Length"))
	if err != nil {
		return 0, err
	}
	return length, nil
}

func resourceLibvirtVolumeCreate(d *schema.ResourceData, meta interface{}) error {
	virConn := meta.(*Client).libvirt
	if virConn == nil {
		return fmt.Errorf("The libvirt connection was nil.")
	}

	poolName := "default"
	if _, ok := d.GetOk("pool"); ok {
		poolName = d.Get("pool").(string)
	}

	pool, err := virConn.LookupStoragePoolByName(poolName)
	if err != nil {
		return fmt.Errorf("can't find storage pool '%s'", poolName)
	}

	volumeDef := newDefVolume()
	volumeDef.Name = d.Get("name").(string)

	// an existing image was given, this mean we can't choose size
	if url, ok := d.GetOk("source"); ok {
		// source and size conflict
		if _, ok := d.GetOk("size"); ok {
			return fmt.Errorf("'size' can't be specified when also 'source' is given (the size will be set to the size of the source image.")
		}

		size, err := remoteImageSize(url.(string))
		if err != nil {
			return err
		}
		log.Printf("Remote image is: %d bytes", size)
		volumeDef.Capacity.Unit = "B"
		volumeDef.Capacity.Amount = size
	} else {
		_, noSize := d.GetOk("size")
		_, noBaseVol := d.GetOk("base_volume")

		if noSize && noBaseVol {
			return fmt.Errorf("'size' needs to be specified if no 'source' or 'base_vol' is given.")
		}
		volumeDef.Capacity.Amount = d.Get("size").(int)
	}

	if baseVolumeId, ok := d.GetOk("base_volume"); ok {
		if _, ok := d.GetOk("size"); ok {
			return fmt.Errorf("'size' can't be specified when also 'base_volume' is given (the size will be set to the size of the backing image.")
		}

		volumeDef.BackingStore = new(defBackingStore)
		volumeDef.BackingStore.Format.Type = "qcow2"
		baseVolume, err := virConn.LookupStorageVolByKey(baseVolumeId.(string))
		if err != nil {
			return fmt.Errorf("Can't retrieve volume %s", baseVolumeId.(string))
		}
		baseVolPath, err := baseVolume.GetPath()
		if err != nil {
			return fmt.Errorf("can't get name for base image '%s'", baseVolumeId)
		}
		volumeDef.BackingStore.Path = baseVolPath
	}

	volumeDefXml, err := xml.Marshal(volumeDef)
	if err != nil {
		return fmt.Errorf("Error serializing libvirt volume: %s", err)
	}

	// create the volume
	volume, err := pool.StorageVolCreateXML(string(volumeDefXml), 0)
	if err != nil {
		return fmt.Errorf("Error creating libvirt volume: %s", err)
	}
	// we use the key as the id
	key, err := volume.GetKey()
	if err != nil {
		return fmt.Errorf("Error retrieving volume key: %s", err)
	}
	d.SetId(key)
	log.Printf("[INFO] Volume ID: %s", d.Id())

	// upload source if present
	if url, ok := d.GetOk("source"); ok {
		stream, err := libvirt.NewVirStream(virConn, 0)
		defer stream.Close()

		volume.Upload(stream, 0, uint64(volumeDef.Capacity.Amount), 0)
		response, err := http.Get(url.(string))
		defer response.Body.Close()
		if err != nil {
			return fmt.Errorf("Error while downloading %s: %s", url.(string), err)
		}

		n, err := io.Copy(stream, response.Body)
		if err != nil {
			return fmt.Errorf("Error while downloading %s: %s", url.(string), err)
		}
		log.Printf("%d bytes uploaded\n", n)
	}

	return resourceLibvirtVolumeRead(d, meta)
}

func resourceLibvirtVolumeRead(d *schema.ResourceData, meta interface{}) error {
	virConn := meta.(*Client).libvirt
	if virConn == nil {
		return fmt.Errorf("The libvirt connection was nil.")
	}

	_, err := virConn.LookupStorageVolByKey(d.Id())
	if err != nil {
		return fmt.Errorf("Can't retrieve volume %s", d.Id())
	}

	return nil
}

func resourceLibvirtVolumeUpdate(d *schema.ResourceData, meta interface{}) error {
	return fmt.Errorf("Couldn't update libvirt domain")
}

func resourceLibvirtVolumeDelete(d *schema.ResourceData, meta interface{}) error {
	virConn := meta.(*Client).libvirt
	if virConn == nil {
		return fmt.Errorf("The libvirt connection was nil.")
	}

	volume, err := virConn.LookupStorageVolByKey(d.Id())
	if err != nil {
		return fmt.Errorf("Can't retrieve volume %s", d.Id())
	}

	err = volume.Delete(0)
	if err != nil {
		return fmt.Errorf("Can't delete volume %s", d.Id())
	}

	return nil
}
<commit_msg>refresh the storage pool so that libvirt does not think volume is used<commit_after>package libvirt

import (
	"encoding/xml"
	"fmt"
	"io"
	"log"
	"net/http"
	"strconv"
	//"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
	//libvirt "gopkg.in/alexzorin/libvirt-go.v2"
	libvirt "github.com/dmacvicar/libvirt-go"
)

func volumeCommonSchema() map[string]*schema.Schema {
	return map[string]*schema.Schema{
		"name": &schema.Schema{
			Type:     schema.TypeString,
			Required: true,
		},
		"pool": &schema.Schema{
			Type:     schema.TypeString,
			Optional: true,
			Default:  "default",
		},
		"source": &schema.Schema{
			Type:     schema.TypeString,
			Optional: true,
		},
		"size": &schema.Schema{
			Type:     schema.TypeInt,
			Optional: true,
			Computed: true,
		},
		"base_volume": &schema.Schema{
			Type:     schema.TypeString,
			Optional: true,
		},
	}
}

func resourceLibvirtVolume() *schema.Resource {
	return &schema.Resource{
		Create: resourceLibvirtVolumeCreate,
		Read:   resourceLibvirtVolumeRead,
		Update: resourceLibvirtVolumeUpdate,
		Delete: resourceLibvirtVolumeDelete,
		Schema: volumeCommonSchema(),
	}
}

func remoteImageSize(url string) (int, error) {
	response, err := http.Head(url)
	if err != nil {
		return 0, err
	}
	length, err := strconv.Atoi(response.Header.Get("Content-Length"))
	if err != nil {
		return 0, err
	}
	return length, nil
}

func resourceLibvirtVolumeCreate(d *schema.ResourceData, meta interface{}) error {
	virConn := meta.(*Client).libvirt
	if virConn == nil {
		return fmt.Errorf("The libvirt connection was nil.")
	}

	poolName := "default"
	if _, ok := d.GetOk("pool"); ok {
		poolName = d.Get("pool").(string)
	}

	pool, err := virConn.LookupStoragePoolByName(poolName)
	if err != nil {
		return fmt.Errorf("can't find storage pool '%s'", poolName)
	}

	// Refresh the pool of the volume so that libvirt knows it is
	// not longer in use.
	err = pool.Refresh(0)
	if err != nil {
		return fmt.Errorf("Error refreshing pool for volume: %s", err)
	}

	volumeDef := newDefVolume()
	volumeDef.Name = d.Get("name").(string)

	// an existing image was given, this mean we can't choose size
	if url, ok := d.GetOk("source"); ok {
		// source and size conflict
		if _, ok := d.GetOk("size"); ok {
			return fmt.Errorf("'size' can't be specified when also 'source' is given (the size will be set to the size of the source image.")
		}

		size, err := remoteImageSize(url.(string))
		if err != nil {
			return err
		}
		log.Printf("Remote image is: %d bytes", size)
		volumeDef.Capacity.Unit = "B"
		volumeDef.Capacity.Amount = size
	} else {
		_, noSize := d.GetOk("size")
		_, noBaseVol := d.GetOk("base_volume")

		if noSize && noBaseVol {
			return fmt.Errorf("'size' needs to be specified if no 'source' or 'base_vol' is given.")
		}
		volumeDef.Capacity.Amount = d.Get("size").(int)
	}

	if baseVolumeId, ok := d.GetOk("base_volume"); ok {
		if _, ok := d.GetOk("size"); ok {
			return fmt.Errorf("'size' can't be specified when also 'base_volume' is given (the size will be set to the size of the backing image.")
		}

		volumeDef.BackingStore = new(defBackingStore)
		volumeDef.BackingStore.Format.Type = "qcow2"
		baseVolume, err := virConn.LookupStorageVolByKey(baseVolumeId.(string))
		if err != nil {
			return fmt.Errorf("Can't retrieve volume %s", baseVolumeId.(string))
		}
		baseVolPath, err := baseVolume.GetPath()
		if err != nil {
			return fmt.Errorf("can't get name for base image '%s'", baseVolumeId)
		}
		volumeDef.BackingStore.Path = baseVolPath
	}

	volumeDefXml, err := xml.Marshal(volumeDef)
	if err != nil {
		return fmt.Errorf("Error serializing libvirt volume: %s", err)
	}

	// create the volume
	volume, err := pool.StorageVolCreateXML(string(volumeDefXml), 0)
	if err != nil {
		return fmt.Errorf("Error creating libvirt volume: %s", err)
	}
	// we use the key as the id
	key, err := volume.GetKey()
	if err != nil {
		return fmt.Errorf("Error retrieving volume key: %s", err)
	}
	d.SetId(key)
	log.Printf("[INFO] Volume ID: %s", d.Id())

	// upload source if present
	if url, ok := d.GetOk("source"); ok {
		stream, err := libvirt.NewVirStream(virConn, 0)
		defer stream.Close()

		volume.Upload(stream, 0, uint64(volumeDef.Capacity.Amount), 0)
		response, err := http.Get(url.(string))
		defer response.Body.Close()
		if err != nil {
			return fmt.Errorf("Error while downloading %s: %s", url.(string), err)
		}

		n, err := io.Copy(stream, response.Body)
		if err != nil {
			return fmt.Errorf("Error while downloading %s: %s", url.(string), err)
		}
		log.Printf("%d bytes uploaded\n", n)
	}

	return resourceLibvirtVolumeRead(d, meta)
}

func resourceLibvirtVolumeRead(d *schema.ResourceData, meta interface{}) error {
	virConn := meta.(*Client).libvirt
	if virConn == nil {
		return fmt.Errorf("The libvirt connection was nil.")
	}

	_, err := virConn.LookupStorageVolByKey(d.Id())
	if err != nil {
		return fmt.Errorf("Can't retrieve volume %s", d.Id())
	}

	return nil
}

func resourceLibvirtVolumeUpdate(d *schema.ResourceData, meta interface{}) error {
	return fmt.Errorf("Couldn't update libvirt domain")
}

func resourceLibvirtVolumeDelete(d *schema.ResourceData, meta interface{}) error {
	virConn := meta.(*Client).libvirt
	if virConn == nil {
		return fmt.Errorf("The libvirt connection was nil.")
	}

	volume, err := virConn.LookupStorageVolByKey(d.Id())
	if err != nil {
		return fmt.Errorf("Can't retrieve volume %s", d.Id())
	}

	// Refresh the pool of the volume so that libvirt knows it is
	// not longer in use.
	volPool, err := volume.LookupPoolByVolume()
	if err != nil {
		return fmt.Errorf("Error retrieving pool for volume: %s", err)
	}

	err = volPool.Refresh(0)
	if err != nil {
		return fmt.Errorf("Error refreshing pool for volume: %s", err)
	}

	err = volume.Delete(0)
	if err != nil {
		return fmt.Errorf("Can't delete volume %s", d.Id())
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2011 Aaron Jacobs. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package serial

import "io"

func openInternal(options OpenOptions) (io.ReadWriteCloser, error) {
	return nil, "Not implemented on this OS."
}
<commit_msg>Added windows support<commit_after>// Copyright 2011 Aaron Jacobs. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package serial

import (
	"fmt"
	//"log"
	//"github.com/hotei/bits"
	"io"
	"os"
	//"strconv"
	"sync"
	"syscall"
	"unsafe"
)

type serialPort struct {
	f  *os.File
	fd syscall.Handle
	rl sync.Mutex
	wl sync.Mutex
	ro *syscall.Overlapped
	wo *syscall.Overlapped
}

type structDCB struct {
	DCBlength, BaudRate                            uint32
	flags                                          [4]byte
	wReserved, XonLim, XoffLim                     uint16
	ByteSize, Parity, StopBits                     byte
	XonChar, XoffChar, ErrorChar, EofChar, EvtChar byte
	wReserved1                                     uint16
}

/*
type _DCB struct {
  DWORD DCBlength
  DWORD BaudRate
  DWORD fBinary  :1
  DWORD fParity  :1
  DWORD fOutxCtsFlow  :1
  DWORD fOutxDsrFlow  :1
  DWORD fDtrControl  :2
  DWORD fDsrSensitivity  :1
  DWORD fTXContinueOnXoff  :1
  DWORD fOutX  :1
  DWORD fInX  :1
  DWORD fErrorChar  :1
  DWORD fNull  :1
  DWORD fRtsControl  :2 /* 13 and 14th bit, so [12:13]
  DWORD fAbortOnError  :1
  DWORD fDummy2  :17
  WORD  wReserved
  WORD  XonLim
  WORD  XoffLim
  BYTE  ByteSize
  BYTE  Parity
  BYTE  StopBits
  char  XonChar
  char  XoffChar
  char  ErrorChar
  char  EofChar
  char  EvtChar
  WORD  wReserved1
}
*/

type structTimeouts struct {
	ReadIntervalTimeout         uint32
	ReadTotalTimeoutMultiplier  uint32
	ReadTotalTimeoutConstant    uint32
	WriteTotalTimeoutMultiplier uint32
	WriteTotalTimeoutConstant   uint32
}
func openInternal(options OpenOptions) (io.ReadWriteCloser, error) {
	if len(options.PortName) > 0 && options.PortName[0] != '\\' {
		options.PortName = "\\\\.\\" + options.PortName
	}

	h, err := syscall.CreateFile(syscall.StringToUTF16Ptr(options.PortName),
		syscall.GENERIC_READ|syscall.GENERIC_WRITE,
		0,
		nil,
		syscall.OPEN_EXISTING,
		syscall.FILE_ATTRIBUTE_NORMAL|syscall.FILE_FLAG_OVERLAPPED,
		0)
	if err != nil {
		return nil, err
	}
	f := os.NewFile(uintptr(h), options.PortName)
	defer func() {
		if err != nil {
			f.Close()
		}
	}()

	if err = setCommState(h, options); err != nil {
		return nil, err
	}
	if err = setupComm(h, 64, 64); err != nil {
		return nil, err
	}
	if err = setCommTimeouts(h); err != nil {
		return nil, err
	}
	if err = setCommMask(h); err != nil {
		return nil, err
	}

	ro, err := newOverlapped()
	if err != nil {
		return nil, err
	}
	wo, err := newOverlapped()
	if err != nil {
		return nil, err
	}
	port := new(serialPort)
	port.f = f
	port.fd = h
	port.ro = ro
	port.wo = wo

	return port, nil
}

func (p *serialPort) Close() error {
	return p.f.Close()
}

func (p *serialPort) Write(buf []byte) (int, error) {
	p.wl.Lock()
	defer p.wl.Unlock()

	if err := resetEvent(p.wo.HEvent); err != nil {
		return 0, err
	}
	var n uint32
	err := syscall.WriteFile(p.fd, buf, &n, p.wo)
	if err != nil && err != syscall.ERROR_IO_PENDING {
		return int(n), err
	}
	return getOverlappedResult(p.fd, p.wo)
}

func (p *serialPort) Read(buf []byte) (int, error) {
	if p == nil || p.f == nil {
		return 0, fmt.Errorf("Invalid port on read %v %v", p, p.f)
	}

	p.rl.Lock()
	defer p.rl.Unlock()

	if err := resetEvent(p.ro.HEvent); err != nil {
		return 0, err
	}
	var done uint32
	err := syscall.ReadFile(p.fd, buf, &done, p.ro)
	if err != nil && err != syscall.ERROR_IO_PENDING {
		return int(done), err
	}
	return getOverlappedResult(p.fd, p.ro)
}

var (
	nSetCommState,
	nSetCommTimeouts,
	nSetCommMask,
	nSetupComm,
	nGetOverlappedResult,
	nCreateEvent,
	nResetEvent uintptr
)

func init() {
	k32, err := syscall.LoadLibrary("kernel32.dll")
	if err != nil {
		panic("LoadLibrary " + err.Error())
	}
	defer syscall.FreeLibrary(k32)

	nSetCommState = getProcAddr(k32, "SetCommState")
	nSetCommTimeouts = getProcAddr(k32, "SetCommTimeouts")
	nSetCommMask = getProcAddr(k32, "SetCommMask")
	nSetupComm = getProcAddr(k32, "SetupComm")
	nGetOverlappedResult = getProcAddr(k32, "GetOverlappedResult")
	nCreateEvent = getProcAddr(k32, "CreateEventW")
	nResetEvent = getProcAddr(k32, "ResetEvent")
}

func getProcAddr(lib syscall.Handle, name string) uintptr {
	addr, err := syscall.GetProcAddress(lib, name)
	if err != nil {
		panic(name + " " + err.Error())
	}
	return addr
}

func setCommState(h syscall.Handle, options OpenOptions) error {
	var params structDCB
	params.DCBlength = uint32(unsafe.Sizeof(params))

	params.flags[0] = 0x01  // fBinary
	params.flags[0] |= 0x10 // Assert DSR
	//params.flags[1] = 0x10  // RTS is on
	//log.Println("Byte val of commstat flags[0]:", strconv.FormatInt(int64(params.flags[0]), 2))
	//log.Println("Byte val of commstat flags[1]:", strconv.FormatInt(int64(params.flags[1]), 2))

	if options.ParityMode != PARITY_NONE {
		params.flags[0] |= 0x03 // fParity
		params.Parity = byte(options.ParityMode)
	}

	if options.StopBits == 1 {
		params.StopBits = 0
	} else if options.StopBits == 2 {
		params.StopBits = 2
	}

	params.BaudRate = uint32(options.BaudRate)
	params.ByteSize = byte(options.DataBits)

	r, _, err := syscall.Syscall(nSetCommState, 2, uintptr(h), uintptr(unsafe.Pointer(&params)), 0)
	if r == 0 {
		return err
	}
	return nil
}

func setCommTimeouts(h syscall.Handle) error {
	var timeouts structTimeouts
	const MAXDWORD = 1<<32 - 1
	timeouts.ReadIntervalTimeout = MAXDWORD
	timeouts.ReadTotalTimeoutMultiplier = MAXDWORD
	timeouts.ReadTotalTimeoutConstant = MAXDWORD - 1

	/* From http://msdn.microsoft.com/en-us/library/aa363190(v=VS.85).aspx

		 For blocking I/O see below:

		 Remarks:

		 If an application sets ReadIntervalTimeout and
		 ReadTotalTimeoutMultiplier to MAXDWORD and sets
		 ReadTotalTimeoutConstant to a value greater than zero and
		 less than MAXDWORD, one of the following occurs when the
		 ReadFile function is called:

		 If there are any bytes in the input buffer, ReadFile returns
		       immediately with the bytes in the buffer.

		 If there are no bytes in the input buffer, ReadFile waits
	               until a byte arrives and then returns immediately.

		 If no bytes arrive within the time specified by
		       ReadTotalTimeoutConstant, ReadFile times out.
	*/

	r, _, err := syscall.Syscall(nSetCommTimeouts, 2, uintptr(h), uintptr(unsafe.Pointer(&timeouts)), 0)
	if r == 0 {
		return err
	}
	return nil
}

func setupComm(h syscall.Handle, in, out int) error {
	r, _, err := syscall.Syscall(nSetupComm, 3, uintptr(h), uintptr(in), uintptr(out))
	if r == 0 {
		return err
	}
	return nil
}

func setCommMask(h syscall.Handle) error {
	const EV_RXCHAR = 0x0001
	r, _, err := syscall.Syscall(nSetCommMask, 2, uintptr(h), EV_RXCHAR, 0)
	if r == 0 {
		return err
	}
	return nil
}

func resetEvent(h syscall.Handle) error {
	r, _, err := syscall.Syscall(nResetEvent, 1, uintptr(h), 0, 0)
	if r == 0 {
		return err
	}
	return nil
}

func newOverlapped() (*syscall.Overlapped, error) {
	var overlapped syscall.Overlapped
	r, _, err := syscall.Syscall6(nCreateEvent, 4, 0, 1, 0, 0, 0, 0)
	if r == 0 {
		return nil, err
	}
	overlapped.HEvent = syscall.Handle(r)
	return &overlapped, nil
}

func getOverlappedResult(h syscall.Handle, overlapped *syscall.Overlapped) (int, error) {
	var n int
	r, _, err := syscall.Syscall6(nGetOverlappedResult, 4,
		uintptr(h),
		uintptr(unsafe.Pointer(overlapped)),
		uintptr(unsafe.Pointer(&n)), 1, 0, 0)
	if r == 0 {
		return n, err
	}

	return n, nil
}<|endoftext|>