<commit_before>package blocks

import (
	"github.com/tendermint/tendermint/p2p"
)

const (
	BlocksCh = "block"

	msgTypeUnknown = Byte(0x00)
	msgTypeState   = Byte(0x01)
	msgTypeRequest = Byte(0x02)
	msgTypeData    = Byte(0x03)

	dataTypeAll        = byte(0x00)
	dataTypeValidation = byte(0x01)
	dataTypeTxs        = byte(0x02)
	// dataTypeCheckpoint = byte(0x04)
)

/*
 */
type BlockManager struct {
	quit    chan struct{}
	started uint32
	stopped uint32
}

func NewBlockManager() *BlockManager {
	bm := &BlockManager{
		sw:   sw,
		quit: make(chan struct{}),
	}
	return bm
}

func (bm *BlockManager) Start() {
	if atomic.CompareAndSwapUint32(&bm.started, 0, 1) {
		log.Info("Starting BlockManager")
		go bm.XXX()
	}
}

func (bm *BlockManager) Stop() {
	if atomic.CompareAndSwapUint32(&bm.stopped, 0, 1) {
		log.Info("Stopping BlockManager")
		close(bm.quit)
	}
}

func (bm *BlockManager) XXX() {
}

//-----------------------------------------------------------------------------

/* Messages */

// TODO: check for unnecessary extra bytes at the end.
func decodeMessage(bz ByteSlice) (msg Message) {
	// log.Debug("decoding msg bytes: %X", bz)
	switch Byte(bz[0]) {
	case msgTypeState:
		return &StateMessage{}
	case msgTypeRequest:
		return readRequestMessage(bytes.NewReader(bz[1:]))
	case msgTypeData:
		return readDataMessage(bytes.NewReader(bz[1:]))
	default:
		return nil
	}
}

/*
A StateMessage declares what (contiguous) blocks & headers are known.

LastValidationHeight >= LastBlockHeight.
*/
type StateMessage struct {
	LastBlockHeight      UInt64
	LastValidationHeight UInt64
}

func readStateMessage(r io.Reader) *StateMessage {
	lastBlockHeight := ReadUInt64(r)
	lastValidationHeight := ReadUInt64(r)
	return &StateMessage{
		LastBlockHeight:      lastBlockHeight,
		LastValidationHeight: lastValidationHeight,
	}
}

func (m *StateMessage) WriteTo(w io.Writer) (n int64, err error) {
	n, err = WriteTo(msgTypeState, w, n, err)
	n, err = WriteTo(m.LastBlockHeight, w, n, err)
	n, err = WriteTo(m.LastValidationHeight, w, n, err)
	return
}

func (m *StateMessage) String() string {
	return fmt.Sprintf("[State %v/%v]",
		m.LastBlockHeight, m.LastValidationHeight)
}

/*
A RequestMessage requests a block and/or header at a given height.
*/
type RequestMessage struct {
	Type   Byte
	Height UInt64
}

func readRequestMessage(r io.Reader) *RequestMessage {
	requestType := ReadByte(r)
	height := ReadUInt64(r)
	return &RequestMessage{
		Type:   requestType,
		Height: height,
	}
}

func (m *RequestMessage) WriteTo(w io.Writer) (n int64, err error) {
	n, err = WriteTo(msgTypeRequest, w, n, err)
	n, err = WriteTo(m.Type, w, n, err)
	n, err = WriteTo(m.Height, w, n, err)
	return
}

func (m *RequestMessage) String() string {
	return fmt.Sprintf("[Request %X@%v]", m.Type, m.Height)
}

/*
A DataMessage contains block data, maybe requested.
The data can be a Validation, Txs, or whole Block object.
*/
type DataMessage struct {
	Type   Byte
	Height UInt64
	Bytes  ByteSlice
}

func readDataMessage(r io.Reader) *DataMessage {
	dataType := ReadByte(r)
	height := ReadUInt64(r)
	bytes := ReadByteSlice(r)
	return &DataMessage{
		Type:   dataType,
		Height: height,
		Bytes:  bytes,
	}
}

func (m *DataMessage) WriteTo(w io.Writer) (n int64, err error) {
	n, err = WriteTo(msgTypeData, w, n, err)
	n, err = WriteTo(m.Type, w, n, err)
	n, err = WriteTo(m.Height, w, n, err)
	n, err = WriteTo(m.Bytes, w, n, err)
	return
}

func (m *DataMessage) String() string {
	return fmt.Sprintf("[Data %X@%v]", m.Type, m.Height)
}
<commit_msg>implementing BlockManager<commit_after>package blocks

import (
	db_ "github.com/tendermint/tendermint/db"
	"github.com/tendermint/tendermint/p2p"
)

const (
	blocksCh = "block"

	msgTypeUnknown = Byte(0x00)
	msgTypeState   = Byte(0x01)
	msgTypeRequest = Byte(0x02)
	msgTypeData    = Byte(0x03)

	DataTypeFullBlock  = byte(0x00)
	DataTypeValidation = byte(0x01)
	DataTypeTxs        = byte(0x02)
	// dataTypeCheckpoint = byte(0x04)

	dbKeyState = "state"
)

/*
 */
type BlockManager struct {
	db         *db_.LevelDB
	sw         *p2p.Switch
	swEvents   chan interface{}
	state      blockManagerState
	peerStates map[string]*blockManagerState
	quit       chan struct{}
	started    uint32
	stopped    uint32
}

func NewBlockManager(sw *p2p.Switch, db *db_.LevelDB) *BlockManager {
	swEvents := make(chan interface{})
	sw.AddEventListener("BlockManager.swEvents", swEvents)
	bm := &BlockManager{
		db:         db,
		sw:         sw,
		swEvents:   swEvents,
		peerStates: make(map[string]*blockManagerState),
		quit:       make(chan struct{}),
	}
	bm.loadState()
	return bm
}

func (bm *BlockManager) Start() {
	if atomic.CompareAndSwapUint32(&bm.started, 0, 1) {
		log.Info("Starting BlockManager")
		go bm.switchEventsHandler()
	}
}

func (bm *BlockManager) Stop() {
	if atomic.CompareAndSwapUint32(&bm.stopped, 0, 1) {
		log.Info("Stopping BlockManager")
		close(bm.quit)
		close(bm.swEvents)
	}
}

func (bm *BlockManager) StoreData(dataType byte, dataObj interface{}) {
	// Validate data if possible.
	// If we have new data that extends our contiguous range, then announce it.
}

func (bm *BlockManager) LoadData(dataType byte) interface{} {
	// NOTE: who's calling?
}

func (bm *BlockManager) loadState() {
	// Load the state
	stateBytes := bm.db.Get(dbKeyState)
	if stateBytes == nil {
		log.Info("New BlockManager with no state")
	} else {
		err := json.Unmarshal(stateBytes, &bm.state)
		if err != nil {
			panic("Could not unmarshal state bytes: %X", stateBytes)
		}
	}
}

func (bm *BlockManager) saveState() {
	stateBytes, err := json.Marshal(&bm.state)
	if err != nil {
		panic("Could not marshal state bytes")
	}
	bm.db.Set(dbKeyState, stateBytes)
}

func (bm *BlockManager) switchEventsHandler() {
	for {
		swEvent, ok := <-bm.swEvents
		if !ok {
			break
		}
		switch swEvent.(type) {
		case p2p.SwitchEventNewPeer:
			event := swEvent.(p2p.SwitchEventNewPeer)
			// Create entry in .peerStates
			// Share our state with event.Peer
			event.Peer
		case p2p.SwitchEventDonePeer:
			// Remove entry from .peerStates
		}
	}
}

//-----------------------------------------------------------------------------

/* This is just to persist the block manager state in the db. */
type blockManagerState struct {
	LastHeaderHeight       uint64 // Last contiguous header height
	OtherHeaderHeights     []uint64
	LastValidationHeight   uint64 // Last contiguous validation height
	OtherValidationHeights []uint64
	LastTxsHeight          uint64 // Last contiguous txs height
	OtherTxsHeights        []uint64
}

//-----------------------------------------------------------------------------

/*
Each part of a block are stored separately in the db.
*/

func headerKey(height int) {
	return fmt.Sprintf("B%v", height)
}

func validationKey(height int) {
	return fmt.Sprintf("V%v", height)
}

func txsKey(height int) {
	return fmt.Sprintf("T%v", height)
}

func checkpointKey(height int) {
	return fmt.Sprintf("C%v", height)
}

//-----------------------------------------------------------------------------

/* Messages */

// TODO: check for unnecessary extra bytes at the end.
func decodeMessage(bz ByteSlice) (msg Message) {
	// log.Debug("decoding msg bytes: %X", bz)
	switch Byte(bz[0]) {
	case msgTypeState:
		return &StateMessage{}
	case msgTypeRequest:
		return readRequestMessage(bytes.NewReader(bz[1:]))
	case msgTypeData:
		return readDataMessage(bytes.NewReader(bz[1:]))
	default:
		return nil
	}
}

/*
A StateMessage declares what (contiguous) blocks & headers are known.

LastValidationHeight >= LastBlockHeight.
*/
type StateMessage struct {
	LastBlockHeight      UInt64
	LastValidationHeight UInt64
}

func readStateMessage(r io.Reader) *StateMessage {
	lastBlockHeight := ReadUInt64(r)
	lastValidationHeight := ReadUInt64(r)
	return &StateMessage{
		LastBlockHeight:      lastBlockHeight,
		LastValidationHeight: lastValidationHeight,
	}
}

func (m *StateMessage) WriteTo(w io.Writer) (n int64, err error) {
	n, err = WriteTo(msgTypeState, w, n, err)
	n, err = WriteTo(m.LastBlockHeight, w, n, err)
	n, err = WriteTo(m.LastValidationHeight, w, n, err)
	return
}

func (m *StateMessage) String() string {
	return fmt.Sprintf("[State %v/%v]",
		m.LastBlockHeight, m.LastValidationHeight)
}

/*
A RequestMessage requests a block and/or header at a given height.
*/
type RequestMessage struct {
	Type   Byte
	Height UInt64
}

func readRequestMessage(r io.Reader) *RequestMessage {
	requestType := ReadByte(r)
	height := ReadUInt64(r)
	return &RequestMessage{
		Type:   requestType,
		Height: height,
	}
}

func (m *RequestMessage) WriteTo(w io.Writer) (n int64, err error) {
	n, err = WriteTo(msgTypeRequest, w, n, err)
	n, err = WriteTo(m.Type, w, n, err)
	n, err = WriteTo(m.Height, w, n, err)
	return
}

func (m *RequestMessage) String() string {
	return fmt.Sprintf("[Request %X@%v]", m.Type, m.Height)
}

/*
A DataMessage contains block data, maybe requested.
The data can be a Validation, Txs, or whole Block object.
*/
type DataMessage struct {
	Type   Byte
	Height UInt64
	Bytes  ByteSlice
}

func readDataMessage(r io.Reader) *DataMessage {
	dataType := ReadByte(r)
	height := ReadUInt64(r)
	bytes := ReadByteSlice(r)
	return &DataMessage{
		Type:   dataType,
		Height: height,
		Bytes:  bytes,
	}
}

func (m *DataMessage) WriteTo(w io.Writer) (n int64, err error) {
	n, err = WriteTo(msgTypeData, w, n, err)
	n, err = WriteTo(m.Type, w, n, err)
	n, err = WriteTo(m.Height, w, n, err)
	n, err = WriteTo(m.Bytes, w, n, err)
	return
}

func (m *DataMessage) String() string {
	return fmt.Sprintf("[Data %X@%v]", m.Type, m.Height)
}
<|endoftext|>


<commit_before>package goscp

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/cheggaaa/pb"
	"golang.org/x/crypto/ssh"
)

var (
	// SCP messages
	fileCopyRx  = regexp.MustCompile(`C(?P<mode>\d{4}) (?P<length>\d+) (?P<filename>.+)`)
	dirCopyRx   = regexp.MustCompile(`D(?P<mode>\d{4}) (?P<length>\d+) (?P<dirname>.+)`)
	timestampRx = regexp.MustCompile(`T(?P<mtime>\d+) 0 (?P<atime>\d+) 0`)
	endDir      = "E"
)

type Client struct {
	SSHClient       *ssh.Client
	DestinationPath []string

	// Errors that have occurred while communicating with host
	errors []error

	// Verbose output when communicating with host
	Verbose bool

	// Stop transfer on OS error - occurs during filepath.Walk
	StopOnOSError bool

	// Show progress bar
	ShowProgressBar bool

	// Control progress bar output
	ProgressCallback func(out string)

	// Stdin for SSH session
	scpStdinPipe io.WriteCloser

	// Stdout for SSH session
	scpStdoutPipe *readCanceller
}

// Returns a ssh.Client wrapper.
// DestinationPath is set to the current directory by default.
func NewClient(c *ssh.Client) *Client {
	return &Client{
		SSHClient:       c,
		DestinationPath: []string{"."},
		ShowProgressBar: true,
	}
}

// Set where content will be sent
func (c *Client) SetDestinationPath(path string) {
	c.DestinationPath = []string{path}
}

func (c *Client) addError(err error) {
	c.errors = append(c.errors, err)
}

// GetLastError should be queried after a call to Download() or Upload().
func (c *Client) GetLastError() error {
	if len(c.errors) > 0 {
		return c.errors[len(c.errors)-1]
	}
	return nil
}

// GetErrorStack returns all errors that have occurred so far
func (c *Client) GetErrorStack() []error {
	return c.errors
}

// Cancel an ongoing operation
func (c *Client) Cancel() {
	if c.scpStdoutPipe != nil {
		close(c.scpStdoutPipe.cancel)
	}
}

// Download remotePath to c.DestinationPath
func (c *Client) Download(remotePath string) {
	session, err := c.SSHClient.NewSession()
	if err != nil {
		c.addError(err)
		return
	}
	defer session.Close()

	go func() {
		c.scpStdinPipe, err = session.StdinPipe()
		if err != nil {
			c.addError(err)
			return
		}
		defer c.scpStdinPipe.Close()

		r, err := session.StdoutPipe()
		if err != nil {
			c.addError(err)
			return
		}

		// Initialise transfer
		c.sendAck(c.scpStdinPipe)

		// Wrapper to support cancellation
		c.scpStdoutPipe = &readCanceller{
			Reader: bufio.NewReader(r),
			cancel: make(chan struct{}, 1),
		}

		for {
			c.outputInfo("Reading message from source")
			msg, err := c.scpStdoutPipe.ReadString('\n')
			if err != nil {
				if err != io.EOF {
					c.addError(err)
				}
				return
			}

			// Strip nulls and new lines
			msg = strings.TrimSpace(strings.Trim(msg, "\x00"))
			c.outputInfo(fmt.Sprintf("Received: %s", msg))

			// Confirm message
			c.sendAck(c.scpStdinPipe)

			switch {
			case c.isFileCopyMsg(msg):
				// Handle incoming file
				err := c.file(msg)
				if err != nil {
					c.addError(err)
					return
				}
			case c.isDirCopyMsg(msg):
				// Handling incoming directory
				err := c.directory(msg)
				if err != nil {
					c.addError(err)
					return
				}
			case msg == endDir:
				// Directory finished, go up a directory
				c.upDirectory()
			case c.isWarningMsg(msg):
				c.addError(fmt.Errorf("Warning message: [%q]\n", msg))
				return
			case c.isErrorMsg(msg):
				c.addError(fmt.Errorf("Error message: [%q]\n", msg))
				return
			default:
				c.addError(fmt.Errorf("Unhandled message: [%q]\n", msg))
				return
			}

			// Confirm message
			c.sendAck(c.scpStdinPipe)
		}
	}()

	cmd := fmt.Sprintf("scp -rf %s", remotePath)
	if err := session.Run(cmd); err != nil {
		c.addError(err)
		return
	}

	return
}

// Upload localPath to c.DestinationPath
func (c *Client) Upload(localPath string) {
	session, err := c.SSHClient.NewSession()
	if err != nil {
		c.addError(err)
		return
	}
	defer session.Close()

	go func() {
		c.scpStdinPipe, err = session.StdinPipe()
		if err != nil {
			c.addError(err)
			return
		}
		defer c.scpStdinPipe.Close()

		r, err := session.StdoutPipe()
		if err != nil {
			c.addError(err)
			return
		}

		// Wrapper to support cancellation
		c.scpStdoutPipe = &readCanceller{
			Reader: bufio.NewReader(r),
			cancel: make(chan struct{}, 1),
		}

		// This has already been used in the cmd call below
		// so it can be reused for 'end of directory' message handling
		c.DestinationPath = []string{}

		err = filepath.Walk(localPath, c.handleItem)
		if err != nil {
			c.addError(err)
			return
		}

		// End transfer
		paths := strings.Split(c.DestinationPath[0], "/")
		for range paths {
			c.sendEndOfDirectoryMessage(c.scpStdinPipe)
		}
	}()

	cmd := fmt.Sprintf("scp -rt %s", filepath.Join(c.DestinationPath...))
	if err := session.Run(cmd); err != nil {
		c.addError(err)
		return
	}

	return
}

// Send an acknowledgement message
func (c *Client) sendAck(w io.Writer) {
	fmt.Fprint(w, "\x00")
}

// Send an error message
func (c *Client) sendErr(w io.Writer) {
	fmt.Fprint(w, "\x02")
}

// Check if an incoming message is a file copy message
func (c *Client) isFileCopyMsg(s string) bool {
	return strings.HasPrefix(s, "C")
}

// Check if an incoming message is a directory copy message
func (c *Client) isDirCopyMsg(s string) bool {
	return strings.HasPrefix(s, "D")
}

// Check if an incoming message is a warning
func (c *Client) isWarningMsg(s string) bool {
	return strings.HasPrefix(s, "\x01")
}

// Check if an incoming message is an error
func (c *Client) isErrorMsg(s string) bool {
	return strings.HasPrefix(s, "\x02")
}

// Send a directory message while in source mode
func (c *Client) sendDirectoryMessage(w io.Writer, mode os.FileMode, dirname string) {
	msg := fmt.Sprintf("D0%o 0 %s", mode, dirname)
	fmt.Fprintln(w, msg)
	c.outputInfo(fmt.Sprintf("Sent: %s", msg))
}

// Send a end of directory message while in source mode
func (c *Client) sendEndOfDirectoryMessage(w io.Writer) {
	msg := endDir
	fmt.Fprintln(w, msg)
	c.outputInfo(fmt.Sprintf("Sent: %s", msg))
}

// Send a file message while in source mode
func (c *Client) sendFileMessage(w io.Writer, mode os.FileMode, size int64, filename string) {
	msg := fmt.Sprintf("C0%o %d %s", mode, size, filename)
	fmt.Fprintln(w, msg)
	c.outputInfo(fmt.Sprintf("Sent: %s", msg))
}

// Handle directory copy message in sink mode
func (c *Client) directory(msg string) error {
	parts, err := c.parseMessage(msg, dirCopyRx)
	if err != nil {
		return err
	}

	err = os.Mkdir(filepath.Join(c.DestinationPath...)+string(filepath.Separator)+parts["dirname"], 0755)
	if err != nil {
		return err
	}

	// Traverse into directory
	c.DestinationPath = append(c.DestinationPath, parts["dirname"])

	return nil
}

// Handle file copy message in sink mode
func (c *Client) file(msg string) error {
	parts, err := c.parseMessage(msg, fileCopyRx)
	if err != nil {
		return err
	}

	fileLen, _ := strconv.Atoi(parts["length"])

	// Create local file
	localFile, err := os.Create(filepath.Join(c.DestinationPath...) + string(filepath.Separator) + parts["filename"])
	if err != nil {
		return err
	}
	defer localFile.Close()

	var w io.Writer
	if c.ShowProgressBar {
		bar := c.newProgressBar(fileLen)
		bar.Start()
		defer bar.Finish()

		w = io.MultiWriter(localFile, bar)
	} else {
		w = localFile
	}

	if n, err := io.CopyN(w, c.scpStdoutPipe, int64(fileLen)); err != nil || n < int64(fileLen) {
		c.sendErr(c.scpStdinPipe)
		return err
	}

	return nil
}

// Break down incoming protocol messages
func (c *Client) parseMessage(msg string, rx *regexp.Regexp) (map[string]string, error) {
	parts := make(map[string]string)
	matches := rx.FindStringSubmatch(msg)
	if len(matches) == 0 {
		return parts, errors.New("Could not parse protocol message: " + msg)
	}

	for i, name := range rx.SubexpNames() {
		parts[name] = matches[i]
	}
	return parts, nil
}

// Go back up one directory
func (c *Client) upDirectory() {
	if len(c.DestinationPath) > 0 {
		c.DestinationPath = c.DestinationPath[:len(c.DestinationPath)-1]
	}
}

// Handle each item coming through filepath.Walk
func (c *Client) handleItem(path string, info os.FileInfo, err error) error {
	if err != nil {
		// OS error
		c.outputInfo(fmt.Sprintf("Item error: %s", err))

		if c.StopOnOSError {
			return err
		}
		return nil
	}

	if info.IsDir() {
		// Handle directories
		if len(c.DestinationPath) != 0 {
			// If not first directory
			currentPath := strings.Split(filepath.Join(c.DestinationPath...), "/")
			newPath := strings.Split(path, "/")

			// <= slashes = going back up
			if len(newPath) <= len(currentPath) {
				// Send EOD messages for the amount of directories we go up
				for i := len(newPath) - 1; i < len(currentPath); i++ {
					c.sendEndOfDirectoryMessage(c.scpStdinPipe)
				}
			}
		}
		c.DestinationPath = []string{path}
		c.sendDirectoryMessage(c.scpStdinPipe, 0644, filepath.Base(path))
	} else {
		// Handle regular files
		targetItem, err := os.Open(path)
		if err != nil {
			return err
		}

		c.sendFileMessage(c.scpStdinPipe, 0644, info.Size(), filepath.Base(path))

		if info.Size() > 0 {
			var w io.Writer
			if c.ShowProgressBar {
				bar := c.newProgressBar(int(info.Size()))
				bar.Start()
				defer bar.Finish()

				w = io.MultiWriter(c.scpStdinPipe, bar)
			} else {
				w = c.scpStdinPipe
			}

			c.outputInfo(fmt.Sprintf("Sending file: %s", path))
			if _, err := io.Copy(w, targetItem); err != nil {
				c.sendErr(c.scpStdinPipe)
				return err
			}

			c.sendAck(c.scpStdinPipe)
		} else {
			c.outputInfo(fmt.Sprintf("Sending empty file: %s", path))
			c.sendAck(c.scpStdinPipe)
		}
	}

	return nil
}

func (c *Client) outputInfo(s ...string) {
	if c.Verbose {
		log.Println(s)
	}
}

// Create progress bar
func (c *Client) newProgressBar(fileLength int) *pb.ProgressBar {
	bar := pb.New(fileLength)
	bar.Callback = c.ProgressCallback
	bar.ShowSpeed = true
	bar.ShowTimeLeft = true
	bar.ShowCounters = true
	bar.Units = pb.U_BYTES
	bar.SetRefreshRate(time.Second)
	bar.SetWidth(80)
	bar.SetMaxWidth(80)

	return bar
}

// Wrapper to support cancellation
type readCanceller struct {
	*bufio.Reader

	// Cancel an ongoing transfer
	cancel chan struct{}
}

// Additional cancellation check
func (r *readCanceller) Read(p []byte) (n int, err error) {
	select {
	case <-r.cancel:
		return 0, errors.New("Transfer cancelled")
	default:
		return r.Reader.Read(p)
	}
}
<commit_msg>Exposed the underlying progress bar.<commit_after>package goscp

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/cheggaaa/pb"
	"golang.org/x/crypto/ssh"
)

var (
	// SCP messages
	fileCopyRx  = regexp.MustCompile(`C(?P<mode>\d{4}) (?P<length>\d+) (?P<filename>.+)`)
	dirCopyRx   = regexp.MustCompile(`D(?P<mode>\d{4}) (?P<length>\d+) (?P<dirname>.+)`)
	timestampRx = regexp.MustCompile(`T(?P<mtime>\d+) 0 (?P<atime>\d+) 0`)
	endDir      = "E"
)

type Client struct {
	SSHClient       *ssh.Client
	DestinationPath []string

	// Errors that have occurred while communicating with host
	errors []error

	// Verbose output when communicating with host
	Verbose bool

	// Stop transfer on OS error - occurs during filepath.Walk
	StopOnOSError bool

	// Show progress bar
	ShowProgressBar bool

	// Configurable progress bar
	ProgressBar *pb.ProgressBar

	// Stdin for SSH session
	scpStdinPipe io.WriteCloser

	// Stdout for SSH session
	scpStdoutPipe *readCanceller
}

// Returns a ssh.Client wrapper.
// DestinationPath is set to the current directory by default.
func NewClient(c *ssh.Client) *Client {
	scpc := &Client{
		SSHClient:       c,
		DestinationPath: []string{"."},
		ShowProgressBar: true,
	}

	// Total is set before progress starts
	scpc.ProgressBar = scpc.newDefaultProgressBar(0)

	return scpc
}

// Set where content will be sent
func (c *Client) SetDestinationPath(path string) {
	c.DestinationPath = []string{path}
}

func (c *Client) addError(err error) {
	c.errors = append(c.errors, err)
}

// GetLastError should be queried after a call to Download() or Upload().
func (c *Client) GetLastError() error {
	if len(c.errors) > 0 {
		return c.errors[len(c.errors)-1]
	}
	return nil
}

// GetErrorStack returns all errors that have occurred so far
func (c *Client) GetErrorStack() []error {
	return c.errors
}

// Cancel an ongoing operation
func (c *Client) Cancel() {
	if c.scpStdoutPipe != nil {
		close(c.scpStdoutPipe.cancel)
	}
}

// Download remotePath to c.DestinationPath
func (c *Client) Download(remotePath string) {
	session, err := c.SSHClient.NewSession()
	if err != nil {
		c.addError(err)
		return
	}
	defer session.Close()

	go func() {
		c.scpStdinPipe, err = session.StdinPipe()
		if err != nil {
			c.addError(err)
			return
		}
		defer c.scpStdinPipe.Close()

		r, err := session.StdoutPipe()
		if err != nil {
			c.addError(err)
			return
		}

		// Initialise transfer
		c.sendAck(c.scpStdinPipe)

		// Wrapper to support cancellation
		c.scpStdoutPipe = &readCanceller{
			Reader: bufio.NewReader(r),
			cancel: make(chan struct{}, 1),
		}

		for {
			c.outputInfo("Reading message from source")
			msg, err := c.scpStdoutPipe.ReadString('\n')
			if err != nil {
				if err != io.EOF {
					c.addError(err)
				}
				return
			}

			// Strip nulls and new lines
			msg = strings.TrimSpace(strings.Trim(msg, "\x00"))
			c.outputInfo(fmt.Sprintf("Received: %s", msg))

			// Confirm message
			c.sendAck(c.scpStdinPipe)

			switch {
			case c.isFileCopyMsg(msg):
				// Handle incoming file
				err := c.file(msg)
				if err != nil {
					c.addError(err)
					return
				}
			case c.isDirCopyMsg(msg):
				// Handling incoming directory
				err := c.directory(msg)
				if err != nil {
					c.addError(err)
					return
				}
			case msg == endDir:
				// Directory finished, go up a directory
				c.upDirectory()
			case c.isWarningMsg(msg):
				c.addError(fmt.Errorf("Warning message: [%q]\n", msg))
				return
			case c.isErrorMsg(msg):
				c.addError(fmt.Errorf("Error message: [%q]\n", msg))
				return
			default:
				c.addError(fmt.Errorf("Unhandled message: [%q]\n", msg))
				return
			}

			// Confirm message
			c.sendAck(c.scpStdinPipe)
		}
	}()

	cmd := fmt.Sprintf("scp -rf %s", remotePath)
	if err := session.Run(cmd); err != nil {
		c.addError(err)
		return
	}

	return
}

// Upload localPath to c.DestinationPath
func (c *Client) Upload(localPath string) {
	session, err := c.SSHClient.NewSession()
	if err != nil {
		c.addError(err)
		return
	}
	defer session.Close()

	go func() {
		c.scpStdinPipe, err = session.StdinPipe()
		if err != nil {
			c.addError(err)
			return
		}
		defer c.scpStdinPipe.Close()

		r, err := session.StdoutPipe()
		if err != nil {
			c.addError(err)
			return
		}

		// Wrapper to support cancellation
		c.scpStdoutPipe = &readCanceller{
			Reader: bufio.NewReader(r),
			cancel: make(chan struct{}, 1),
		}

		// This has already been used in the cmd call below
		// so it can be reused for 'end of directory' message handling
		c.DestinationPath = []string{}

		err = filepath.Walk(localPath, c.handleItem)
		if err != nil {
			c.addError(err)
			return
		}

		// End transfer
		paths := strings.Split(c.DestinationPath[0], "/")
		for range paths {
			c.sendEndOfDirectoryMessage(c.scpStdinPipe)
		}
	}()

	cmd := fmt.Sprintf("scp -rt %s", filepath.Join(c.DestinationPath...))
	if err := session.Run(cmd); err != nil {
		c.addError(err)
		return
	}

	return
}

// Send an acknowledgement message
func (c *Client) sendAck(w io.Writer) {
	fmt.Fprint(w, "\x00")
}

// Send an error message
func (c *Client) sendErr(w io.Writer) {
	fmt.Fprint(w, "\x02")
}

// Check if an incoming message is a file copy message
func (c *Client) isFileCopyMsg(s string) bool {
	return strings.HasPrefix(s, "C")
}

// Check if an incoming message is a directory copy message
func (c *Client) isDirCopyMsg(s string) bool {
	return strings.HasPrefix(s, "D")
}

// Check if an incoming message is a warning
func (c *Client) isWarningMsg(s string) bool {
	return strings.HasPrefix(s, "\x01")
}

// Check if an incoming message is an error
func (c *Client) isErrorMsg(s string) bool {
	return strings.HasPrefix(s, "\x02")
}

// Send a directory message while in source mode
func (c *Client) sendDirectoryMessage(w io.Writer, mode os.FileMode, dirname string) {
	msg := fmt.Sprintf("D0%o 0 %s", mode, dirname)
	fmt.Fprintln(w, msg)
	c.outputInfo(fmt.Sprintf("Sent: %s", msg))
}

// Send a end of directory message while in source mode
func (c *Client) sendEndOfDirectoryMessage(w io.Writer) {
	msg := endDir
	fmt.Fprintln(w, msg)
	c.outputInfo(fmt.Sprintf("Sent: %s", msg))
}

// Send a file message while in source mode
func (c *Client) sendFileMessage(w io.Writer, mode os.FileMode, size int64, filename string) {
	msg := fmt.Sprintf("C0%o %d %s", mode, size, filename)
	fmt.Fprintln(w, msg)
	c.outputInfo(fmt.Sprintf("Sent: %s", msg))
}

// Handle directory copy message in sink mode
func (c *Client) directory(msg string) error {
	parts, err := c.parseMessage(msg, dirCopyRx)
	if err != nil {
		return err
	}

	err = os.Mkdir(filepath.Join(c.DestinationPath...)+string(filepath.Separator)+parts["dirname"], 0755)
	if err != nil {
		return err
	}

	// Traverse into directory
	c.DestinationPath = append(c.DestinationPath, parts["dirname"])

	return nil
}

// Handle file copy message in sink mode
func (c *Client) file(msg string) error {
	parts, err := c.parseMessage(msg, fileCopyRx)
	if err != nil {
		return err
	}

	fileLen, _ := strconv.Atoi(parts["length"])

	// Create local file
	localFile, err := os.Create(filepath.Join(c.DestinationPath...) + string(filepath.Separator) + parts["filename"])
	if err != nil {
		return err
	}
	defer localFile.Close()

	var w io.Writer
	if c.ShowProgressBar {
		c.ProgressBar.Total = int64(fileLen)
		c.ProgressBar.Start()
		defer c.ProgressBar.Finish()

		w = io.MultiWriter(localFile, c.ProgressBar)
	} else {
		w = localFile
	}

	if n, err := io.CopyN(w, c.scpStdoutPipe, int64(fileLen)); err != nil || n < int64(fileLen) {
		c.sendErr(c.scpStdinPipe)
		return err
	}

	return nil
}

// Break down incoming protocol messages
func (c *Client) parseMessage(msg string, rx *regexp.Regexp) (map[string]string, error) {
	parts := make(map[string]string)
	matches := rx.FindStringSubmatch(msg)
	if len(matches) == 0 {
		return parts, errors.New("Could not parse protocol message: " + msg)
	}

	for i, name := range rx.SubexpNames() {
		parts[name] = matches[i]
	}
	return parts, nil
}

// Go back up one directory
func (c *Client) upDirectory() {
	if len(c.DestinationPath) > 0 {
		c.DestinationPath = c.DestinationPath[:len(c.DestinationPath)-1]
	}
}

// Handle each item coming through filepath.Walk
func (c *Client) handleItem(path string, info os.FileInfo, err error) error {
	if err != nil {
		// OS error
		c.outputInfo(fmt.Sprintf("Item error: %s", err))

		if c.StopOnOSError {
			return err
		}
		return nil
	}

	if info.IsDir() {
		// Handle directories
		if len(c.DestinationPath) != 0 {
			// If not first directory
			currentPath := strings.Split(filepath.Join(c.DestinationPath...), "/")
			newPath := strings.Split(path, "/")

			// <= slashes = going back up
			if len(newPath) <= len(currentPath) {
				// Send EOD messages for the amount of directories we go up
				for i := len(newPath) - 1; i < len(currentPath); i++ {
					c.sendEndOfDirectoryMessage(c.scpStdinPipe)
				}
			}
		}
		c.DestinationPath = []string{path}
		c.sendDirectoryMessage(c.scpStdinPipe, 0644, filepath.Base(path))
	} else {
		// Handle regular files
		targetItem, err := os.Open(path)
		if err != nil {
			return err
		}

		c.sendFileMessage(c.scpStdinPipe, 0644, info.Size(), filepath.Base(path))

		if info.Size() > 0 {
			var w io.Writer
			if c.ShowProgressBar {
				c.ProgressBar.Total = info.Size()
				c.ProgressBar.Start()
				defer c.ProgressBar.Finish()

				w = io.MultiWriter(c.scpStdinPipe, c.ProgressBar)
			} else {
				w = c.scpStdinPipe
			}

			c.outputInfo(fmt.Sprintf("Sending file: %s", path))
			if _, err := io.Copy(w, targetItem); err != nil {
				c.sendErr(c.scpStdinPipe)
				return err
			}

			c.sendAck(c.scpStdinPipe)
		} else {
			c.outputInfo(fmt.Sprintf("Sending empty file: %s", path))
			c.sendAck(c.scpStdinPipe)
		}
	}

	return nil
}

func (c *Client) outputInfo(s ...string) {
	if c.Verbose {
		log.Println(s)
	}
}

// Create progress bar
func (c *Client) newDefaultProgressBar(fileLength int) *pb.ProgressBar {
	bar := pb.New(fileLength)
	bar.ShowSpeed = true
	bar.ShowTimeLeft = true
	bar.ShowCounters = true
	bar.Units = pb.U_BYTES
	bar.SetRefreshRate(time.Second)
	bar.SetWidth(80)
	bar.SetMaxWidth(80)

	return bar
}

// Wrapper to support cancellation
type readCanceller struct {
	*bufio.Reader

	// Cancel an ongoing transfer
	cancel chan struct{}
}

// Additional cancellation check
func (r *readCanceller) Read(p []byte) (n int, err error) {
	select {
	case <-r.cancel:
		return 0, errors.New("Transfer cancelled")
	default:
		return r.Reader.Read(p)
	}
}
<|endoftext|>


<commit_before>
package main

import (
	"image/color"
	"image/jpeg"
	"log"
	"os"
	"fmt"
	"reflect"
)

type Pixel struct {
    color color.Color
}

type Blob struct {
	color color.Color
    x1 int
    x2 int
    y1 int
    y2 int
}

func getPixelsFromImage(imagePath string) [800][530]color.Color {
	file, err := os.Open(imagePath)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	img, err := jpeg.Decode(file)
	if err != nil {
		log.Fatal(os.Stderr, "%s: %v\n", imagePath, err)
	}

	b := img.Bounds()

	//Defining a Fixed array
	var imgSet [800][530]color.Color

	for y := b.Min.Y; y < b.Max.Y; y++ {
		for x := b.Min.X; x < b.Max.X; x++ {
			oldPixel := img.At(x, y)
			_, g, _, a := oldPixel.RGBA()
			pixel := color.RGBA{uint8(g), uint8(g), uint8(g), uint8(a)}
			imgSet[x][y] = pixel
		}
	}

	return imgSet
}

/*
func getImageDimension([800][530]Pixel imageData) (x int, y int) {

	for _, h := range imageData {
        y := (len(h)) // each one prints 5
    }
    x = len(imageData)

    return x, y
}
*/

func getBlobs(imageData [800][530]color.Color, colorPattern color.Color ) Blob {

	//https://golang.org/pkg/image/color/

	var counter int
	counter = 1

	for y := 0; y < 530; y++ {
		for x := 0; x < 800; x++ {
			pixel := imageData[x][y]

			if (reflect.DeepEqual(pixel, colorPattern)){
				fmt.Println(counter, x, y, "FOUND")
				counter++
			}

		}
	}

	a := Blob{color.RGBA{uint8(255), uint8(255), uint8(255), uint8(255)},1,1,1,1}
	return a
}

func main() {

	imageFramePath := "./psmove.jpg"
	imageData := getPixelsFromImage(imageFramePath)
	//x, y := getImageDimension(imageData)
	//x := 800
	//y := 530

	//Cyan
	colorPattern := color.RGBA{uint8(143), uint8(143), uint8(143), uint8(255)}
	getBlobs(imageData, colorPattern)

}<commit_msg>minor change<commit_after>/*
package main

import (
	"image/color"
	"image/jpeg"
	"log"
	"os"
	"fmt"
	"reflect"
)

type Pixel struct {
    color color.Color
}

type Blob struct {
	color color.Color
    x1 int
    x2 int
    y1 int
    y2 int
}

func getPixelsFromImage(imagePath string) [800][530]color.Color {
	file, err := os.Open(imagePath)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	img, err := jpeg.Decode(file)
	if err != nil {
		log.Fatal(os.Stderr, "%s: %v\n", imagePath, err)
	}

	b := img.Bounds()

	//Defining a Fixed array
	var imgSet [800][530]color.Color

	for y := b.Min.Y; y < b.Max.Y; y++ {
		for x := b.Min.X; x < b.Max.X; x++ {
			oldPixel := img.At(x, y)
			_, g, _, a := oldPixel.RGBA()
			pixel := color.RGBA{uint8(g), uint8(g), uint8(g), uint8(a)}
			imgSet[x][y] = pixel
		}
	}

	return imgSet
}

func getBlobs(imageData [800][530]color.Color, colorPattern color.Color ) Blob {

	//https://golang.org/pkg/image/color/

	var counter int
	counter = 1

	for y := 0; y < 530; y++ {
		for x := 0; x < 800; x++ {
			pixel := imageData[x][y]

			if (reflect.DeepEqual(pixel, colorPattern)){
				fmt.Println(counter, x, y, "FOUND")
				counter++
			}

		}
	}

	a := Blob{color.RGBA{uint8(255), uint8(255), uint8(255), uint8(255)},1,1,1,1}
	return a
}

func main() {

	imageFramePath := "./psmove.jpg"
	imageData := getPixelsFromImage(imageFramePath)
	//x, y := getImageDimension(imageData)
	//x := 800
	//y := 530

	//Cyan
	colorPattern := color.RGBA{uint8(143), uint8(143), uint8(143), uint8(255)}
	getBlobs(imageData, colorPattern)

}

*/<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package deployer

import (
	"errors"
	"fmt"
	osexec "os/exec"
	"strings"
	"time"

	"github.com/google/shlex"

	"k8s.io/klog/v2"
	"k8s.io/kops/tests/e2e/kubetest2-kops/aws"
	"k8s.io/kops/tests/e2e/kubetest2-kops/do"
	"k8s.io/kops/tests/e2e/kubetest2-kops/gce"
	"k8s.io/kops/tests/e2e/pkg/util"
	"k8s.io/kops/tests/e2e/pkg/version"
	"sigs.k8s.io/kubetest2/pkg/exec"
)

func (d *deployer) Up() error {
	if err := d.init(); err != nil {
		return err
	}

	if d.terraform == nil {
		klog.Info("Cleaning up any leaked resources from previous cluster")
		// Intentionally ignore errors:
		// Either the cluster didn't exist or something failed that the next cluster creation will catch
		_ = d.Down()
	}

	if d.CloudProvider == "gce" && d.createBucket {
		if err := gce.EnsureGCSBucket(d.stateStore(), d.GCPProject); err != nil {
			return err
		}
	}

	adminAccess := d.AdminAccess
	if adminAccess == "" {
		publicIP, err := util.ExternalIPRange()
		if err != nil {
			return err
		}

		adminAccess = publicIP
	}

	zones, err := d.zones()
	if err != nil {
		return err
	}

	if d.TemplatePath != "" {
		values, err := d.templateValues(zones, adminAccess)
		if err != nil {
			return err
		}
		if err := d.renderTemplate(values); err != nil {
			return err
		}
		if err := d.replace(); err != nil {
			return err
		}
	} else {
		err := d.createCluster(zones, adminAccess)
		if err != nil {
			return err
		}
	}
	isUp, err := d.IsUp()
	if err != nil {
		return err
	} else if isUp {
		klog.V(1).Infof("cluster reported as up")
	} else {
		klog.Errorf("cluster reported as down")
	}
	return nil
}

func (d *deployer) createCluster(zones []string, adminAccess string) error {

	args := []string{
		d.KopsBinaryPath, "create", "cluster",
		"--name", d.ClusterName,
		"--cloud", d.CloudProvider,
		"--kubernetes-version", d.KubernetesVersion,
		"--ssh-public-key", d.SSHPublicKeyPath,
		"--override", "cluster.spec.nodePortAccess=0.0.0.0/0",
		"--yes",
	}

	if d.CreateArgs != "" {
		createArgs, err := shlex.Split(d.CreateArgs)
		if err != nil {
			return err
		}
		args = append(args, createArgs...)
	}
	args = appendIfUnset(args, "--admin-access", adminAccess)
	args = appendIfUnset(args, "--master-count", "1")
	args = appendIfUnset(args, "--master-volume-size", "48")
	args = appendIfUnset(args, "--node-count", "4")
	args = appendIfUnset(args, "--node-volume-size", "48")
	args = appendIfUnset(args, "--override", adminAccess)
	args = appendIfUnset(args, "--admin-access", adminAccess)
	args = appendIfUnset(args, "--zones", strings.Join(zones, ","))

	switch d.CloudProvider {
	case "aws":
		args = appendIfUnset(args, "--master-size", "c5.large")
	case "gce":
		args = appendIfUnset(args, "--master-size", "e2-standard-2")
		if d.GCPProject != "" {
			args = appendIfUnset(args, "--project", d.GCPProject)
		}
		args = appendIfUnset(args, "--vpc", strings.Split(d.ClusterName, ".")[0])
	case "digitalocean":
		args = appendIfUnset(args, "--master-size", "s-8vcpu-16gb")
		args = appendIfUnset(args, "--node-size", "s-8vcpu-16gb")
	}

	if d.terraform != nil {
		args = append(args, "--target", "terraform", "--out", d.terraform.Dir())
	}

	klog.Info(strings.Join(args, " "))
	cmd := exec.Command(args[0], args[1:]...)
	cmd.SetEnv(d.env()...)

	exec.InheritOutput(cmd)
	err := cmd.Run()
	if err != nil {
		return err
	}

	if d.terraform != nil {
		if err := d.terraform.InitApply(); err != nil {
			return err
		}
	}

	return nil
}

func (d *deployer) IsUp() (bool, error) {
	wait := d.ValidationWait
	if wait == 0 {
		if d.TerraformVersion != "" || d.CloudProvider == "digitalocean" {
			// `--target terraform` doesn't precreate the API DNS records,
			// so kops is more likely to hit negative TTLs during validation.
			// Digital Ocean also occasionally takes longer to validate.
			wait = time.Duration(20) * time.Minute
		} else {
			wait = time.Duration(15) * time.Minute
		}
	}
	args := []string{
		d.KopsBinaryPath, "validate", "cluster",
		"--name", d.ClusterName,
		"--count", "10",
		"--wait", wait.String(),
	}
	klog.Info(strings.Join(args, " "))

	cmd := exec.Command(args[0], args[1:]...)
	cmd.SetEnv(d.env()...)

	exec.InheritOutput(cmd)
	err := cmd.Run()
	// `kops validate cluster` exits 2 if validation failed
	if exitErr, ok := err.(*osexec.ExitError); ok && exitErr.ExitCode() == 2 {
		return false, nil
	}
	if err == nil && d.TerraformVersion != "" && d.commonOptions.ShouldTest() {
		klog.Info("Waiting 5 minutes for DNS TTLs before starting tests")
		time.Sleep(5 * time.Minute)
	}
	return err == nil, err
}

// verifyUpFlags ensures fields are set for creation of the cluster
func (d *deployer) verifyUpFlags() error {
	if d.KubernetesVersion == "" {
		return errors.New("missing required --kubernetes-version flag")
	}

	v, err := version.ParseKubernetesVersion(d.KubernetesVersion)
	if err != nil {
		return err
	}
	d.KubernetesVersion = v

	return nil
}

func (d *deployer) zones() ([]string, error) {
	switch d.CloudProvider {
	case "aws":
		return aws.RandomZones(1)
	case "gce":
		return gce.RandomZones(1)
	case "digitalocean":
		return do.RandomZones(1)
	}
	return nil, fmt.Errorf("unsupported CloudProvider: %v", d.CloudProvider)
}

// appendIfUnset will append an argument and its value to args if the arg is not already present
// This shouldn't be used for arguments that can be specified multiple times like --override
func appendIfUnset(args []string, arg, value string) []string {
	for _, existingArg := range args {
		existingKey := strings.Split(existingArg, "=")
		if existingKey[0] == arg {
			return args
		}
	}
	args = append(args, arg, value)
	return args
}
<commit_msg>Increase droplet size<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package deployer

import (
	"errors"
	"fmt"
	osexec "os/exec"
	"strings"
	"time"

	"github.com/google/shlex"

	"k8s.io/klog/v2"
	"k8s.io/kops/tests/e2e/kubetest2-kops/aws"
	"k8s.io/kops/tests/e2e/kubetest2-kops/do"
	"k8s.io/kops/tests/e2e/kubetest2-kops/gce"
	"k8s.io/kops/tests/e2e/pkg/util"
	"k8s.io/kops/tests/e2e/pkg/version"
	"sigs.k8s.io/kubetest2/pkg/exec"
)

func (d *deployer) Up() error {
	if err := d.init(); err != nil {
		return err
	}

	if d.terraform == nil {
		klog.Info("Cleaning up any leaked resources from previous cluster")
		// Intentionally ignore errors:
		// Either the cluster didn't exist or something failed that the next cluster creation will catch
		_ = d.Down()
	}

	if d.CloudProvider == "gce" && d.createBucket {
		if err := gce.EnsureGCSBucket(d.stateStore(), d.GCPProject); err != nil {
			return err
		}
	}

	adminAccess := d.AdminAccess
	if adminAccess == "" {
		publicIP, err := util.ExternalIPRange()
		if err != nil {
			return err
		}

		adminAccess = publicIP
	}

	zones, err := d.zones()
	if err != nil {
		return err
	}

	if d.TemplatePath != "" {
		values, err := d.templateValues(zones, adminAccess)
		if err != nil {
			return err
		}
		if err := d.renderTemplate(values); err != nil {
			return err
		}
		if err := d.replace(); err != nil {
			return err
		}
	} else {
		err := d.createCluster(zones, adminAccess)
		if err != nil {
			return err
		}
	}
	isUp, err := d.IsUp()
	if err != nil {
		return err
	} else if isUp {
		klog.V(1).Infof("cluster reported as up")
	} else {
		klog.Errorf("cluster reported as down")
	}
	return nil
}

func (d *deployer) createCluster(zones []string, adminAccess string) error {

	args := []string{
		d.KopsBinaryPath, "create", "cluster",
		"--name", d.ClusterName,
		"--cloud", d.CloudProvider,
		"--kubernetes-version", d.KubernetesVersion,
		"--ssh-public-key", d.SSHPublicKeyPath,
		"--override", "cluster.spec.nodePortAccess=0.0.0.0/0",
		"--yes",
	}

	if d.CreateArgs != "" {
		createArgs, err := shlex.Split(d.CreateArgs)
		if err != nil {
			return err
		}
		args = append(args, createArgs...)
	}
	args = appendIfUnset(args, "--admin-access", adminAccess)
	args = appendIfUnset(args, "--master-count", "1")
	args = appendIfUnset(args, "--master-volume-size", "48")
	args = appendIfUnset(args, "--node-count", "4")
	args = appendIfUnset(args, "--node-volume-size", "48")
	args = appendIfUnset(args, "--override", adminAccess)
	args = appendIfUnset(args, "--admin-access", adminAccess)
	args = appendIfUnset(args, "--zones", strings.Join(zones, ","))

	switch d.CloudProvider {
	case "aws":
		args = appendIfUnset(args, "--master-size", "c5.large")
	case "gce":
		args = appendIfUnset(args, "--master-size", "e2-standard-2")
		if d.GCPProject != "" {
			args = appendIfUnset(args, "--project", d.GCPProject)
		}
		args = appendIfUnset(args, "--vpc", strings.Split(d.ClusterName, ".")[0])
	case "digitalocean":
		args = appendIfUnset(args, "--master-size", "c2-16vcpu-32gb")
		args = appendIfUnset(args, "--node-size", "c2-16vcpu-32gb")
	}

	if d.terraform != nil {
		args = append(args, "--target", "terraform", "--out", d.terraform.Dir())
	}

	klog.Info(strings.Join(args, " "))
	cmd := exec.Command(args[0], args[1:]...)
	cmd.SetEnv(d.env()...)

	exec.InheritOutput(cmd)
	err := cmd.Run()
	if err != nil {
		return err
	}

	if d.terraform != nil {
		if err := d.terraform.InitApply(); err != nil {
			return err
		}
	}

	return nil
}

func (d *deployer) IsUp() (bool, error) {
	wait := d.ValidationWait
	if wait == 0 {
		if d.TerraformVersion != "" || d.CloudProvider == "digitalocean" {
			// `--target terraform` doesn't precreate the API DNS records,
			// so kops is more likely to hit negative TTLs during validation.
			// Digital Ocean also occasionally takes longer to validate.
			wait = time.Duration(20) * time.Minute
		} else {
			wait = time.Duration(15) * time.Minute
		}
	}
	args := []string{
		d.KopsBinaryPath, "validate", "cluster",
		"--name", d.ClusterName,
		"--count", "10",
		"--wait", wait.String(),
	}
	klog.Info(strings.Join(args, " "))

	cmd := exec.Command(args[0], args[1:]...)
	cmd.SetEnv(d.env()...)

	exec.InheritOutput(cmd)
	err := cmd.Run()
	// `kops validate cluster` exits 2 if validation failed
	if exitErr, ok := err.(*osexec.ExitError); ok && exitErr.ExitCode() == 2 {
		return false, nil
	}
	if err == nil && d.TerraformVersion != "" && d.commonOptions.ShouldTest() {
		klog.Info("Waiting 5 minutes for DNS TTLs before starting tests")
		time.Sleep(5 * time.Minute)
	}
	return err == nil, err
}

// verifyUpFlags ensures fields are set for creation of the cluster
func (d *deployer) verifyUpFlags() error {
	if d.KubernetesVersion == "" {
		return errors.New("missing required --kubernetes-version flag")
	}

	v, err := version.ParseKubernetesVersion(d.KubernetesVersion)
	if err != nil {
		return err
	}
	d.KubernetesVersion = v

	return nil
}

func (d *deployer) zones() ([]string, error) {
	switch d.CloudProvider {
	case "aws":
		return aws.RandomZones(1)
	case "gce":
		return gce.RandomZones(1)
	case "digitalocean":
		return do.RandomZones(1)
	}
	return nil, fmt.Errorf("unsupported CloudProvider: %v", d.CloudProvider)
}

// appendIfUnset will append an argument and its value to args if the arg is not already present
// This shouldn't be used for arguments that can be specified multiple times like --override
func appendIfUnset(args []string, arg, value string) []string {
	for _, existingArg := range args {
		existingKey := strings.Split(existingArg, "=")
		if existingKey[0] == arg {
			return args
		}
	}
	args = append(args, arg, value)
	return args
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package strings

// Count UTF-8 sequences in s.
// Assumes s is well-formed.
export func utflen(s string) int {
	n := 0;
	for i := 0; i < len(s); i++ {
		return int(s[i]);
		if s[i]&0xC0 != 0x80 {
			n++
		}
	}
	return n
}

// Split string into array of UTF-8 sequences (still strings)
export func explode(s string) *[]string {
	a := new([]string, utflen(s));
	j := 0;
	for i := 0; i < len(a); i++ {
		ej := j;
		ej++;
		for ej < len(s) && (s[ej]&0xC0) == 0x80 {
			ej++
		}
		a[i] = s[j:ej];
		j = ej
	}
	return a
}

// Count non-overlapping instances of sep in s.
export func count(s, sep string) int {
	if sep == "" {
		return utflen(s)+1
	}
	c := sep[0];
	n := 0;
	for i := 0; i+len(sep) <= len(s); i++ {
		if s[i] == c && (len(sep) == 1 || s[i:i+len(sep)] == sep) {
			n++;
			i += len(sep)-1
		}
	}
	return n
}

// Return index of first instance of sep in s.
export func index(s, sep string) int {
	if sep == "" {
		return 0
	}
	c := sep[0];
	for i := 0; i+len(sep) <= len(s); i++ {
		if s[i] == c && (len(sep) == 1 || s[i:i+len(sep)] == sep) {
			return i
		}
	}
	return -1
}

// Split string into list of strings at separators
export func split(s, sep string) *[]string {
	if sep == "" {
		return explode(s)
	}
	c := sep[0];
	start := 0;
	n := count(s, sep)+1;
	a := new([]string, n);
	na := 0;
	for i := 0; i+len(sep) <= len(s); i++ {
		if s[i] == c && (len(sep) == 1 || s[i:i+len(sep)] == sep) {
			a[na] = s[start:i];
			na++;
			start = i+len(sep);
			i += len(sep)-1
		}
	}
	a[na] = s[start:len(s)];
	return a
}

// Join list of strings with separators between them.
export func join(a *[]string, sep string) string {
	if len(a) == 0 {
		return ""
	}
	if len(a) == 1 {
		return a[0]
	}
	n := len(sep) * (len(a)-1);
	for i := 0; i < len(a); i++ {
		n += len(a[i])
	}

	b := new([]byte, n);
	bp := 0;
	for i := 0; i < len(a); i++ {
		s := a[i];
		for j := 0; j < len(s); j++ {
			b[bp] = s[j];
			bp++
		}
		if i + 1 < len(a) {
			s = sep;
			for j := 0; j < len(s); j++ {
				b[bp] = s[j];
				bp++
			}
		}
	}
	return string(b)
}

// Convert decimal string to integer.
// TODO: Doesn't check for overflow.
export func atol(s string) (i int64, ok bool) {
	// empty string bad
	if len(s) == 0 {
		return 0, false
	}

	// pick off leading sign
	neg := false;
	if s[0] == '+' {
		s = s[1:len(s)]
	} else if s[0] == '-' {
		neg = true;
		s = s[1:len(s)]
	}

	// empty string bad
	if len(s) == 0 {
		return 0, false
	}

	// pick off zero
	if s == "0" {
		return 0, true
	}

	// otherwise, leading zero bad
	if s[0] == '0' {
		return 0, false
	}

	// parse number
	n := int64(0);
	for i := 0; i < len(s); i++ {
		if s[i] < '0' || s[i] > '9' {
			return 0, false
		}
		n = n*10 + int64(s[i] - '0')
	}
	if neg {
		n = -n
	}
	return n, true
}

export func atoi(s string) (i int, ok bool) {
	ii, okok := atol(s);
	i = int(ii);
	return i, okok
}

export func ltoa(i int64) string {
	if i == 0 {
		return "0"
	}

	neg := false;	// negative
	u := uint(i);
	if i < 0 {
		neg = true;
		u = -u;
	}

	// Assemble decimal in reverse order.
	var b [32]byte;
	bp := len(b);
	for ; u > 0; u /= 10 {
		bp--;
		b[bp] = byte(u%10) + '0'
	}
	if neg {	// add sign
		bp--;
		b[bp] = '-'
	}

	// BUG return string(b[bp:len(b)])
	return string((&b)[bp:len(b)])
}

export func itoa(i int) string {
	return ltoa(int64(i));
}

// Convert float64 to string.  No control over format.
// Result not great; only useful for simple debugging.
export func dtoa(v float64) string {
	var buf [20]byte;

	const n = 7;	// digits printed
	e := 0;	// exp
	var sign byte = '+';
	if(v != 0) {
		// sign
		if(v < 0) {
			v = -v;
			sign = '-';
		}

		// normalize
		for v >= 10 {
			e++;
			v /= 10;
		}
		for v < 1 {
			e--;
			v *= 10;
		}

		// round
		var h float64 = 5;
		for i := 0; i < n; i++ {
			h /= 10;
		}
		v += h;
		if v >= 10 {
			e++;
			v /= 10;
		}
	}

	// format +d.dddd+edd
	buf[0] = sign;
	for i := 0; i < n; i++ {
		s := int64(v);
		buf[i+2] = byte(s)+'0';
		v -= float64(s);
		v *= 10;
	}
	buf[1] = buf[2];
	buf[2] = '.';

	buf[n+2] = 'e';
	buf[n+3] = '+';
	if e < 0 {
		e = -e;
		buf[n+3] = '-';
	}

	// TODO: exponents > 99?
	buf[n+4] = byte((e/10) + '0');
	buf[n+5] = byte((e%10) + '0');
	return string(buf)[0:n+6];	// TODO: should be able to slice buf
}

export func ftoa(v float) string {
	return dtoa(float64(v));
}
<commit_msg>fix strings bug from CL 18024<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package strings

// Count UTF-8 sequences in s.
// Assumes s is well-formed.
export func utflen(s string) int {
	n := 0;
	for i := 0; i < len(s); i++ {
		if s[i]&0xC0 != 0x80 {
			n++
		}
	}
	return n
}

// Split string into array of UTF-8 sequences (still strings)
export func explode(s string) *[]string {
	a := new([]string, utflen(s));
	j := 0;
	for i := 0; i < len(a); i++ {
		ej := j;
		ej++;
		for ej < len(s) && (s[ej]&0xC0) == 0x80 {
			ej++
		}
		a[i] = s[j:ej];
		j = ej
	}
	return a
}

// Count non-overlapping instances of sep in s.
export func count(s, sep string) int {
	if sep == "" {
		return utflen(s)+1
	}
	c := sep[0];
	n := 0;
	for i := 0; i+len(sep) <= len(s); i++ {
		if s[i] == c && (len(sep) == 1 || s[i:i+len(sep)] == sep) {
			n++;
			i += len(sep)-1
		}
	}
	return n
}

// Return index of first instance of sep in s.
export func index(s, sep string) int {
	if sep == "" {
		return 0
	}
	c := sep[0];
	for i := 0; i+len(sep) <= len(s); i++ {
		if s[i] == c && (len(sep) == 1 || s[i:i+len(sep)] == sep) {
			return i
		}
	}
	return -1
}

// Split string into list of strings at separators
export func split(s, sep string) *[]string {
	if sep == "" {
		return explode(s)
	}
	c := sep[0];
	start := 0;
	n := count(s, sep)+1;
	a := new([]string, n);
	na := 0;
	for i := 0; i+len(sep) <= len(s); i++ {
		if s[i] == c && (len(sep) == 1 || s[i:i+len(sep)] == sep) {
			a[na] = s[start:i];
			na++;
			start = i+len(sep);
			i += len(sep)-1
		}
	}
	a[na] = s[start:len(s)];
	return a
}

// Join list of strings with separators between them.
export func join(a *[]string, sep string) string {
	if len(a) == 0 {
		return ""
	}
	if len(a) == 1 {
		return a[0]
	}
	n := len(sep) * (len(a)-1);
	for i := 0; i < len(a); i++ {
		n += len(a[i])
	}

	b := new([]byte, n);
	bp := 0;
	for i := 0; i < len(a); i++ {
		s := a[i];
		for j := 0; j < len(s); j++ {
			b[bp] = s[j];
			bp++
		}
		if i + 1 < len(a) {
			s = sep;
			for j := 0; j < len(s); j++ {
				b[bp] = s[j];
				bp++
			}
		}
	}
	return string(b)
}

// Convert decimal string to integer.
// TODO: Doesn't check for overflow.
export func atol(s string) (i int64, ok bool) {
	// empty string bad
	if len(s) == 0 {
		return 0, false
	}

	// pick off leading sign
	neg := false;
	if s[0] == '+' {
		s = s[1:len(s)]
	} else if s[0] == '-' {
		neg = true;
		s = s[1:len(s)]
	}

	// empty string bad
	if len(s) == 0 {
		return 0, false
	}

	// pick off zero
	if s == "0" {
		return 0, true
	}

	// otherwise, leading zero bad
	if s[0] == '0' {
		return 0, false
	}

	// parse number
	n := int64(0);
	for i := 0; i < len(s); i++ {
		if s[i] < '0' || s[i] > '9' {
			return 0, false
		}
		n = n*10 + int64(s[i] - '0')
	}
	if neg {
		n = -n
	}
	return n, true
}

export func atoi(s string) (i int, ok bool) {
	ii, okok := atol(s);
	i = int(ii);
	return i, okok
}

export func ltoa(i int64) string {
	if i == 0 {
		return "0"
	}

	neg := false;	// negative
	u := uint(i);
	if i < 0 {
		neg = true;
		u = -u;
	}

	// Assemble decimal in reverse order.
	var b [32]byte;
	bp := len(b);
	for ; u > 0; u /= 10 {
		bp--;
		b[bp] = byte(u%10) + '0'
	}
	if neg {	// add sign
		bp--;
		b[bp] = '-'
	}

	// BUG return string(b[bp:len(b)])
	return string((&b)[bp:len(b)])
}

export func itoa(i int) string {
	return ltoa(int64(i));
}

// Convert float64 to string.  No control over format.
// Result not great; only useful for simple debugging.
export func dtoa(v float64) string {
	var buf [20]byte;

	const n = 7;	// digits printed
	e := 0;	// exp
	var sign byte = '+';
	if(v != 0) {
		// sign
		if(v < 0) {
			v = -v;
			sign = '-';
		}

		// normalize
		for v >= 10 {
			e++;
			v /= 10;
		}
		for v < 1 {
			e--;
			v *= 10;
		}

		// round
		var h float64 = 5;
		for i := 0; i < n; i++ {
			h /= 10;
		}
		v += h;
		if v >= 10 {
			e++;
			v /= 10;
		}
	}

	// format +d.dddd+edd
	buf[0] = sign;
	for i := 0; i < n; i++ {
		s := int64(v);
		buf[i+2] = byte(s)+'0';
		v -= float64(s);
		v *= 10;
	}
	buf[1] = buf[2];
	buf[2] = '.';

	buf[n+2] = 'e';
	buf[n+3] = '+';
	if e < 0 {
		e = -e;
		buf[n+3] = '-';
	}

	// TODO: exponents > 99?
	buf[n+4] = byte((e/10) + '0');
	buf[n+5] = byte((e%10) + '0');
	return string(buf)[0:n+6];	// TODO: should be able to slice buf
}

export func ftoa(v float) string {
	return dtoa(float64(v));
}
<|endoftext|>


<commit_before>package main

import (
	"testing"
	// "time"

	// "database/sql"
	// "reflect"
	// "strings"
	// "time"
	// "fmt"

	// "github.com/coopernurse/gorp"
	// "github.com/mattn/go-sqlite3"
)

func Test_createLogRecord_good(t *testing.T) {
	prepareDb()
	record, err := createLogRecord("demo", "msg", 5, []string{"foo", "bar"})

	if err != nil {
		t.Error()
	}

	if record.Message != "msg" {
		t.Error()
	}
	if record.Level != 5 {
		t.Error()
	}
	if len(record.Tags) != 2 {
		t.Error()
	}
	if record.Tags[0].Name != "foo" {
		t.Error()
	}
	if record.Tags[1].Name != "bar" {
		t.Error()
	}
}

func Test_createLogRecord_fail(t *testing.T) {
	prepareDb()
	_, err := createLogRecord("", "msg", 5, []string{"foo", "bar"})

	if err == nil {
		t.Error()
	}
}

func prepareDb() {
	config = Config{}
	config.Database.Path = "test.sqlite"
	config.Database.LockTimeout = 1
	config.Database.RetryDelay = 10
	config.Database.MaxIdleConnections = 5

	config.Log.Path = "test.log"
	config.Log.LogDatabase = false

	initDB()
}
<commit_msg>Fix tests<commit_after>package main

import (
	"testing"
	// "time"

	// "database/sql"
	// "reflect"
	// "strings"
	// "time"
	// "fmt"

	// "github.com/coopernurse/gorp"
	// "github.com/mattn/go-sqlite3"
)

func Test_createLogRecord_good(t *testing.T) {
	prepareDb()
	record, err := createLogRecord("demo", "msg", 5, []string{"foo", "bar"})

	if err != nil {
		t.Error()
	}

	if record.Message != "msg" {
		t.Error()
	}
	if record.Level != 5 {
		t.Error()
	}
	if len(record.Tags) != 2 {
		t.Error()
	}
	if record.Tags[0].Name != "foo" {
		t.Error()
	}
	if record.Tags[1].Name != "bar" {
		t.Error()
	}
}

func prepareDb() {
	config = Config{}
	config.Database.Path = "test.sqlite"
	config.Database.LockTimeout = 1
	config.Database.RetryDelay = 10
	config.Database.MaxIdleConnections = 5

	config.Log.Path = "test.log"
	config.Log.LogDatabase = false

	initDB()
}
<|endoftext|>


<commit_before>package test

import (
	"fmt"

	storagev1 "k8s.io/api/storage/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/kubernetes/test/e2e/framework"
	e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
	"k8s.io/kubernetes/test/e2e/storage/testpatterns"
	"k8s.io/kubernetes/test/e2e/storage/testsuites"
)

type cinderDriver struct {
	driverInfo testsuites.DriverInfo
	manifests  []string
}

var Cinderdriver = InitCinderDriver

type cinderVolume struct {
	ID               string
	Name             string
	Status           string
	AvailabilityZone string
	f                *framework.Framework
}

// initCinderDriver returns cinderDriver that implements TestDriver interface
func initCinderDriver(name string, manifests ...string) testsuites.TestDriver {
	return &cinderDriver{
		driverInfo: testsuites.DriverInfo{
			Name:        name,
			MaxFileSize: testpatterns.FileSizeLarge,
			SupportedFsType: sets.NewString(
				"", // Default fsType
				"ext2",
				"ext3",
				"ext4",
				"xfs",
			),
			SupportedSizeRange: e2evolume.SizeRange{
				Min: "1Gi",
			},
			Capabilities: map[testsuites.Capability]bool{
				testsuites.CapPersistence:        true,
				testsuites.CapFsGroup:            true,
				testsuites.CapExec:               true,
				testsuites.CapMultiPODs:          true,
				testsuites.CapBlock:              true,
				testsuites.CapSnapshotDataSource: true,
			},
		},
		manifests: manifests,
	}
}

func InitCinderDriver() testsuites.TestDriver {

	return initCinderDriver("cinder.csi.openstack.org",
		"cinder-csi-controllerplugin.yaml",
		"cinder-csi-controllerplugin-rbac.yaml",
		"cinder-csi-nodeplugin.yaml",
		"cinder-csi-nodeplugin-rbac.yaml",
		"csi-secret-cinderplugin.yaml")

}

var _ testsuites.TestDriver = &cinderDriver{}

// var _ testsuites.PreprovisionedVolumeTestDriver = &cinderDriver{}
// var _ testsuites.PreprovisionedPVTestDriver = &cinderDriver{}
var _ testsuites.DynamicPVTestDriver = &cinderDriver{}
var _ testsuites.SnapshottableTestDriver = &cinderDriver{}

func (d *cinderDriver) GetDriverInfo() *testsuites.DriverInfo {
	return &d.driverInfo
}

func (d *cinderDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
}

func (d *cinderDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
	provisioner := "cinder.csi.openstack.org"
	parameters := map[string]string{}
	if fsType != "" {
		parameters["fsType"] = fsType
	}
	ns := config.Framework.Namespace.Name
	suffix := fmt.Sprintf("%s-sc", d.driverInfo.Name)

	return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix)
}

func (d *cinderDriver) GetSnapshotClass(config *testsuites.PerTestConfig) *unstructured.Unstructured {
	parameters := map[string]string{}
	snapshotter := d.driverInfo.Name
	suffix := fmt.Sprintf("%s-vsc", snapshotter)
	ns := config.Framework.Namespace.Name
	return testsuites.GetSnapshotClass(snapshotter, parameters, ns, suffix)
}

func (d *cinderDriver) GetClaimSize() string {
	return "2Gi"
}

func (d *cinderDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
	config := &testsuites.PerTestConfig{
		Driver:    d,
		Prefix:    "cinder",
		Framework: f,
	}

	return config, func() {}
}
<commit_msg>Enable volume cloning tests to run (#1422)<commit_after>package test

import (
	"fmt"

	storagev1 "k8s.io/api/storage/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/kubernetes/test/e2e/framework"
	e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
	"k8s.io/kubernetes/test/e2e/storage/testpatterns"
	"k8s.io/kubernetes/test/e2e/storage/testsuites"
)

type cinderDriver struct {
	driverInfo testsuites.DriverInfo
	manifests  []string
}

var Cinderdriver = InitCinderDriver

type cinderVolume struct {
	ID               string
	Name             string
	Status           string
	AvailabilityZone string
	f                *framework.Framework
}

// initCinderDriver returns cinderDriver that implements TestDriver interface
func initCinderDriver(name string, manifests ...string) testsuites.TestDriver {
	return &cinderDriver{
		driverInfo: testsuites.DriverInfo{
			Name:        name,
			MaxFileSize: testpatterns.FileSizeLarge,
			SupportedFsType: sets.NewString(
				"", // Default fsType
				"ext2",
				"ext3",
				"ext4",
				"xfs",
			),
			SupportedSizeRange: e2evolume.SizeRange{
				Min: "1Gi",
			},
			Capabilities: map[testsuites.Capability]bool{
				testsuites.CapPersistence:        true,
				testsuites.CapFsGroup:            true,
				testsuites.CapExec:               true,
				testsuites.CapMultiPODs:          true,
				testsuites.CapBlock:              true,
				testsuites.CapSnapshotDataSource: true,
				testsuites.CapPVCDataSource:      true,
			},
		},
		manifests: manifests,
	}
}

func InitCinderDriver() testsuites.TestDriver {

	return initCinderDriver("cinder.csi.openstack.org",
		"cinder-csi-controllerplugin.yaml",
		"cinder-csi-controllerplugin-rbac.yaml",
		"cinder-csi-nodeplugin.yaml",
		"cinder-csi-nodeplugin-rbac.yaml",
		"csi-secret-cinderplugin.yaml")

}

var _ testsuites.TestDriver = &cinderDriver{}

// var _ testsuites.PreprovisionedVolumeTestDriver = &cinderDriver{}
// var _ testsuites.PreprovisionedPVTestDriver = &cinderDriver{}
var _ testsuites.DynamicPVTestDriver = &cinderDriver{}
var _ testsuites.SnapshottableTestDriver = &cinderDriver{}

func (d *cinderDriver) GetDriverInfo() *testsuites.DriverInfo {
	return &d.driverInfo
}

func (d *cinderDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
}

func (d *cinderDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
	provisioner := "cinder.csi.openstack.org"
	parameters := map[string]string{}
	if fsType != "" {
		parameters["fsType"] = fsType
	}
	ns := config.Framework.Namespace.Name
	suffix := fmt.Sprintf("%s-sc", d.driverInfo.Name)

	return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix)
}

func (d *cinderDriver) GetSnapshotClass(config *testsuites.PerTestConfig) *unstructured.Unstructured {
	parameters := map[string]string{}
	snapshotter := d.driverInfo.Name
	suffix := fmt.Sprintf("%s-vsc", snapshotter)
	ns := config.Framework.Namespace.Name
	return testsuites.GetSnapshotClass(snapshotter, parameters, ns, suffix)
}

func (d *cinderDriver) GetClaimSize() string {
	return "2Gi"
}

func (d *cinderDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
	config := &testsuites.PerTestConfig{
		Driver:    d,
		Prefix:    "cinder",
		Framework: f,
	}

	return config, func() {}
}
<|endoftext|>


<commit_before>package posts

import (
	"errors"
	"github.com/go-fsnotify/fsnotify"
	"github.com/naoina/toml"
	"io/ioutil"
	"log"
	"os"
	"sort"
	"strings"
	"time"
)

// Info of a post
type PostInfo struct {
	Title    string
	Ignore   bool
	Subtitle string
	Author   string
	Created  time.Time
	Modified time.Time
}

// Post (PostInfo and the actual text content)
type Post struct {
	PostInfo
	Content []string
}

// Returns just info of a post
func (p Post) Info() PostInfo {
	return p.PostInfo
}

// for sorting
type postSlice []*Post

func (p postSlice) Len() int {
	return len(p)
}

func (p postSlice) Less(i, j int) bool {
	return p[i].Created.After(p[j].Created)
}

func (p postSlice) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

var postMap map[string]*Post

// Gets post by title
func GetPost(id string) (*Post, error) {
	post, ok := postMap[id]
	if !ok {
		return nil, errors.New(id + " not found")
	}
	return post, nil
}

// Returns a list of all the posts sorted by time modified
func GetPostList() []string {
	posts := make(postSlice, len(postMap))
	i := 0
	for _, post := range postMap {
		posts[i] = post
		i += 1
	}

	sort.Stable(posts)

	postTitles := make([]string, len(posts))
	for i, post := range posts {
		postTitles[i] = post.Title
	}

	return postTitles
}

// Reads the folder and parses all the posts
// Also sets up a watcher so the posts will automatically
//   be updated if any of the files change
func Init(path string) error {
	err := readFolder(path)
	if err != nil {
		return err
	}
	doneChan := make(chan bool)
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return err
	}

	go func() {
		<-doneChan
		watcher.Close()
	}()

	err = watcher.Add(path)
	if err != nil {
		return err
	}
	go func() {
		for {
			select {
			case event := <-watcher.Events:
				if strings.HasSuffix(event.Name, ".toml") {
					log.Println("event:", event)
					readFolder(path)
					log.Println("Updated posts")
				}
			case err := <-watcher.Errors:
				log.Println("error:", err)
			}
		}
	}()

	return nil
}

// Reads the folder and parses all the files into posts
func readFolder(path string) error {

	tempPostMap := make(map[string]*Post)
	files, err := ioutil.ReadDir(path)
	if err != nil {
		log.Println("Can't read at %s", path)
		return err
	}

	for _, file := range files {
		if !strings.HasSuffix(file.Name(), ".toml") || file.IsDir() || file.Name()[0] == '.' {
			continue
		}
		post, err := readPostFile(path + "/" + file.Name())
		if err != nil {
			log.Println(err)
			continue
		}
		if post.Ignore {
			continue
		}
		if _, ok := tempPostMap[post.Title]; !ok {
			tempPostMap[post.Title] = post
		} else {
			log.Println(post.Title + " encountered twice")
		}

	}

	postMap = tempPostMap

	return nil
}

// Reads a file and returns a post
func readPostFile(path string) (*Post, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	buf, err := ioutil.ReadAll(f)
	if err != nil {
		return nil, err
	}
	var post Post
	if err := toml.Unmarshal(buf, &post); err != nil {
		return nil, err
	}

	if post.Title == "" {
		return nil, errors.New("No Title specified")
	}
	return &post, nil
}
<commit_msg>fixed image error<commit_after>package posts

import (
	"errors"
	"github.com/go-fsnotify/fsnotify"
	"github.com/naoina/toml"
	"io/ioutil"
	"log"
	"os"
	"sort"
	"strings"
	"time"
)

// Info of a post
type PostInfo struct {
	Title    string
	Ignore   bool
	Subtitle string
	Author   string
	Created  time.Time
	Modified time.Time
}

// Post (PostInfo and the actual text content)
type Post struct {
	PostInfo
	Image   string
	Content []string
}

// Returns just info of a post
func (p Post) Info() PostInfo {
	return p.PostInfo
}

// for sorting
type postSlice []*Post

func (p postSlice) Len() int {
	return len(p)
}

func (p postSlice) Less(i, j int) bool {
	return p[i].Created.After(p[j].Created)
}

func (p postSlice) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

var postMap map[string]*Post

// Gets post by title
func GetPost(id string) (*Post, error) {
	post, ok := postMap[id]
	if !ok {
		return nil, errors.New(id + " not found")
	}
	return post, nil
}

// Returns a list of all the posts sorted by time modified
func GetPostList() []string {
	posts := make(postSlice, len(postMap))
	i := 0
	for _, post := range postMap {
		posts[i] = post
		i += 1
	}

	sort.Stable(posts)

	postTitles := make([]string, len(posts))
	for i, post := range posts {
		postTitles[i] = post.Title
	}

	return postTitles
}

// Reads the folder and parses all the posts
// Also sets up a watcher so the posts will automatically
//   be updated if any of the files change
func Init(path string) error {
	err := readFolder(path)
	if err != nil {
		return err
	}
	doneChan := make(chan bool)
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return err
	}

	go func() {
		<-doneChan
		watcher.Close()
	}()

	err = watcher.Add(path)
	if err != nil {
		return err
	}
	go func() {
		for {
			select {
			case event := <-watcher.Events:
				if strings.HasSuffix(event.Name, ".toml") {
					log.Println("event:", event)
					readFolder(path)
					log.Println("Updated posts")
				}
			case err := <-watcher.Errors:
				log.Println("error:", err)
			}
		}
	}()

	return nil
}

// Reads the folder and parses all the files into posts
func readFolder(path string) error {

	tempPostMap := make(map[string]*Post)
	files, err := ioutil.ReadDir(path)
	if err != nil {
		log.Println("Can't read at %s", path)
		return err
	}

	for _, file := range files {
		if !strings.HasSuffix(file.Name(), ".toml") || file.IsDir() || file.Name()[0] == '.' {
			continue
		}
		post, err := readPostFile(path + "/" + file.Name())
		if err != nil {
			log.Println(err)
			continue
		}
		if post.Ignore {
			continue
		}
		if _, ok := tempPostMap[post.Title]; !ok {
			tempPostMap[post.Title] = post
		} else {
			log.Println(post.Title + " encountered twice")
		}

	}

	postMap = tempPostMap

	return nil
}

// Reads a file and returns a post
func readPostFile(path string) (*Post, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	buf, err := ioutil.ReadAll(f)
	if err != nil {
		return nil, err
	}
	var post Post
	if err := toml.Unmarshal(buf, &post); err != nil {
		return nil, err
	}

	if post.Title == "" {
		return nil, errors.New("No Title specified")
	}
	return &post, nil
}
<|endoftext|>


<commit_before>//
// daemon.go
//
package srnd
import (
  "log"
  "net"
  "net/textproto"
  "strconv"
  "strings"
  "os"
  "time"
)

type NNTPDaemon struct {
  instance_name string
  bind_addr string
  conf *SRNdConfig
  store ArticleStore
  database Database
  mod ModEngine
  expire ExpirationCore
  listener net.Listener
  debug bool
  sync_on_start bool
  // anon settings
  allow_anon bool
  allow_anon_attachments bool

  running bool
  // http frontend
  frontend Frontend

  // map of addr -> NNTPConnection
  feeds map[string]nntpConnection
  // for registering and deregistering outbound feeds
  register_outfeed chan nntpConnection
  deregister_outfeed chan nntpConnection
  // infeed for articles
  infeed chan NNTPMessage
  // channel to load messages to infeed given their message id
  infeed_load chan string
  // channel for broadcasting a message to all feeds given their newsgroup, message_id
  send_all_feeds chan ArticleEntry
  // channel for broadcasting an ARTICLE command to all feeds in reader mode
  ask_for_article chan ArticleEntry
}

func (self NNTPDaemon) End() {
  self.listener.Close()
}

func (self NNTPDaemon) persistFeed(conf FeedConfig, mode string) {
  for {
    if self.running {

      var conn net.Conn
      var err error
      proxy_type := strings.ToLower(conf.proxy_type)

      if proxy_type ==  "" || proxy_type == "none" {
        // connect out without proxy
        log.Println("dial out to ", conf.addr)
        conn, err = net.Dial("tcp", conf.addr)
        if err != nil {
          log.Println("cannot connect to outfeed", conf.addr, err)
					time.Sleep(time.Second * 5)
          continue
        }
      } else if proxy_type == "socks4a" {
        // connect via socks4a
        log.Println("dial out via proxy", conf.proxy_addr)
        conn, err = net.Dial("tcp", conf.proxy_addr)
        if err != nil {
          log.Println("cannot connect to proxy", conf.proxy_addr)
					time.Sleep(time.Second * 5)
          continue
        }
        // generate request
        idx := strings.LastIndex(conf.addr, ":")
        if idx == -1 {
          log.Fatal("invalid outfeed address")
        }
        var port uint64
        addr := conf.addr[:idx]
        port, err = strconv.ParseUint(conf.addr[idx+1:], 10, 16)
        if port >= 25536 {
          log.Fatal("bad proxy port" , port)
        }
        var proxy_port uint16
        proxy_port = uint16(port)
        proxy_ident := "srndv2"
        req_len := len(addr) + 1 + len(proxy_ident) + 1 + 8

        req := make([]byte, req_len)
        // pack request
        req[0] = '\x04'
        req[1] = '\x01'
        req[2] = byte(proxy_port & 0xff00 >> 8)
        req[3] = byte(proxy_port & 0x00ff)
        req[7] = '\x01'
        idx = 8

        proxy_ident_b := []byte(proxy_ident)
        addr_b := []byte(addr)

        var bi int
        for bi = range proxy_ident_b {
          req[idx] = proxy_ident_b[bi]
          idx += 1
        }
        idx += 1
        for bi = range addr_b {
          req[idx] = addr_b[bi]
          idx += 1
        }

        // send request
        conn.Write(req)
        resp := make([]byte, 8)

        // receive response
        conn.Read(resp)
        if resp[1] == '\x5a' {
          // success
          log.Println("connected to", conf.addr)
        } else {
          log.Println("failed to connect to", conf.addr)
					time.Sleep(time.Second * 5)
          continue
        }
      }
      nntp := createNNTPConnection()
      nntp.policy = conf.policy
      nntp.name = conf.addr
      c := textproto.NewConn(conn)
      stream, reader, err := nntp.outboundHandshake(c)
      if err == nil {
        if self.sync_on_start {
          go func() {
            log.Println(nntp.name, "will do full sync")
            for _, article := range self.database.GetAllArticles() {
              if nntp.policy.AllowsNewsgroup(article.Newsgroup()) {
                nntp.check <- article.MessageID()
              }
            }

          }()
        }
        // don't use streaming if we have set mode reader
        ok, err := nntp.modeSwitch(mode, c)
        if ok {
          self.register_outfeed <- nntp
          nntp.runConnection(self, false, stream, reader, c)
        } else {
          log.Println("failed to switch modes", err)
        }
        self.deregister_outfeed <- nntp

      } else {
        log.Println("error doing outbound hanshake", err)
      }
    }
  }
  time.Sleep(1 * time.Second)
}

// run daemon
func (self NNTPDaemon) Run() {

  self.bind_addr = self.conf.daemon["bind"]

  listener , err := net.Listen("tcp", self.bind_addr)
  if err != nil {
    log.Fatal("failed to bind to", self.bind_addr, err)
  }
  self.listener = listener
  log.Printf("SRNd NNTPD bound at %s", listener.Addr())

  self.register_outfeed = make(chan nntpConnection)
  self.deregister_outfeed = make(chan nntpConnection)
  self.infeed = make(chan NNTPMessage, 8)
  self.infeed_load = make(chan string)
  self.send_all_feeds = make(chan ArticleEntry, 64)
  self.feeds = make(map[string]nntpConnection)
  self.ask_for_article = make(chan ArticleEntry, 64)

  self.expire = createExpirationCore(self.database, self.store)
  self.sync_on_start = self.conf.daemon["sync_on_start"] == "1"
  self.debug = self.conf.daemon["log"] == "debug"
  self.instance_name = self.conf.daemon["instance_name"]
  self.allow_anon = self.conf.daemon["allow_anon"] == "1"
  self.allow_anon_attachments = self.conf.daemon["allow_anon_attachments"] == "1"

  if self.debug {
    log.Println("debug mode activated")
  }

  // do we enable the frontend?
  if self.conf.frontend["enable"] == "1" {
    log.Printf("frontend %s enabled", self.conf.frontend["name"])
    http_frontend := NewHTTPFrontend(&self, self.conf.frontend, self.conf.worker["url"])
    nntp_frontend := NewNNTPFrontend(&self, self.conf.frontend["nntp"])
    self.frontend = MuxFrontends(http_frontend, nntp_frontend)
    go self.frontend.Mainloop()
  }

  // set up admin user if it's specified in the config
  pubkey , ok := self.conf.frontend["admin_key"]
  if ok {
    // TODO: check for valid format
    log.Println("add admin key", pubkey)
    err = self.database.MarkModPubkeyGlobal(pubkey)
    if err != nil {
      log.Printf("failed to add admin mod key, %s", err)
    }
  }

  defer self.listener.Close()
  // run expiration mainloop
  go self.expire.Mainloop()
  // we are now running
  self.running = true

  // persist outfeeds
  for idx := range self.conf.feeds {
    go self.persistFeed(self.conf.feeds[idx], "reader")
    go self.persistFeed(self.conf.feeds[idx], "stream")
  }

  // start accepting incoming connections
  go self.acceptloop()

  go func () {
    // if we have no initial posts create one
    if self.database.ArticleCount() == 0 {
      nntp := newPlaintextArticle("welcome to nntpchan, this post was inserted on startup automatically", "system@"+self.instance_name, "Welcome to NNTPChan", "system", self.instance_name, genMessageID(self.instance_name), "overchan.test")
      nntp.Pack()
      file := self.store.CreateTempFile(nntp.MessageID())
      if file != nil {
        err := self.store.WriteMessage(nntp, file)
        file.Close()
        if err == nil {
          self.infeed <- nntp
        } else {
          log.Println("failed to create startup messge?", err)
        }
      }
    }
  }()

  // get all pending articles from infeed and load them
  go func() {
    f, err := os.Open(self.store.TempDir())
    if err == nil {
      names, err := f.Readdirnames(0)
      if err == nil {
        for _, name := range names {
          self.infeed_load <- name
        }
      }
    }

  }()

  // if we have no frontend this does nothing
  if self.frontend != nil {
    go self.pollfrontend()
  }
  go self.pollinfeed()
  go self.pollmessages()
  self.polloutfeeds()
}

func (self NNTPDaemon) pollfrontend() {
  chnl := self.frontend.NewPostsChan()
  for {
    nntp := <- chnl
    // new post from frontend
    log.Println("frontend post", nntp.MessageID())
    self.infeed <- nntp
  }
}
func (self NNTPDaemon) pollinfeed() {
  for {
    msgid := <- self.infeed_load
    log.Println("load from infeed", msgid)
    msg := self.store.ReadTempMessage(msgid)
    if msg != nil {
      self.infeed <- msg
    }
  }
}

func (self NNTPDaemon) polloutfeeds() {

  for {
    select {

    case outfeed := <- self.register_outfeed:
      log.Println("outfeed", outfeed.name, "registered")
      self.feeds[outfeed.name] = outfeed
    case outfeed := <- self.deregister_outfeed:
      log.Println("outfeed", outfeed.name, "de-registered")
      delete(self.feeds, outfeed.name)
    case nntp := <- self.send_all_feeds:
      log.Println("federate", nntp.MessageID())
      for _, feed := range self.feeds {
        log.Println("mode", feed.mode)
        if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) && feed.mode == "STREAM" {
          feed.check <- nntp.MessageID()
        }
      }
    case nntp := <- self.ask_for_article:
      for _, feed := range self.feeds {
        if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) && feed.mode == "READER" {
          log.Println("asking", feed.name, "for", nntp.MessageID())
          feed.article <- nntp.MessageID()
        }
      }
    }
  }
}

func (self NNTPDaemon) pollmessages() {
  var chnl chan NNTPMessage
  modchnl := self.mod.MessageChan()
  if self.frontend != nil {
    chnl = self.frontend.PostsChan()
  }
  for {

    nntp := <- self.infeed
    // ammend path
    nntp.AppendPath(self.instance_name)
    msgid := nntp.MessageID()
    log.Println("daemon got", msgid)

    // store article and attachments
    // register with database
    // this also generates thumbnails
    self.store.StorePost(nntp)

    ref := nntp.Reference()
    if ref != "" && ValidMessageID(ref) && ! self.database.HasArticleLocal(ref) {
      // we don't have the root post
      // generate it
      //log.Println("creating temp root post for", ref , "in", nntp.Newsgroup())
      //root := newPlaintextArticle("temporary placeholder", "lol@lol", "root post "+ref+" not found", "system", "temp", ref, nntp.Newsgroup())
      //self.store.StorePost(root)
    }

    // prepare for content rollover
    // fallback rollover
    rollover := 100

    group := nntp.Newsgroup()
    tpp, err := self.database.GetThreadsPerPage(group)
    ppb, err := self.database.GetPagesPerBoard(group)
    if err == nil {
      rollover = tpp * ppb
    }

    // roll over old content
    self.expire.ExpireGroup(group, rollover)
    // handle mod events
    if group == "ctl" {
      modchnl <- nntp
    }

    // queue to all outfeeds
    // XXX: blocking ?
    self.send_all_feeds <- ArticleEntry{msgid, group}
    // tell frontend
    // XXX: blocking ?
    if chnl != nil {
      if self.frontend.AllowNewsgroup(group) {
        chnl <- nntp
      } else {
        log.Println("frontend does not allow", group, "not sending")
      }
    }
  }
}

func (self NNTPDaemon) acceptloop() {
  for {
    // accept
    conn, err := self.listener.Accept()
    if err != nil {
      log.Fatal(err)
    }
    // make a new inbound nntp connection handler
    nntp := createNNTPConnection()
    c := textproto.NewConn(conn)
    // send banners and shit
    err = nntp.inboundHandshake(c)
    if err == nil {
      // run, we support stream and reader
      go nntp.runConnection(self, true, true, true, c)
    } else {
      log.Println("failed to send banners", err)
      c.Close()
    }
  }
}

func (self NNTPDaemon) Setup() NNTPDaemon {
  log.Println("checking for configs...")
  // check that are configs exist
  CheckConfig()
  log.Println("loading config...")
  // read the config
  self.conf = ReadConfig()
  if self.conf == nil {
    log.Fatal("failed to load config")
  }
  // validate the config
  log.Println("validating configs...")
  self.conf.Validate()
  log.Println("configs are valid")

  db_host := self.conf.database["host"]
  db_port := self.conf.database["port"]
  db_user := self.conf.database["user"]
  db_passwd := self.conf.database["password"]

  // set up database stuff
  log.Println("connecting to database...")
  self.database = NewDatabase(self.conf.database["type"], self.conf.database["schema"], db_host, db_port, db_user, db_passwd)
  log.Println("ensure that the database is created...")
  self.database.CreateTables()

  // set up store
  log.Println("set up article store...")
  self.store = createArticleStore(self.conf.store, self.database)

  self.mod = modEngine{
    store: self.store,
    database:  self.database,
    chnl: make(chan NNTPMessage),
  }
  return self
}
<commit_msg>remove logging<commit_after>//
// daemon.go
//
package srnd
import (
  "log"
  "net"
  "net/textproto"
  "strconv"
  "strings"
  "os"
  "time"
)

type NNTPDaemon struct {
  instance_name string
  bind_addr string
  conf *SRNdConfig
  store ArticleStore
  database Database
  mod ModEngine
  expire ExpirationCore
  listener net.Listener
  debug bool
  sync_on_start bool
  // anon settings
  allow_anon bool
  allow_anon_attachments bool

  running bool
  // http frontend
  frontend Frontend

  // map of addr -> NNTPConnection
  feeds map[string]nntpConnection
  // for registering and deregistering outbound feeds
  register_outfeed chan nntpConnection
  deregister_outfeed chan nntpConnection
  // infeed for articles
  infeed chan NNTPMessage
  // channel to load messages to infeed given their message id
  infeed_load chan string
  // channel for broadcasting a message to all feeds given their newsgroup, message_id
  send_all_feeds chan ArticleEntry
  // channel for broadcasting an ARTICLE command to all feeds in reader mode
  ask_for_article chan ArticleEntry
}

func (self NNTPDaemon) End() {
  self.listener.Close()
}

func (self NNTPDaemon) persistFeed(conf FeedConfig, mode string) {
  for {
    if self.running {

      var conn net.Conn
      var err error
      proxy_type := strings.ToLower(conf.proxy_type)

      if proxy_type ==  "" || proxy_type == "none" {
        // connect out without proxy
        log.Println("dial out to ", conf.addr)
        conn, err = net.Dial("tcp", conf.addr)
        if err != nil {
          log.Println("cannot connect to outfeed", conf.addr, err)
					time.Sleep(time.Second * 5)
          continue
        }
      } else if proxy_type == "socks4a" {
        // connect via socks4a
        log.Println("dial out via proxy", conf.proxy_addr)
        conn, err = net.Dial("tcp", conf.proxy_addr)
        if err != nil {
          log.Println("cannot connect to proxy", conf.proxy_addr)
					time.Sleep(time.Second * 5)
          continue
        }
        // generate request
        idx := strings.LastIndex(conf.addr, ":")
        if idx == -1 {
          log.Fatal("invalid outfeed address")
        }
        var port uint64
        addr := conf.addr[:idx]
        port, err = strconv.ParseUint(conf.addr[idx+1:], 10, 16)
        if port >= 25536 {
          log.Fatal("bad proxy port" , port)
        }
        var proxy_port uint16
        proxy_port = uint16(port)
        proxy_ident := "srndv2"
        req_len := len(addr) + 1 + len(proxy_ident) + 1 + 8

        req := make([]byte, req_len)
        // pack request
        req[0] = '\x04'
        req[1] = '\x01'
        req[2] = byte(proxy_port & 0xff00 >> 8)
        req[3] = byte(proxy_port & 0x00ff)
        req[7] = '\x01'
        idx = 8

        proxy_ident_b := []byte(proxy_ident)
        addr_b := []byte(addr)

        var bi int
        for bi = range proxy_ident_b {
          req[idx] = proxy_ident_b[bi]
          idx += 1
        }
        idx += 1
        for bi = range addr_b {
          req[idx] = addr_b[bi]
          idx += 1
        }

        // send request
        conn.Write(req)
        resp := make([]byte, 8)

        // receive response
        conn.Read(resp)
        if resp[1] == '\x5a' {
          // success
          log.Println("connected to", conf.addr)
        } else {
          log.Println("failed to connect to", conf.addr)
					time.Sleep(time.Second * 5)
          continue
        }
      }
      nntp := createNNTPConnection()
      nntp.policy = conf.policy
      nntp.name = conf.addr
      c := textproto.NewConn(conn)
      stream, reader, err := nntp.outboundHandshake(c)
      if err == nil {
        if self.sync_on_start {
          go func() {
            log.Println(nntp.name, "will do full sync")
            for _, article := range self.database.GetAllArticles() {
              if nntp.policy.AllowsNewsgroup(article.Newsgroup()) {
                nntp.check <- article.MessageID()
              }
            }

          }()
        }
        // don't use streaming if we have set mode reader
        ok, err := nntp.modeSwitch(mode, c)
        if ok {
          self.register_outfeed <- nntp
          nntp.runConnection(self, false, stream, reader, c)
        } else {
          log.Println("failed to switch modes", err)
        }
        self.deregister_outfeed <- nntp

      } else {
        log.Println("error doing outbound hanshake", err)
      }
    }
  }
  time.Sleep(1 * time.Second)
}

// run daemon
func (self NNTPDaemon) Run() {

  self.bind_addr = self.conf.daemon["bind"]

  listener , err := net.Listen("tcp", self.bind_addr)
  if err != nil {
    log.Fatal("failed to bind to", self.bind_addr, err)
  }
  self.listener = listener
  log.Printf("SRNd NNTPD bound at %s", listener.Addr())

  self.register_outfeed = make(chan nntpConnection)
  self.deregister_outfeed = make(chan nntpConnection)
  self.infeed = make(chan NNTPMessage, 8)
  self.infeed_load = make(chan string)
  self.send_all_feeds = make(chan ArticleEntry, 64)
  self.feeds = make(map[string]nntpConnection)
  self.ask_for_article = make(chan ArticleEntry, 64)

  self.expire = createExpirationCore(self.database, self.store)
  self.sync_on_start = self.conf.daemon["sync_on_start"] == "1"
  self.debug = self.conf.daemon["log"] == "debug"
  self.instance_name = self.conf.daemon["instance_name"]
  self.allow_anon = self.conf.daemon["allow_anon"] == "1"
  self.allow_anon_attachments = self.conf.daemon["allow_anon_attachments"] == "1"

  if self.debug {
    log.Println("debug mode activated")
  }

  // do we enable the frontend?
  if self.conf.frontend["enable"] == "1" {
    log.Printf("frontend %s enabled", self.conf.frontend["name"])
    http_frontend := NewHTTPFrontend(&self, self.conf.frontend, self.conf.worker["url"])
    nntp_frontend := NewNNTPFrontend(&self, self.conf.frontend["nntp"])
    self.frontend = MuxFrontends(http_frontend, nntp_frontend)
    go self.frontend.Mainloop()
  }

  // set up admin user if it's specified in the config
  pubkey , ok := self.conf.frontend["admin_key"]
  if ok {
    // TODO: check for valid format
    log.Println("add admin key", pubkey)
    err = self.database.MarkModPubkeyGlobal(pubkey)
    if err != nil {
      log.Printf("failed to add admin mod key, %s", err)
    }
  }

  defer self.listener.Close()
  // run expiration mainloop
  go self.expire.Mainloop()
  // we are now running
  self.running = true

  // persist outfeeds
  for idx := range self.conf.feeds {
    go self.persistFeed(self.conf.feeds[idx], "reader")
    go self.persistFeed(self.conf.feeds[idx], "stream")
  }

  // start accepting incoming connections
  go self.acceptloop()

  go func () {
    // if we have no initial posts create one
    if self.database.ArticleCount() == 0 {
      nntp := newPlaintextArticle("welcome to nntpchan, this post was inserted on startup automatically", "system@"+self.instance_name, "Welcome to NNTPChan", "system", self.instance_name, genMessageID(self.instance_name), "overchan.test")
      nntp.Pack()
      file := self.store.CreateTempFile(nntp.MessageID())
      if file != nil {
        err := self.store.WriteMessage(nntp, file)
        file.Close()
        if err == nil {
          self.infeed <- nntp
        } else {
          log.Println("failed to create startup messge?", err)
        }
      }
    }
  }()

  // get all pending articles from infeed and load them
  go func() {
    f, err := os.Open(self.store.TempDir())
    if err == nil {
      names, err := f.Readdirnames(0)
      if err == nil {
        for _, name := range names {
          self.infeed_load <- name
        }
      }
    }

  }()

  // if we have no frontend this does nothing
  if self.frontend != nil {
    go self.pollfrontend()
  }
  go self.pollinfeed()
  go self.pollmessages()
  self.polloutfeeds()
}

func (self NNTPDaemon) pollfrontend() {
  chnl := self.frontend.NewPostsChan()
  for {
    nntp := <- chnl
    // new post from frontend
    log.Println("frontend post", nntp.MessageID())
    self.infeed <- nntp
  }
}
func (self NNTPDaemon) pollinfeed() {
  for {
    msgid := <- self.infeed_load
    log.Println("load from infeed", msgid)
    msg := self.store.ReadTempMessage(msgid)
    if msg != nil {
      self.infeed <- msg
    }
  }
}

func (self NNTPDaemon) polloutfeeds() {

  for {
    select {

    case outfeed := <- self.register_outfeed:
      log.Println("outfeed", outfeed.name, "registered")
      self.feeds[outfeed.name] = outfeed
    case outfeed := <- self.deregister_outfeed:
      log.Println("outfeed", outfeed.name, "de-registered")
      delete(self.feeds, outfeed.name)
    case nntp := <- self.send_all_feeds:
      log.Println("federate", nntp.MessageID())
      for _, feed := range self.feeds {
        if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) && feed.mode == "STREAM" {
          feed.check <- nntp.MessageID()
        }
      }
    case nntp := <- self.ask_for_article:
      for _, feed := range self.feeds {
        if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) && feed.mode == "READER" {
          log.Println("asking", feed.name, "for", nntp.MessageID())
          feed.article <- nntp.MessageID()
        }
      }
    }
  }
}

func (self NNTPDaemon) pollmessages() {
  var chnl chan NNTPMessage
  modchnl := self.mod.MessageChan()
  if self.frontend != nil {
    chnl = self.frontend.PostsChan()
  }
  for {

    nntp := <- self.infeed
    // ammend path
    nntp.AppendPath(self.instance_name)
    msgid := nntp.MessageID()
    log.Println("daemon got", msgid)

    // store article and attachments
    // register with database
    // this also generates thumbnails
    self.store.StorePost(nntp)

    ref := nntp.Reference()
    if ref != "" && ValidMessageID(ref) && ! self.database.HasArticleLocal(ref) {
      // we don't have the root post
      // generate it
      //log.Println("creating temp root post for", ref , "in", nntp.Newsgroup())
      //root := newPlaintextArticle("temporary placeholder", "lol@lol", "root post "+ref+" not found", "system", "temp", ref, nntp.Newsgroup())
      //self.store.StorePost(root)
    }

    // prepare for content rollover
    // fallback rollover
    rollover := 100

    group := nntp.Newsgroup()
    tpp, err := self.database.GetThreadsPerPage(group)
    ppb, err := self.database.GetPagesPerBoard(group)
    if err == nil {
      rollover = tpp * ppb
    }

    // roll over old content
    self.expire.ExpireGroup(group, rollover)
    // handle mod events
    if group == "ctl" {
      modchnl <- nntp
    }

    // queue to all outfeeds
    // XXX: blocking ?
    self.send_all_feeds <- ArticleEntry{msgid, group}
    // tell frontend
    // XXX: blocking ?
    if chnl != nil {
      if self.frontend.AllowNewsgroup(group) {
        chnl <- nntp
      } else {
        log.Println("frontend does not allow", group, "not sending")
      }
    }
  }
}

func (self NNTPDaemon) acceptloop() {
  for {
    // accept
    conn, err := self.listener.Accept()
    if err != nil {
      log.Fatal(err)
    }
    // make a new inbound nntp connection handler
    nntp := createNNTPConnection()
    c := textproto.NewConn(conn)
    // send banners and shit
    err = nntp.inboundHandshake(c)
    if err == nil {
      // run, we support stream and reader
      go nntp.runConnection(self, true, true, true, c)
    } else {
      log.Println("failed to send banners", err)
      c.Close()
    }
  }
}

func (self NNTPDaemon) Setup() NNTPDaemon {
  log.Println("checking for configs...")
  // check that are configs exist
  CheckConfig()
  log.Println("loading config...")
  // read the config
  self.conf = ReadConfig()
  if self.conf == nil {
    log.Fatal("failed to load config")
  }
  // validate the config
  log.Println("validating configs...")
  self.conf.Validate()
  log.Println("configs are valid")

  db_host := self.conf.database["host"]
  db_port := self.conf.database["port"]
  db_user := self.conf.database["user"]
  db_passwd := self.conf.database["password"]

  // set up database stuff
  log.Println("connecting to database...")
  self.database = NewDatabase(self.conf.database["type"], self.conf.database["schema"], db_host, db_port, db_user, db_passwd)
  log.Println("ensure that the database is created...")
  self.database.CreateTables()

  // set up store
  log.Println("set up article store...")
  self.store = createArticleStore(self.conf.store, self.database)

  self.mod = modEngine{
    store: self.store,
    database:  self.database,
    chnl: make(chan NNTPMessage),
  }
  return self
}
<|endoftext|>