<commit_before>package models

import (
	"fmt"
	"regexp"
	"time"

	"github.com/gocql/gocql"
	"github.com/skeswa/gophr/common/errors"
)

// Constants directly related to interacting with the package model in the
// cassandra database.
const (
	// TableNamePackages is the name of the table containing the package model.
	TableNamePackages = "gophr.packages"
	// IndexNamePackages is the name of the lucene index
	IndexNamePackages             = "packages_index"
	ColumnNamePackagesRepo        = "repo"
	ColumnNamePackagesStars       = "stars"
	ColumnNamePackagesExists      = "exists"
	ColumnNamePackagesAuthor      = "author"
	ColumnNamePackagesVersions    = "versions"
	ColumnNamePackagesGodocURL    = "godoc_url"
	ColumnNamePackagesIndexTime   = "index_time"
	ColumnNamePackagesAwesomeGo   = "awesome_go"
	ColumnNamePackagesSearchBlob  = "search_blob"
	ColumnNamePackagesDescription = "description"
)

const (
	packagesSearchBlobTemplate = "%s %s %s"
)

var (
	cqlQueryFuzzySearchPackagesTemplate = fmt.Sprintf(
		`SELECT %s,%s,%s FROM %s WHERE expr(%s,'{query:{type:"fuzzy",field:"%s",value:"%s"}}') LIMIT 10`,
		ColumnNamePackagesRepo,
		ColumnNamePackagesAuthor,
		ColumnNamePackagesDescription,
		TableNamePackages,
		IndexNamePackages,
		ColumnNamePackagesSearchBlob,
		"%s",
	)

	cqlQuerySelectPackageVersions = fmt.Sprintf(
		`SELECT %s FROM %s WHERE %s = ? AND %s = ? LIMIT 1`,
		ColumnNamePackagesVersions,
		TableNamePackages,
		ColumnNamePackagesAuthor,
		ColumnNamePackagesRepo,
	)

	cqlQueryInsertPackage = fmt.Sprintf(
		`INSERT INTO %s (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) VALUES (?,?,?,?,?,?,?,?,?,?)`,
		TableNamePackages,
		ColumnNamePackagesRepo,
		ColumnNamePackagesStars,
		ColumnNamePackagesExists,
		ColumnNamePackagesAuthor,
		ColumnNamePackagesVersions,
		ColumnNamePackagesGodocURL,
		ColumnNamePackagesIndexTime,
		ColumnNamePackagesAwesomeGo,
		ColumnNamePackagesSearchBlob,
		ColumnNamePackagesDescription,
	)

	cqlQueryDeletePackage = fmt.Sprintf(
		`DELETE FROM %s WHERE %s = ? AND %s = ?`,
		TableNamePackages,
		ColumnNamePackagesAuthor,
		ColumnNamePackagesRepo,
	)
)

var (
	alphanumericFilterRegex = regexp.MustCompile(`[^\sa-zA-Z0-9\-_]+`)
)

// PackageModel is a struct representing one individual package in the database.
type PackageModel struct {
	Repo        *string
	Stars       *int
	Exists      *bool
	Author      *string
	Versions    []string
	GodocURL    *string
	IndexTime   *time.Time
	AwesomeGo   *bool
	SearchBlob  *string
	Description *string
}

// NewPackageModelForInsert creates an instance of PackageModel that is
// optimized and validated for the insert operation in the database.
func NewPackageModelForInsert(
	author string,
	exists bool,
	repo string,
	versions []string,
	godocURL string,
	indexTime time.Time,
	awesomeGo bool,
	description string,
	stars int,
) (*PackageModel, error) {
	if len(repo) < 1 {
		return nil, errors.NewInvalidParameterError("repo", repo)
	}
	if len(author) < 1 {
		return nil, errors.NewInvalidParameterError("author", author)
	}
	if len(godocURL) < 1 {
		return nil, errors.NewInvalidParameterError("godocURL", godocURL)
	}

	searchBlob := fmt.Sprintf(
		packagesSearchBlobTemplate,
		author,
		repo,
		description,
	)

	return &PackageModel{
		Repo:        &repo,
		Stars:       &stars,
		Exists:      &exists,
		Author:      &author,
		Versions:    versions,
		GodocURL:    &godocURL,
		IndexTime:   &indexTime,
		AwesomeGo:   &awesomeGo,
		SearchBlob:  &searchBlob,
		Description: &description,
	}, nil
}

// NewPackageModelFromBulkSelect creates an instance of PackageModel that is
// optimized and validated for a select operation designed to get data about
// multiple packages from the database.
func NewPackageModelFromBulkSelect(
	author string,
	repo string,
	description string,
) (*PackageModel, error) {
	if len(repo) < 1 {
		return nil, errors.NewInvalidParameterError("repo", repo)
	}
	if len(author) < 1 {
		return nil, errors.NewInvalidParameterError("author", author)
	}

	return &PackageModel{
		Repo:        &repo,
		Author:      &author,
		Description: &description,
	}, nil
}

// TODO(Shikkic): get your shit together.
func NewPackageModelTest(
	author string,
	repo string,
	awesome_go bool,
	description string,
	exists bool,
	godoc_url string,
	index_time time.Time,
	search_blob string,
	versions []string,
	stars int,
) *PackageModel {
	return &PackageModel{
		Repo:        &repo,
		Stars:       &stars,
		Exists:      &exists,
		Author:      &author,
		Versions:    versions,
		GodocURL:    &godoc_url,
		IndexTime:   &index_time,
		AwesomeGo:   &awesome_go,
		SearchBlob:  &search_blob,
		Description: &description,
	}
}

// NewPackageModelFromSingleSelect creates an instance of PackageModel that is
// optimized and validated for a select operation designed to get data about
// a single package from the database.
func NewPackageModelFromSingleSelect(
	author string,
	exists bool,
	repo string,
	versions []string,
	godocURL string,
	awesomeGo bool,
	description string,
) (*PackageModel, error) {
	if len(repo) < 1 {
		return nil, errors.NewInvalidParameterError("repo", repo)
	}
	if len(author) < 1 {
		return nil, errors.NewInvalidParameterError("author", author)
	}
	if len(godocURL) < 1 {
		return nil, errors.NewInvalidParameterError("godocURL", godocURL)
	}

	return &PackageModel{
		Repo:        &repo,
		Exists:      &exists,
		Author:      &author,
		Versions:    versions,
		GodocURL:    &godocURL,
		AwesomeGo:   &awesomeGo,
		Description: &description,
	}, nil
}

// FindPackageVersions gets the versions of a package from the database. If
// no such package exists, or there were no versions for said package, then nil
// is returned.
func FindPackageVersions(session *gocql.Session, author string, repo string) ([]string, error) {
	var (
		err      error
		versions []string
	)

	iter := session.Query(cqlQuerySelectPackageVersions, author, repo).Iter()

	if !iter.Scan(&versions) {
		return nil, nil
	}

	if err = iter.Close(); err != nil {
		return nil, errors.NewQueryScanError(nil, err)
	}

	return versions, nil
}

// FuzzySearchPackages finds a list of packages relevant to a query phrase
// string. The search takes author, package and description into account.
func FuzzySearchPackages(
	session *gocql.Session,
	searchText string,
) ([]*PackageModel, error) {
	// First, remove all non-essential characters
	searchText = alphanumericFilterRegex.ReplaceAllString(searchText, "")
	// Next put the search text into a query string
	query := fmt.Sprintf(cqlQueryFuzzySearchPackagesTemplate, searchText)
	// Return the processed results of the query
	return scanPackageModels(session.Query(query))
}

// InsertPackage inserts an individual package into the database.
func InsertPackage(
	session *gocql.Session,
	packageModel *PackageModel,
) error {
	err := session.Query(cqlQueryInsertPackage,
		*packageModel.Repo,
		*packageModel.Stars,
		*packageModel.Exists,
		*packageModel.Author,
		packageModel.Versions,
		*packageModel.GodocURL,
		*packageModel.IndexTime,
		*packageModel.AwesomeGo,
		*packageModel.SearchBlob,
		*packageModel.Description,
	).Exec()

	return err
}

// InsertPackages inserts a slice of package models into the database.
func InsertPackages(
	session *gocql.Session,
	packageModels []*PackageModel,
) error {
	batch := gocql.NewBatch(gocql.LoggedBatch)

	if packageModels == nil || len(packageModels) == 0 {
		return errors.NewInvalidParameterError("packageModels", packageModels)
	}

	for _, packageModel := range packageModels {
		if packageModel != nil &&
			packageModel.Repo != nil &&
			packageModel.Exists != nil &&
			packageModel.Author != nil &&
			packageModel.GodocURL != nil &&
			packageModel.IndexTime != nil &&
			packageModel.AwesomeGo != nil &&
			packageModel.SearchBlob != nil &&
			packageModel.Description != nil {
			batch.Query(
				cqlQueryInsertPackage,
				*packageModel.Repo,
				*packageModel.Exists,
				*packageModel.Author,
				packageModel.Versions,
				*packageModel.GodocURL,
				*packageModel.IndexTime,
				*packageModel.AwesomeGo,
				*packageModel.SearchBlob,
				*packageModel.Description,
			)
		} else {
			return errors.NewInvalidParameterError(
				"packageModels",
				fmt.Sprintf("[ ..., %v, ... ]", packageModel),
			)
		}
	}

	err := session.ExecuteBatch(batch)
	if err != nil {
		return err
	}

	return nil
}

/********************************** HELPERS ***********************************/

// TODO(skeswa): implement this for querying single packages
func scanPackageModel(query *gocql.Query) ([]*PackageModel, error) {
	return nil, nil
}

func scanPackageModels(query *gocql.Query) ([]*PackageModel, error) {
	var (
		err          error
		scanError    error
		closeError   error
		packageModel *PackageModel

		repo        string
		author      string
		description string

		iter          = query.Iter()
		packageModels = make([]*PackageModel, 0)
	)

	for iter.Scan(&repo, &author, &description) {
		packageModel, err = NewPackageModelFromBulkSelect(author, repo, description)
		if err != nil {
			scanError = err
			break
		} else {
			packageModels = append(packageModels, packageModel)
		}
	}

	if err = iter.Close(); err != nil {
		closeError = err
	}

	if scanError != nil || closeError != nil {
		return nil, errors.NewQueryScanError(scanError, closeError)
	}

	return packageModels, nil
}

func ScanAllPackageModels(session *gocql.Session) ([]*PackageModel, error) {
	var (
		err          error
		scanError    error
		closeError   error
		packageModel *PackageModel

		author      string
		repo        string
		awesome_go  bool
		description string
		exists      bool
		godoc_url   string
		index_time  time.Time
		search_blob string
		versions    []string
		stars       int

		query = session.Query(`SELECT
			author,
			repo,
			awesome_go,
			description,
			exists,
			godoc_url,
			index_time,
			search_blob,
			versions,
			stars
			FROM gophr.packages`)
		iter          = query.Iter()
		packageModels = make([]*PackageModel, 0)
	)

	for iter.Scan(&author, &repo, &awesome_go, &description, &exists, &godoc_url, &index_time, &search_blob, &versions, &stars) {
		packageModel = NewPackageModelTest(author, repo, awesome_go, description, exists, godoc_url, index_time, search_blob, versions, stars)
		packageModels = append(packageModels, packageModel)
	}

	if err = iter.Close(); err != nil {
		closeError = err
	}

	if scanError != nil || closeError != nil {
		return nil, errors.NewQueryScanError(scanError, closeError)
	}

	return packageModels, nil
}

func DeletePackageModel(session *gocql.Session, packageModel *PackageModel) error {
	author := *packageModel.Author
	repo := *packageModel.Repo
	query := session.Query(cqlQueryDeletePackage, author, repo)
	err := query.Exec()

	return err
}
<commit_msg>Update package.go<commit_after>package models

import (
	"fmt"
	"regexp"
	"time"

	"github.com/gocql/gocql"
	"github.com/skeswa/gophr/common/errors"
)

// Constants directly related to interacting with the package model in the
// cassandra database.
const (
	// TableNamePackages is the name of the table containing the package model.
	TableNamePackages = "gophr.packages"
	// IndexNamePackages is the name of the lucene index
	IndexNamePackages             = "packages_index"
	ColumnNamePackagesRepo        = "repo"
	ColumnNamePackagesStars       = "stars"
	ColumnNamePackagesExists      = "exists"
	ColumnNamePackagesAuthor      = "author"
	ColumnNamePackagesVersions    = "versions"
	ColumnNamePackagesGodocURL    = "godoc_url"
	ColumnNamePackagesIndexTime   = "index_time"
	ColumnNamePackagesAwesomeGo   = "awesome_go"
	ColumnNamePackagesSearchBlob  = "search_blob"
	ColumnNamePackagesDescription = "description"
)

const (
	packagesSearchBlobTemplate = "%s %s %s"
)

var (
	cqlQueryFuzzySearchPackagesTemplate = fmt.Sprintf(
		`SELECT %s,%s,%s FROM %s WHERE expr(%s,'{query:{type:"fuzzy",field:"%s",value:"%s"}}') LIMIT 10`,
		ColumnNamePackagesRepo,
		ColumnNamePackagesAuthor,
		ColumnNamePackagesDescription,
		TableNamePackages,
		IndexNamePackages,
		ColumnNamePackagesSearchBlob,
		"%s",
	)

	cqlQuerySelectPackageVersions = fmt.Sprintf(
		`SELECT %s FROM %s WHERE %s = ? AND %s = ? LIMIT 1`,
		ColumnNamePackagesVersions,
		TableNamePackages,
		ColumnNamePackagesAuthor,
		ColumnNamePackagesRepo,
	)

	cqlQueryInsertPackage = fmt.Sprintf(
		`INSERT INTO %s (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) VALUES (?,?,?,?,?,?,?,?,?,?)`,
		TableNamePackages,
		ColumnNamePackagesRepo,
		ColumnNamePackagesStars,
		ColumnNamePackagesExists,
		ColumnNamePackagesAuthor,
		ColumnNamePackagesVersions,
		ColumnNamePackagesGodocURL,
		ColumnNamePackagesIndexTime,
		ColumnNamePackagesAwesomeGo,
		ColumnNamePackagesSearchBlob,
		ColumnNamePackagesDescription,
	)

	cqlQueryDeletePackage = fmt.Sprintf(
		`DELETE FROM %s WHERE %s = ? AND %s = ?`,
		TableNamePackages,
		ColumnNamePackagesAuthor,
		ColumnNamePackagesRepo,
	)
)

var (
	alphanumericFilterRegex = regexp.MustCompile(`[^\sa-zA-Z0-9\-_]+`)
)

// PackageModel is a struct representing one individual package in the database.
type PackageModel struct {
	Repo        *string
	Stars       *int
	Exists      *bool
	Author      *string
	Versions    []string
	GodocURL    *string
	IndexTime   *time.Time
	AwesomeGo   *bool
	SearchBlob  *string
	Description *string
}

// NewPackageModelForInsert creates an instance of PackageModel that is
// optimized and validated for the insert operation in the database.
func NewPackageModelForInsert(
	author string,
	exists bool,
	repo string,
	versions []string,
	godocURL string,
	indexTime time.Time,
	awesomeGo bool,
	description string,
	stars int,
) (*PackageModel, error) {
	if len(repo) < 1 {
		return nil, errors.NewInvalidParameterError("repo", repo)
	}
	if len(author) < 1 {
		return nil, errors.NewInvalidParameterError("author", author)
	}
	if len(godocURL) < 1 {
		return nil, errors.NewInvalidParameterError("godocURL", godocURL)
	}

	searchBlob := fmt.Sprintf(
		packagesSearchBlobTemplate,
		author,
		repo,
		description,
	)

	return &PackageModel{
		Repo:        &repo,
		Stars:       &stars,
		Exists:      &exists,
		Author:      &author,
		Versions:    versions,
		GodocURL:    &godocURL,
		IndexTime:   &indexTime,
		AwesomeGo:   &awesomeGo,
		SearchBlob:  &searchBlob,
		Description: &description,
	}, nil
}

// NewPackageModelFromBulkSelect creates an instance of PackageModel that is
// optimized and validated for a select operation designed to get data about
// multiple packages from the database.
func NewPackageModelFromBulkSelect(
	author string,
	repo string,
	description string,
) (*PackageModel, error) {
	if len(repo) < 1 {
		return nil, errors.NewInvalidParameterError("repo", repo)
	}
	if len(author) < 1 {
		return nil, errors.NewInvalidParameterError("author", author)
	}

	return &PackageModel{
		Repo:        &repo,
		Author:      &author,
		Description: &description,
	}, nil
}

// TODO(Shikkic): Fix func name and attribute naming scheme
func NewPackageModelTest(
	author string,
	repo string,
	awesome_go bool,
	description string,
	exists bool,
	godoc_url string,
	index_time time.Time,
	search_blob string,
	versions []string,
	stars int,
) *PackageModel {
	return &PackageModel{
		Repo:        &repo,
		Stars:       &stars,
		Exists:      &exists,
		Author:      &author,
		Versions:    versions,
		GodocURL:    &godoc_url,
		IndexTime:   &index_time,
		AwesomeGo:   &awesome_go,
		SearchBlob:  &search_blob,
		Description: &description,
	}
}

// NewPackageModelFromSingleSelect creates an instance of PackageModel that is
// optimized and validated for a select operation designed to get data about
// a single package from the database.
func NewPackageModelFromSingleSelect(
	author string,
	exists bool,
	repo string,
	versions []string,
	godocURL string,
	awesomeGo bool,
	description string,
) (*PackageModel, error) {
	if len(repo) < 1 {
		return nil, errors.NewInvalidParameterError("repo", repo)
	}
	if len(author) < 1 {
		return nil, errors.NewInvalidParameterError("author", author)
	}
	if len(godocURL) < 1 {
		return nil, errors.NewInvalidParameterError("godocURL", godocURL)
	}

	return &PackageModel{
		Repo:        &repo,
		Exists:      &exists,
		Author:      &author,
		Versions:    versions,
		GodocURL:    &godocURL,
		AwesomeGo:   &awesomeGo,
		Description: &description,
	}, nil
}

// FindPackageVersions gets the versions of a package from the database. If
// no such package exists, or there were no versions for said package, then nil
// is returned.
func FindPackageVersions(session *gocql.Session, author string, repo string) ([]string, error) {
	var (
		err      error
		versions []string
	)

	iter := session.Query(cqlQuerySelectPackageVersions, author, repo).Iter()

	if !iter.Scan(&versions) {
		return nil, nil
	}

	if err = iter.Close(); err != nil {
		return nil, errors.NewQueryScanError(nil, err)
	}

	return versions, nil
}

// FuzzySearchPackages finds a list of packages relevant to a query phrase
// string. The search takes author, package and description into account.
func FuzzySearchPackages(
	session *gocql.Session,
	searchText string,
) ([]*PackageModel, error) {
	// First, remove all non-essential characters
	searchText = alphanumericFilterRegex.ReplaceAllString(searchText, "")
	// Next put the search text into a query string
	query := fmt.Sprintf(cqlQueryFuzzySearchPackagesTemplate, searchText)
	// Return the processed results of the query
	return scanPackageModels(session.Query(query))
}

// InsertPackage inserts an individual package into the database.
func InsertPackage(
	session *gocql.Session,
	packageModel *PackageModel,
) error {
	err := session.Query(cqlQueryInsertPackage,
		*packageModel.Repo,
		*packageModel.Stars,
		*packageModel.Exists,
		*packageModel.Author,
		packageModel.Versions,
		*packageModel.GodocURL,
		*packageModel.IndexTime,
		*packageModel.AwesomeGo,
		*packageModel.SearchBlob,
		*packageModel.Description,
	).Exec()

	return err
}

// InsertPackages inserts a slice of package models into the database.
func InsertPackages(
	session *gocql.Session,
	packageModels []*PackageModel,
) error {
	batch := gocql.NewBatch(gocql.LoggedBatch)

	if packageModels == nil || len(packageModels) == 0 {
		return errors.NewInvalidParameterError("packageModels", packageModels)
	}

	for _, packageModel := range packageModels {
		if packageModel != nil &&
			packageModel.Repo != nil &&
			packageModel.Exists != nil &&
			packageModel.Author != nil &&
			packageModel.GodocURL != nil &&
			packageModel.IndexTime != nil &&
			packageModel.AwesomeGo != nil &&
			packageModel.SearchBlob != nil &&
			packageModel.Description != nil {
			batch.Query(
				cqlQueryInsertPackage,
				*packageModel.Repo,
				*packageModel.Exists,
				*packageModel.Author,
				packageModel.Versions,
				*packageModel.GodocURL,
				*packageModel.IndexTime,
				*packageModel.AwesomeGo,
				*packageModel.SearchBlob,
				*packageModel.Description,
			)
		} else {
			return errors.NewInvalidParameterError(
				"packageModels",
				fmt.Sprintf("[ ..., %v, ... ]", packageModel),
			)
		}
	}

	err := session.ExecuteBatch(batch)
	if err != nil {
		return err
	}

	return nil
}

/********************************** HELPERS ***********************************/

// TODO(skeswa): implement this for querying single packages
func scanPackageModel(query *gocql.Query) ([]*PackageModel, error) {
	return nil, nil
}

func scanPackageModels(query *gocql.Query) ([]*PackageModel, error) {
	var (
		err          error
		scanError    error
		closeError   error
		packageModel *PackageModel

		repo        string
		author      string
		description string

		iter          = query.Iter()
		packageModels = make([]*PackageModel, 0)
	)

	for iter.Scan(&repo, &author, &description) {
		packageModel, err = NewPackageModelFromBulkSelect(author, repo, description)
		if err != nil {
			scanError = err
			break
		} else {
			packageModels = append(packageModels, packageModel)
		}
	}

	if err = iter.Close(); err != nil {
		closeError = err
	}

	if scanError != nil || closeError != nil {
		return nil, errors.NewQueryScanError(scanError, closeError)
	}

	return packageModels, nil
}

func ScanAllPackageModels(session *gocql.Session) ([]*PackageModel, error) {
	var (
		err          error
		scanError    error
		closeError   error
		packageModel *PackageModel

		author      string
		repo        string
		awesome_go  bool
		description string
		exists      bool
		godoc_url   string
		index_time  time.Time
		search_blob string
		versions    []string
		stars       int

		query = session.Query(`SELECT
			author,
			repo,
			awesome_go,
			description,
			exists,
			godoc_url,
			index_time,
			search_blob,
			versions,
			stars
			FROM gophr.packages`)
		iter          = query.Iter()
		packageModels = make([]*PackageModel, 0)
	)

	for iter.Scan(&author, &repo, &awesome_go, &description, &exists, &godoc_url, &index_time, &search_blob, &versions, &stars) {
		packageModel = NewPackageModelTest(author, repo, awesome_go, description, exists, godoc_url, index_time, search_blob, versions, stars)
		packageModels = append(packageModels, packageModel)
	}

	if err = iter.Close(); err != nil {
		closeError = err
	}

	if scanError != nil || closeError != nil {
		return nil, errors.NewQueryScanError(scanError, closeError)
	}

	return packageModels, nil
}

func DeletePackageModel(session *gocql.Session, packageModel *PackageModel) error {
	author := *packageModel.Author
	repo := *packageModel.Repo
	query := session.Query(cqlQueryDeletePackage, author, repo)
	err := query.Exec()

	return err
}
<|endoftext|>


<commit_before>// +build windows

package loadavg

import (
	"testing"
)

func TestGetLoadavg(t *testing.T) {
	loadavg, err := Get()
	if err == nil {
		t.Errorf("error should occur for Windows")
	}
}
<commit_msg>fix tests for Windows<commit_after>// +build windows

package loadavg

import (
	"testing"
)

func TestGetLoadavg(t *testing.T) {
	loadavg, err := Get()
	if err == nil {
		t.Errorf("error should occur for Windows")
	}
	if loadavg != nil {
		t.Errorf("loadavg should be nil")
	}
}
<|endoftext|>


<commit_before>package rfc5424

import (
	"bufio"
	"io"
	"strings"
)

// Delimiter splits incoming data on RFC5424 headers.
type Delimiter struct {
	r      *bufio.Reader
	buf    []byte
	priLen int
	state  fsmState
}

// NewDelimiter returns an instance of a Delimiter.
func NewDelimiter(r io.Reader) *Delimiter {
	return &Delimiter{
		r: bufio.NewReader(r),
	}
}

// ReadLine returns a line beginning with an RFC5424 header, and includes
// all characters up to and including the last character before the start
// of the next RFC5424 header. Any trailing newline characters are stripped
// from the line before it is returned.
func (d *Delimiter) ReadLine() (string, error) {
	for {
		b, err := d.r.ReadByte()
		if err != nil {
			return d.line(false), err
		}
		d.buf = append(d.buf, b)

		switch d.state {
		case newline:
			if b == '\n' {
				d.state = priStart
			}
		case priStart:
			if b == '<' {
				d.state = priVal0
			}
		case priVal0:
			if isDigit(b) {
				d.priLen = 1
				d.state = priVal1
			} else {
				// Invalid, reset parser.
				d.state = priStart
			}
		case priVal1:
			if isDigit(b) {
				d.priLen = 2
				d.state = priVal2
			} else if b == '>' {
				d.state = version
			}
		case priVal2:
			if isDigit(b) {
				d.priLen = 3
				d.state = priVal3
			} else if b == '>' {
				d.state = version
			}
		case priVal3:
			if isDigit(b) {
				d.priLen = 4
				d.state = priEnd
			} else if b == '>' {
				d.state = version
			}
		case priEnd:
			if b == '>' {
				d.state = version
			} else {
				// Invalid, reset parser.
				d.state = priStart
			}
		case version:
			if isDigit(b) {
				d.state = postVersion
			} else {
				// Invalid, reset parser.
				d.state = priStart
			}
		case postVersion:
			if b == ' ' {
				return d.line(true), nil
			} else {
				// Invalid, reset parser.
				d.state = priStart
			}
		}
	}
}

// line returns the line in the current buffer. The returned string is then removed
// from the buffer.
func (d *Delimiter) line(stripDelim bool) string {
	d.state = priStart

	var line string
	if stripDelim {
		line = string(d.buf[:len(d.buf)-d.priLen-4])
	} else {
		line = string(d.buf[:len(d.buf)])
	}
	d.buf = d.buf[len(line):]

	d.priLen = 0
	return strings.TrimRight(line, "\r\n")
}

func isDigit(b byte) bool {
	return b >= '0' && b <= '9'
}

// fsmState represents the state of the parser and what it is expecting next.
type fsmState int

const (
	newline fsmState = iota
	priStart
	priEnd
	priVal0
	priVal1
	priVal2
	priVal3
	version
	postVersion
)
<commit_msg>Remove redundant else<commit_after>package rfc5424

import (
	"bufio"
	"io"
	"strings"
)

// Delimiter splits incoming data on RFC5424 headers.
type Delimiter struct {
	r      *bufio.Reader
	buf    []byte
	priLen int
	state  fsmState
}

// NewDelimiter returns an instance of a Delimiter.
func NewDelimiter(r io.Reader) *Delimiter {
	return &Delimiter{
		r: bufio.NewReader(r),
	}
}

// ReadLine returns a line beginning with an RFC5424 header, and includes
// all characters up to and including the last character before the start
// of the next RFC5424 header. Any trailing newline characters are stripped
// from the line before it is returned.
func (d *Delimiter) ReadLine() (string, error) {
	for {
		b, err := d.r.ReadByte()
		if err != nil {
			return d.line(false), err
		}
		d.buf = append(d.buf, b)

		switch d.state {
		case newline:
			if b == '\n' {
				d.state = priStart
			}
		case priStart:
			if b == '<' {
				d.state = priVal0
			}
		case priVal0:
			if isDigit(b) {
				d.priLen = 1
				d.state = priVal1
			} else {
				// Invalid, reset parser.
				d.state = priStart
			}
		case priVal1:
			if isDigit(b) {
				d.priLen = 2
				d.state = priVal2
			} else if b == '>' {
				d.state = version
			}
		case priVal2:
			if isDigit(b) {
				d.priLen = 3
				d.state = priVal3
			} else if b == '>' {
				d.state = version
			}
		case priVal3:
			if isDigit(b) {
				d.priLen = 4
				d.state = priEnd
			} else if b == '>' {
				d.state = version
			}
		case priEnd:
			if b == '>' {
				d.state = version
			} else {
				// Invalid, reset parser.
				d.state = priStart
			}
		case version:
			if isDigit(b) {
				d.state = postVersion
			} else {
				// Invalid, reset parser.
				d.state = priStart
			}
		case postVersion:
			if b == ' ' {
				return d.line(true), nil
			} else {
			// Invalid, reset parser.
			d.state = priStart
		}
	}
}

// line returns the line in the current buffer. The returned string is then removed
// from the buffer.
func (d *Delimiter) line(stripDelim bool) string {
	d.state = priStart

	var line string
	if stripDelim {
		line = string(d.buf[:len(d.buf)-d.priLen-4])
	} else {
		line = string(d.buf[:len(d.buf)])
	}
	d.buf = d.buf[len(line):]

	d.priLen = 0
	return strings.TrimRight(line, "\r\n")
}

func isDigit(b byte) bool {
	return b >= '0' && b <= '9'
}

// fsmState represents the state of the parser and what it is expecting next.
type fsmState int

const (
	newline fsmState = iota
	priStart
	priEnd
	priVal0
	priVal1
	priVal2
	priVal3
	version
	postVersion
)
<|endoftext|>


<commit_before>package netman_cf_upgrade_test

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"os"
	"strings"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("apps remain available during an upgrade deploy", func() {
	var (
		NoASGTargetIP string
		ASGTargetIP   string
		ASGFilepath   string
	)

	AfterEach(func() {
		os.Remove(ASGFilepath)
	})

	It("upgrades CF with no downtime", func() {
		org, space := "upgrade-org", "upgrade-space"

		baseManifest := os.Getenv("BASE_MANIFEST")
		upgradeManifest := os.Getenv("UPGRADE_MANIFEST")
		By("deleting the deployment")
		boshDeleteDeployment()

		By("deploying base manifest")
		boshDeploy(baseManifest)

		ASGTargetIP = boshIPFor("router")
		NoASGTargetIP = boshIPFor("uaa")
		By(fmt.Sprintf("found ASG Target IPs (allow %s) (deny %s)", ASGTargetIP, NoASGTargetIP))

		Expect(cli.SetApiWithoutSsl(config.ApiEndpoint)).To(Succeed())
		Expect(cli.Auth(config.AdminUser, config.AdminPassword)).To(Succeed())
		Expect(cli.CreateOrg(org)).To(Succeed())
		Expect(cli.TargetOrg(org)).To(Succeed())
		Expect(cli.CreateSpace(space)).To(Succeed())
		Expect(cli.TargetSpace(space)).To(Succeed())

		By("create and bind security group")

		asg := `[
		 {
		 "protocol": "tcp",
		 "destination": "` + ASGTargetIP + `",
		 "ports": "80"
		 }
		 ]
		 `
		ASGFilepath = createASGFile(asg)
		Expect(cli.CreateSecurityGroup("test-running-asg", ASGFilepath)).To(Succeed())
		Expect(cli.BindSecurityGroup("test-running-asg", org, space)).To(Succeed())

		By("pushing the proxy app")
		Expect(cli.Push("proxy-upgrade", "../example-apps/proxy", "../example-apps/proxy/manifest.yml")).To(Succeed())
		Expect(cli.Scale("proxy-upgrade", 3)).To(Succeed())

		By("checking the app has started")
		Eventually(checkStatusCode).Should(Equal(http.StatusOK))

		By("checking the app continuously")
		var failures []string
		go checkStatusCodeContinuously(ASGTargetIP, NoASGTargetIP, &failures)

		By("deploying upgrade manifest")
		boshDeploy(upgradeManifest)
		fmt.Printf("\n\n### Got %d failures ###\n\n", len(failures))
		fmt.Println(strings.Join(failures, "\n"))
		Expect(len(failures)).To(BeNumerically("<", 5))
	})
})

func checkASG(ip string) (int, string) {
	resp, err := http.Get(fmt.Sprintf("http://proxy-upgrade.%s/proxy/%s", config.AppsDomain, ip))
	dump, err := httputil.DumpResponse(resp, true)
	if err != nil {
		return http.StatusTeapot, string(dump)
	}
	defer resp.Body.Close()
	return resp.StatusCode, string(dump)
}

func checkApp() (int, string) {
	resp, err := http.Get("http://proxy-upgrade." + config.AppsDomain)
	dump, err := httputil.DumpResponse(resp, true)
	if err != nil {
		return http.StatusTeapot, string(dump)
	}
	defer resp.Body.Close()
	return resp.StatusCode, string(dump)
}

func checkStatusCode() int {
	resp, err := http.Get("http://proxy-upgrade." + config.AppsDomain)
	if err != nil {
		return http.StatusTeapot
	}
	defer resp.Body.Close()
	return resp.StatusCode
}

func checkStatusCodeContinuously(allowIP, denyIP string, failures *[]string) {
	defer GinkgoRecover()
	for {
		sc, dump := checkApp()
		if sc != http.StatusOK {
			*failures = append(*failures, dump)
		}
		sc, dump = checkASG(allowIP)
		if sc != http.StatusOK {
			*failures = append(*failures, dump)
		}
		sc, dump = checkASG(denyIP)
		if sc != http.StatusInternalServerError {
			*failures = append(*failures, dump)
		}
		time.Sleep(1 * time.Second)
	}
}

func createASGFile(asg string) string {
	asgFile, err := ioutil.TempFile("", "")
	Expect(err).NotTo(HaveOccurred())
	path := asgFile.Name()
	Expect(ioutil.WriteFile(path, []byte(asg), os.ModePerm))
	return path
}
<commit_msg>Separate logs for each type of failure<commit_after>package netman_cf_upgrade_test

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"os"
	"strings"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("apps remain available during an upgrade deploy", func() {
	var (
		ASGFilepath string
	)

	AfterEach(func() {
		os.Remove(ASGFilepath)
	})

	It("upgrades CF with no downtime", func() {
		org, space := "upgrade-org", "upgrade-space"

		baseManifest := os.Getenv("BASE_MANIFEST")
		upgradeManifest := os.Getenv("UPGRADE_MANIFEST")
		By("deleting the deployment")
		boshDeleteDeployment()

		By("deploying base manifest")
		boshDeploy(baseManifest)

		ASGTargetIP := boshIPFor("router")
		noASGTargetIP := boshIPFor("uaa")
		By(fmt.Sprintf("found ASG Target IPs (allow %s) (deny %s)", ASGTargetIP, noASGTargetIP))

		Expect(cli.SetApiWithoutSsl(config.ApiEndpoint)).To(Succeed())
		Expect(cli.Auth(config.AdminUser, config.AdminPassword)).To(Succeed())
		Expect(cli.CreateOrg(org)).To(Succeed())
		Expect(cli.TargetOrg(org)).To(Succeed())
		Expect(cli.CreateSpace(space)).To(Succeed())
		Expect(cli.TargetSpace(space)).To(Succeed())

		By("create and bind security group")
		asg := `[
		 {
		 "protocol": "tcp",
		 "destination": "` + ASGTargetIP + `",
		 "ports": "80"
		 }
		 ]
		 `
		ASGFilepath = createASGFile(asg)
		Expect(cli.CreateSecurityGroup("test-running-asg", ASGFilepath)).To(Succeed())
		Expect(cli.BindSecurityGroup("test-running-asg", org, space)).To(Succeed())

		By("pushing the proxy app")
		Expect(cli.Push("proxy-upgrade", "../example-apps/proxy", "../example-apps/proxy/manifest.yml")).To(Succeed())
		Expect(cli.Scale("proxy-upgrade", 3)).To(Succeed())

		By("checking the app has started")
		Eventually(checkStatusCode).Should(Equal(http.StatusOK))

		By("checking the app continuously")
		var appFailures []string
		var ASGFailures []string
		var noASGFailures []string
		go checkContinuously("http://proxy-upgrade."+config.AppsDomain, http.StatusOK, &appFailures)
		go checkContinuously(fmt.Sprintf("http://proxy-upgrade.%s/proxy/%s", config.AppsDomain, ASGTargetIP), http.StatusOK, &ASGFailures)
		go checkContinuously(fmt.Sprintf("http://proxy-upgrade.%s/proxy/%s", config.AppsDomain, noASGTargetIP), http.StatusInternalServerError, &noASGFailures)

		By("deploying upgrade manifest")
		boshDeploy(upgradeManifest)

		fmt.Printf("\n\n### Got %d app failures ###\n\n", len(appFailures))
		fmt.Println(strings.Join(appFailures, "\n"))
		fmt.Printf("\n\n### Got %d ASG failures ###\n\n", len(ASGFailures))
		fmt.Println(strings.Join(ASGFailures, "\n"))
		fmt.Printf("\n\n### Got %d no ASG failures ###\n\n", len(noASGFailures))
		fmt.Println(strings.Join(noASGFailures, "\n"))

		Expect(len(appFailures)).To(BeNumerically("<", 5))
		Expect(len(ASGFailures)).To(BeNumerically("<", 5))
		Expect(len(noASGFailures)).To(BeNumerically("<", 5))
	})
})

func check(url string) (int, string) {
	resp, err := http.Get(url)
	defer resp.Body.Close()
	if err != nil {
		return http.StatusTeapot, ""
	}

	dump, err := httputil.DumpResponse(resp, true)
	if err != nil {
		return http.StatusTeapot, string(dump)
	}

	return resp.StatusCode, string(dump)
}

func checkStatusCode() int {
	sc, _ := check("http://proxy-upgrade." + config.AppsDomain)
	return sc
}

func checkContinuously(url string, statusCode int, failures *[]string) {
	defer GinkgoRecover()
	for {
		sc, dump := check(url)
		if sc != statusCode {
			*failures = append(*failures, dump)
		}
		time.Sleep(1 * time.Second)
	}
}

func createASGFile(asg string) string {
	asgFile, err := ioutil.TempFile("", "")
	Expect(err).NotTo(HaveOccurred())
	path := asgFile.Name()
	Expect(ioutil.WriteFile(path, []byte(asg), os.ModePerm))
	return path
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package math

import "math"

// x^y: exponentation
export func Pow(x, y float64) float64 {
	// TODO: x or y NaN, ±Inf, maybe ±0.
	switch {
	case y == 0:
		return 1;
	case y == 1:
		return x;
	case x == 0 && y > 0:
		return 0;
	case x == 0 && y < 0:
		return sys.Inf(1);
	case y == 0.5:
		return Sqrt(x);
	case y == -0.5:
		return 1 / Sqrt(x);
	}

	absy := y;
	flip := false;
	if absy < 0 {
		absy = -absy;
		flip = true;
	}
	yi, yf := sys.modf(absy);
	if yf != 0 && x < 0 {
		return sys.NaN();
	}
	if yi >= 1<<63 {
		return Exp(y * Log(x));
	}

	ans := float64(1);

	// ans *= x^yf
	if yf != 0 {
		if yf > 0.5 {
			yf--;
			yi++;
		}
		ans = Exp(yf * Log(x));
	}

	// ans *= x^yi
	// by multiplying in successive squarings
	// of x according to bits of yi.
	// accumulate powers of two into exp.
	// will still have to do ans *= 2^exp later.
	x1, xe := sys.frexp(x);
	exp := 0;
	if i := int64(yi); i != 0 {
		for {
			if i&1 == 1 {
				ans *= x1;
				exp += xe;
			}
			i >>= 1;
			if i == 0 {
				break;
			}
			x1 *= x1;
			xe <<= 1;
			if x1 < .5 {
				x1 += x1;
				xe--;
			}
		}
	}

	// ans *= 2^exp
	// if flip { ans = 1 / ans }
	// but in the opposite order
	if flip {
		ans = 1 / ans;
		exp = -exp;
	}
	return sys.ldexp(ans, exp);
}

<commit_msg>slightly simpler math.Pow per gri's suggestion<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package math

import "math"

// x^y: exponentiation
export func Pow(x, y float64) float64 {
	// TODO: x or y NaN, ±Inf, maybe ±0.
	switch {
	case y == 0:
		return 1;
	case y == 1:
		return x;
	case x == 0 && y > 0:
		return 0;
	case x == 0 && y < 0:
		return sys.Inf(1);
	case y == 0.5:
		return Sqrt(x);
	case y == -0.5:
		return 1 / Sqrt(x);
	}

	absy := y;
	flip := false;
	if absy < 0 {
		absy = -absy;
		flip = true;
	}
	yi, yf := sys.modf(absy);
	if yf != 0 && x < 0 {
		return sys.NaN();
	}
	if yi >= 1<<63 {
		return Exp(y * Log(x));
	}

	// ans = a1 * 2^ae (= 1 for now).
	a1 := float64(1);
	ae := 0;

	// ans *= x^yf
	if yf != 0 {
		if yf > 0.5 {
			yf--;
			yi++;
		}
		a1 = Exp(yf * Log(x));
	}

	// ans *= x^yi
	// by multiplying in successive squarings
	// of x according to bits of yi.
	// accumulate powers of two into exp.
	x1, xe := sys.frexp(x);
	for i := int64(yi); i != 0; i >>= 1 {
		if i&1 == 1 {
			a1 *= x1;
			ae += xe;
		}
		x1 *= x1;
		xe <<= 1;
		if x1 < .5 {
			x1 += x1;
			xe--;
		}
	}

	// ans = a1*2^ae
	// if flip { ans = 1 / ans }
	// but in the opposite order
	if flip {
		a1 = 1 / a1;
		ae = -ae;
	}
	return sys.ldexp(a1, ae);
}
<|endoftext|>


<commit_before>// Copyright 2012 - 2014 Alex Palaistras. All rights reserved.
// Use of this source code is governed by the MIT License, the
// full text of which can be found in the LICENSE file.

package server

import (
	"bufio"
	"fmt"
	"io"
	"net"
	"os"
	"strconv"
	"strings"

	"github.com/thoughtmonster/sleepy/core/user"
)

type ftpSession struct {
	conn net.Conn
	data net.Listener
	user *user.User
}

func ServeFTP(addr string) error {
	ln, err := net.Listen("tcp", addr)
	if err != nil {
		return err
	}

	defer ln.Close()
	for {
		conn, err := ln.Accept()
		if err != nil {
			continue
		}

		session := &ftpSession{conn, nil, nil}
		go session.serve()
	}
}

func (s *ftpSession) serve() {
	buf := bufio.NewReader(s.conn)
	s.respond("220 Connection established")

	for {
		line, err := buf.ReadString('\n')
		if err != nil {
			continue
		}

		params := strings.Fields(line)
		if len(params) == 0 {
			continue
		}

		switch strings.ToUpper(params[0]) {
		case "TYPE", "MODE", "STRU":
			s.respond("200 Command OK")
		case "PWD":
			s.respond("257 \"/\" is the current directory")
		case "PORT":
			s.respond("421 Cannot use active mode, use passive mode instead")
		case "PASV":
			s.data, err = net.Listen("tcp", ":0")
			if err != nil {
				s.respond("421 Could not start in passive mode, creating socket failed")
				goto quit
			}

			_, port, _ := net.SplitHostPort(s.data.Addr().String())
			t, _ := strconv.ParseInt(port, 10, 64)
			p := strconv.FormatInt(t/256, 10) + "," + strconv.FormatInt(t%256, 10)
			s.respond("227 Entering Passive Mode (127,0,0,1," + p + ")")
		case "USER":
			if len(params) < 2 {
				s.respond("501 USER expects an SHA1 authkey, none given")
				break
			}

			u, _ := user.Auth(params[1])
			if u == nil {
				s.respond("530 Login failed")
				break
			}

			s.user = u
			s.respond("230 Login successful")
		case "STOR":
			if s.user == nil {
				s.respond("532 You need to login to access this command")
				break
			}

			if len(params) < 2 {
				s.respond("501 STOR expects a name for the file, none given")
				break
			}

			s.respond("150 File transfer starting")
			go s.storeFile(params[1])
		case "QUIT":
			s.respond("221 Closing connection")
			goto quit
		default:
			s.respond("502 Command not implemented")
		}
	}

quit:
	if s.data != nil {
		s.data.Close()
	}

	s.conn.Close()
}

func (s *ftpSession) respond(msg string) {
	fmt.Fprintln(s.conn, msg)
}

func (s *ftpSession) storeFile(name string) {
	defer s.data.Close()

	conn, err := s.data.Accept()
	if err != nil {
		s.respond("451 Could not establish connection to server")
		return
	}

	defer conn.Close()
	path := os.TempDir() + "/sleepy/" + strconv.FormatInt(int64(s.user.Id), 10)

	err = os.MkdirAll(path, 0755)
	if err != nil {
		s.respond("451 Could not create temporary directory")
		return
	}

	file, err := os.Create(path + "/" + name)
	if err != nil {
		s.respond("451 Could not create remote file")
		return
	}

	io.Copy(file, conn)
	s.respond("226 File transfer successful")
}
<commit_msg>Close created file after opening for upload<commit_after>// Copyright 2012 - 2014 Alex Palaistras. All rights reserved.
// Use of this source code is governed by the MIT License, the
// full text of which can be found in the LICENSE file.

package server

import (
	"bufio"
	"fmt"
	"io"
	"net"
	"os"
	"strconv"
	"strings"

	"github.com/thoughtmonster/sleepy/core/user"
)

type ftpSession struct {
	conn net.Conn
	data net.Listener
	user *user.User
}

func ServeFTP(addr string) error {
	ln, err := net.Listen("tcp", addr)
	if err != nil {
		return err
	}

	defer ln.Close()
	for {
		conn, err := ln.Accept()
		if err != nil {
			continue
		}

		session := &ftpSession{conn, nil, nil}
		go session.serve()
	}
}

func (s *ftpSession) serve() {
	buf := bufio.NewReader(s.conn)
	s.respond("220 Connection established")

	for {
		line, err := buf.ReadString('\n')
		if err != nil {
			continue
		}

		params := strings.Fields(line)
		if len(params) == 0 {
			continue
		}

		switch strings.ToUpper(params[0]) {
		case "TYPE", "MODE", "STRU":
			s.respond("200 Command OK")
		case "PWD":
			s.respond("257 \"/\" is the current directory")
		case "PORT":
			s.respond("421 Cannot use active mode, use passive mode instead")
		case "PASV":
			s.data, err = net.Listen("tcp", ":0")
			if err != nil {
				s.respond("421 Could not start in passive mode, creating socket failed")
				goto quit
			}

			_, port, _ := net.SplitHostPort(s.data.Addr().String())
			t, _ := strconv.ParseInt(port, 10, 64)
			p := strconv.FormatInt(t/256, 10) + "," + strconv.FormatInt(t%256, 10)
			s.respond("227 Entering Passive Mode (127,0,0,1," + p + ")")
		case "USER":
			if len(params) < 2 {
				s.respond("501 USER expects an SHA1 authkey, none given")
				break
			}

			u, _ := user.Auth(params[1])
			if u == nil {
				s.respond("530 Login failed")
				break
			}

			s.user = u
			s.respond("230 Login successful")
		case "STOR":
			if s.user == nil {
				s.respond("532 You need to login to access this command")
				break
			}

			if len(params) < 2 {
				s.respond("501 STOR expects a name for the file, none given")
				break
			}

			s.respond("150 File transfer starting")
			go s.storeFile(params[1])
		case "QUIT":
			s.respond("221 Closing connection")
			goto quit
		default:
			s.respond("502 Command not implemented")
		}
	}

quit:
	if s.data != nil {
		s.data.Close()
	}

	s.conn.Close()
}

func (s *ftpSession) respond(msg string) {
	fmt.Fprintln(s.conn, msg)
}

func (s *ftpSession) storeFile(name string) {
	defer s.data.Close()

	conn, err := s.data.Accept()
	if err != nil {
		s.respond("451 Could not establish connection to server")
		return
	}

	defer conn.Close()
	path := os.TempDir() + "/sleepy/" + strconv.FormatInt(int64(s.user.Id), 10)

	err = os.MkdirAll(path, 0755)
	if err != nil {
		s.respond("451 Could not create temporary directory")
		return
	}

	file, err := os.Create(path + "/" + name)
	if err != nil {
		s.respond("451 Could not create remote file")
		return
	}

	defer file.Close()

	io.Copy(file, conn)
	s.respond("226 File transfer successful")
}
<|endoftext|>


<commit_before>package viewLib

import (
	"encoding/json"
	"log"
	"os"
	"strconv"
	"sync"
	"time"

	"github.com/boltdb/bolt"
)

//Counter is an instance of a [pageName]pageView hash map. This is implemented
//with a mutex RW lock for concurrent R/W safety
var Counter = struct {
	sync.RWMutex
	M map[string]int
}{M: make(map[string]int)}

//IPs is an instance of a [ipAdress]bool hash map. hash offers a
//easy implementation of a set with quick insertion. This is implemented
//with a mutex RW lock for concurrent R/W safety
var IPs = struct {
	sync.RWMutex
	M map[string]bool
}{M: make(map[string]bool)}

//RefreshTime is how often the page views and IPs are saved to disk, set at a
//default of 1 minute
var SaveDuration = time.Minute * 1

//IPList struct is used to marshal/unmarshal IP visitor data into JSON
//for disk storage
type IPList struct {
	IPs map[string]bool
}

//SavePoint struct is used to marshal/unmarshal pageview data into JSON
//for disk storage
type SavePoint struct {
	PageCounts  map[string]int
	UniqueViews int
}

//init checks checks for previos data and then initiates the HTTP server.
//init() does not need to be called, it runs on startup automatically.
func init() {
	checkForRecords()
	go periodicMemoryWriter()
}

//ViewInc locks the Counter and ip set mutexes, writes to both then unlocks
func ViewInc(ip string, page string) {
	log.Println(ip + " requests " + page)

	Counter.Lock()
	Counter.M[page]++
	Counter.Unlock()
	Counter.RLock()
	Counter.RUnlock()

	IPs.Lock()
	IPs.M[ip] = true
	IPs.Unlock()
}

//PageExists checks the Counter map to see if that page is present
func PageExists(page string) bool {
	Counter.RLock()
	_, ok := Counter.M[page]
	Counter.RUnlock()

	return ok
}

//periodicMemoryWriter initiates a BoltDB client, sets up a ticker and
//then wrties the maps to disk. Should be called as a go routine.
func periodicMemoryWriter() {
	//start the bolt client
	boltClient, err := bolt.Open("viewCounter.db", 0600, nil)
	if err != nil {
		log.Fatal(err)
	}
	defer boltClient.Close()

	//check and create a bucket in bolt to store the data
	boltClient.Update(func(tx *bolt.Tx) error {
		tx.CreateBucketIfNotExists([]byte("historicData"))
		return nil
	})

	//start a ticker for period between disk writes
	ticker := time.NewTicker(SaveDuration)

	for {

		<-ticker.C
		//Debug fmt.Println("Save start time: ", time.Now())

		//The date is made of the day number concatinated with the year, e.g. the
		//05/01/2015 would be 52015
		date := strconv.Itoa((time.Now().YearDay() * 10000) + time.Now().Year())

		Counter.RLock()
		IPs.RLock()

		m1 := SavePoint{
			PageCounts:  Counter.M,
			UniqueViews: len(IPs.M),
		}

		m2 := IPList{
			IPs: IPs.M,
		}

		Counter.RUnlock()
		IPs.RUnlock()

		m1json, err := json.Marshal(m1)
		errLog(err)
		m2json, err := json.Marshal(m2)
		errLog(err)
		boltClient.Update(func(tx *bolt.Tx) error {

			err = tx.Bucket([]byte("historicData")).Put([]byte(date), []byte(m1json))
			errLog(err)

			err = tx.Bucket([]byte("historicData")).Put([]byte("current"), []byte(m1json))
			errLog(err)

			err = tx.Bucket([]byte("historicData")).Put([]byte("IPs"), []byte(m2json))
			errLog(err)
			return nil
		})
		//Debug fmt.Println("Save finish time: ", time.Now())
	}
}

//checkForRecords is used to see if a BoltDB database is present in the file system,
//and if it is then to load the IP and pageview sets into program memory.
func checkForRecords() {
	if _, err := os.Stat("viewCounter.db"); err == nil {
		log.Println("viewCount.db database already exists; processing old entries")

		boltClient, err := bolt.Open("viewCounter.db", 0600, nil)
		if err != nil {
			log.Fatal(err)
		}
		defer boltClient.Close()

		var b1, b2 []byte
		boltClient.View(func(tx *bolt.Tx) error {
			b1 = tx.Bucket([]byte("historicData")).Get([]byte("current"))
			errLog(err)

			b2 = tx.Bucket([]byte("historicData")).Get([]byte("IPs"))
			errLog(err)

			return nil
		})

		var mjson1 SavePoint
		err = json.Unmarshal(b1, &mjson1)
		errLog(err)

		for k, v := range mjson1.PageCounts {
			Counter.M[k] = v
		}

		var mjson2 IPList
		err = json.Unmarshal(b2, &mjson2)
		errLog(err)

		for k := range mjson2.IPs {
			IPs.M[k] = true
		}

	} else {
		log.Println("viewCount.db not present; creating database")

	}
}

func errLog(err error) {
	if err != nil {
		log.Print(err)
	}
}
<commit_msg>added functions so the user doesn't use the maps directly<commit_after>package viewLib

import (
	"encoding/json"
	"log"
	"os"
	"strconv"
	"sync"
	"time"

	"github.com/boltdb/bolt"
)

//Counter is an instance of a [pageName]pageView hash map. This is implemented
//with a mutex RW lock for concurrent R/W safety
var Counter = struct {
	sync.RWMutex
	M map[string]int
}{M: make(map[string]int)}

//IPs is an instance of a [ipAdress]bool hash map. hash offers a
//easy implementation of a set with quick insertion. This is implemented
//with a mutex RW lock for concurrent R/W safety
var IPs = struct {
	sync.RWMutex
	M map[string]bool
}{M: make(map[string]bool)}

//RefreshTime is how often the page views and IPs are saved to disk, set at a
//default of 1 minute
var SaveDuration = time.Minute * 1

//IPList struct is used to marshal/unmarshal IP visitor data into JSON
//for disk storage
type IPList struct {
	IPs map[string]bool
}

//SavePoint struct is used to marshal/unmarshal pageview data into JSON
//for disk storage
type SavePoint struct {
	PageCounts  map[string]int
	UniqueViews int
}

//init checks checks for previos data and then initiates the HTTP server.
//init() does not need to be called, it runs on startup automatically.
func init() {
	checkForRecords()
	go periodicMemoryWriter()
}

//ViewInc locks the Counter and ip set mutexes, writes to both then unlocks
func ViewInc(ip string, page string) {
	log.Println(ip + " requests " + page)

	Counter.Lock()
	Counter.M[page]++
	Counter.Unlock()

	IPs.Lock()
	IPs.M[ip] = true
	IPs.Unlock()
}

//AddPage adds a new page to the Counter with 0 views
func AddPage(page string) {
	Counter.Lock()
	Counter.M[page] = 0
	Counter.Unlock()
}

//DeletePage deletes the page and its views from the counter
func DeletePage(page string) {
	Counter.Lock()
	delete(Counter.M, page)
	Counter.Unlock()
}

//GetPageViews returns a boolean to indicate if a page is present in the
//counter. If it is it also returns the count, else count = 0
func GetPageViews(page string) (count int, exists bool) {
	Counter.RLock()
	count, exists = Counter.M[page]
	Counter.RUnlock()
	return
}

//GetNumberOfUniqueIPs returns the number of unique IPs
func GetNumberOfUniqueIPs(page string) (numberOfUniqueIPs int) {
	IPs.RLock()
	numberOfUniqueIPs = len(IPs.M)
	IPs.RUnlock()
	return
}

//periodicMemoryWriter initiates a BoltDB client, sets up a ticker and
//then wrties the maps to disk. Should be called as a go routine.
func periodicMemoryWriter() {
	//start the bolt client
	boltClient, err := bolt.Open("viewCounter.db", 0600, nil)
	if err != nil {
		log.Fatal(err)
	}
	defer boltClient.Close()

	//check and create a bucket in bolt to store the data
	boltClient.Update(func(tx *bolt.Tx) error {
		tx.CreateBucketIfNotExists([]byte("historicData"))
		return nil
	})

	//start a ticker for period between disk writes
	ticker := time.NewTicker(SaveDuration)

	for {

		<-ticker.C
		//Debug fmt.Println("Save start time: ", time.Now())

		//The date is made of the day number concatinated with the year, e.g. the
		//05/01/2015 would be 52015
		date := strconv.Itoa((time.Now().YearDay() * 10000) + time.Now().Year())

		Counter.RLock()
		IPs.RLock()

		m1 := SavePoint{
			PageCounts:  Counter.M,
			UniqueViews: len(IPs.M),
		}

		m2 := IPList{
			IPs: IPs.M,
		}

		Counter.RUnlock()
		IPs.RUnlock()

		m1json, err := json.Marshal(m1)
		errLog(err)
		m2json, err := json.Marshal(m2)
		errLog(err)
		boltClient.Update(func(tx *bolt.Tx) error {

			err = tx.Bucket([]byte("historicData")).Put([]byte(date), []byte(m1json))
			errLog(err)

			err = tx.Bucket([]byte("historicData")).Put([]byte("current"), []byte(m1json))
			errLog(err)

			err = tx.Bucket([]byte("historicData")).Put([]byte("IPs"), []byte(m2json))
			errLog(err)
			return nil
		})
		//Debug fmt.Println("Save finish time: ", time.Now())
	}
}

//checkForRecords is used to see if a BoltDB database is present in the file system,
//and if it is then to load the IP and pageview sets into program memory.
func checkForRecords() {
	if _, err := os.Stat("viewCounter.db"); err == nil {
		log.Println("viewCount.db database already exists; processing old entries")

		boltClient, err := bolt.Open("viewCounter.db", 0600, nil)
		if err != nil {
			log.Fatal(err)
		}
		defer boltClient.Close()

		var b1, b2 []byte
		boltClient.View(func(tx *bolt.Tx) error {
			b1 = tx.Bucket([]byte("historicData")).Get([]byte("current"))
			errLog(err)

			b2 = tx.Bucket([]byte("historicData")).Get([]byte("IPs"))
			errLog(err)

			return nil
		})

		var mjson1 SavePoint
		err = json.Unmarshal(b1, &mjson1)
		errLog(err)

		for k, v := range mjson1.PageCounts {
			Counter.M[k] = v
		}

		var mjson2 IPList
		err = json.Unmarshal(b2, &mjson2)
		errLog(err)

		for k := range mjson2.IPs {
			IPs.M[k] = true
		}

	} else {
		log.Println("viewCount.db not present; creating database")

	}
}

func errLog(err error) {
	if err != nil {
		log.Print(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package all

import (
	"reflect"

	"github.com/juju/cmd"
	"github.com/juju/errors"
	"github.com/juju/names"

	"github.com/juju/juju/apiserver/common"
	"github.com/juju/juju/process"
	"github.com/juju/juju/process/api"
	"github.com/juju/juju/process/api/server"
	"github.com/juju/juju/process/context"
	"github.com/juju/juju/process/plugin"
	procstate "github.com/juju/juju/process/state"
	"github.com/juju/juju/state"
	"github.com/juju/juju/worker/uniter/runner"
	"github.com/juju/juju/worker/uniter/runner/jujuc"
)

type workloadProcesses struct{}

func (c workloadProcesses) registerForServer() error {
	c.registerHookContext()
	c.registerState()
	return nil
}

func (c workloadProcesses) registerForClient() error {
	return nil
}

func (c workloadProcesses) registerHookContext() {
	if !markRegistered(process.ComponentName, "hook-context") {
		return
	}

	runner.RegisterComponentFunc(process.ComponentName,
		func() (jujuc.ContextComponent, error) {
			// TODO(ericsnow) The API client or facade should be passed
			// in to the factory func and passed to NewInternalClient.
			client, err := api.NewInternalClient()
			if err != nil {
				return nil, errors.Trace(err)
			}
			component, err := context.NewContextAPI(client)
			if err != nil {
				return nil, errors.Trace(err)
			}
			return component, nil
		},
	)

	c.registerHookContextCommands()
	c.registerHookContextFacades()
}

func (c workloadProcesses) registerHookContextFacades() {

	newHookContextApi := func(st *state.State, _ *state.Unit) (interface{}, error) {
		if st == nil {
			return nil, errors.NewNotValid(nil, "st is nil")
		}
		// TODO(natefinch): uncomment when the appropriate state functions exist.
		return &server.HookContextAPI{ /* st */ }, nil
	}

	common.RegisterHookContextFacade(
		process.ComponentName,
		0,
		newHookContextApi,
		reflect.TypeOf(server.HookContextAPI{}),
	)
}

type workloadProcessesHookContext struct {
	jujuc.Context
}

// Component implements context.HookContext.
func (c workloadProcessesHookContext) Component(name string) (context.Component, error) {
	found, err := c.Context.Component(name)
	if err != nil {
		return nil, errors.Trace(err)
	}
	compCtx, ok := found.(context.Component)
	if !ok && found != nil {
		return nil, errors.Errorf("wrong component context type registered: %T", found)
	}
	return compCtx, nil
}

func (workloadProcesses) registerHookContextCommands() {
	if !markRegistered(process.ComponentName, "hook-context-commands") {
		return
	}

	jujuc.RegisterCommand("register", func(ctx jujuc.Context) cmd.Command {
		compCtx := workloadProcessesHookContext{ctx}
		cmd, err := context.NewProcRegistrationCommand(compCtx)
		if err != nil {
			// TODO(ericsnow) Return an error instead.
			panic(err)
		}
		return cmd
	})

	jujuc.RegisterCommand("launch", func(ctx jujuc.Context) cmd.Command {
		compCtx := workloadProcessesHookContext{ctx}
		cmd, err := context.NewProcLaunchCommand(plugin.Find, plugin.Plugin.Launch, compCtx)
		if err != nil {
			// TODO(ericsnow) Return an error instead.
			panic(err)
		}
		return cmd
	})
}

func (c workloadProcesses) registerState() {
	newUnitProcesses := func(persist state.Persistence, unit names.UnitTag, charm names.CharmTag) (state.UnitProcesses, error) {
		return procstate.NewUnitProcesses(persist, unit, &charm), nil
	}
	newProcessDefinitions := func(persist state.Persistence, charm names.CharmTag) (state.ProcessDefinitions, error) {
		return procstate.NewDefinitions(persist, charm), nil
	}
	state.SetProcessesComponent(newUnitProcesses, newProcessDefinitions)
}
<commit_msg>Addressed review comments.<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package all

import (
	"reflect"

	"github.com/juju/cmd"
	"github.com/juju/errors"
	"github.com/juju/names"

	"github.com/juju/juju/apiserver/common"
	"github.com/juju/juju/process"
	"github.com/juju/juju/process/api"
	"github.com/juju/juju/process/api/server"
	"github.com/juju/juju/process/context"
	"github.com/juju/juju/process/plugin"
	procstate "github.com/juju/juju/process/state"
	"github.com/juju/juju/state"
	"github.com/juju/juju/worker/uniter/runner"
	"github.com/juju/juju/worker/uniter/runner/jujuc"
)

type workloadProcesses struct{}

func (c workloadProcesses) registerForServer() error {
	c.registerHookContext()
	c.registerState()
	return nil
}

func (c workloadProcesses) registerForClient() error {
	return nil
}

func (c workloadProcesses) registerHookContext() {
	if !markRegistered(process.ComponentName, "hook-context") {
		return
	}

	runner.RegisterComponentFunc(process.ComponentName,
		func() (jujuc.ContextComponent, error) {
			// TODO(ericsnow) The API client or facade should be passed
			// in to the factory func and passed to NewInternalClient.
			client, err := api.NewInternalClient()
			if err != nil {
				return nil, errors.Trace(err)
			}
			component, err := context.NewContextAPI(client)
			if err != nil {
				return nil, errors.Trace(err)
			}
			return component, nil
		},
	)

	c.registerHookContextCommands()
	c.registerHookContextFacade()
}

func (c workloadProcesses) registerHookContextFacade() {

	newHookContextApi := func(st *state.State, _ *state.Unit) (interface{}, error) {
		if st == nil {
			return nil, errors.NewNotValid(nil, "st is nil")
		}
		// TODO(natefinch): uncomment when the appropriate state functions exist.
		return &server.HookContextAPI{ /* st */ }, nil
	}

	common.RegisterHookContextFacade(
		process.ComponentName,
		0,
		newHookContextApi,
		reflect.TypeOf(server.HookContextAPI{}),
	)
}

type workloadProcessesHookContext struct {
	jujuc.Context
}

// Component implements context.HookContext.
func (c workloadProcessesHookContext) Component(name string) (context.Component, error) {
	found, err := c.Context.Component(name)
	if err != nil {
		return nil, errors.Trace(err)
	}
	compCtx, ok := found.(context.Component)
	if !ok && found != nil {
		return nil, errors.Errorf("wrong component context type registered: %T", found)
	}
	return compCtx, nil
}

func (workloadProcesses) registerHookContextCommands() {
	if !markRegistered(process.ComponentName, "hook-context-commands") {
		return
	}

	jujuc.RegisterCommand("register", func(ctx jujuc.Context) cmd.Command {
		compCtx := workloadProcessesHookContext{ctx}
		cmd, err := context.NewProcRegistrationCommand(compCtx)
		if err != nil {
			// TODO(ericsnow) Return an error instead.
			panic(err)
		}
		return cmd
	})

	jujuc.RegisterCommand("launch", func(ctx jujuc.Context) cmd.Command {
		compCtx := workloadProcessesHookContext{ctx}
		cmd, err := context.NewProcLaunchCommand(plugin.Find, plugin.Plugin.Launch, compCtx)
		if err != nil {
			// TODO(ericsnow) Return an error instead.
			panic(err)
		}
		return cmd
	})
}

func (c workloadProcesses) registerState() {
	newUnitProcesses := func(persist state.Persistence, unit names.UnitTag, charm names.CharmTag) (state.UnitProcesses, error) {
		return procstate.NewUnitProcesses(persist, unit, &charm), nil
	}
	newProcessDefinitions := func(persist state.Persistence, charm names.CharmTag) (state.ProcessDefinitions, error) {
		return procstate.NewDefinitions(persist, charm), nil
	}
	state.SetProcessesComponent(newUnitProcesses, newProcessDefinitions)
}
<|endoftext|>


<commit_before>package stackit

import (
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudformation"
	"time"
	"log"
	"fmt"
	"strings"
)

type TailStackEvent struct {
	cloudformation.StackEvent
}

func PollStackEvents(sess *session.Session, stackId string, startEventId *string, channel chan<- TailStackEvent) error {
	cfn := cloudformation.New(sess)

	go func() {

		for {
			time.Sleep(3*time.Second)

			events := []*cloudformation.StackEvent{}

			cfn.DescribeStackEventsPages(&cloudformation.DescribeStackEventsInput{
				StackName: &stackId,
			}, func(page *cloudformation.DescribeStackEventsOutput, lastPage bool) bool {
				for _, event := range page.StackEvents {
					if *event.EventId == *startEventId {
						return false
					}

					events = append(events, event)
				}
				return true
			})

			if len(events) == 0 {
				continue
			}

			startEventId = events[0].EventId

			resp, err := cfn.DescribeStacks(&cloudformation.DescribeStacksInput{StackName: &stackId})

			if err != nil {
				log.Fatal(err.Error())
			}

			status := *resp.Stacks[0].StackStatus

			for ev_i := len(events) - 1; ev_i >= 0; ev_i-- {
				done := IsTerminalStatus(status) && ev_i == 0
				if done {
					close(channel)
					return
				}

				event := events[ev_i]
				tailEvent := TailStackEvent{*event}
				channel <- tailEvent
			}
		}
	}()

	return nil
}

func fixedLengthString(length int, str string) string {
	verb := fmt.Sprintf("%%%d.%ds", length, length)
	return fmt.Sprintf(verb, str)
}

func isBadStatus(status string) bool {
	return strings.HasSuffix(status, "_FAILED")
}

func IsTerminalStatus(status string) bool {
	switch status {
	case
		"CREATE_COMPLETE",
		"DELETE_COMPLETE",
		"CREATE_FAILED",
		"DELETE_FAILED",
		"ROLLBACK_COMPLETE",
		"ROLLBACK_FAILED",
		"UPDATE_COMPLETE",
		"UPDATE_FAILED",
		"UPDATE_ROLLBACK_COMPLETE",
		"UPDATE_ROLLBACK_FAILED":
		return true
	default:
		return false
	}
}<commit_msg>Don't abort on rate limit exceeded (closes #14)<commit_after>package stackit

import (
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudformation"
	"time"
	"fmt"
	"strings"
	"github.com/aws/aws-sdk-go/aws/awserr"
)

type TailStackEvent struct {
	cloudformation.StackEvent
}

func describe(cfn *cloudformation.CloudFormation, stackId string) *cloudformation.DescribeStacksOutput {
	resp, err := cfn.DescribeStacks(&cloudformation.DescribeStacksInput{StackName: &stackId})
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "ThrottlingException" {
				return describe(cfn, stackId)
			}
		}
		panic(err)
	}
	return resp
}

func PollStackEvents(sess *session.Session, stackId string, startEventId *string, channel chan<- TailStackEvent) error {
	cfn := cloudformation.New(sess)

	go func() {

		for {
			time.Sleep(3 * time.Second)

			events := []*cloudformation.StackEvent{}

			cfn.DescribeStackEventsPages(&cloudformation.DescribeStackEventsInput{
				StackName: &stackId,
			}, func(page *cloudformation.DescribeStackEventsOutput, lastPage bool) bool {
				for _, event := range page.StackEvents {
					if *event.EventId == *startEventId {
						return false
					}

					events = append(events, event)
				}
				return true
			})

			if len(events) == 0 {
				continue
			}

			startEventId = events[0].EventId
			resp := describe(cfn, stackId)
			status := *resp.Stacks[0].StackStatus

			for ev_i := len(events) - 1; ev_i >= 0; ev_i-- {
				done := IsTerminalStatus(status) && ev_i == 0
				if done {
					close(channel)
					return
				}

				event := events[ev_i]
				tailEvent := TailStackEvent{*event}
				channel <- tailEvent
			}
		}
	}()

	return nil
}

func fixedLengthString(length int, str string) string {
	verb := fmt.Sprintf("%%%d.%ds", length, length)
	return fmt.Sprintf(verb, str)
}

func isBadStatus(status string) bool {
	return strings.HasSuffix(status, "_FAILED")
}

func IsTerminalStatus(status string) bool {
	switch status {
	case
		"CREATE_COMPLETE",
		"DELETE_COMPLETE",
		"CREATE_FAILED",
		"DELETE_FAILED",
		"ROLLBACK_COMPLETE",
		"ROLLBACK_FAILED",
		"UPDATE_COMPLETE",
		"UPDATE_FAILED",
		"UPDATE_ROLLBACK_COMPLETE",
		"UPDATE_ROLLBACK_FAILED":
		return true
	default:
		return false
	}
}
<|endoftext|>


<commit_before>package hypervisor

import (
	"bufio"
	"bytes"
	"encoding/json"
	"io"
	"log"
	"strings"
	"sync"

	"github.com/gfrey/gconn"
	"github.com/gfrey/glog"
	"github.com/pkg/errors"
)

type smartOS struct {
	addr   string
	user   string
	client gconn.Client
}

func NewSmartOSHypervisor(addr string) (Client, error) {
	var err error
	hp := new(smartOS)
	hp.addr = addr
	hp.user = "root"
	hp.client, err = gconn.NewSSHClient(hp.addr, hp.user)
	return hp, err
}

func (hp *smartOS) ConnectVRes(uuid string) (gconn.Client, error) {
	// determine the vm brand
	brand, err := hp.Brand(uuid)
	if err != nil {
		return nil, err
	}

	switch brand {
	case "kvm":
		ip, err := hp.KVMIP(uuid)
		if err != nil {
			return nil, err
		}
		return gconn.NewSSHProxyClient(hp.client, ip, "root")
	case "joyent":
		return gconn.NewWrappedClient(hp.client, "zlogin "+uuid), nil
	default:
		return nil, errors.Errorf("unknown VM brand: %s", brand)
	}
}

func (hp *smartOS) KVMIP(uuid string) (string, error) {
	sess, err := hp.client.NewSession("vmadm get " + uuid + " | json nics[0].ip")
	if err != nil {
		return "", err
	}
	defer sess.Close()

	stdout, err := sess.StdoutPipe()
	if err != nil {
		return "", err
	}

	if err := sess.Start(); err != nil {
		return "", err
	}

	buf := bytes.NewBuffer(nil)
	if _, err := io.Copy(buf, stdout); err != nil {
		return "", err
	}

	if err := sess.Wait(); err != nil {
		return "", err
	}

	return strings.TrimSpace(buf.String()), nil
}

func (hp *smartOS) Brand(uuid string) (string, error) {
	// determine whether the VM in question already exists
	sess, err := hp.client.NewSession("vmadm get " + uuid + " | json brand")
	if err != nil {
		return "", err
	}
	defer sess.Close()

	stdout, err := sess.StdoutPipe()
	if err != nil {
		return "", err
	}

	if err := sess.Start(); err != nil {
		return "", err
	}

	buf := bytes.NewBuffer(nil)
	if _, err := io.Copy(buf, stdout); err != nil {
		return "", err
	}

	if err := sess.Wait(); err != nil {
		return "", err
	}

	return strings.TrimSpace(buf.String()), nil
}

func (hp *smartOS) UUID(alias string) (string, error) {
	// determine whether the VM in question already exists
	sess, err := hp.client.NewSession("vmadm", "list", "-p")
	if err != nil {
		return "", err
	}
	defer sess.Close()

	stdout, err := sess.StdoutPipe()
	if err != nil {
		return "", err
	}
	if err := sess.Start(); err != nil {
		return "", err
	}

	buf := bytes.NewBuffer(nil)
	if _, err := io.Copy(buf, stdout); err != nil {
		return "", err
	}

	if err := sess.Wait(); err != nil {
		return "", err
	}

	sc := bufio.NewScanner(buf)
	for sc.Scan() {
		fields := strings.Split(sc.Text(), ":")
		if fields[4] == alias {
			return fields[0], nil
		}
	}

	if err := sc.Err(); err != nil {
		return "", errors.Wrap(err, "failed to scan output")
	}

	return "", nil
}

func image_uuid(m map[string]interface{}) (string, error) {
	u, found := m["image_uuid"]
	if found {
		uuid, ok := u.(string)
		if !ok {
			return "", errors.Errorf("image_uuid not a string")
		}
		return uuid, nil
	}
	return "", nil
}

func image_uuids(m map[string]interface{}) ([]string, error) {
	uuids := []string{}
	switch u, err := image_uuid(m); {
	case err != nil:
		return nil, err
	case u != "":
		uuids = append(uuids, u)
	}

	disksR, found := m["disks"]
	if found {
		disks, ok := disksR.([]map[string]interface{})
		if !ok {
			return uuids, nil
		}

		for _, disk := range disks {
			switch u, err := image_uuid(disk); {
			case err != nil:
				return nil, err
			case u != "":
				uuids = append(uuids, u)
			}
		}
	}

	return uuids, nil
}

func (hp *smartOS) Create(l glog.Logger, blueprint string) (string, error) {
	m := map[string]interface{}{}
	if err := json.Unmarshal([]byte(blueprint), &m); err != nil {
		return "", errors.Wrap(err, "failed to unmarshal the blueprint")
	}

	l.Printf("updating the image database")
	if err := runCommand(hp.client, "imgadm update"); err != nil {
		return "", err
	}

	imgUUIDs, err := image_uuids(m)
	if err != nil {
		return "", err
	}

	for _, imgUUID := range imgUUIDs {
		l.Printf("importing image %s", imgUUID)
		if err := runCommand(hp.client, "imgadm import -q "+imgUUID); err != nil {
			return "", err
		}
	}

	// determine whether the VM in question already exists
	sess, err := hp.client.NewSession("vmadm", "create")
	if err != nil {
		return "", err
	}
	defer sess.Close()

	wg := new(sync.WaitGroup)
	wg.Add(2)

	outBuf := bytes.NewBuffer(nil)
	stderr, err := sess.StderrPipe()
	if err != nil {
		return "", errors.Wrap(err, "failed to retrieve stderr pipe")
	}
	go func() {
		defer wg.Done()
		_, _ = io.Copy(outBuf, stderr)
	}()

	stdin, err := sess.StdinPipe()
	if err != nil {
		return "", errors.Wrap(err, "failed to retrieve stdin pipe")
	}
	go func() {
		defer wg.Done()
		_, _ = io.WriteString(stdin, blueprint)
		stdin.Close()
	}()

	l.Printf("creating the virtual resource")
	if err := sess.Run(); err != nil {
		log.Printf("failed: %s", outBuf.String())
		return "", err
	}

	wg.Wait()

	output := strings.TrimSpace(outBuf.String())
	expResponsePrefix := "Successfully created VM "
	if !strings.HasPrefix(output, expResponsePrefix) {
		return "", errors.Errorf("wrong response received: %s", output)
	}

	vmID := strings.TrimPrefix(output, expResponsePrefix)

	if autostart, ok := m["autostart"].(bool); ok && !autostart {
		sess, err := hp.client.NewSession("vmadm", "start", vmID)
		if err != nil {
			return "", err
		}
		defer sess.Close()

		l.Printf("starting the virtual resource")
		if err := sess.Run(); err != nil {
			log.Printf("failed to start VM %s", vmID)
			return "", err
		}
	}

	return vmID, nil
}
<commit_msg>Wait for waitgroup in case of error<commit_after>package hypervisor

import (
	"bufio"
	"bytes"
	"encoding/json"
	"io"
	"log"
	"strings"
	"sync"

	"github.com/gfrey/gconn"
	"github.com/gfrey/glog"
	"github.com/pkg/errors"
)

type smartOS struct {
	addr   string
	user   string
	client gconn.Client
}

func NewSmartOSHypervisor(addr string) (Client, error) {
	var err error
	hp := new(smartOS)
	hp.addr = addr
	hp.user = "root"
	hp.client, err = gconn.NewSSHClient(hp.addr, hp.user)
	return hp, err
}

func (hp *smartOS) ConnectVRes(uuid string) (gconn.Client, error) {
	// determine the vm brand
	brand, err := hp.Brand(uuid)
	if err != nil {
		return nil, err
	}

	switch brand {
	case "kvm":
		ip, err := hp.KVMIP(uuid)
		if err != nil {
			return nil, err
		}
		return gconn.NewSSHProxyClient(hp.client, ip, "root")
	case "joyent":
		return gconn.NewWrappedClient(hp.client, "zlogin "+uuid), nil
	default:
		return nil, errors.Errorf("unknown VM brand: %s", brand)
	}
}

func (hp *smartOS) KVMIP(uuid string) (string, error) {
	sess, err := hp.client.NewSession("vmadm get " + uuid + " | json nics[0].ip")
	if err != nil {
		return "", err
	}
	defer sess.Close()

	stdout, err := sess.StdoutPipe()
	if err != nil {
		return "", err
	}

	if err := sess.Start(); err != nil {
		return "", err
	}

	buf := bytes.NewBuffer(nil)
	if _, err := io.Copy(buf, stdout); err != nil {
		return "", err
	}

	if err := sess.Wait(); err != nil {
		return "", err
	}

	return strings.TrimSpace(buf.String()), nil
}

func (hp *smartOS) Brand(uuid string) (string, error) {
	// determine whether the VM in question already exists
	sess, err := hp.client.NewSession("vmadm get " + uuid + " | json brand")
	if err != nil {
		return "", err
	}
	defer sess.Close()

	stdout, err := sess.StdoutPipe()
	if err != nil {
		return "", err
	}

	if err := sess.Start(); err != nil {
		return "", err
	}

	buf := bytes.NewBuffer(nil)
	if _, err := io.Copy(buf, stdout); err != nil {
		return "", err
	}

	if err := sess.Wait(); err != nil {
		return "", err
	}

	return strings.TrimSpace(buf.String()), nil
}

func (hp *smartOS) UUID(alias string) (string, error) {
	// determine whether the VM in question already exists
	sess, err := hp.client.NewSession("vmadm", "list", "-p")
	if err != nil {
		return "", err
	}
	defer sess.Close()

	stdout, err := sess.StdoutPipe()
	if err != nil {
		return "", err
	}
	if err := sess.Start(); err != nil {
		return "", err
	}

	buf := bytes.NewBuffer(nil)
	if _, err := io.Copy(buf, stdout); err != nil {
		return "", err
	}

	if err := sess.Wait(); err != nil {
		return "", err
	}

	sc := bufio.NewScanner(buf)
	for sc.Scan() {
		fields := strings.Split(sc.Text(), ":")
		if fields[4] == alias {
			return fields[0], nil
		}
	}

	if err := sc.Err(); err != nil {
		return "", errors.Wrap(err, "failed to scan output")
	}

	return "", nil
}

func image_uuid(m map[string]interface{}) (string, error) {
	u, found := m["image_uuid"]
	if found {
		uuid, ok := u.(string)
		if !ok {
			return "", errors.Errorf("image_uuid not a string")
		}
		return uuid, nil
	}
	return "", nil
}

func image_uuids(m map[string]interface{}) ([]string, error) {
	uuids := []string{}
	switch u, err := image_uuid(m); {
	case err != nil:
		return nil, err
	case u != "":
		uuids = append(uuids, u)
	}

	disksR, found := m["disks"]
	if found {
		disks, ok := disksR.([]map[string]interface{})
		if !ok {
			return uuids, nil
		}

		for _, disk := range disks {
			switch u, err := image_uuid(disk); {
			case err != nil:
				return nil, err
			case u != "":
				uuids = append(uuids, u)
			}
		}
	}

	return uuids, nil
}

func (hp *smartOS) Create(l glog.Logger, blueprint string) (string, error) {
	m := map[string]interface{}{}
	if err := json.Unmarshal([]byte(blueprint), &m); err != nil {
		return "", errors.Wrap(err, "failed to unmarshal the blueprint")
	}

	l.Printf("updating the image database")
	if err := runCommand(hp.client, "imgadm update"); err != nil {
		return "", err
	}

	imgUUIDs, err := image_uuids(m)
	if err != nil {
		return "", err
	}

	for _, imgUUID := range imgUUIDs {
		l.Printf("importing image %s", imgUUID)
		if err := runCommand(hp.client, "imgadm import -q "+imgUUID); err != nil {
			return "", err
		}
	}

	// determine whether the VM in question already exists
	sess, err := hp.client.NewSession("vmadm", "create")
	if err != nil {
		return "", err
	}
	defer sess.Close()

	wg := new(sync.WaitGroup)
	wg.Add(2)

	outBuf := bytes.NewBuffer(nil)
	stderr, err := sess.StderrPipe()
	if err != nil {
		return "", errors.Wrap(err, "failed to retrieve stderr pipe")
	}
	go func() {
		defer wg.Done()
		_, _ = io.Copy(outBuf, stderr)
	}()

	stdin, err := sess.StdinPipe()
	if err != nil {
		return "", errors.Wrap(err, "failed to retrieve stdin pipe")
	}
	go func() {
		defer wg.Done()
		_, _ = io.WriteString(stdin, blueprint)
		stdin.Close()
	}()

	l.Printf("creating the virtual resource")
	if err := sess.Run(); err != nil {
		wg.Wait()
		log.Printf("failed: %s", outBuf.String())
		return "", err
	}
	wg.Wait()

	output := strings.TrimSpace(outBuf.String())
	expResponsePrefix := "Successfully created VM "
	if !strings.HasPrefix(output, expResponsePrefix) {
		return "", errors.Errorf("wrong response received: %s", output)
	}

	vmID := strings.TrimPrefix(output, expResponsePrefix)

	if autostart, ok := m["autostart"].(bool); ok && !autostart {
		sess, err := hp.client.NewSession("vmadm", "start", vmID)
		if err != nil {
			return "", err
		}
		defer sess.Close()

		l.Printf("starting the virtual resource")
		if err := sess.Run(); err != nil {
			log.Printf("failed to start VM %s", vmID)
			return "", err
		}
	}

	return vmID, nil
}
<|endoftext|>


<commit_before>// Copyright © 2017 Alexander Sosna <alexander@xxor.de>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
package wal

import (
	"errors"
	"io/ioutil"
	"os"
	"regexp"
	"sync"
	"sync/atomic"

	log "github.com/Sirupsen/logrus"
	minio "github.com/minio/minio-go"
)

const (
	// MaxWalSize maximum size a WAL can have
	MaxWalSize = int64(16777216)

	// MinArchiveSize minimal size for files to archive
	MinArchiveSize = int64(100)
	// Regex to represent the ...
	regFullWal    = `^[0-9A-Za-z]{24}`                   // ... name of a WAL file
	regWalWithExt = `^([0-9A-Za-z]{24})(.*)`             // ... name of a WAL file wit extension
	regTimeline   = `^[0-9A-Za-z]{8}`                    // ... timeline of a given WAL file name
	regCounter    = `^([0-9A-Za-z]{8})([0-9A-Za-z]{16})` // ... segment counter in the given timeline
)

var (
	nameFinder      = regexp.MustCompile(regWalWithExt) // *Regexp to extract the name from a WAL file with extension
	fulWalValidator = regexp.MustCompile(regFullWal)    // *Regexp to identify a WAL file
	counterFinder   = regexp.MustCompile(regCounter)    // *Regexp to get the segment counter

)

// Wal is a struct to represent a WAL file
type Wal struct {
	Name      string
	Extension string
	Size      int64
	Archive   *Archive
}

// ImportName imports a WAL file by name (including extension)
func (w *Wal) ImportName(nameWithExtension string) (err error) {
	nameRaw := nameFinder.FindStringSubmatch(nameWithExtension)

	if len(nameRaw) < 2 {
		return errors.New("WAL name does not parse: " + nameWithExtension)
	}

	// 0 contains full string
	// 1 contains name
	// 2 contains extension
	w.Name = string(nameRaw[1])
	w.Extension = string(nameRaw[2])
	return nil
}

// Sane returns if the WAL file seems sane
func (w *Wal) Sane() (sane bool) {
	return w.SaneName()
}

// SaneName returns if the WAL file name seems sane
func (w *Wal) SaneName() (saneName bool) {
	if fulWalValidator.MatchString(w.Name) {
		return true
	}
	return false
}

// Timeline returns the timeline of the WAL file
func (w *Wal) Timeline() (timeline string) {
	timelineFinder := regexp.MustCompile(regTimeline)
	timelineRaw := timelineFinder.Find([]byte(w.Name))

	timeline = string(timelineRaw)
	return timeline
}

// Counter returns the counter / postion in the current timeline
func (w *Wal) Counter() (counter string) {
	counterRaw := counterFinder.FindStringSubmatch(w.Name)

	// 0 contains full string
	// 1 contains timeline
	// 2 contains counter
	counter = string(counterRaw[2])
	return counter
}

// OlderThan returns if *Wal is older than newWal
func (w *Wal) OlderThan(newWal Wal) (isOlderThan bool, err error) {
	if w.Sane() != true {
		return false, errors.New("WAL not sane: " + w.Name)
	}

	if newWal.Sane() != true {
		return false, errors.New("WAL not sane: " + newWal.Name)
	}

	if newWal.Name > w.Name {
		return true, nil
	}
	return false, nil
}

// Delete delets the WAL file
func (w *Wal) Delete() (err error) {
	switch w.Archive.StorageType() {
	case "file":
		err = os.Remove(w.Archive.Path + "/" + w.Name + w.Extension)
		if err != nil {
			log.Warn(err)
		}
		return err
	case "s3":
		err = w.Archive.MinioClient.RemoveObject(w.Archive.Bucket, w.Name+w.Extension)
	default:
		return errors.New("Not supported StorageType: " + w.Archive.StorageType())
	}
	if err != nil {
		log.Warn(err)
	}
	return err
}

// Archive is a struct to represent an WAL archive
type Archive struct {
	walFile     []Wal
	Path        string
	Bucket      string
	MinioClient minio.Client
}

// StorageType returns the type of storage the backup is on
func (w *Archive) StorageType() (storageType string) {
	if w.Path > "" {
		return "file"
	}
	if w.Bucket > "" {
		return "s3"
	}
	// Not defined
	return ""
}

func (w *Archive) DeleteOldWalFromFile(lastWalToKeep Wal) (count int, err error) {
	// WAL files are deleted sequential from file system.
	// Due to the file system architecture parallel delete
	// from the filesystem will not bring great benefit.
	files, _ := ioutil.ReadDir(w.Path)
	for _, f := range files {
		wal := Wal{Archive: w}
		err := wal.ImportName(f.Name())
		if err != nil {
			log.Warn(err)
			continue
		}
		old, err := wal.OlderThan(lastWalToKeep)
		if err != nil {
			log.Warn(err)
			continue
		}
		if old {
			log.Debugf("Older than %s => going to delete: %s", lastWalToKeep.Name, wal.Name)
			err := wal.Delete()
			if err != nil {
				log.Warn(err)
				continue
			}
			count++
		}
	}
	return count, nil
}

func (w *Archive) DeleteOldWalFromS3(lastWalToKeep Wal) (count int, err error) {
	// Object storage has the potential to process operations parallel.
	// Therefor we are going to delete WAL files in parallel.

	// Create a done channel to control 'ListObjects' go routine.
	doneCh := make(chan struct{})
	defer close(doneCh)
	atomicCounter := int32(0)
	var wg sync.WaitGroup

	isRecursive := true
	objectCh := w.MinioClient.ListObjects(w.Bucket, "", isRecursive, doneCh)
	for object := range objectCh {
		go func(object minio.ObjectInfo) {
			wg.Add(1)
			defer wg.Done()
			if object.Err != nil {
				log.Error(object.Err)
			}
			wal := Wal{Archive: w}
			err := wal.ImportName(object.Key)
			if err != nil {
				log.Warn(err)
				return
			}
			old, err := wal.OlderThan(lastWalToKeep)
			if err != nil {
				log.Warn(err)
				return
			}
			if old {
				log.Debugf("Older than %s => going to delete: %s", lastWalToKeep.Name, wal.Name)
				err := wal.Delete()
				if err != nil {
					log.Warn(err)
					return
				}
				// Count up
				atomic.AddInt32(&atomicCounter, 1)
			}
		}(object)
	}
	// Wait for all goroutines to finish
	wg.Wait()
	return int(atomicCounter), nil
}

func (w *Archive) DeleteOldWal(lastWalToKeep Wal) (count int, err error) {
	switch w.StorageType() {
	case "file":
		return w.DeleteOldWalFromFile(lastWalToKeep)
	case "s3":
		return w.DeleteOldWalFromS3(lastWalToKeep)
	default:
		return 0, errors.New("Not supported StorageType: " + w.StorageType())
	}
}
<commit_msg>Add comments to wal functions<commit_after>// Copyright © 2017 Alexander Sosna <alexander@xxor.de>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
package wal

import (
	"errors"
	"io/ioutil"
	"os"
	"regexp"
	"sync"
	"sync/atomic"

	log "github.com/Sirupsen/logrus"
	minio "github.com/minio/minio-go"
)

const (
	// MaxWalSize maximum size a WAL can have
	MaxWalSize = int64(16777216)

	// MinArchiveSize minimal size for files to archive
	MinArchiveSize = int64(100)
	// Regex to represent the ...
	regFullWal    = `^[0-9A-Za-z]{24}`                   // ... name of a WAL file
	regWalWithExt = `^([0-9A-Za-z]{24})(.*)`             // ... name of a WAL file wit extension
	regTimeline   = `^[0-9A-Za-z]{8}`                    // ... timeline of a given WAL file name
	regCounter    = `^([0-9A-Za-z]{8})([0-9A-Za-z]{16})` // ... segment counter in the given timeline
)

var (
	nameFinder      = regexp.MustCompile(regWalWithExt) // *Regexp to extract the name from a WAL file with extension
	fulWalValidator = regexp.MustCompile(regFullWal)    // *Regexp to identify a WAL file
	counterFinder   = regexp.MustCompile(regCounter)    // *Regexp to get the segment counter

)

// Wal is a struct to represent a WAL file
type Wal struct {
	Name      string
	Extension string
	Size      int64
	Archive   *Archive
}

// ImportName imports a WAL file by name (including extension)
func (w *Wal) ImportName(nameWithExtension string) (err error) {
	nameRaw := nameFinder.FindStringSubmatch(nameWithExtension)

	if len(nameRaw) < 2 {
		return errors.New("WAL name does not parse: " + nameWithExtension)
	}

	// 0 contains full string
	// 1 contains name
	// 2 contains extension
	w.Name = string(nameRaw[1])
	w.Extension = string(nameRaw[2])
	return nil
}

// Sane returns if the WAL file seems sane
func (w *Wal) Sane() (sane bool) {
	return w.SaneName()
}

// SaneName returns if the WAL file name seems sane
func (w *Wal) SaneName() (saneName bool) {
	if fulWalValidator.MatchString(w.Name) {
		return true
	}
	return false
}

// Timeline returns the timeline of the WAL file
func (w *Wal) Timeline() (timeline string) {
	timelineFinder := regexp.MustCompile(regTimeline)
	timelineRaw := timelineFinder.Find([]byte(w.Name))

	timeline = string(timelineRaw)
	return timeline
}

// Counter returns the counter / postion in the current timeline
func (w *Wal) Counter() (counter string) {
	counterRaw := counterFinder.FindStringSubmatch(w.Name)

	// 0 contains full string
	// 1 contains timeline
	// 2 contains counter
	counter = string(counterRaw[2])
	return counter
}

// OlderThan returns if *Wal is older than newWal
func (w *Wal) OlderThan(newWal Wal) (isOlderThan bool, err error) {
	if w.Sane() != true {
		return false, errors.New("WAL not sane: " + w.Name)
	}

	if newWal.Sane() != true {
		return false, errors.New("WAL not sane: " + newWal.Name)
	}

	if newWal.Name > w.Name {
		return true, nil
	}
	return false, nil
}

// Delete delets the WAL file
func (w *Wal) Delete() (err error) {
	switch w.Archive.StorageType() {
	case "file":
		err = os.Remove(w.Archive.Path + "/" + w.Name + w.Extension)
		if err != nil {
			log.Warn(err)
		}
		return err
	case "s3":
		err = w.Archive.MinioClient.RemoveObject(w.Archive.Bucket, w.Name+w.Extension)
	default:
		return errors.New("Not supported StorageType: " + w.Archive.StorageType())
	}
	if err != nil {
		log.Warn(err)
	}
	return err
}

// Archive is a struct to represent an WAL archive
type Archive struct {
	walFile     []Wal
	Path        string
	Bucket      string
	MinioClient minio.Client
}

// StorageType returns the type of storage the backup is on
func (w *Archive) StorageType() (storageType string) {
	if w.Path > "" {
		return "file"
	}
	if w.Bucket > "" {
		return "s3"
	}
	// Not defined
	return ""
}

// DeleteOldWalFromFile deletes all WAL files from filesystem that are older than lastWalToKeep
func (w *Archive) DeleteOldWalFromFile(lastWalToKeep Wal) (count int, err error) {
	// WAL files are deleted sequential from file system.
	// Due to the file system architecture parallel delete
	// from the filesystem will not bring great benefit.
	files, _ := ioutil.ReadDir(w.Path)
	for _, f := range files {
		wal := Wal{Archive: w}
		err := wal.ImportName(f.Name())
		if err != nil {
			log.Warn(err)
			continue
		}
		old, err := wal.OlderThan(lastWalToKeep)
		if err != nil {
			log.Warn(err)
			continue
		}
		if old {
			log.Debugf("Older than %s => going to delete: %s", lastWalToKeep.Name, wal.Name)
			err := wal.Delete()
			if err != nil {
				log.Warn(err)
				continue
			}
			count++
		}
	}
	return count, nil
}

// DeleteOldWalFromS3 deletes all WAL files from S3 that are older than lastWalToKeep
func (w *Archive) DeleteOldWalFromS3(lastWalToKeep Wal) (count int, err error) {
	// Object storage has the potential to process operations parallel.
	// Therefor we are going to delete WAL files in parallel.

	// Create a done channel to control 'ListObjects' go routine.
	doneCh := make(chan struct{})
	defer close(doneCh)
	atomicCounter := int32(0)
	var wg sync.WaitGroup

	isRecursive := true
	objectCh := w.MinioClient.ListObjects(w.Bucket, "", isRecursive, doneCh)
	for object := range objectCh {
		go func(object minio.ObjectInfo) {
			wg.Add(1)
			defer wg.Done()
			if object.Err != nil {
				log.Error(object.Err)
			}
			wal := Wal{Archive: w}
			err := wal.ImportName(object.Key)
			if err != nil {
				log.Warn(err)
				return
			}
			old, err := wal.OlderThan(lastWalToKeep)
			if err != nil {
				log.Warn(err)
				return
			}
			if old {
				log.Debugf("Older than %s => going to delete: %s", lastWalToKeep.Name, wal.Name)
				err := wal.Delete()
				if err != nil {
					log.Warn(err)
					return
				}
				// Count up
				atomic.AddInt32(&atomicCounter, 1)
			}
		}(object)
	}
	// Wait for all goroutines to finish
	wg.Wait()
	return int(atomicCounter), nil
}

// DeleteOldWal deletes all WAL files that are older than lastWalToKeep
func (w *Archive) DeleteOldWal(lastWalToKeep Wal) (count int, err error) {
	switch w.StorageType() {
	case "file":
		return w.DeleteOldWalFromFile(lastWalToKeep)
	case "s3":
		return w.DeleteOldWalFromS3(lastWalToKeep)
	default:
		return 0, errors.New("Not supported StorageType: " + w.StorageType())
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package fuchsia

import (
	"github.com/google/syzkaller/prog"
)

func initTarget(target *prog.Target) {
	arch := &arch{}

	target.PageSize = pageSize
	target.DataOffset = dataOffset
	target.MmapSyscall = arch.mmapSyscall
	target.MakeMmap = arch.makeMmap
	target.AnalyzeMmap = arch.analyzeMmap
}

const (
	pageSize   = 4 << 10
	dataOffset = 512 << 20
)

type arch struct {
}

// createMmapCall creates a "normal" mmap call that maps [start, start+npages) page range.
func (arch *arch) makeMmap(start, npages uint64) *prog.Call {
	return nil
}

func (arch *arch) analyzeMmap(c *prog.Call) (start, npages uint64, mapped bool) {
	return
}
<commit_msg>sys/fuchsia: fix build<commit_after>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package fuchsia

import (
	"github.com/google/syzkaller/prog"
)

func initTarget(target *prog.Target) {
	arch := &arch{}

	target.PageSize = pageSize
	target.DataOffset = dataOffset
	target.MmapSyscall = arch.mmapSyscall
	target.MakeMmap = arch.makeMmap
	target.AnalyzeMmap = arch.analyzeMmap
}

const (
	pageSize   = 4 << 10
	dataOffset = 512 << 20
)

type arch struct {
	mmapSyscall *prog.Syscall
}

// createMmapCall creates a "normal" mmap call that maps [start, start+npages) page range.
func (arch *arch) makeMmap(start, npages uint64) *prog.Call {
	return nil
}

func (arch *arch) analyzeMmap(c *prog.Call) (start, npages uint64, mapped bool) {
	return
}
<|endoftext|>