<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package state

import (
	"fmt"
	"io"
	"os"
	"path"
	"time"

	"github.com/juju/errors"
	"github.com/juju/utils/filestorage"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"gopkg.in/mgo.v2/txn"

	"github.com/juju/juju/environs/storage"
	"github.com/juju/juju/state/backups/metadata"
	"github.com/juju/juju/version"
)

/*
Backups are not a part of juju state nor of normal state operations.
However, they certainly are tightly coupled with state (the very
subject of backups).  This puts backups in an odd position,
particularly with regard to the storage of backup metadata and
archives.  As a result, here are a couple concerns worth mentioning.

First, as noted above backup is about state but not a part of state.
So exposing backup-related methods on State would imply the wrong
thing.  Thus the backup functionality here in the state package (not
state/backups) is exposed as functions to which you pass a state
object.

Second, backup creates an archive file containing a dump of state's
mongo DB.  Storing backup metadata/archives in mongo is thus a
somewhat circular proposition that has the potential to cause
problems.  That may need further attention.

Note that state (and juju as a whole) currently does not have a
persistence layer abstraction to facilitate separating different
persistence needs and implementations.  As a consequence, state's
data, whether about how an environment should look or about existing
resources within an environment, is dumped essentially straight into
State's mongo connection.  The code in the state package does not
make any distinction between the two (nor does the package clearly
distinguish between state-related abstractions and state-related
data).

Backup adds yet another category, merely taking advantage of
State's DB.  In the interest of making the distinction clear, the
code that directly interacts with State (and its DB) lives in this
file.  As mentioned previously, the functionality here is exposed
through functions that take State, rather than as methods on State.
Furthermore, the bulk of the backup-related code, which does not need
direct interaction with State, lives in the state/backups package.
*/

// backupMetadataDoc is a mirror of metadata.Metadata, used just for DB storage.
type backupMetadataDoc struct {
	ID             string `bson:"_id"`
	Started        int64  `bson:"started,minsize"`
	Finished       int64  `bson:"finished,minsize"`
	Checksum       string `bson:"checksum"`
	ChecksumFormat string `bson:"checksumformat"`
	Size           int64  `bson:"size,minsize"`
	Stored         bool   `bson:"stored"`
	Notes          string `bson:"notes,omitempty"`

	// origin
	Environment string         `bson:"environment"`
	Machine     string         `bson:"machine"`
	Hostname    string         `bson:"hostname"`
	Version     version.Number `bson:"version"`
}

func (doc *backupMetadataDoc) fileSet() bool {
	if doc.Finished == 0 {
		return false
	}
	if doc.Checksum == "" {
		return false
	}
	if doc.ChecksumFormat == "" {
		return false
	}
	if doc.Size == 0 {
		return false
	}
	return true
}

func (doc *backupMetadataDoc) validate() error {
	if doc.ID == "" {
		return errors.New("missing ID")
	}
	if doc.Started == 0 {
		return errors.New("missing Started")
	}
	if doc.Environment == "" {
		return errors.New("missing Environment")
	}
	if doc.Machine == "" {
		return errors.New("missing Machine")
	}
	if doc.Hostname == "" {
		return errors.New("missing Hostname")
	}
	if doc.Version.Major == 0 {
		return errors.New("missing Version")
	}

	// Check the file-related fields.
	if !doc.fileSet() {
		if doc.Stored {
			return errors.New(`"Stored" flag is unexpectedly true`)
		}
		// Don't check the file-related fields.
		return nil
	}
	if doc.Finished == 0 {
		return errors.New("missing Finished")
	}
	if doc.Checksum == "" {
		return errors.New("missing Checksum")
	}
	if doc.ChecksumFormat == "" {
		return errors.New("missing ChecksumFormat")
	}
	if doc.Size == 0 {
		return errors.New("missing Size")
	}

	return nil
}

// asMetadata returns a new metadata.Metadata based on the backupMetadataDoc.
func (doc *backupMetadataDoc) asMetadata() *metadata.Metadata {
	// Create a new Metadata.
	origin := metadata.ExistingOrigin(
		doc.Environment,
		doc.Machine,
		doc.Hostname,
		doc.Version,
	)

	started := time.Unix(doc.Started, 0).UTC()
	meta := metadata.NewMetadata(
		*origin,
		doc.Notes,
		&started,
	)

	// The ID is already set.
	meta.SetID(doc.ID)

	// Exit early if file-related fields not set.
	if !doc.fileSet() {
		return meta
	}

	// Set the file-related fields.
	var finished *time.Time
	if doc.Finished != 0 {
		val := time.Unix(doc.Finished, 0).UTC()
		finished = &val
	}
	err := meta.Finish(doc.Size, doc.Checksum, doc.ChecksumFormat, finished)
	if err != nil {
		// The doc should have already been validated.  An error here
		// indicates that Metadata changed and backupMetadataDoc did not
		// accommodate the change.  Thus an error here indicates a
		// developer "error".  A caller should not need to worry about
		// that case so we panic instead of passing the error out.
		panic(fmt.Sprintf("unexpectedly invalid metadata doc: %v", err))
	}
	if doc.Stored {
		meta.SetStored()
	}
	return meta
}

// updateFromMetadata copies the corresponding data from the backup
// Metadata into the backupMetadataDoc.
func (doc *backupMetadataDoc) updateFromMetadata(metadata *metadata.Metadata) {
	finished := metadata.Finished()
	// Ignore metadata.ID.
	doc.Started = metadata.Started().Unix()
	if finished != nil {
		doc.Finished = finished.Unix()
	}
	doc.Checksum = metadata.Checksum()
	doc.ChecksumFormat = metadata.ChecksumFormat()
	doc.Size = metadata.Size()
	doc.Stored = metadata.Stored()
	doc.Notes = metadata.Notes()

	origin := metadata.Origin()
	doc.Environment = origin.Environment()
	doc.Machine = origin.Machine()
	doc.Hostname = origin.Hostname()
	doc.Version = origin.Version()
}

//---------------------------
// DB operations

// getBackupMetadata returns the backup metadata associated with "id".
// If "id" does not match any stored records, an error satisfying
// juju/errors.IsNotFound() is returned.
func getBackupMetadata(st *State, id string) (*metadata.Metadata, error) {
	collection, closer := st.getCollection(backupsMetaC)
	defer closer()

	var doc backupMetadataDoc
	// There can only be one!
	err := collection.FindId(id).One(&doc)
	if err == mgo.ErrNotFound {
		return nil, errors.NotFoundf("backup metadata %q", id)
	} else if err != nil {
		return nil, errors.Annotate(err, "error getting backup metadata")
	}

	if err := doc.validate(); err != nil {
		return nil, errors.Trace(err)
	}
	return doc.asMetadata(), nil
}

// newBackupID returns a new ID for a state backup.  The format is the
// UTC timestamp from the metadata followed by the environment ID:
// "YYYYMMDD-hhmmss.<env ID>".  This makes the ID a little more human-
// consumable (in contrast to a plain UUID string).  Ideally we would
// use some form of environment name rather than the UUID, but for now
// the raw env ID is sufficient.
func newBackupID(metadata *metadata.Metadata) string {
	rawts := metadata.Started()
	Y, M, D := rawts.Date()
	h, m, s := rawts.Clock()
	timestamp := fmt.Sprintf("%04d%02d%02d-%02d%02d%02d", Y, M, D, h, m, s)
	origin := metadata.Origin()
	env := origin.Environment()
	return timestamp + "." + env
}

// addBackupMetadata stores metadata for a backup where it can be
// accessed later.  It returns a new ID that is associated with the
// backup.  If the provided metadata already has an ID set, it is
// ignored.
func addBackupMetadata(st *State, metadata *metadata.Metadata) (string, error) {
	// We use our own mongo _id value since the auto-generated one from
	// mongo may contain sensitive data (see bson.ObjectID).
	id := newBackupID(metadata)
	return id, addBackupMetadataID(st, metadata, id)
}

func addBackupMetadataID(st *State, metadata *metadata.Metadata, id string) error {
	var doc backupMetadataDoc
	doc.updateFromMetadata(metadata)
	doc.ID = id
	if err := doc.validate(); err != nil {
		return errors.Trace(err)
	}

	ops := []txn.Op{{
		C:      backupsMetaC,
		Id:     doc.ID,
		Assert: txn.DocMissing,
		Insert: doc,
	}}
	if err := st.runTransaction(ops); err != nil {
		if err == txn.ErrAborted {
			return errors.AlreadyExistsf("backup metadata %q", doc.ID)
		}
		return errors.Annotate(err, "error running transaction")
	}

	return nil
}

// setBackupStored updates the backup metadata associated with "id"
// to indicate that a backup archive has been stored.  If "id" does
// not match any stored records, an error satisfying
// juju/errors.IsNotFound() is returned.
func setBackupStored(st *State, id string) error {
	ops := []txn.Op{{
		C:      backupsMetaC,
		Id:     id,
		Assert: txn.DocExists,
		Update: bson.D{{"$set", bson.D{
			{"stored", true},
		}}},
	}}
	if err := st.runTransaction(ops); err != nil {
		if err == txn.ErrAborted {
			return errors.NotFoundf(id)
		}
		return errors.Annotate(err, "error running transaction")
	}
	return nil
}

//---------------------------
// metadata storage

// NewBackupsOrigin returns a snapshot of where backup was run.  That
// snapshot is a new backup Origin value, for use in a backup's
// metadata.  Every value except for the machine name is populated
// either from juju state or some other implicit mechanism.
func NewBackupsOrigin(st *State, machine string) *metadata.Origin {
	// hostname could be derived from the environment...
	hostname, err := os.Hostname()
	if err != nil {
		// If os.Hostname() is not working, something is woefully wrong.
		// Run for the hills.
		panic(fmt.Sprintf("could not get hostname (system unstable?): %v", err))
	}
	origin := metadata.NewOrigin(
		st.EnvironTag().Id(),
		machine,
		hostname,
	)
	return origin
}

// Ensure we satisfy the interface.
var _ = filestorage.MetadataStorage((*backupMetadataStorage)(nil))

type backupMetadataStorage struct {
	state *State
}

func newBackupMetadataStorage(st *State) filestorage.MetadataStorage {
	stor := backupMetadataStorage{
		state: st,
	}
	return &stor
}

func (s *backupMetadataStorage) AddDoc(doc interface{}) (string, error) {
	meta, ok := doc.(*metadata.Metadata)
	if !ok {
		return "", errors.Errorf("doc must be of type state.backups.metadata.Metadata")
	}
	return addBackupMetadata(s.state, meta)
}

func (s *backupMetadataStorage) Doc(id string) (interface{}, error) {
	return s.Metadata(id)
}

func (s *backupMetadataStorage) Metadata(id string) (filestorage.Metadata, error) {
	metadata, err := getBackupMetadata(s.state, id)
	if err != nil {
		return nil, errors.Trace(err)
	}
	return metadata, nil
}

func (s *backupMetadataStorage) ListDocs() ([]interface{}, error) {
	metas, err := s.ListMetadata()
	if err != nil {
		return nil, errors.Trace(err)
	}
	docs := []interface{}{}
	for _, meta := range metas {
		docs = append(docs, meta)
	}
	return docs, nil
}

func (s *backupMetadataStorage) ListMetadata() ([]filestorage.Metadata, error) {
	// This will be implemented when backups needs this functionality.
	// For now the method is stubbed out for the same of the
	// MetadataStorage interface.
	return nil, errors.NotImplementedf("ListMetadata")
}

func (s *backupMetadataStorage) RemoveDoc(id string) error {
	// This will be implemented when backups needs this functionality.
	// For now the method is stubbed out for the same of the
	// MetadataStorage interface.
	return errors.NotImplementedf("RemoveDoc")
}

func (s *backupMetadataStorage) New() filestorage.Metadata {
	origin := NewBackupsOrigin(s.state, "")
	return metadata.NewMetadata(*origin, "", nil)
}

func (s *backupMetadataStorage) SetStored(meta filestorage.Metadata) error {
	err := setBackupStored(s.state, meta.ID())
	if err != nil {
		return errors.Trace(err)
	}
	meta.SetStored()
	return nil
}

//---------------------------
// raw file storage

const backupStorageRoot = "/"

// Ensure we satisfy the interface.
var _ filestorage.RawFileStorage = (*envFileStorage)(nil)

type envFileStorage struct {
	envStor storage.Storage
	root    string
}

func newBackupFileStorage(envStor storage.Storage, root string) filestorage.RawFileStorage {
	// Due to circular imports we cannot simply get the storage from
	// State using environs.GetStorage().
	stor := envFileStorage{
		envStor: envStor,
		root:    root,
	}
	return &stor
}

func (s *envFileStorage) path(id string) string {
	// Use of path.Join instead of filepath.Join is intentional - this
	// is an environment storage path not a filesystem path.
	return path.Join(s.root, id)
}

func (s *envFileStorage) File(id string) (io.ReadCloser, error) {
	return s.envStor.Get(s.path(id))
}

func (s *envFileStorage) AddFile(id string, file io.Reader, size int64) error {
	return s.envStor.Put(s.path(id), file, size)
}

func (s *envFileStorage) RemoveFile(id string) error {
	return s.envStor.Remove(s.path(id))
}

//---------------------------
// backup storage

// NewBackupsStorage returns a new FileStorage to use for storing backup
// archives (and metadata).
func NewBackupsStorage(st *State, envStor storage.Storage) filestorage.FileStorage {
	files := newBackupFileStorage(envStor, backupStorageRoot)
	docs := newBackupMetadataStorage(st)
	return filestorage.NewFileStorage(docs, files)
}
<commit_msg>Do not prepend "/" to the storage name (httpstorage cannot deal with it).<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package state

import (
	"fmt"
	"io"
	"os"
	"path"
	"time"

	"github.com/juju/errors"
	"github.com/juju/utils/filestorage"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"gopkg.in/mgo.v2/txn"

	"github.com/juju/juju/environs/storage"
	"github.com/juju/juju/state/backups/metadata"
	"github.com/juju/juju/version"
)

/*
Backups are not a part of juju state nor of normal state operations.
However, they certainly are tightly coupled with state (the very
subject of backups).  This puts backups in an odd position,
particularly with regard to the storage of backup metadata and
archives.  As a result, here are a couple concerns worth mentioning.

First, as noted above backup is about state but not a part of state.
So exposing backup-related methods on State would imply the wrong
thing.  Thus the backup functionality here in the state package (not
state/backups) is exposed as functions to which you pass a state
object.

Second, backup creates an archive file containing a dump of state's
mongo DB.  Storing backup metadata/archives in mongo is thus a
somewhat circular proposition that has the potential to cause
problems.  That may need further attention.

Note that state (and juju as a whole) currently does not have a
persistence layer abstraction to facilitate separating different
persistence needs and implementations.  As a consequence, state's
data, whether about how an environment should look or about existing
resources within an environment, is dumped essentially straight into
State's mongo connection.  The code in the state package does not
make any distinction between the two (nor does the package clearly
distinguish between state-related abstractions and state-related
data).

Backup adds yet another category, merely taking advantage of
State's DB.  In the interest of making the distinction clear, the
code that directly interacts with State (and its DB) lives in this
file.  As mentioned previously, the functionality here is exposed
through functions that take State, rather than as methods on State.
Furthermore, the bulk of the backup-related code, which does not need
direct interaction with State, lives in the state/backups package.
*/

// backupMetadataDoc is a mirror of metadata.Metadata, used just for DB storage.
type backupMetadataDoc struct {
	ID             string `bson:"_id"`
	Started        int64  `bson:"started,minsize"`
	Finished       int64  `bson:"finished,minsize"`
	Checksum       string `bson:"checksum"`
	ChecksumFormat string `bson:"checksumformat"`
	Size           int64  `bson:"size,minsize"`
	Stored         bool   `bson:"stored"`
	Notes          string `bson:"notes,omitempty"`

	// origin
	Environment string         `bson:"environment"`
	Machine     string         `bson:"machine"`
	Hostname    string         `bson:"hostname"`
	Version     version.Number `bson:"version"`
}

func (doc *backupMetadataDoc) fileSet() bool {
	if doc.Finished == 0 {
		return false
	}
	if doc.Checksum == "" {
		return false
	}
	if doc.ChecksumFormat == "" {
		return false
	}
	if doc.Size == 0 {
		return false
	}
	return true
}

func (doc *backupMetadataDoc) validate() error {
	if doc.ID == "" {
		return errors.New("missing ID")
	}
	if doc.Started == 0 {
		return errors.New("missing Started")
	}
	if doc.Environment == "" {
		return errors.New("missing Environment")
	}
	if doc.Machine == "" {
		return errors.New("missing Machine")
	}
	if doc.Hostname == "" {
		return errors.New("missing Hostname")
	}
	if doc.Version.Major == 0 {
		return errors.New("missing Version")
	}

	// Check the file-related fields.
	if !doc.fileSet() {
		if doc.Stored {
			return errors.New(`"Stored" flag is unexpectedly true`)
		}
		// Don't check the file-related fields.
		return nil
	}
	if doc.Finished == 0 {
		return errors.New("missing Finished")
	}
	if doc.Checksum == "" {
		return errors.New("missing Checksum")
	}
	if doc.ChecksumFormat == "" {
		return errors.New("missing ChecksumFormat")
	}
	if doc.Size == 0 {
		return errors.New("missing Size")
	}

	return nil
}

// asMetadata returns a new metadata.Metadata based on the backupMetadataDoc.
func (doc *backupMetadataDoc) asMetadata() *metadata.Metadata {
	// Create a new Metadata.
	origin := metadata.ExistingOrigin(
		doc.Environment,
		doc.Machine,
		doc.Hostname,
		doc.Version,
	)

	started := time.Unix(doc.Started, 0).UTC()
	meta := metadata.NewMetadata(
		*origin,
		doc.Notes,
		&started,
	)

	// The ID is already set.
	meta.SetID(doc.ID)

	// Exit early if file-related fields not set.
	if !doc.fileSet() {
		return meta
	}

	// Set the file-related fields.
	var finished *time.Time
	if doc.Finished != 0 {
		val := time.Unix(doc.Finished, 0).UTC()
		finished = &val
	}
	err := meta.Finish(doc.Size, doc.Checksum, doc.ChecksumFormat, finished)
	if err != nil {
		// The doc should have already been validated.  An error here
		// indicates that Metadata changed and backupMetadataDoc did not
		// accommodate the change.  Thus an error here indicates a
		// developer "error".  A caller should not need to worry about
		// that case so we panic instead of passing the error out.
		panic(fmt.Sprintf("unexpectedly invalid metadata doc: %v", err))
	}
	if doc.Stored {
		meta.SetStored()
	}
	return meta
}

// updateFromMetadata copies the corresponding data from the backup
// Metadata into the backupMetadataDoc.
func (doc *backupMetadataDoc) updateFromMetadata(metadata *metadata.Metadata) {
	finished := metadata.Finished()
	// Ignore metadata.ID.
	doc.Started = metadata.Started().Unix()
	if finished != nil {
		doc.Finished = finished.Unix()
	}
	doc.Checksum = metadata.Checksum()
	doc.ChecksumFormat = metadata.ChecksumFormat()
	doc.Size = metadata.Size()
	doc.Stored = metadata.Stored()
	doc.Notes = metadata.Notes()

	origin := metadata.Origin()
	doc.Environment = origin.Environment()
	doc.Machine = origin.Machine()
	doc.Hostname = origin.Hostname()
	doc.Version = origin.Version()
}

//---------------------------
// DB operations

// getBackupMetadata returns the backup metadata associated with "id".
// If "id" does not match any stored records, an error satisfying
// juju/errors.IsNotFound() is returned.
func getBackupMetadata(st *State, id string) (*metadata.Metadata, error) {
	collection, closer := st.getCollection(backupsMetaC)
	defer closer()

	var doc backupMetadataDoc
	// There can only be one!
	err := collection.FindId(id).One(&doc)
	if err == mgo.ErrNotFound {
		return nil, errors.NotFoundf("backup metadata %q", id)
	} else if err != nil {
		return nil, errors.Annotate(err, "error getting backup metadata")
	}

	if err := doc.validate(); err != nil {
		return nil, errors.Trace(err)
	}
	return doc.asMetadata(), nil
}

// newBackupID returns a new ID for a state backup.  The format is the
// UTC timestamp from the metadata followed by the environment ID:
// "YYYYMMDD-hhmmss.<env ID>".  This makes the ID a little more human-
// consumable (in contrast to a plain UUID string).  Ideally we would
// use some form of environment name rather than the UUID, but for now
// the raw env ID is sufficient.
func newBackupID(metadata *metadata.Metadata) string {
	rawts := metadata.Started()
	Y, M, D := rawts.Date()
	h, m, s := rawts.Clock()
	timestamp := fmt.Sprintf("%04d%02d%02d-%02d%02d%02d", Y, M, D, h, m, s)
	origin := metadata.Origin()
	env := origin.Environment()
	return timestamp + "." + env
}

// addBackupMetadata stores metadata for a backup where it can be
// accessed later.  It returns a new ID that is associated with the
// backup.  If the provided metadata already has an ID set, it is
// ignored.
func addBackupMetadata(st *State, metadata *metadata.Metadata) (string, error) {
	// We use our own mongo _id value since the auto-generated one from
	// mongo may contain sensitive data (see bson.ObjectID).
	id := newBackupID(metadata)
	return id, addBackupMetadataID(st, metadata, id)
}

func addBackupMetadataID(st *State, metadata *metadata.Metadata, id string) error {
	var doc backupMetadataDoc
	doc.updateFromMetadata(metadata)
	doc.ID = id
	if err := doc.validate(); err != nil {
		return errors.Trace(err)
	}

	ops := []txn.Op{{
		C:      backupsMetaC,
		Id:     doc.ID,
		Assert: txn.DocMissing,
		Insert: doc,
	}}
	if err := st.runTransaction(ops); err != nil {
		if err == txn.ErrAborted {
			return errors.AlreadyExistsf("backup metadata %q", doc.ID)
		}
		return errors.Annotate(err, "error running transaction")
	}

	return nil
}

// setBackupStored updates the backup metadata associated with "id"
// to indicate that a backup archive has been stored.  If "id" does
// not match any stored records, an error satisfying
// juju/errors.IsNotFound() is returned.
func setBackupStored(st *State, id string) error {
	ops := []txn.Op{{
		C:      backupsMetaC,
		Id:     id,
		Assert: txn.DocExists,
		Update: bson.D{{"$set", bson.D{
			{"stored", true},
		}}},
	}}
	if err := st.runTransaction(ops); err != nil {
		if err == txn.ErrAborted {
			return errors.NotFoundf(id)
		}
		return errors.Annotate(err, "error running transaction")
	}
	return nil
}

//---------------------------
// metadata storage

// NewBackupsOrigin returns a snapshot of where backup was run.  That
// snapshot is a new backup Origin value, for use in a backup's
// metadata.  Every value except for the machine name is populated
// either from juju state or some other implicit mechanism.
func NewBackupsOrigin(st *State, machine string) *metadata.Origin {
	// hostname could be derived from the environment...
	hostname, err := os.Hostname()
	if err != nil {
		// If os.Hostname() is not working, something is woefully wrong.
		// Run for the hills.
		panic(fmt.Sprintf("could not get hostname (system unstable?): %v", err))
	}
	origin := metadata.NewOrigin(
		st.EnvironTag().Id(),
		machine,
		hostname,
	)
	return origin
}

// Ensure we satisfy the interface.
var _ = filestorage.MetadataStorage((*backupMetadataStorage)(nil))

type backupMetadataStorage struct {
	state *State
}

func newBackupMetadataStorage(st *State) filestorage.MetadataStorage {
	stor := backupMetadataStorage{
		state: st,
	}
	return &stor
}

func (s *backupMetadataStorage) AddDoc(doc interface{}) (string, error) {
	meta, ok := doc.(*metadata.Metadata)
	if !ok {
		return "", errors.Errorf("doc must be of type state.backups.metadata.Metadata")
	}
	return addBackupMetadata(s.state, meta)
}

func (s *backupMetadataStorage) Doc(id string) (interface{}, error) {
	return s.Metadata(id)
}

func (s *backupMetadataStorage) Metadata(id string) (filestorage.Metadata, error) {
	metadata, err := getBackupMetadata(s.state, id)
	if err != nil {
		return nil, errors.Trace(err)
	}
	return metadata, nil
}

func (s *backupMetadataStorage) ListDocs() ([]interface{}, error) {
	metas, err := s.ListMetadata()
	if err != nil {
		return nil, errors.Trace(err)
	}
	docs := []interface{}{}
	for _, meta := range metas {
		docs = append(docs, meta)
	}
	return docs, nil
}

func (s *backupMetadataStorage) ListMetadata() ([]filestorage.Metadata, error) {
	// This will be implemented when backups needs this functionality.
	// For now the method is stubbed out for the same of the
	// MetadataStorage interface.
	return nil, errors.NotImplementedf("ListMetadata")
}

func (s *backupMetadataStorage) RemoveDoc(id string) error {
	// This will be implemented when backups needs this functionality.
	// For now the method is stubbed out for the same of the
	// MetadataStorage interface.
	return errors.NotImplementedf("RemoveDoc")
}

func (s *backupMetadataStorage) New() filestorage.Metadata {
	origin := NewBackupsOrigin(s.state, "")
	return metadata.NewMetadata(*origin, "", nil)
}

func (s *backupMetadataStorage) SetStored(meta filestorage.Metadata) error {
	err := setBackupStored(s.state, meta.ID())
	if err != nil {
		return errors.Trace(err)
	}
	meta.SetStored()
	return nil
}

//---------------------------
// raw file storage

const backupStorageRoot = ""

// Ensure we satisfy the interface.
var _ filestorage.RawFileStorage = (*envFileStorage)(nil)

type envFileStorage struct {
	envStor storage.Storage
	root    string
}

func newBackupFileStorage(envStor storage.Storage, root string) filestorage.RawFileStorage {
	// Due to circular imports we cannot simply get the storage from
	// State using environs.GetStorage().
	stor := envFileStorage{
		envStor: envStor,
		root:    root,
	}
	return &stor
}

func (s *envFileStorage) path(id string) string {
	// Use of path.Join instead of filepath.Join is intentional - this
	// is an environment storage path not a filesystem path.
	return path.Join(s.root, id)
}

func (s *envFileStorage) File(id string) (io.ReadCloser, error) {
	return s.envStor.Get(s.path(id))
}

func (s *envFileStorage) AddFile(id string, file io.Reader, size int64) error {
	return s.envStor.Put(s.path(id), file, size)
}

func (s *envFileStorage) RemoveFile(id string) error {
	return s.envStor.Remove(s.path(id))
}

//---------------------------
// backup storage

// NewBackupsStorage returns a new FileStorage to use for storing backup
// archives (and metadata).
func NewBackupsStorage(st *State, envStor storage.Storage) filestorage.FileStorage {
	files := newBackupFileStorage(envStor, backupStorageRoot)
	docs := newBackupMetadataStorage(st)
	return filestorage.NewFileStorage(docs, files)
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package state

import (
	"github.com/juju/errors"
	"gopkg.in/mgo.v2/bson"
	"gopkg.in/mgo.v2/txn"
)

// SubnetInfo describes a single subnet.
type SubnetInfo struct {
	// ProviderId is a provider-specific network id. This may be empty.
	ProviderId string

	// CIDR of the network, in 123.45.67.89/24 format.
	CIDR string

	// VLANTag needs to be between 1 and 4094 for VLANs and 0 for normal
	// networks. It's defined by IEEE 802.1Q standard.
	VLANTag int

	// AllocatableIPHigh and Low describe the allocatable portion of the
	// subnet. The remainder, if any, is reserved by the provider.
	// Either both of these must be set or neither, if they're empty it
	// means that none of the subnet is allocatable.
	AllocatableIPHigh string
	AllocatableIPLow  string

	AvailabilityZone string
}

type Subnet struct {
	st  *State
	doc subnetDoc
}

type subnetDoc struct {
	DocID             string `bson:"_id"`
	EnvUUID           string `bson:"env-uuid"`
	Life              Life
	ProviderId        string
	CIDR              string
	AllocatableIPHigh string
	AllocatableIPLow  string

	VLANTag          int    `bson:",omitempty"`
	AvailabilityZone string `bson:",omitempty"`
}

// Life returns whether the subnet is Alive, Dying or Dead.
func (s *Subnet) Life() Life {
	return s.doc.Life
}

func (s *Subnet) EnsureDead() (err error) {
	defer errors.DeferredAnnotatef(&err, "cannot destroy subnet %v", s)
	if s.doc.Life == Dead {
		return nil
	}

	defer func() {
		if err == nil {
			s.doc.Life = Dead
		}
	}()

	ops := []txn.Op{{
		C:  subnetsC,
		Id: s.doc.DocID,
		Update: bson.D{{"$set", bson.D{
			{"Life", Dead},
		}}},
	}}
	return s.st.runTransaction(ops)
}

func (s *Subnet) Remove() (err error) {
	defer errors.DeferredAnnotatef(&err, "cannot destroy subnet %v", s)
	if s.doc.Life != Dead {
		return errors.New("subnet is not dead")
	}
	ops := []txn.Op{{
		C:      subnetsC,
		Id:     s.doc.DocID,
		Remove: true,
	}}
	return s.st.runTransaction(ops)
}

// ProviderId returns the provider-specific id of the subnet.
func (s *Subnet) ProviderId() string {
	return s.doc.ProviderId
}

// CIDR returns the subnet CIDR (e.g. 192.168.50.0/24).
func (s *Subnet) CIDR() string {
	return s.doc.CIDR
}

// VLANTag returns the subnet VLAN tag. It's a number between 1 and
// 4094 for VLANs and 0 if the network is not a VLAN.
func (s *Subnet) VLANTag() int {
	return s.doc.VLANTag
}

// AllocatableIPLow returns the lowest allocatable IP address in the subnet
func (s *Subnet) AllocatableIPLow() string {
	return s.doc.AllocatableIPLow
}

// AllocatableIPHigh returns the hightest allocatable IP address in the subnet.
func (s *Subnet) AllocatableIPHigh() string {
	return s.doc.AllocatableIPHigh
}

// AvailabilityZone returns the availability zone of the subnet. If the subnet
// is not associated with an availability zone it will be the empty string.
func (s *Subnet) AvailabilityZone() string {
	return s.doc.AvailabilityZone
}
<commit_msg>Minor tweaks<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package state

import (
	"github.com/juju/errors"
	"gopkg.in/mgo.v2/bson"
	"gopkg.in/mgo.v2/txn"
)

// SubnetInfo describes a single subnet.
type SubnetInfo struct {
	// ProviderId is a provider-specific network id. This may be empty.
	ProviderId string

	// CIDR of the network, in 123.45.67.89/24 format.
	CIDR string

	// VLANTag needs to be between 1 and 4094 for VLANs and 0 for normal
	// networks. It's defined by IEEE 802.1Q standard.
	VLANTag int

	// AllocatableIPHigh and Low describe the allocatable portion of the
	// subnet. The remainder, if any, is reserved by the provider.
	// Either both of these must be set or neither, if they're empty it
	// means that none of the subnet is allocatable. If present they must
	// be valid IP addresses within the subnet CIDR.
	AllocatableIPHigh string
	AllocatableIPLow  string

	// AvailabilityZone describes which availability zone this subnet is in. It can
	// be empty if the provider does not support availability zones.
	AvailabilityZone string
}

type Subnet struct {
	st  *State
	doc subnetDoc
}

type subnetDoc struct {
	DocID             string `bson:"_id"`
	EnvUUID           string `bson:"env-uuid"`
	Life              Life
	ProviderId        string `bson:",omitempty"`
	CIDR              string
	AllocatableIPHigh string `bson:",omitempty"`
	AllocatableIPLow  string `bson:",omitempty"`

	VLANTag          int    `bson:",omitempty"`
	AvailabilityZone string `bson:",omitempty"`
}

// Life returns whether the subnet is Alive, Dying or Dead.
func (s *Subnet) Life() Life {
	return s.doc.Life
}

func (s *Subnet) EnsureDead() (err error) {
	defer errors.DeferredAnnotatef(&err, "cannot destroy subnet %v", s)
	if s.doc.Life == Dead {
		return nil
	}

	defer func() {
		if err == nil {
			s.doc.Life = Dead
		}
	}()

	ops := []txn.Op{{
		C:  subnetsC,
		Id: s.doc.DocID,
		Update: bson.D{{"$set", bson.D{
			{"Life", Dead},
		}}},
	}}
	return s.st.runTransaction(ops)
}

func (s *Subnet) Remove() (err error) {
	defer errors.DeferredAnnotatef(&err, "cannot destroy subnet %v", s)
	if s.doc.Life != Dead {
		return errors.New("subnet is not dead")
	}
	ops := []txn.Op{{
		C:      subnetsC,
		Id:     s.doc.DocID,
		Remove: true,
	}}
	return s.st.runTransaction(ops)
}

// ProviderId returns the provider-specific id of the subnet.
func (s *Subnet) ProviderId() string {
	return s.doc.ProviderId
}

// CIDR returns the subnet CIDR (e.g. 192.168.50.0/24).
func (s *Subnet) CIDR() string {
	return s.doc.CIDR
}

// VLANTag returns the subnet VLAN tag. It's a number between 1 and
// 4094 for VLANs and 0 if the network is not a VLAN.
func (s *Subnet) VLANTag() int {
	return s.doc.VLANTag
}

// AllocatableIPLow returns the lowest allocatable IP address in the subnet
func (s *Subnet) AllocatableIPLow() string {
	return s.doc.AllocatableIPLow
}

// AllocatableIPHigh returns the hightest allocatable IP address in the subnet.
func (s *Subnet) AllocatableIPHigh() string {
	return s.doc.AllocatableIPHigh
}

// AvailabilityZone returns the availability zone of the subnet. If the subnet
// is not associated with an availability zone it will be the empty string.
func (s *Subnet) AvailabilityZone() string {
	return s.doc.AvailabilityZone
}
<|endoftext|>


<commit_before>package main

import "github.com/limetext/qml-go"

type linesList struct {
	qml.ItemModel
	qml.ItemModelDefaultImpl
	lines    []*lineStruct
	internal qml.ItemModelInternal
}

var _ qml.ItemModelImpl = &linesList{}

func NewLinesList(engine *qml.Engine, parent qml.Object) *linesList {
	ll := &linesList{}
	ll.ItemModel, ll.internal = qml.NewItemModel(engine, parent, ll)
	return ll
}

func (l *linesList) RowCount(parent qml.ModelIndex) int {
	return len(l.lines)
}

func (l *linesList) Data(index qml.ModelIndex, role qml.Role) interface{} {
	if index.IsValid() {
		return l.lines[index.Row()]
	}
	return nil
}

func (l *linesList) Index(row int, column int, parent qml.ModelIndex) qml.ModelIndex {
	if !parent.IsValid() && column == 0 && row >= 0 && row < len(l.lines) {
		return l.internal.CreateIndex(row, column, 0)
	}
	return nil
}

func (l *linesList) get(i int) *lineStruct {
	return l.lines[i]
}

func (l *linesList) len() int {
	return len(l.lines)
}

func (l *linesList) insertLines(pos int, newLines []*lineStruct) {
	add := len(newLines)
	nn := make([]*lineStruct, len(l.lines)+add)
	copy(nn, l.lines[:pos])
	copy(nn[pos+add:], l.lines[pos:])
	copy(nn[pos:pos+add], newLines)

	qml.RunMain(func() {
		l.internal.BeginInsertRows(nil, pos, pos+add-1)
		l.lines = nn
		l.internal.EndInsertRows()
	})
}

func (l *linesList) deleteLines(pos int, count int) {

	qml.RunMain(func() {
		l.internal.BeginRemoveRows(nil, pos, pos+count-1)
		copy(l.lines[pos:], l.lines[pos+count:])
		l.lines = l.lines[:len(l.lines)-count]
		l.internal.EndRemoveRows()
	})
}

// This allows us to trigger a qml.Changed on a specific line in the view so
// that only it is re-rendered by qml
type lineStruct struct {
	Text     string
	Chunks   []lineChunk
	Width    int
	Measured bool
}

func (l *lineStruct) ChunksLen() int {
	if l == nil {
		return 0
	}
	return len(l.Chunks)
}

func (l *lineStruct) Chunk(i int) *lineChunk {
	return &l.Chunks[i]
}

type lineChunk struct {
	Text       string
	Background string
	Foreground string
	SkipWidth  int
	Width      int
	Measured   bool
}
<commit_msg>added missing license to lineslist.go<commit_after>// Copyright 2016 The lime Authors.
// Use of this source code is governed by a 2-clause
// BSD-style license that can be found in the LICENSE file.

package main

import "github.com/limetext/qml-go"

type linesList struct {
	qml.ItemModel
	qml.ItemModelDefaultImpl
	lines    []*lineStruct
	internal qml.ItemModelInternal
}

var _ qml.ItemModelImpl = &linesList{}

func NewLinesList(engine *qml.Engine, parent qml.Object) *linesList {
	ll := &linesList{}
	ll.ItemModel, ll.internal = qml.NewItemModel(engine, parent, ll)
	return ll
}

func (l *linesList) RowCount(parent qml.ModelIndex) int {
	return len(l.lines)
}

func (l *linesList) Data(index qml.ModelIndex, role qml.Role) interface{} {
	if index.IsValid() {
		return l.lines[index.Row()]
	}
	return nil
}

func (l *linesList) Index(row int, column int, parent qml.ModelIndex) qml.ModelIndex {
	if !parent.IsValid() && column == 0 && row >= 0 && row < len(l.lines) {
		return l.internal.CreateIndex(row, column, 0)
	}
	return nil
}

func (l *linesList) get(i int) *lineStruct {
	return l.lines[i]
}

func (l *linesList) len() int {
	return len(l.lines)
}

func (l *linesList) insertLines(pos int, newLines []*lineStruct) {
	add := len(newLines)
	nn := make([]*lineStruct, len(l.lines)+add)
	copy(nn, l.lines[:pos])
	copy(nn[pos+add:], l.lines[pos:])
	copy(nn[pos:pos+add], newLines)

	qml.RunMain(func() {
		l.internal.BeginInsertRows(nil, pos, pos+add-1)
		l.lines = nn
		l.internal.EndInsertRows()
	})
}

func (l *linesList) deleteLines(pos int, count int) {

	qml.RunMain(func() {
		l.internal.BeginRemoveRows(nil, pos, pos+count-1)
		copy(l.lines[pos:], l.lines[pos+count:])
		l.lines = l.lines[:len(l.lines)-count]
		l.internal.EndRemoveRows()
	})
}

// This allows us to trigger a qml.Changed on a specific line in the view so
// that only it is re-rendered by qml
type lineStruct struct {
	Text     string
	Chunks   []lineChunk
	Width    int
	Measured bool
}

func (l *lineStruct) ChunksLen() int {
	if l == nil {
		return 0
	}
	return len(l.Chunks)
}

func (l *lineStruct) Chunk(i int) *lineChunk {
	return &l.Chunks[i]
}

type lineChunk struct {
	Text       string
	Background string
	Foreground string
	SkipWidth  int
	Width      int
	Measured   bool
}
<|endoftext|>


<commit_before>package psdock

import (
	"os"
	"os/signal"
	"syscall"
)

//ManageSignals awaits for incoming signals and triggers a http request when one
//is received. Signals listened to are SIGINT, SIGQUIT, SIGTERM, SIGHUP, SIGALRM and SIGPIPE
func ManageSignals(p *Process) {
	signalChannel := make(chan os.Signal)
	signal.Notify(signalChannel, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGHUP, syscall.SIGALRM, syscall.SIGPIPE)
	_ = <-signalChannel

	//Terminate the process and notify
	p.eofChannel <- true
	_ = p.Terminate(5)
}
<commit_msg>Now redirects almost all the signals received to the child<commit_after>package psdock

import (
	"os"
	"os/signal"
	"syscall"
)

//ManageSignals awaits for incoming signals and triggers a http request when one
//is received. Signals listened to are SIGINT, SIGQUIT, SIGTERM, SIGHUP, SIGALRM and SIGPIPE
func ManageSignals(p *Process) {
	termSignalChannel := make(chan os.Signal)
	otherSignalChannel := make(chan os.Signal)
	signal.Notify(termSignalChannel, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGHUP, syscall.SIGALRM, syscall.SIGPIPE)
	otherSignals := []os.Signal{syscall.SIGABRT, syscall.SIGBUS, syscall.SIGCHLD, syscall.SIGCONT, syscall.SIGEMT, syscall.SIGFPE,
		syscall.SIGILL, syscall.SIGIO, syscall.SIGIOT, syscall.SIGPROF, syscall.SIGSEGV, syscall.SIGSTOP, syscall.SIGSYS,
		syscall.SIGTRAP, syscall.SIGTSTP, syscall.SIGTTIN, syscall.SIGTTOU, syscall.SIGURG, syscall.SIGUSR1,
		syscall.SIGUSR2, syscall.SIGVTALRM, syscall.SIGWINCH, syscall.SIGXCPU, syscall.SIGXFSZ}

	signal.Notify(otherSignalChannel, otherSignals...)
	go func() {
		for {
			s := <-otherSignalChannel
			p.Cmd.Process.Signal(s)
		}
	}()
	_ = <-termSignalChannel

	//Terminate the process and notify
	p.eofChannel <- true
	_ = p.Terminate(5)
}
<|endoftext|>


<commit_before>package scenario

import (
	"errors"
	"io"
	"strconv"

	"encoding/json"

	"fmt"
	"os"

	"net/url"

	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/catatsuy/isucon6-final/bench/fails"
	"github.com/catatsuy/isucon6-final/bench/http"
	"github.com/catatsuy/isucon6-final/bench/score"
	"github.com/catatsuy/isucon6-final/bench/session"
	"github.com/catatsuy/isucon6-final/seed"
)

var (
	IndexGetScore     int64 = 2
	SVGGetScore       int64 = 1
	CreateRoomScore   int64 = 20
	CreateStrokeScore int64 = 20
)

func makeDocument(r io.Reader) (*goquery.Document, error) {
	doc, err := goquery.NewDocumentFromReader(r)
	if err != nil {
		return nil, errors.New(fails.Add("ページのHTMLがパースできませんでした"))
	}
	return doc, nil
}

func extractImages(doc *goquery.Document) []string {
	imageUrls := []string{}

	doc.Find("img").Each(func(_ int, selection *goquery.Selection) {
		if url, ok := selection.Attr("src"); ok {
			imageUrls = append(imageUrls, url)
		}
	})

	return imageUrls
}

func extractCsrfToken(doc *goquery.Document) string {
	var token string

	doc.Find("html").Each(func(_ int, selection *goquery.Selection) {
		if t, ok := selection.Attr("data-csrf-token"); ok {
			token = t
		}
	})

	return token
}

func loadImages(s *session.Session, images []string) error {
	var lastErr error
	for _, image := range images {
		err := s.Get(image, func(status int, body io.Reader) error {
			if status != 200 {
				return errors.New(fails.Add("GET /, ステータスが200ではありません: " + strconv.Itoa(status)))
			}
			score.Increment(SVGGetScore)
			return nil
		})
		if err != nil {
			lastErr = err
		}
	}
	return lastErr

	// TODO: 画像を並列リクエストするようにしてみたが、 connection reset by peer というエラーが出るので直列に戻した
	// もしかすると s.Transport.MaxIdleConnsPerHost ずつ処理するといけるのかも
	//errs := make(chan error, len(images))
	//for _, image := range images {
	//	go func(image string) {
	//		err := s.Get(image, func(status int, body io.Reader) error {
	//			if status != 200 {
	//				return errors.New(fails.Add("GET /, ステータスが200ではありません: " + strconv.Itoa(status)))
	//			}
	//			score.Increment(SVGGetScore)
	//			return nil
	//		})
	//		errs <- err
	//	}(image)
	//}
	//var lastErr error
	//for i := 0; i < len(images); i++ {
	//	err := <-errs
	//	if err != nil {
	//		lastErr = err
	//	}
	//}
	//return lastErr
}

// トップページと画像に負荷をかける
func LoadIndexPage(s *session.Session) {
	var token string
	var images []string

	err := s.Get("/", func(status int, body io.Reader) error {
		if status != 200 {
			return errors.New(fails.Add("GET /, ステータスが200ではありません: " + strconv.Itoa(status)))
		}
		doc, err := makeDocument(body)
		if err != nil {
			return err
		}

		token = extractCsrfToken(doc)

		if token == "" {
			return errors.New(fails.Add("GET /, csrf_tokenが取得できませんでした"))
		}

		images = extractImages(doc)
		if len(images) < 100 {
			return errors.New(fails.Add("GET /, 画像の枚数が少なすぎます"))
		}

		score.Increment(IndexGetScore)

		return nil
	})
	if err != nil {
		return
	}

	err = loadImages(s, images)
	if err != nil {
		return
	}
}

// ページ内のCSRFトークンが毎回変わっていることをチェック
func CheckCSRFTokenRefreshed(s *session.Session) {
	var token string

	err := s.Get("/", func(status int, body io.Reader) error {
		if status != 200 {
			return errors.New(fails.Add("GET /, ステータスが200ではありません: " + strconv.Itoa(status)))
		}
		doc, err := makeDocument(body)
		if err != nil {
			return err
		}

		token = extractCsrfToken(doc)

		if token == "" {
			return errors.New(fails.Add("GET /, csrf_tokenが取得できませんでした"))
		}

		score.Increment(IndexGetScore)

		return nil
	})
	if err != nil {
		return
	}

	_ = s.Get("/", func(status int, body io.Reader) error {
		if status != 200 {
			return errors.New(fails.Add("GET /, ステータスが200ではありません: " + strconv.Itoa(status)))
		}
		doc, err := makeDocument(body)
		if err != nil {
			return err
		}

		t := extractCsrfToken(doc)

		if t == token {
			return errors.New(fails.Add("GET /, csrf_tokenが使いまわされています"))
		}

		score.Increment(IndexGetScore)

		return nil
	})
}

// 一人がroomを作る→大勢がそのroomをwatchする
func MatsuriRoom(s *session.Session, aud string) {
	var token string

	err := s.Get("/", func(status int, body io.Reader) error {
		if status != 200 {
			return errors.New(fails.Add("GET /, ステータスが200ではありません: " + strconv.Itoa(status)))
		}
		doc, err := makeDocument(body)
		if err != nil {
			return err
		}

		token = extractCsrfToken(doc)

		if token == "" {
			return errors.New(fails.Add("GET /, csrf_tokenが取得できませんでした"))
		}

		score.Increment(IndexGetScore)

		return nil
	})
	if err != nil {
		return
	}

	postBody, _ := json.Marshal(struct {
		Name         string `json:"name"`
		CanvasWidth  int    `json:"canvas_width"`
		CanvasHeight int    `json:"canvas_height"`
	}{
		Name:         "ひたすら椅子を描く部屋",
		CanvasWidth:  1024,
		CanvasHeight: 768,
	})

	headers := map[string]string{
		"Content-Type": "application/json",
		"x-csrf-token": token,
	}

	var RoomID int64

	err = s.Post("/api/rooms", postBody, headers, func(status int, body io.Reader) error {
		if status != 200 {
			return errors.New(fails.Add("GET /api/rooms, ステータスが200ではありません: " + strconv.Itoa(status)))
		}

		var res Response
		err := json.NewDecoder(body).Decode(&res)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			return errors.New(fails.Add("GET /api/rooms, レスポンス内容が正しくありません"))
		}
		fmt.Println(res)
		if res.Room == nil || res.Room.ID <= 0 {
			return errors.New(fails.Add("GET /api/rooms, レスポンス内容が正しくありません"))
		}
		RoomID = res.Room.ID

		score.Increment(CreateRoomScore)

		return nil
	})

	if err != nil {
		return
	}

	// TODO: strokeを順次postしていく
	seedStroke := seed.GetStroke("main001")

	type StrokeTime struct {
		ID           int64
		PostTime     time.Time
		ResponseTime time.Time
	}
	strokeTimes := make([]StrokeTime, 0)

	go func() {
		for _, str := range seedStroke {
			// TODO: 指定時間以上たったら終わる

			postBody, _ := json.Marshal(struct {
				RoomID int64 `json:"room_id"`
				seed.Stroke
			}{
				RoomID: RoomID,
				Stroke: str,
			})

			postTime := time.Now()

			err := s.Post("/api/strokes/rooms/"+strconv.FormatInt(RoomID, 10), postBody, headers, func(status int, body io.Reader) error {
				responseTime := time.Now()

				if status != 200 {
					return errors.New(fails.Add("POST /api/strokes/rooms/" + strconv.FormatInt(RoomID, 10) + ", ステータスが200ではありません: " + strconv.Itoa(status)))
				}

				var res Response
				err := json.NewDecoder(body).Decode(&res)
				if err != nil || res.Room == nil || res.Room.ID <= 0 {
					return errors.New(fails.Add("GET /api/strokes/rooms/" + strconv.FormatInt(RoomID, 10) + ", レスポンス内容が正しくありません"))
				}

				timeTaken := responseTime.Sub(postTime).Seconds()
				if timeTaken < 1 {
					score.Increment(CreateStrokeScore * 2)
				} else if timeTaken < 3 {
					score.Increment(CreateStrokeScore)
				}

				strokeTimes = append(strokeTimes, StrokeTime{
					ID:           res.Stroke.ID,
					PostTime:     postTime,
					ResponseTime: responseTime,
				})

				return nil
			})
			if err != nil {
				break
			}
		}
	}()

	fmt.Println(strokeTimes)

	resp, err := http.Get(aud + "?scheme=" + url.QueryEscape(s.Scheme) + "&host=" + url.QueryEscape(s.Host))
	if err != nil {
		fmt.Fprintln(os.Stderr, "failed to call audience "+aud+" :"+err.Error())
	}
	defer resp.Body.Close()
	// TODO: audienceのresponse処理
}
<commit_msg>Change import according the new seed directory<commit_after>package scenario

import (
	"errors"
	"io"
	"strconv"

	"encoding/json"

	"fmt"
	"os"

	"net/url"

	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/catatsuy/isucon6-final/bench/fails"
	"github.com/catatsuy/isucon6-final/bench/http"
	"github.com/catatsuy/isucon6-final/bench/score"
	"github.com/catatsuy/isucon6-final/bench/seed"
	"github.com/catatsuy/isucon6-final/bench/session"
)

var (
	IndexGetScore     int64 = 2
	SVGGetScore       int64 = 1
	CreateRoomScore   int64 = 20
	CreateStrokeScore int64 = 20
)

func makeDocument(r io.Reader) (*goquery.Document, error) {
	doc, err := goquery.NewDocumentFromReader(r)
	if err != nil {
		return nil, errors.New(fails.Add("ページのHTMLがパースできませんでした"))
	}
	return doc, nil
}

func extractImages(doc *goquery.Document) []string {
	imageUrls := []string{}

	doc.Find("img").Each(func(_ int, selection *goquery.Selection) {
		if url, ok := selection.Attr("src"); ok {
			imageUrls = append(imageUrls, url)
		}
	})

	return imageUrls
}

func extractCsrfToken(doc *goquery.Document) string {
	var token string

	doc.Find("html").Each(func(_ int, selection *goquery.Selection) {
		if t, ok := selection.Attr("data-csrf-token"); ok {
			token = t
		}
	})

	return token
}

func loadImages(s *session.Session, images []string) error {
	var lastErr error
	for _, image := range images {
		err := s.Get(image, func(status int, body io.Reader) error {
			if status != 200 {
				return errors.New(fails.Add("GET /, ステータスが200ではありません: " + strconv.Itoa(status)))
			}
			score.Increment(SVGGetScore)
			return nil
		})
		if err != nil {
			lastErr = err
		}
	}
	return lastErr

	// TODO: 画像を並列リクエストするようにしてみたが、 connection reset by peer というエラーが出るので直列に戻した
	// もしかすると s.Transport.MaxIdleConnsPerHost ずつ処理するといけるのかも
	//errs := make(chan error, len(images))
	//for _, image := range images {
	//	go func(image string) {
	//		err := s.Get(image, func(status int, body io.Reader) error {
	//			if status != 200 {
	//				return errors.New(fails.Add("GET /, ステータスが200ではありません: " + strconv.Itoa(status)))
	//			}
	//			score.Increment(SVGGetScore)
	//			return nil
	//		})
	//		errs <- err
	//	}(image)
	//}
	//var lastErr error
	//for i := 0; i < len(images); i++ {
	//	err := <-errs
	//	if err != nil {
	//		lastErr = err
	//	}
	//}
	//return lastErr
}

// トップページと画像に負荷をかける
func LoadIndexPage(s *session.Session) {
	var token string
	var images []string

	err := s.Get("/", func(status int, body io.Reader) error {
		if status != 200 {
			return errors.New(fails.Add("GET /, ステータスが200ではありません: " + strconv.Itoa(status)))
		}
		doc, err := makeDocument(body)
		if err != nil {
			return err
		}

		token = extractCsrfToken(doc)

		if token == "" {
			return errors.New(fails.Add("GET /, csrf_tokenが取得できませんでした"))
		}

		images = extractImages(doc)
		if len(images) < 100 {
			return errors.New(fails.Add("GET /, 画像の枚数が少なすぎます"))
		}

		score.Increment(IndexGetScore)

		return nil
	})
	if err != nil {
		return
	}

	err = loadImages(s, images)
	if err != nil {
		return
	}
}

// ページ内のCSRFトークンが毎回変わっていることをチェック
func CheckCSRFTokenRefreshed(s *session.Session) {
	var token string

	err := s.Get("/", func(status int, body io.Reader) error {
		if status != 200 {
			return errors.New(fails.Add("GET /, ステータスが200ではありません: " + strconv.Itoa(status)))
		}
		doc, err := makeDocument(body)
		if err != nil {
			return err
		}

		token = extractCsrfToken(doc)

		if token == "" {
			return errors.New(fails.Add("GET /, csrf_tokenが取得できませんでした"))
		}

		score.Increment(IndexGetScore)

		return nil
	})
	if err != nil {
		return
	}

	_ = s.Get("/", func(status int, body io.Reader) error {
		if status != 200 {
			return errors.New(fails.Add("GET /, ステータスが200ではありません: " + strconv.Itoa(status)))
		}
		doc, err := makeDocument(body)
		if err != nil {
			return err
		}

		t := extractCsrfToken(doc)

		if t == token {
			return errors.New(fails.Add("GET /, csrf_tokenが使いまわされています"))
		}

		score.Increment(IndexGetScore)

		return nil
	})
}

// 一人がroomを作る→大勢がそのroomをwatchする
func MatsuriRoom(s *session.Session, aud string) {
	var token string

	err := s.Get("/", func(status int, body io.Reader) error {
		if status != 200 {
			return errors.New(fails.Add("GET /, ステータスが200ではありません: " + strconv.Itoa(status)))
		}
		doc, err := makeDocument(body)
		if err != nil {
			return err
		}

		token = extractCsrfToken(doc)

		if token == "" {
			return errors.New(fails.Add("GET /, csrf_tokenが取得できませんでした"))
		}

		score.Increment(IndexGetScore)

		return nil
	})
	if err != nil {
		return
	}

	postBody, _ := json.Marshal(struct {
		Name         string `json:"name"`
		CanvasWidth  int    `json:"canvas_width"`
		CanvasHeight int    `json:"canvas_height"`
	}{
		Name:         "ひたすら椅子を描く部屋",
		CanvasWidth:  1024,
		CanvasHeight: 768,
	})

	headers := map[string]string{
		"Content-Type": "application/json",
		"x-csrf-token": token,
	}

	var RoomID int64

	err = s.Post("/api/rooms", postBody, headers, func(status int, body io.Reader) error {
		if status != 200 {
			return errors.New(fails.Add("GET /api/rooms, ステータスが200ではありません: " + strconv.Itoa(status)))
		}

		var res Response
		err := json.NewDecoder(body).Decode(&res)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			return errors.New(fails.Add("GET /api/rooms, レスポンス内容が正しくありません"))
		}
		fmt.Println(res)
		if res.Room == nil || res.Room.ID <= 0 {
			return errors.New(fails.Add("GET /api/rooms, レスポンス内容が正しくありません"))
		}
		RoomID = res.Room.ID

		score.Increment(CreateRoomScore)

		return nil
	})

	if err != nil {
		return
	}

	// TODO: strokeを順次postしていく
	seedStroke := seed.GetStroke("main001")

	type StrokeTime struct {
		ID           int64
		PostTime     time.Time
		ResponseTime time.Time
	}
	strokeTimes := make([]StrokeTime, 0)

	go func() {
		for _, str := range seedStroke {
			// TODO: 指定時間以上たったら終わる

			postBody, _ := json.Marshal(struct {
				RoomID int64 `json:"room_id"`
				seed.Stroke
			}{
				RoomID: RoomID,
				Stroke: str,
			})

			postTime := time.Now()

			err := s.Post("/api/strokes/rooms/"+strconv.FormatInt(RoomID, 10), postBody, headers, func(status int, body io.Reader) error {
				responseTime := time.Now()

				if status != 200 {
					return errors.New(fails.Add("POST /api/strokes/rooms/" + strconv.FormatInt(RoomID, 10) + ", ステータスが200ではありません: " + strconv.Itoa(status)))
				}

				var res Response
				err := json.NewDecoder(body).Decode(&res)
				if err != nil || res.Room == nil || res.Room.ID <= 0 {
					return errors.New(fails.Add("GET /api/strokes/rooms/" + strconv.FormatInt(RoomID, 10) + ", レスポンス内容が正しくありません"))
				}

				timeTaken := responseTime.Sub(postTime).Seconds()
				if timeTaken < 1 {
					score.Increment(CreateStrokeScore * 2)
				} else if timeTaken < 3 {
					score.Increment(CreateStrokeScore)
				}

				strokeTimes = append(strokeTimes, StrokeTime{
					ID:           res.Stroke.ID,
					PostTime:     postTime,
					ResponseTime: responseTime,
				})

				return nil
			})
			if err != nil {
				break
			}
		}
	}()

	fmt.Println(strokeTimes)

	resp, err := http.Get(aud + "?scheme=" + url.QueryEscape(s.Scheme) + "&host=" + url.QueryEscape(s.Host))
	if err != nil {
		fmt.Fprintln(os.Stderr, "failed to call audience "+aud+" :"+err.Error())
	}
	defer resp.Body.Close()
	// TODO: audienceのresponse処理
}
<|endoftext|>


<commit_before>package sender

import (
	"bytes"
	"encoding/json"
	"net/http"
	"net/http/httputil"
	"time"

	log "github.com/Sirupsen/logrus"

	"github.com/Cepave/open-falcon-backend/modules/transfer/g"
	"github.com/Cepave/open-falcon-backend/modules/transfer/proc"
	cmodel "github.com/open-falcon/common/model"
	nsema "github.com/toolkits/concurrent/semaphore"
	"github.com/toolkits/container/list"
)

// send
const (
	DefaultSendTaskSleepInterval = time.Millisecond * 50 //默认睡眠间隔为50ms
)

// TODO 添加对发送任务的控制,比如stop等
func startSendTasks() {
	cfg := g.Config()
	// init semaphore
	judgeConcurrent := cfg.Judge.MaxConns
	graphConcurrent := cfg.Graph.MaxConns
	tsdbConcurrent := cfg.Tsdb.MaxConns
	influxdbConcurrent := cfg.Influxdb.MaxIdle

	if tsdbConcurrent < 1 {
		tsdbConcurrent = 1
	}

	if judgeConcurrent < 1 {
		judgeConcurrent = 1
	}

	if graphConcurrent < 1 {
		graphConcurrent = 1
	}
	if influxdbConcurrent < 1 {
		influxdbConcurrent = 1
	}

	// init send go-routines
	for node, _ := range cfg.Judge.Cluster {
		queue := JudgeQueues[node]
		go forward2JudgeTask(queue, node, judgeConcurrent)
	}

	for node, nitem := range cfg.Graph.ClusterList {
		for _, addr := range nitem.Addrs {
			queue := GraphQueues[node+addr]
			go forward2GraphTask(queue, node, addr, graphConcurrent)
		}
	}

	if cfg.Tsdb.Enabled {
		go forward2TsdbTask(tsdbConcurrent)
	}

	go forward2InfluxdbTask(InfluxdbQueues["default"], influxdbConcurrent)

	if cfg.NqmRest.Enabled {
		go forward2NqmIcmpTask()
		go forward2NqmTcpTask()
		go forward2NqmTcpconnTask()
	}

	if cfg.Staging.Enabled {
		go forward2StagingTask()
	}
}

// Judge定时任务, 将 Judge发送缓存中的数据 通过rpc连接池 发送到Judge
func forward2JudgeTask(Q *list.SafeListLimited, node string, concurrent int) {
	batch := g.Config().Judge.Batch // 一次发送,最多batch条数据
	addr := g.Config().Judge.Cluster[node]
	sema := nsema.NewSemaphore(concurrent)

	for {
		items := Q.PopBackBy(batch)
		count := len(items)
		if count == 0 {
			time.Sleep(DefaultSendTaskSleepInterval)
			continue
		}

		judgeItems := make([]*cmodel.JudgeItem, count)
		for i := 0; i < count; i++ {
			judgeItems[i] = items[i].(*cmodel.JudgeItem)
		}

		//	同步Call + 有限并发 进行发送
		sema.Acquire()
		go func(addr string, judgeItems []*cmodel.JudgeItem, count int) {
			defer sema.Release()

			resp := &cmodel.SimpleRpcResponse{}
			var err error
			sendOk := false
			for i := 0; i < 3; i++ { //最多重试3次
				err = JudgeConnPools.Call(addr, "Judge.Send", judgeItems, resp)
				if err == nil {
					sendOk = true
					break
				}
				time.Sleep(time.Millisecond * 10)
			}

			// statistics
			if !sendOk {
				log.Printf("send judge %s:%s fail: %v", node, addr, err)
				proc.SendToJudgeFailCnt.IncrBy(int64(count))
			} else {
				proc.SendToJudgeCnt.IncrBy(int64(count))
			}
		}(addr, judgeItems, count)
	}
}

// Graph定时任务, 将 Graph发送缓存中的数据 通过rpc连接池 发送到Graph
func forward2GraphTask(Q *list.SafeListLimited, node string, addr string, concurrent int) {
	batch := g.Config().Graph.Batch // 一次发送,最多batch条数据
	sema := nsema.NewSemaphore(concurrent)

	for {
		items := Q.PopBackBy(batch)
		count := len(items)
		if count == 0 {
			time.Sleep(DefaultSendTaskSleepInterval)
			continue
		}

		graphItems := make([]*cmodel.GraphItem, count)
		for i := 0; i < count; i++ {
			graphItems[i] = items[i].(*cmodel.GraphItem)
		}

		sema.Acquire()
		go func(addr string, graphItems []*cmodel.GraphItem, count int) {
			defer sema.Release()

			resp := &cmodel.SimpleRpcResponse{}
			var err error
			sendOk := false
			for i := 0; i < 3; i++ { //最多重试3次
				err = GraphConnPools.Call(addr, "Graph.Send", graphItems, resp)
				if err == nil {
					sendOk = true
					break
				}
				time.Sleep(time.Millisecond * 10)
			}

			// statistics
			if !sendOk {
				log.Printf("send to graph %s:%s fail: %v", node, addr, err)
				proc.SendToGraphFailCnt.IncrBy(int64(count))
			} else {
				proc.SendToGraphCnt.IncrBy(int64(count))
			}
		}(addr, graphItems, count)
	}
}

// Tsdb定时任务, 将数据通过api发送到tsdb
func forward2TsdbTask(concurrent int) {
	batch := g.Config().Tsdb.Batch // 一次发送,最多batch条数据
	retry := g.Config().Tsdb.MaxRetry
	sema := nsema.NewSemaphore(concurrent)

	for {
		items := TsdbQueue.PopBackBy(batch)
		if len(items) == 0 {
			time.Sleep(DefaultSendTaskSleepInterval)
			continue
		}
		//  同步Call + 有限并发 进行发送
		sema.Acquire()
		go func(itemList []interface{}) {
			defer sema.Release()

			var tsdbBuffer bytes.Buffer
			for i := 0; i < len(itemList); i++ {
				tsdbItem := itemList[i].(*cmodel.TsdbItem)
				tsdbBuffer.WriteString(tsdbItem.TsdbString())
				tsdbBuffer.WriteString("\n")
			}

			var err error
			for i := 0; i < retry; i++ {
				err = TsdbConnPoolHelper.Send(tsdbBuffer.Bytes())
				if err == nil {
					proc.SendToTsdbCnt.IncrBy(int64(len(itemList)))
					break
				}
				time.Sleep(100 * time.Millisecond)
			}

			if err != nil {
				proc.SendToTsdbFailCnt.IncrBy(int64(len(itemList)))
				log.Println(err)
				return
			}
		}(items)
	}
}

// Influxdb schedule
func forward2InfluxdbTask(Q *list.SafeListLimited, concurrent int) {
	cfg := g.Config().Influxdb
	batch := cfg.Batch // 一次发送,最多batch条数据
	conn, err := parseDSN(cfg.Address)
	if err != nil {
		log.Print("syntax of influxdb address is wrong")
		return
	}
	addr := conn.Address

	sema := nsema.NewSemaphore(concurrent)

	for {
		items := Q.PopBackBy(batch)
		count := len(items)
		if count == 0 {
			time.Sleep(DefaultSendTaskSleepInterval)
			continue
		}

		influxdbItems := make([]*cmodel.JudgeItem, count)
		for i := 0; i < count; i++ {
			influxdbItems[i] = items[i].(*cmodel.JudgeItem)
		}

		//	同步Call + 有限并发 进行发送
		sema.Acquire()
		go func(addr string, influxdbItems []*cmodel.JudgeItem, count int) {
			defer sema.Release()

			var err error
			sendOk := false
			for i := 0; i < 3; i++ { //最多重试3次
				err = InfluxdbConnPools.Call(addr, influxdbItems)
				if err == nil {
					sendOk = true
					break
				}
				time.Sleep(time.Millisecond * 10)
			}

			// statistics
			if !sendOk {
				log.Printf("send influxdb %s fail: %v", addr, err)
				proc.SendToInfluxdbFailCnt.IncrBy(int64(count))
			} else {
				proc.SendToInfluxdbCnt.IncrBy(int64(count))
			}
		}(addr, influxdbItems, count)
	}
}

func forward2NqmIcmpTask() {
	batch := g.Config().NqmRest.Batch // 一次发送,最多batch条数据
	concurrent := g.Config().NqmRest.MaxConns
	sema := nsema.NewSemaphore(concurrent)

	for {
		items := NqmIcmpQueue.PopBackBy(batch)
		if len(items) == 0 {
			time.Sleep(DefaultSendTaskSleepInterval)
			continue
		}

		nqmIcmpItems := make([]*nqmPingItem, len(items))
		for i, v := range items {
			nqmIcmpItems[i] = v.(*nqmPingItem)
		}

		//  同步Call + 有限并发 进行发送
		sema.Acquire()
		go func(itemList []*nqmPingItem) {
			defer sema.Release()

			for _, v := range itemList {
				//resp := &cmodel.SimpleRpcResponse{}
				var err error
				sendOk := false
				for i := 0; i < 3; i++ { //最多重试3次
					//err = NqmRpcConnPoolHelper.Call("NqmEndpoint.AddIcmp", []*nqmRpcItem{v}, resp)
					vv := []nqmPingItem{*v}
					paramsBody, err := json.Marshal(vv[0])
					postReq, err := http.NewRequest("POST", g.Config().NqmRest.Fping, bytes.NewBuffer(paramsBody))

					postReq.Header.Set("Content-Type", "application/json; charset=UTF-8")
					postReq.Header.Set("Connection", "close")

					b, e := httputil.DumpRequest(postReq, true)
					if e != nil {
						log.Errorln("fuckya:", e)
					}
					log.Debugf("[ Cassandra ] Body: %q", b)

					httpClient := &http.Client{}
					postResp, err := httpClient.Do(postReq)
					if err != nil {
						log.Errorln("[ Cassandra ] Error on push:", err)
						time.Sleep(time.Millisecond * 10)
						continue
					}
					defer postResp.Body.Close()
					b, e = httputil.DumpResponse(postResp, true)
					if e != nil {
						log.Errorln("fuckya:", e)
					}
					log.Debugf("Resp Body: %q", b)
					sendOk = true
					break
				}

				// statistics
				if !sendOk {
					log.Printf("send to /nqm/icmp fail: %v", err)
					proc.SendToNqmIcmpFailCnt.IncrBy(1)
				} else {
					proc.SendToNqmIcmpCnt.IncrBy(1)
				}
			}
		}(nqmIcmpItems)
	}
}

func forward2NqmTcpTask() {
	batch := g.Config().NqmRest.Batch // 一次发送,最多batch条数据
	concurrent := g.Config().NqmRest.MaxConns
	sema := nsema.NewSemaphore(concurrent)

	for {
		items := NqmTcpQueue.PopBackBy(batch)
		if len(items) == 0 {
			time.Sleep(DefaultSendTaskSleepInterval)
			continue
		}

		nqmTcpItems := make([]*nqmPingItem, len(items))
		for i, v := range items {
			nqmTcpItems[i] = v.(*nqmPingItem)
		}

		//  同步Call + 有限并发 进行发送
		sema.Acquire()
		go func(itemList []*nqmPingItem) {
			defer sema.Release()

			for _, v := range itemList {
				//resp := &cmodel.SimpleRpcResponse{}
				var err error
				sendOk := false
				for i := 0; i < 3; i++ { //最多重试3次
					//err = NqmRpcConnPoolHelper.Call("NqmEndpoint.AddIcmp", []*nqmRpcItem{v}, resp)
					vv := []nqmPingItem{*v}
					paramsBody, err := json.Marshal(vv[0])
					postReq, err := http.NewRequest("POST", g.Config().NqmRest.Tcpping, bytes.NewBuffer(paramsBody))

					postReq.Header.Set("Content-Type", "application/json; charset=UTF-8")
					postReq.Header.Set("Connection", "close")

					b, e := httputil.DumpRequest(postReq, true)
					if e != nil {
						log.Errorln("fuckya:", e)
					}
					log.Debugf("[ Cassandra ] Body: %q", b)

					httpClient := &http.Client{}
					postResp, err := httpClient.Do(postReq)
					if err != nil {
						log.Errorln("[ Cassandra ] Error on push:", err)
						time.Sleep(time.Millisecond * 10)
						continue
					}
					defer postResp.Body.Close()
					b, e = httputil.DumpResponse(postResp, true)
					if e != nil {
						log.Errorln("fuckya:", e)
					}
					log.Debugf("Resp Body: %q", b)
					sendOk = true
					break
				}

				// statistics
				if !sendOk {
					log.Printf("send to /nqm/tcp fail: %v", err)
					proc.SendToNqmTcpFailCnt.IncrBy(1)
				} else {
					proc.SendToNqmTcpCnt.IncrBy(1)
				}
			}
		}(nqmTcpItems)
	}
}

func forward2NqmTcpconnTask() {
	batch := g.Config().NqmRest.Batch // 一次发送,最多batch条数据
	concurrent := g.Config().NqmRest.MaxConns
	sema := nsema.NewSemaphore(concurrent)

	for {
		items := NqmTcpconnQueue.PopBackBy(batch)
		if len(items) == 0 {
			time.Sleep(DefaultSendTaskSleepInterval)
			continue
		}

		nqmTcpconnItems := make([]*nqmConnItem, len(items))
		for i, v := range items {
			nqmTcpconnItems[i] = v.(*nqmConnItem)
		}

		//  同步Call + 有限并发 进行发送
		sema.Acquire()
		go func(itemList []*nqmConnItem) {
			defer sema.Release()

			for _, v := range itemList {
				//resp := &cmodel.SimpleRpcResponse{}
				var err error
				sendOk := false
				for i := 0; i < 3; i++ { //最多重试3次
					//err = NqmRpcConnPoolHelper.Call("NqmEndpoint.AddIcmp", []*nqmRpcItem{v}, resp)
					vv := []nqmConnItem{*v}
					paramsBody, err := json.Marshal(vv[0])
					postReq, err := http.NewRequest("POST", g.Config().NqmRest.Tcpconn, bytes.NewBuffer(paramsBody))

					postReq.Header.Set("Content-Type", "application/json; charset=UTF-8")
					postReq.Header.Set("Connection", "close")

					b, e := httputil.DumpRequest(postReq, true)
					if e != nil {
						log.Errorln("fuckya:", e)
					}
					log.Debugf("[ Cassandra ] Body: %q", b)

					httpClient := &http.Client{}
					postResp, err := httpClient.Do(postReq)
					if err != nil {
						log.Errorln("[ Cassandra ] Error on push:", err)
						time.Sleep(time.Millisecond * 10)
						continue
					}
					defer postResp.Body.Close()
					b, e = httputil.DumpResponse(postResp, true)
					if e != nil {
						log.Errorln("fuckya:", e)
					}
					log.Debugf("Resp Body: %q", b)
					sendOk = true
					break
				}

				// statistics
				if !sendOk {
					log.Printf("send to /nqm/tcpconn fail: %v", err)
					proc.SendToNqmTcpconnFailCnt.IncrBy(1)
				} else {
					proc.SendToNqmTcpconnCnt.IncrBy(1)
				}
			}
		}(nqmTcpconnItems)
	}
}

func forward2StagingTask() {
	batch := g.Config().Staging.Batch
	retry := g.Config().Staging.MaxRetry
	concurrent := g.Config().Staging.MaxConns
	sema := nsema.NewSemaphore(concurrent)

	for {
		items := StagingQueue.PopBackBy(batch)
		count := len(items)
		if count == 0 {
			time.Sleep(DefaultSendTaskSleepInterval)
			continue
		}

		stagingItems := make([]*cmodel.MetricValue, count)
		for i := 0; i < count; i++ {
			stagingItems[i] = items[i].(*cmodel.MetricValue)
		}

		//	A synchronous call with limited concurrence
		sema.Acquire()
		go func(stagingItems []*cmodel.MetricValue, count int) {
			defer sema.Release()

			resp := &cmodel.SimpleRpcResponse{}
			var err error
			sendOk := false
			for i := 0; i < retry; i++ {
				err = StagingConnPoolHelper.Call("Transfer.Update", stagingItems, resp)
				if err == nil {
					sendOk = true
					break
				}
				time.Sleep(time.Millisecond * 10)
			}

			// statistics
			if !sendOk {
				log.Printf("send staging fail: %v", err)
				proc.SendToStagingFailCnt.IncrBy(int64(count))
			} else {
				proc.SendToStagingCnt.IncrBy(int64(count))
			}
		}(stagingItems, count)
	}
}
<commit_msg>[OWL-865][transfer] Code refactoring & refinement<commit_after>package sender

import (
	"bytes"
	"encoding/json"
	"net/http"
	"path"
	"time"

	log "github.com/Sirupsen/logrus"

	"github.com/Cepave/open-falcon-backend/modules/transfer/g"
	"github.com/Cepave/open-falcon-backend/modules/transfer/proc"
	cmodel "github.com/open-falcon/common/model"
	nsema "github.com/toolkits/concurrent/semaphore"
	"github.com/toolkits/container/list"
)

// send
const (
	DefaultSendTaskSleepInterval = time.Millisecond * 50 //默认睡眠间隔为50ms
)

// TODO 添加对发送任务的控制,比如stop等
func startSendTasks() {
	cfg := g.Config()
	// init semaphore
	judgeConcurrent := cfg.Judge.MaxConns
	graphConcurrent := cfg.Graph.MaxConns
	tsdbConcurrent := cfg.Tsdb.MaxConns
	influxdbConcurrent := cfg.Influxdb.MaxIdle

	if tsdbConcurrent < 1 {
		tsdbConcurrent = 1
	}

	if judgeConcurrent < 1 {
		judgeConcurrent = 1
	}

	if graphConcurrent < 1 {
		graphConcurrent = 1
	}
	if influxdbConcurrent < 1 {
		influxdbConcurrent = 1
	}

	// init send go-routines
	for node, _ := range cfg.Judge.Cluster {
		queue := JudgeQueues[node]
		go forward2JudgeTask(queue, node, judgeConcurrent)
	}

	for node, nitem := range cfg.Graph.ClusterList {
		for _, addr := range nitem.Addrs {
			queue := GraphQueues[node+addr]
			go forward2GraphTask(queue, node, addr, graphConcurrent)
		}
	}

	if cfg.Tsdb.Enabled {
		go forward2TsdbTask(tsdbConcurrent)
	}

	go forward2InfluxdbTask(InfluxdbQueues["default"], influxdbConcurrent)

	if cfg.NqmRest.Enabled {
		go forward2NqmTask(NqmIcmpQueue, g.Config().NqmRest.Fping)
		go forward2NqmTask(NqmTcpQueue, g.Config().NqmRest.Tcpping)
		go forward2NqmTask(NqmTcpconnQueue, g.Config().NqmRest.Tcpconn)
	}

	if cfg.Staging.Enabled {
		go forward2StagingTask()
	}
}

// Judge定时任务, 将 Judge发送缓存中的数据 通过rpc连接池 发送到Judge
func forward2JudgeTask(Q *list.SafeListLimited, node string, concurrent int) {
	batch := g.Config().Judge.Batch // 一次发送,最多batch条数据
	addr := g.Config().Judge.Cluster[node]
	sema := nsema.NewSemaphore(concurrent)

	for {
		items := Q.PopBackBy(batch)
		count := len(items)
		if count == 0 {
			time.Sleep(DefaultSendTaskSleepInterval)
			continue
		}

		judgeItems := make([]*cmodel.JudgeItem, count)
		for i := 0; i < count; i++ {
			judgeItems[i] = items[i].(*cmodel.JudgeItem)
		}

		//	同步Call + 有限并发 进行发送
		sema.Acquire()
		go func(addr string, judgeItems []*cmodel.JudgeItem, count int) {
			defer sema.Release()

			resp := &cmodel.SimpleRpcResponse{}
			var err error
			sendOk := false
			for i := 0; i < 3; i++ { //最多重试3次
				err = JudgeConnPools.Call(addr, "Judge.Send", judgeItems, resp)
				if err == nil {
					sendOk = true
					break
				}
				time.Sleep(time.Millisecond * 10)
			}

			// statistics
			if !sendOk {
				log.Printf("send judge %s:%s fail: %v", node, addr, err)
				proc.SendToJudgeFailCnt.IncrBy(int64(count))
			} else {
				proc.SendToJudgeCnt.IncrBy(int64(count))
			}
		}(addr, judgeItems, count)
	}
}

// Graph定时任务, 将 Graph发送缓存中的数据 通过rpc连接池 发送到Graph
func forward2GraphTask(Q *list.SafeListLimited, node string, addr string, concurrent int) {
	batch := g.Config().Graph.Batch // 一次发送,最多batch条数据
	sema := nsema.NewSemaphore(concurrent)

	for {
		items := Q.PopBackBy(batch)
		count := len(items)
		if count == 0 {
			time.Sleep(DefaultSendTaskSleepInterval)
			continue
		}

		graphItems := make([]*cmodel.GraphItem, count)
		for i := 0; i < count; i++ {
			graphItems[i] = items[i].(*cmodel.GraphItem)
		}

		sema.Acquire()
		go func(addr string, graphItems []*cmodel.GraphItem, count int) {
			defer sema.Release()

			resp := &cmodel.SimpleRpcResponse{}
			var err error
			sendOk := false
			for i := 0; i < 3; i++ { //最多重试3次
				err = GraphConnPools.Call(addr, "Graph.Send", graphItems, resp)
				if err == nil {
					sendOk = true
					break
				}
				time.Sleep(time.Millisecond * 10)
			}

			// statistics
			if !sendOk {
				log.Printf("send to graph %s:%s fail: %v", node, addr, err)
				proc.SendToGraphFailCnt.IncrBy(int64(count))
			} else {
				proc.SendToGraphCnt.IncrBy(int64(count))
			}
		}(addr, graphItems, count)
	}
}

// Tsdb定时任务, 将数据通过api发送到tsdb
func forward2TsdbTask(concurrent int) {
	batch := g.Config().Tsdb.Batch // 一次发送,最多batch条数据
	retry := g.Config().Tsdb.MaxRetry
	sema := nsema.NewSemaphore(concurrent)

	for {
		items := TsdbQueue.PopBackBy(batch)
		if len(items) == 0 {
			time.Sleep(DefaultSendTaskSleepInterval)
			continue
		}
		//  同步Call + 有限并发 进行发送
		sema.Acquire()
		go func(itemList []interface{}) {
			defer sema.Release()

			var tsdbBuffer bytes.Buffer
			for i := 0; i < len(itemList); i++ {
				tsdbItem := itemList[i].(*cmodel.TsdbItem)
				tsdbBuffer.WriteString(tsdbItem.TsdbString())
				tsdbBuffer.WriteString("\n")
			}

			var err error
			for i := 0; i < retry; i++ {
				err = TsdbConnPoolHelper.Send(tsdbBuffer.Bytes())
				if err == nil {
					proc.SendToTsdbCnt.IncrBy(int64(len(itemList)))
					break
				}
				time.Sleep(100 * time.Millisecond)
			}

			if err != nil {
				proc.SendToTsdbFailCnt.IncrBy(int64(len(itemList)))
				log.Println(err)
				return
			}
		}(items)
	}
}

// Influxdb schedule
func forward2InfluxdbTask(Q *list.SafeListLimited, concurrent int) {
	cfg := g.Config().Influxdb
	batch := cfg.Batch // 一次发送,最多batch条数据
	conn, err := parseDSN(cfg.Address)
	if err != nil {
		log.Print("syntax of influxdb address is wrong")
		return
	}
	addr := conn.Address

	sema := nsema.NewSemaphore(concurrent)

	for {
		items := Q.PopBackBy(batch)
		count := len(items)
		if count == 0 {
			time.Sleep(DefaultSendTaskSleepInterval)
			continue
		}

		influxdbItems := make([]*cmodel.JudgeItem, count)
		for i := 0; i < count; i++ {
			influxdbItems[i] = items[i].(*cmodel.JudgeItem)
		}

		//	同步Call + 有限并发 进行发送
		sema.Acquire()
		go func(addr string, influxdbItems []*cmodel.JudgeItem, count int) {
			defer sema.Release()

			var err error
			sendOk := false
			for i := 0; i < 3; i++ { //最多重试3次
				err = InfluxdbConnPools.Call(addr, influxdbItems)
				if err == nil {
					sendOk = true
					break
				}
				time.Sleep(time.Millisecond * 10)
			}

			// statistics
			if !sendOk {
				log.Printf("send influxdb %s fail: %v", addr, err)
				proc.SendToInfluxdbFailCnt.IncrBy(int64(count))
			} else {
				proc.SendToInfluxdbCnt.IncrBy(int64(count))
			}
		}(addr, influxdbItems, count)
	}
}

func forwardNqmItems(p []byte, nqmUrl string, s *nsema.Semaphore) {
	defer s.Release()

	log.Debugf("[ Cassandra ] JSON data to %s: %s", nqmUrl, string(p))
	postReq, err := http.NewRequest("POST", nqmUrl, bytes.NewBuffer(p))

	postReq.Header.Set("Content-Type", "application/json; charset=UTF-8")
	postReq.Header.Set("Connection", "close")
	httpClient := &http.Client{}
	postResp, err := httpClient.Do(postReq)
	if err != nil {
		log.Errorln("[ Cassandra ] Error on push:", err)
		proc.SendToNqmIcmpFailCnt.IncrBy(1)
		return
	}
	defer postResp.Body.Close()
	proc.SendToNqmIcmpCnt.IncrBy(1)
}

func forward2NqmTask(Q *list.SafeListLimited, apiUrl string) {
	batch := g.Config().NqmRest.Batch // 一次发送,最多batch条数据
	concurrent := g.Config().NqmRest.MaxConns
	sema := nsema.NewSemaphore(concurrent)

	for {
		items := Q.PopBackBy(batch)
		if len(items) == 0 {
			time.Sleep(DefaultSendTaskSleepInterval)
			continue
		}
		var paramsBody []byte
		switch path.Base(apiUrl) {
		case "icmp", "tcp":
			nqmItems := []*nqmPingItem{}
			for _, v := range items {
				nqmItems = append(nqmItems, v.(*nqmPingItem))
			}
			paramsBody, _ = json.Marshal(nqmItems)
		case "tcpconn":
			nqmItems := []*nqmConnItem{}
			for _, v := range items {
				nqmItems = append(nqmItems, v.(*nqmConnItem))
			}
			paramsBody, _ = json.Marshal(nqmItems)
		}

		sema.Acquire()
		go forwardNqmItems(paramsBody, apiUrl, sema)

	}
}

func forward2StagingTask() {
	batch := g.Config().Staging.Batch
	retry := g.Config().Staging.MaxRetry
	concurrent := g.Config().Staging.MaxConns
	sema := nsema.NewSemaphore(concurrent)

	for {
		items := StagingQueue.PopBackBy(batch)
		count := len(items)
		if count == 0 {
			time.Sleep(DefaultSendTaskSleepInterval)
			continue
		}

		stagingItems := make([]*cmodel.MetricValue, count)
		for i := 0; i < count; i++ {
			stagingItems[i] = items[i].(*cmodel.MetricValue)
		}

		//	A synchronous call with limited concurrence
		sema.Acquire()
		go func(stagingItems []*cmodel.MetricValue, count int) {
			defer sema.Release()

			resp := &cmodel.SimpleRpcResponse{}
			var err error
			sendOk := false
			for i := 0; i < retry; i++ {
				err = StagingConnPoolHelper.Call("Transfer.Update", stagingItems, resp)
				if err == nil {
					sendOk = true
					break
				}
				time.Sleep(time.Millisecond * 10)
			}

			// statistics
			if !sendOk {
				log.Printf("send staging fail: %v", err)
				proc.SendToStagingFailCnt.IncrBy(int64(count))
			} else {
				proc.SendToStagingCnt.IncrBy(int64(count))
			}
		}(stagingItems, count)
	}
}
<|endoftext|>


<commit_before>package integration_test

import (
	"net/http"
	"os/exec"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
	"github.com/onsi/gomega/ghttp"
)

var _ = Describe("CheckResource", func() {
	var (
		flyCmd *exec.Cmd
	)

	Context("when ATC request succeeds", func() {
		BeforeEach(func() {
			expectedURL := "/api/v1/teams/main/pipelines/mypipeline/resources/myresource/check"
			atcServer.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("POST", expectedURL),
					ghttp.VerifyJSON(`{"from":{"ref":"fake-ref"}}`),
					ghttp.RespondWithJSONEncoded(http.StatusOK, ""),
				),
			)
		})

		It("sends check resource request to ATC", func() {
			Expect(func() {
				flyCmd = exec.Command(flyPath, "-t", targetName, "check-resource", "-r", "mypipeline/myresource", "-f", "ref:fake-ref")
				sess, err := gexec.Start(flyCmd, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(sess).Should(gexec.Exit(0))

				Expect(sess.Out).To(gbytes.Say("checked 'myresource'"))

			}).To(Change(func() int {
				return len(atcServer.ReceivedRequests())
			}).By(2))
		})
	})

	Context("when version is omitted", func() {
		BeforeEach(func() {
			expectedURL := "/api/v1/teams/main/pipelines/mypipeline/resources/myresource/check"
			atcServer.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("POST", expectedURL),
					ghttp.VerifyJSON(`{"from":null}`),
					ghttp.RespondWithJSONEncoded(http.StatusOK, ""),
				),
			)
		})

		It("sends check resource request to ATC", func() {
			Expect(func() {
				flyCmd = exec.Command(flyPath, "-t", targetName, "check-resource", "-r", "mypipeline/myresource")
				sess, err := gexec.Start(flyCmd, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(sess).Should(gexec.Exit(0))

				Expect(sess.Out).To(gbytes.Say("checked 'myresource'"))

			}).To(Change(func() int {
				return len(atcServer.ReceivedRequests())
			}).By(2))
		})
	})

	Context("when specifying multiple versions", func() {
		BeforeEach(func() {
			expectedURL := "/api/v1/teams/main/pipelines/mypipeline/resources/myresource/check"
			atcServer.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("POST", expectedURL),
					ghttp.VerifyJSON(`{"from":{"ref1":"fake-ref-1","ref2":"fake-ref-2"}}`),
					ghttp.RespondWithJSONEncoded(http.StatusOK, ""),
				),
			)
		})

		It("sends correct check resource request to ATC", func() {
			Expect(func() {
				flyCmd = exec.Command(flyPath, "-t", targetName, "check-resource", "-r", "mypipeline/myresource", "-f", "ref1:fake-ref-1", "-f", "ref2:fake-ref-2")
				sess, err := gexec.Start(flyCmd, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(sess).Should(gexec.Exit(0))

				Expect(sess.Out).To(gbytes.Say("checked 'myresource'"))

			}).To(Change(func() int {
				return len(atcServer.ReceivedRequests())
			}).By(2))
		})
	})

	Context("when pipeline or resource is not found", func() {
		BeforeEach(func() {
			expectedURL := "/api/v1/teams/main/pipelines/mypipeline/resources/myresource/check"
			atcServer.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("POST", expectedURL),
					ghttp.RespondWithJSONEncoded(http.StatusNotFound, ""),
				),
			)
		})

		It("fails with error", func() {
			flyCmd = exec.Command(flyPath, "-t", targetName, "check-resource", "-r", "mypipeline/myresource")
			sess, err := gexec.Start(flyCmd, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			Eventually(sess).Should(gexec.Exit(1))

			Expect(sess.Err).To(gbytes.Say("pipeline 'mypipeline' or resource 'myresource' not found"))
		})
	})
})
<commit_msg>add test for showing internal servern error message for check resource<commit_after>package integration_test

import (
	"net/http"
	"os/exec"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
	"github.com/onsi/gomega/ghttp"
)

var _ = Describe("CheckResource", func() {
	var (
		flyCmd *exec.Cmd
	)

	Context("when ATC request succeeds", func() {
		BeforeEach(func() {
			expectedURL := "/api/v1/teams/main/pipelines/mypipeline/resources/myresource/check"
			atcServer.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("POST", expectedURL),
					ghttp.VerifyJSON(`{"from":{"ref":"fake-ref"}}`),
					ghttp.RespondWithJSONEncoded(http.StatusOK, ""),
				),
			)
		})

		It("sends check resource request to ATC", func() {
			Expect(func() {
				flyCmd = exec.Command(flyPath, "-t", targetName, "check-resource", "-r", "mypipeline/myresource", "-f", "ref:fake-ref")
				sess, err := gexec.Start(flyCmd, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(sess).Should(gexec.Exit(0))

				Expect(sess.Out).To(gbytes.Say("checked 'myresource'"))

			}).To(Change(func() int {
				return len(atcServer.ReceivedRequests())
			}).By(2))
		})
	})

	Context("when version is omitted", func() {
		BeforeEach(func() {
			expectedURL := "/api/v1/teams/main/pipelines/mypipeline/resources/myresource/check"
			atcServer.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("POST", expectedURL),
					ghttp.VerifyJSON(`{"from":null}`),
					ghttp.RespondWithJSONEncoded(http.StatusOK, ""),
				),
			)
		})

		It("sends check resource request to ATC", func() {
			Expect(func() {
				flyCmd = exec.Command(flyPath, "-t", targetName, "check-resource", "-r", "mypipeline/myresource")
				sess, err := gexec.Start(flyCmd, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(sess).Should(gexec.Exit(0))

				Expect(sess.Out).To(gbytes.Say("checked 'myresource'"))

			}).To(Change(func() int {
				return len(atcServer.ReceivedRequests())
			}).By(2))
		})
	})

	Context("when specifying multiple versions", func() {
		BeforeEach(func() {
			expectedURL := "/api/v1/teams/main/pipelines/mypipeline/resources/myresource/check"
			atcServer.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("POST", expectedURL),
					ghttp.VerifyJSON(`{"from":{"ref1":"fake-ref-1","ref2":"fake-ref-2"}}`),
					ghttp.RespondWithJSONEncoded(http.StatusOK, ""),
				),
			)
		})

		It("sends correct check resource request to ATC", func() {
			Expect(func() {
				flyCmd = exec.Command(flyPath, "-t", targetName, "check-resource", "-r", "mypipeline/myresource", "-f", "ref1:fake-ref-1", "-f", "ref2:fake-ref-2")
				sess, err := gexec.Start(flyCmd, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(sess).Should(gexec.Exit(0))

				Expect(sess.Out).To(gbytes.Say("checked 'myresource'"))

			}).To(Change(func() int {
				return len(atcServer.ReceivedRequests())
			}).By(2))
		})
	})

	Context("when pipeline or resource is not found", func() {
		BeforeEach(func() {
			expectedURL := "/api/v1/teams/main/pipelines/mypipeline/resources/myresource/check"
			atcServer.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("POST", expectedURL),
					ghttp.RespondWithJSONEncoded(http.StatusNotFound, ""),
				),
			)
		})

		It("fails with error", func() {
			flyCmd = exec.Command(flyPath, "-t", targetName, "check-resource", "-r", "mypipeline/myresource")
			sess, err := gexec.Start(flyCmd, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			Eventually(sess).Should(gexec.Exit(1))

			Expect(sess.Err).To(gbytes.Say("pipeline 'mypipeline' or resource 'myresource' not found"))
		})
	})

	Context("When resource check returns internal server error", func() {
		BeforeEach(func() {
			expectedURL := "/api/v1/teams/main/pipelines/mypipeline/resources/myresource/check"
			atcServer.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("POST", expectedURL),
					ghttp.RespondWith(http.StatusInternalServerError, "unknown server error"),
				),
			)
		})

		It("outputs error in response body", func() {
			flyCmd = exec.Command(flyPath, "-t", targetName, "check-resource", "-r", "mypipeline/myresource")
			sess, err := gexec.Start(flyCmd, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			Eventually(sess).Should(gexec.Exit(1))

			Expect(sess.Err).To(gbytes.Say("unknown server error"))

		})
	})
})
<|endoftext|>


<commit_before>package push

import (
	"regexp"
	"strings"

	"code.cloudfoundry.org/cli/integration/helpers"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("push with only an app name", func() {
	var (
		appName string
	)

	BeforeEach(func() {
		appName = helpers.NewAppName()
	})

	Describe("app existence", func() {
		Context("when the app does not exist", func() {
			It("creates the app", func() {
				helpers.WithHelloWorldApp(func(dir string) {
					session := helpers.CustomCF(helpers.CFEnv{WorkingDirectory: dir}, PushCommandName, appName)
					Eventually(session).Should(Say("Getting app info\\.\\.\\."))
					Eventually(session).Should(Say("Creating app with these attributes\\.\\.\\."))
					Eventually(session).Should(Say("\\+\\s+name:\\s+%s", appName))
					Eventually(session).Should(Say("\\s+path:\\s+%s", regexp.QuoteMeta(dir)))
					Eventually(session).Should(Say("\\s+routes:"))
					Eventually(session).Should(Say("(?i)\\+\\s+%s.%s", appName, defaultSharedDomain()))
					Eventually(session).Should(Say("Mapping routes\\.\\.\\."))
					Eventually(session).Should(Say("Packaging files to upload\\.\\.\\."))
					Eventually(session).Should(Say("Uploading files\\.\\.\\."))
					Eventually(session).Should(Say("100.00%"))
					Eventually(session).Should(Say("Waiting for API to complete processing files\\.\\.\\."))
					Eventually(session).Should(Say("Downloaded staticfile_buildpack"))
					Eventually(session).Should(Say("Staging complete"))
					Eventually(session).Should(Say("Waiting for app to start\\.\\.\\."))
					Eventually(session).Should(Say("requested state:\\s+started"))
					Eventually(session).Should(Exit(0))
				})

				session := helpers.CF("app", appName)
				Eventually(session).Should(Say("name:\\s+%s", appName))
				Eventually(session).Should(Exit(0))
			})
		})

		Context("when the app exists", func() {
			BeforeEach(func() {
				helpers.WithHelloWorldApp(func(dir string) {
					Eventually(helpers.CustomCF(helpers.CFEnv{WorkingDirectory: dir}, "push", appName)).Should(Exit(0))
				})
			})

			It("updates the app", func() {
				helpers.WithHelloWorldApp(func(dir string) {
					session := helpers.CustomCF(helpers.CFEnv{WorkingDirectory: dir}, PushCommandName, appName)
					Eventually(session).Should(Say("Getting app info\\.\\.\\."))
					Eventually(session).Should(Say("Updating app with these attributes\\.\\.\\."))
					Eventually(session).Should(Say("(?m)^\\s+name:\\s+%s$", appName))
					Eventually(session).Should(Say("\\s+path:\\s+%s", regexp.QuoteMeta(dir)))
					Eventually(session).Should(Say("\\s+routes:"))
					Eventually(session).Should(Say("(?mi)^\\s+%s.%s$", strings.ToLower(appName), defaultSharedDomain()))
					Eventually(session).Should(Say("Mapping routes\\.\\.\\."))
					Eventually(session).Should(Say("Waiting for API to complete processing files\\.\\.\\."))
					Eventually(session).Should(Say("Uploading files\\.\\.\\."))
					Eventually(session).Should(Say("100.00%"))
					Eventually(session).Should(Say("Processing files\\.\\.\\."))
					Eventually(session).Should(Say("Stopping app\\.\\.\\."))
					Eventually(session).Should(Say("Downloaded staticfile_buildpack"))
					Eventually(session).Should(Say("Staging complete"))
					Eventually(session).Should(Say("Waiting for app to start\\.\\.\\."))
					Eventually(session).Should(Say("requested state:\\s+started"))
					Eventually(session).Should(Exit(0))
				})

				session := helpers.CF("app", appName)
				Eventually(session).Should(Say("name:\\s+%s", appName))
				Eventually(session).Should(Exit(0))
			})
		})
	})
})
<commit_msg>fix integration test ordering<commit_after>package push

import (
	"regexp"
	"strings"

	"code.cloudfoundry.org/cli/integration/helpers"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("push with only an app name", func() {
	var (
		appName string
	)

	BeforeEach(func() {
		appName = helpers.NewAppName()
	})

	Describe("app existence", func() {
		Context("when the app does not exist", func() {
			It("creates the app", func() {
				helpers.WithHelloWorldApp(func(dir string) {
					session := helpers.CustomCF(helpers.CFEnv{WorkingDirectory: dir}, PushCommandName, appName)
					Eventually(session).Should(Say("Getting app info\\.\\.\\."))
					Eventually(session).Should(Say("Creating app with these attributes\\.\\.\\."))
					Eventually(session).Should(Say("\\+\\s+name:\\s+%s", appName))
					Eventually(session).Should(Say("\\s+path:\\s+%s", regexp.QuoteMeta(dir)))
					Eventually(session).Should(Say("\\s+routes:"))
					Eventually(session).Should(Say("(?i)\\+\\s+%s.%s", appName, defaultSharedDomain()))
					Eventually(session).Should(Say("Mapping routes\\.\\.\\."))
					Eventually(session).Should(Say("Packaging files to upload\\.\\.\\."))
					Eventually(session).Should(Say("Uploading files\\.\\.\\."))
					Eventually(session).Should(Say("100.00%"))
					Eventually(session).Should(Say("Waiting for API to complete processing files\\.\\.\\."))
					Eventually(session).Should(Say("Downloaded staticfile_buildpack"))
					Eventually(session).Should(Say("Staging complete"))
					Eventually(session).Should(Say("Waiting for app to start\\.\\.\\."))
					Eventually(session).Should(Say("requested state:\\s+started"))
					Eventually(session).Should(Exit(0))
				})

				session := helpers.CF("app", appName)
				Eventually(session).Should(Say("name:\\s+%s", appName))
				Eventually(session).Should(Exit(0))
			})
		})

		Context("when the app exists", func() {
			BeforeEach(func() {
				helpers.WithHelloWorldApp(func(dir string) {
					Eventually(helpers.CustomCF(helpers.CFEnv{WorkingDirectory: dir}, "push", appName)).Should(Exit(0))
				})
			})

			It("updates the app", func() {
				helpers.WithHelloWorldApp(func(dir string) {
					session := helpers.CustomCF(helpers.CFEnv{WorkingDirectory: dir}, PushCommandName, appName)
					Eventually(session).Should(Say("Getting app info\\.\\.\\."))
					Eventually(session).Should(Say("Updating app with these attributes\\.\\.\\."))
					Eventually(session).Should(Say("(?m)^\\s+name:\\s+%s$", appName))
					Eventually(session).Should(Say("\\s+path:\\s+%s", regexp.QuoteMeta(dir)))
					Eventually(session).Should(Say("\\s+routes:"))
					Eventually(session).Should(Say("(?mi)^\\s+%s.%s$", strings.ToLower(appName), defaultSharedDomain()))
					Eventually(session).Should(Say("Mapping routes\\.\\.\\."))
					Eventually(session).Should(Say("Uploading files\\.\\.\\."))
					Eventually(session).Should(Say("100.00%"))
					Eventually(session).Should(Say("Waiting for API to complete processing files\\.\\.\\."))
					Eventually(session).Should(Say("Stopping app\\.\\.\\."))
					Eventually(session).Should(Say("Downloaded staticfile_buildpack"))
					Eventually(session).Should(Say("Staging complete"))
					Eventually(session).Should(Say("Waiting for app to start\\.\\.\\."))
					Eventually(session).Should(Say("requested state:\\s+started"))
					Eventually(session).Should(Exit(0))
				})

				session := helpers.CF("app", appName)
				Eventually(session).Should(Say("name:\\s+%s", appName))
				Eventually(session).Should(Exit(0))
			})
		})
	})
})
<|endoftext|>


<commit_before>package self_test

import (
	"bytes"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"time"

	quic "github.com/lucas-clemente/quic-go"
	"github.com/lucas-clemente/quic-go/h2quic"
	"github.com/lucas-clemente/quic-go/integrationtests/tools/testserver"
	"github.com/lucas-clemente/quic-go/internal/protocol"
	"github.com/lucas-clemente/quic-go/internal/testdata"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
)

var _ = Describe("HTTP tests", func() {
	var client *http.Client

	versions := protocol.SupportedVersions

	BeforeEach(func() {
		testserver.StartQuicServer(versions)
	})

	AfterEach(func() {
		testserver.StopQuicServer()
	})

	for _, v := range versions {
		version := v

		Context(fmt.Sprintf("with QUIC version %s", version), func() {
			BeforeEach(func() {
				client = &http.Client{
					Transport: &h2quic.RoundTripper{
						TLSClientConfig: &tls.Config{
							RootCAs: testdata.GetRootCA(),
						},
						QuicConfig: &quic.Config{
							Versions:    []protocol.VersionNumber{version},
							IdleTimeout: 10 * time.Second,
						},
					},
				}
			})

			It("downloads a hello", func() {
				resp, err := client.Get("https://localhost:" + testserver.Port() + "/hello")
				Expect(err).ToNot(HaveOccurred())
				Expect(resp.StatusCode).To(Equal(200))
				body, err := ioutil.ReadAll(gbytes.TimeoutReader(resp.Body, 3*time.Second))
				Expect(err).ToNot(HaveOccurred())
				Expect(string(body)).To(Equal("Hello, World!\n"))
			})

			It("downloads a small file", func() {
				resp, err := client.Get("https://localhost:" + testserver.Port() + "/prdata")
				Expect(err).ToNot(HaveOccurred())
				Expect(resp.StatusCode).To(Equal(200))
				body, err := ioutil.ReadAll(gbytes.TimeoutReader(resp.Body, 5*time.Second))
				Expect(err).ToNot(HaveOccurred())
				Expect(body).To(Equal(testserver.PRData))
			})

			It("downloads a large file", func() {
				resp, err := client.Get("https://localhost:" + testserver.Port() + "/prdatalong")
				Expect(err).ToNot(HaveOccurred())
				Expect(resp.StatusCode).To(Equal(200))
				body, err := ioutil.ReadAll(gbytes.TimeoutReader(resp.Body, 20*time.Second))
				Expect(err).ToNot(HaveOccurred())
				Expect(body).To(Equal(testserver.PRDataLong))
			})

			It("downloads many files, if the response is not read", func() {
				const num = 150

				for i := 0; i < num; i++ {
					resp, err := client.Get("https://localhost:" + testserver.Port() + "/prdata")
					Expect(err).ToNot(HaveOccurred())
					Expect(resp.StatusCode).To(Equal(200))
					Expect(resp.Body.Close()).To(Succeed())
				}
			})

			It("uploads a file", func() {
				resp, err := client.Post(
					"https://localhost:"+testserver.Port()+"/echo",
					"text/plain",
					bytes.NewReader(testserver.PRData),
				)
				Expect(err).ToNot(HaveOccurred())
				Expect(resp.StatusCode).To(Equal(200))
				body, err := ioutil.ReadAll(gbytes.TimeoutReader(resp.Body, 5*time.Second))
				Expect(err).ToNot(HaveOccurred())
				Expect(bytes.Equal(body, testserver.PRData)).To(BeTrue())
			})
		})
	}
})
<commit_msg>disable failing HTTP integration test<commit_after>package self_test

import (
	"bytes"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"time"

	quic "github.com/lucas-clemente/quic-go"
	"github.com/lucas-clemente/quic-go/h2quic"
	"github.com/lucas-clemente/quic-go/integrationtests/tools/testserver"
	"github.com/lucas-clemente/quic-go/internal/protocol"
	"github.com/lucas-clemente/quic-go/internal/testdata"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
)

var _ = Describe("HTTP tests", func() {
	var client *http.Client

	versions := protocol.SupportedVersions

	BeforeEach(func() {
		testserver.StartQuicServer(versions)
	})

	AfterEach(func() {
		testserver.StopQuicServer()
	})

	for _, v := range versions {
		version := v

		Context(fmt.Sprintf("with QUIC version %s", version), func() {
			BeforeEach(func() {
				client = &http.Client{
					Transport: &h2quic.RoundTripper{
						TLSClientConfig: &tls.Config{
							RootCAs: testdata.GetRootCA(),
						},
						QuicConfig: &quic.Config{
							Versions:    []protocol.VersionNumber{version},
							IdleTimeout: 10 * time.Second,
						},
					},
				}
			})

			It("downloads a hello", func() {
				resp, err := client.Get("https://localhost:" + testserver.Port() + "/hello")
				Expect(err).ToNot(HaveOccurred())
				Expect(resp.StatusCode).To(Equal(200))
				body, err := ioutil.ReadAll(gbytes.TimeoutReader(resp.Body, 3*time.Second))
				Expect(err).ToNot(HaveOccurred())
				Expect(string(body)).To(Equal("Hello, World!\n"))
			})

			It("downloads a small file", func() {
				resp, err := client.Get("https://localhost:" + testserver.Port() + "/prdata")
				Expect(err).ToNot(HaveOccurred())
				Expect(resp.StatusCode).To(Equal(200))
				body, err := ioutil.ReadAll(gbytes.TimeoutReader(resp.Body, 5*time.Second))
				Expect(err).ToNot(HaveOccurred())
				Expect(body).To(Equal(testserver.PRData))
			})

			It("downloads a large file", func() {
				resp, err := client.Get("https://localhost:" + testserver.Port() + "/prdatalong")
				Expect(err).ToNot(HaveOccurred())
				Expect(resp.StatusCode).To(Equal(200))
				body, err := ioutil.ReadAll(gbytes.TimeoutReader(resp.Body, 20*time.Second))
				Expect(err).ToNot(HaveOccurred())
				Expect(body).To(Equal(testserver.PRDataLong))
			})

			// TODO(#1756): this test times out
			PIt("downloads many files, if the response is not read", func() {
				const num = 150

				for i := 0; i < num; i++ {
					resp, err := client.Get("https://localhost:" + testserver.Port() + "/prdata")
					Expect(err).ToNot(HaveOccurred())
					Expect(resp.StatusCode).To(Equal(200))
					Expect(resp.Body.Close()).To(Succeed())
				}
			})

			It("uploads a file", func() {
				resp, err := client.Post(
					"https://localhost:"+testserver.Port()+"/echo",
					"text/plain",
					bytes.NewReader(testserver.PRData),
				)
				Expect(err).ToNot(HaveOccurred())
				Expect(resp.StatusCode).To(Equal(200))
				body, err := ioutil.ReadAll(gbytes.TimeoutReader(resp.Body, 5*time.Second))
				Expect(err).ToNot(HaveOccurred())
				Expect(bytes.Equal(body, testserver.PRData)).To(BeTrue())
			})
		})
	}
})
<|endoftext|>