<commit_before>package nakadi

import (
	"context"
	"time"

	"github.com/cenkalti/backoff"
)

// A Cursor marks the current read position in a stream. It returned along with each received batch of
// events and is furthermore used to commit a batch of events (as well as all previous events).
type Cursor struct {
	Partition      string `json:"partition"`
	Offset         string `json:"offset"`
	EventType      string `json:"event_type"`
	CursorToken    string `json:"cursor_token"`
	NakadiStreamID string `json:"-"`
}

// StreamOptions contains optional parameters that are used to create a StreamAPI.
type StreamOptions struct {
	// The maximum number of Events in each chunk (and therefore per partition) of the stream (default: 1)
	BatchLimit uint
	// Maximum time in seconds to wait for the flushing of each chunk (per partition).(default: 30)
	FlushTimeout uint
	// The amount of uncommitted events Nakadi will stream before pausing the stream. When in paused
	// state and commit comes - the stream will resume. If MaxUncommittedEvents is lower than BatchLimit,
	// effective batch size will be upperbound by MaxUncommittedEvents. (default: 10, minimum: 1)
	MaxUncommittedEvents uint
	// The initial (minimal) retry interval used for the exponential backoff. This value is applied for
	// stream initialization as well as for cursor commits.
	InitialRetryInterval time.Duration
	// MaxRetryInterval the maximum retry interval. Once the exponential backoff reaches this value
	// the retry intervals remain constant. This value is applied for stream initialization as well as
	// for cursor commits.
	MaxRetryInterval time.Duration
	// MaxElapsedTime is the maximum time spent on retries when committing a cursor. Once this value
	// was reached the exponential backoff is halted and the cursor will not be committed.
	CommitMaxElapsedTime time.Duration
	// Whether or not CommitCursor will retry when a request fails. If
	// set to true InitialRetryInterval, MaxRetryInterval, and CommitMaxElapsedTime have
	// no effect for commit requests (default: false).
	CommitRetry bool
	// NotifyErr is called when an error occurs that leads to a retry. This notify function can be used to
	// detect unhealthy streams.
	NotifyErr func(error, time.Duration)
	// NotifyOK is called whenever a successful operation was completed. This notify function can be used
	// to detect that a stream is healthy again.
	NotifyOK func()
}

func (o *StreamOptions) withDefaults() *StreamOptions {
	var copyOptions StreamOptions
	if o != nil {
		copyOptions = *o
	}
	if copyOptions.InitialRetryInterval == 0 {
		copyOptions.InitialRetryInterval = defaultInitialRetryInterval
	}
	if copyOptions.MaxRetryInterval == 0 {
		copyOptions.MaxRetryInterval = defaultMaxRetryInterval
	}
	if copyOptions.CommitMaxElapsedTime == 0 {
		copyOptions.CommitMaxElapsedTime = defaultMaxElapsedTime
	}
	if copyOptions.NotifyErr == nil {
		copyOptions.NotifyErr = func(_ error, _ time.Duration) {}
	}
	if copyOptions.NotifyOK == nil {
		copyOptions.NotifyOK = func() {}
	}
	if copyOptions.MaxUncommittedEvents == 0 {
		copyOptions.MaxUncommittedEvents = 10
	}
	return &copyOptions
}

// NewStream is used to instantiate a new steam processing sub API. As for all sub APIs of the `go-nakadi`
// package NewStream receives a configured Nakadi client. Furthermore a valid subscription ID must be
// provided. Use the SubscriptionAPI in order to obtain subscriptions. The options parameter can be used
// to configure the behavior of the stream. The options may be nil.
func NewStream(client *Client, subscriptionID string, options *StreamOptions) *StreamAPI {
	options = options.withDefaults()

	ctx, cancel := context.WithCancel(context.Background())

	streamAPI := &StreamAPI{
		opener: &simpleStreamOpener{
			client:               client,
			subscriptionID:       subscriptionID,
			batchLimit:           options.BatchLimit,
			flushTimeout:         options.FlushTimeout,
			maxUncommittedEvents: options.MaxUncommittedEvents},
		committer: &simpleCommitter{
			client:         client,
			subscriptionID: subscriptionID},
		eventCh: make(chan eventsOrError, 10),
		ctx:     ctx,
		cancel:  cancel,
		streamBackOffConf: backOffConfiguration{
			Retry:                true,
			InitialRetryInterval: options.InitialRetryInterval,
			MaxRetryInterval:     options.MaxRetryInterval,
		},
		commitBackOffConf: backOffConfiguration{
			Retry:                options.CommitRetry,
			InitialRetryInterval: options.InitialRetryInterval,
			MaxRetryInterval:     options.MaxRetryInterval,
			MaxElapsedTime:       options.CommitMaxElapsedTime,
		},
		notifyErr: options.NotifyErr,
		notifyOK:  options.NotifyOK}

	go streamAPI.startStream()

	return streamAPI
}

// A StreamAPI is a sub API which is used to consume events from a specific subscription using Nakadi's
// high level stream API. In order to ensure that only successfully processed events are committed, it is
// crucial to commit cursors of respective event batches in the same order they were received.
type StreamAPI struct {
	opener            streamOpener
	committer         committer
	eventCh           chan eventsOrError
	ctx               context.Context
	cancel            context.CancelFunc
	commitBackOffConf backOffConfiguration
	streamBackOffConf backOffConfiguration
	notifyErr         func(error, time.Duration)
	notifyOK          func()
}

// NextEvents reads the next batch of events from the stream and returns the encoded events along with the
// respective cursor. It blocks until the batch of events can be read from the stream, or the stream is closed.
func (s *StreamAPI) NextEvents() (Cursor, []byte, error) {
	select {
	case <-s.ctx.Done():
		return Cursor{}, nil, context.Canceled
	case next := <-s.eventCh:
		return next.cursor, next.events, next.err
	}
}

// CommitCursor commits a cursor to Nakadi.
func (s *StreamAPI) CommitCursor(cursor Cursor) error {
	var err error

	commitBackOff := backoff.WithContext(s.commitBackOffConf.create(), s.ctx)
	backoff.RetryNotify(func() error {
		err = s.committer.commitCursor(cursor)
		return err
	}, commitBackOff, s.notifyErr)

	if err == nil {
		s.notifyOK()
	}

	return err
}

// Close ends the stream.
func (s *StreamAPI) Close() error {
	s.cancel()
	return nil
}

// startStream is used to start a background routine which consumes events using a streamOpener and streamer.
// this routine will never terminate (not even on errors) unless the stream is closed.
func (s *StreamAPI) startStream() {
	for {
		var err error
		var stream streamer

		streamBackOff := backoff.WithContext(s.streamBackOffConf.create(), s.ctx)
		backoff.RetryNotify(func() error {
			stream, err = s.opener.openStream()
			return err
		}, streamBackOff, s.notifyErr)

		if err != nil {
			continue
		}
		s.notifyOK()

		var cursor Cursor
		var events []byte
		for {
			select {
			case <-s.ctx.Done():
				err = context.Canceled
			default:
				cursor, events, err = stream.nextEvents()
			}

			if err == nil && len(events) == 0 {
				continue
			}

			select {
			case <-s.ctx.Done():
				err = context.Canceled
			case s.eventCh <- eventsOrError{cursor: cursor, events: events, err: err}:
				// nothing
			}

			if err != nil {
				if err == context.Canceled {
					stream.closeStream()
					close(s.eventCh)
					return
				}
				break
			}
		}

		stream.closeStream()
	}
}

// streamOpener is a internally used interface which is used to establish a new stream.
type streamOpener interface {
	openStream() (streamer, error)
}

// streamer is a internally used interface which is used to consume events from a stream.
type streamer interface {
	nextEvents() (Cursor, []byte, error)
	closeStream() error
}

// committer is a internally used interface which is used to commit cursors.
type committer interface {
	commitCursor(cursor Cursor) error
}

// eventsOrError is used to represent a successful or failed batch read.
type eventsOrError struct {
	cursor Cursor
	events []byte
	err    error
}
<commit_msg>Fix infinite loop when openStream errors<commit_after>package nakadi

import (
	"context"
	"time"

	"github.com/cenkalti/backoff"
)

// A Cursor marks the current read position in a stream. It returned along with each received batch of
// events and is furthermore used to commit a batch of events (as well as all previous events).
type Cursor struct {
	Partition      string `json:"partition"`
	Offset         string `json:"offset"`
	EventType      string `json:"event_type"`
	CursorToken    string `json:"cursor_token"`
	NakadiStreamID string `json:"-"`
}

// StreamOptions contains optional parameters that are used to create a StreamAPI.
type StreamOptions struct {
	// The maximum number of Events in each chunk (and therefore per partition) of the stream (default: 1)
	BatchLimit uint
	// Maximum time in seconds to wait for the flushing of each chunk (per partition).(default: 30)
	FlushTimeout uint
	// The amount of uncommitted events Nakadi will stream before pausing the stream. When in paused
	// state and commit comes - the stream will resume. If MaxUncommittedEvents is lower than BatchLimit,
	// effective batch size will be upperbound by MaxUncommittedEvents. (default: 10, minimum: 1)
	MaxUncommittedEvents uint
	// The initial (minimal) retry interval used for the exponential backoff. This value is applied for
	// stream initialization as well as for cursor commits.
	InitialRetryInterval time.Duration
	// MaxRetryInterval the maximum retry interval. Once the exponential backoff reaches this value
	// the retry intervals remain constant. This value is applied for stream initialization as well as
	// for cursor commits.
	MaxRetryInterval time.Duration
	// MaxElapsedTime is the maximum time spent on retries when committing a cursor. Once this value
	// was reached the exponential backoff is halted and the cursor will not be committed.
	CommitMaxElapsedTime time.Duration
	// Whether or not CommitCursor will retry when a request fails. If
	// set to true InitialRetryInterval, MaxRetryInterval, and CommitMaxElapsedTime have
	// no effect for commit requests (default: false).
	CommitRetry bool
	// NotifyErr is called when an error occurs that leads to a retry. This notify function can be used to
	// detect unhealthy streams.
	NotifyErr func(error, time.Duration)
	// NotifyOK is called whenever a successful operation was completed. This notify function can be used
	// to detect that a stream is healthy again.
	NotifyOK func()
}

func (o *StreamOptions) withDefaults() *StreamOptions {
	var copyOptions StreamOptions
	if o != nil {
		copyOptions = *o
	}
	if copyOptions.InitialRetryInterval == 0 {
		copyOptions.InitialRetryInterval = defaultInitialRetryInterval
	}
	if copyOptions.MaxRetryInterval == 0 {
		copyOptions.MaxRetryInterval = defaultMaxRetryInterval
	}
	if copyOptions.CommitMaxElapsedTime == 0 {
		copyOptions.CommitMaxElapsedTime = defaultMaxElapsedTime
	}
	if copyOptions.NotifyErr == nil {
		copyOptions.NotifyErr = func(_ error, _ time.Duration) {}
	}
	if copyOptions.NotifyOK == nil {
		copyOptions.NotifyOK = func() {}
	}
	if copyOptions.MaxUncommittedEvents == 0 {
		copyOptions.MaxUncommittedEvents = 10
	}
	return &copyOptions
}

// NewStream is used to instantiate a new steam processing sub API. As for all sub APIs of the `go-nakadi`
// package NewStream receives a configured Nakadi client. Furthermore a valid subscription ID must be
// provided. Use the SubscriptionAPI in order to obtain subscriptions. The options parameter can be used
// to configure the behavior of the stream. The options may be nil.
func NewStream(client *Client, subscriptionID string, options *StreamOptions) *StreamAPI {
	options = options.withDefaults()

	ctx, cancel := context.WithCancel(context.Background())

	streamAPI := &StreamAPI{
		opener: &simpleStreamOpener{
			client:               client,
			subscriptionID:       subscriptionID,
			batchLimit:           options.BatchLimit,
			flushTimeout:         options.FlushTimeout,
			maxUncommittedEvents: options.MaxUncommittedEvents},
		committer: &simpleCommitter{
			client:         client,
			subscriptionID: subscriptionID},
		eventCh: make(chan eventsOrError, 10),
		ctx:     ctx,
		cancel:  cancel,
		streamBackOffConf: backOffConfiguration{
			Retry:                true,
			InitialRetryInterval: options.InitialRetryInterval,
			MaxRetryInterval:     options.MaxRetryInterval,
		},
		commitBackOffConf: backOffConfiguration{
			Retry:                options.CommitRetry,
			InitialRetryInterval: options.InitialRetryInterval,
			MaxRetryInterval:     options.MaxRetryInterval,
			MaxElapsedTime:       options.CommitMaxElapsedTime,
		},
		notifyErr: options.NotifyErr,
		notifyOK:  options.NotifyOK}

	go streamAPI.startStream()

	return streamAPI
}

// A StreamAPI is a sub API which is used to consume events from a specific subscription using Nakadi's
// high level stream API. In order to ensure that only successfully processed events are committed, it is
// crucial to commit cursors of respective event batches in the same order they were received.
type StreamAPI struct {
	opener            streamOpener
	committer         committer
	eventCh           chan eventsOrError
	ctx               context.Context
	cancel            context.CancelFunc
	commitBackOffConf backOffConfiguration
	streamBackOffConf backOffConfiguration
	notifyErr         func(error, time.Duration)
	notifyOK          func()
}

// NextEvents reads the next batch of events from the stream and returns the encoded events along with the
// respective cursor. It blocks until the batch of events can be read from the stream, or the stream is closed.
func (s *StreamAPI) NextEvents() (Cursor, []byte, error) {
	select {
	case <-s.ctx.Done():
		return Cursor{}, nil, context.Canceled
	case next := <-s.eventCh:
		return next.cursor, next.events, next.err
	}
}

// CommitCursor commits a cursor to Nakadi.
func (s *StreamAPI) CommitCursor(cursor Cursor) error {
	var err error

	commitBackOff := backoff.WithContext(s.commitBackOffConf.create(), s.ctx)
	backoff.RetryNotify(func() error {
		err = s.committer.commitCursor(cursor)
		return err
	}, commitBackOff, s.notifyErr)

	if err == nil {
		s.notifyOK()
	}

	return err
}

// Close ends the stream.
func (s *StreamAPI) Close() error {
	s.cancel()
	return nil
}

// startStream is used to start a background routine which consumes events using a streamOpener and streamer.
// this routine will never terminate (not even on errors) unless the stream is closed.
func (s *StreamAPI) startStream() {
	for {
		var err error
		var stream streamer

		streamBackOff := backoff.WithContext(s.streamBackOffConf.create(), s.ctx)
		backoff.RetryNotify(func() error {
			stream, err = s.opener.openStream()
			return err
		}, streamBackOff, s.notifyErr)

		if err != nil {
			select {
			case <-s.ctx.Done():
				return
			default:
				continue
			}
		}
		s.notifyOK()

		var cursor Cursor
		var events []byte
		for {
			select {
			case <-s.ctx.Done():
				err = context.Canceled
			default:
				cursor, events, err = stream.nextEvents()
			}

			if err == nil && len(events) == 0 {
				continue
			}

			select {
			case <-s.ctx.Done():
				err = context.Canceled
			case s.eventCh <- eventsOrError{cursor: cursor, events: events, err: err}:
				// nothing
			}

			if err != nil {
				if err == context.Canceled {
					stream.closeStream()
					close(s.eventCh)
					return
				}
				break
			}
		}

		stream.closeStream()
	}
}

// streamOpener is a internally used interface which is used to establish a new stream.
type streamOpener interface {
	openStream() (streamer, error)
}

// streamer is a internally used interface which is used to consume events from a stream.
type streamer interface {
	nextEvents() (Cursor, []byte, error)
	closeStream() error
}

// committer is a internally used interface which is used to commit cursors.
type committer interface {
	commitCursor(cursor Cursor) error
}

// eventsOrError is used to represent a successful or failed batch read.
type eventsOrError struct {
	cursor Cursor
	events []byte
	err    error
}
<|endoftext|>


<commit_before>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package charmstore

import (
	"io"

	"gopkg.in/juju/charm.v6-unstable"
	charmresource "gopkg.in/juju/charm.v6-unstable/resource"
)

// Client exposes the functionality of the charm store, as provided
// by github.com/juju/charmrepo/csclient.Client.
type Client interface {
	// TODO(ericsnow) Replace use of Get with use of more specific API methods?

	// Get makes a GET request to the given path in the charm store. The
	// path must have a leading slash, but must not include the host
	// name or version prefix. The result is parsed as JSON into the
	// given result value, which should be a pointer to the expected
	// data, but may be nil if no result is desired.
	Get(path string, result interface{}) error

	// TODO(ericsnow) Just embed resource/charmstore.BaseClient?

	// ListResources composes, for each of the identified charms, the
	// list of details for each of the charm's resources. Those details
	// are those associated with the specific charm revision. They
	// include the resource's metadata and revision.
	ListResources(charmURLs []charm.URL) ([][]charmresource.Resource, error)

	// GetResource returns a reader for the resource's data. That data
	// is streamed from the charm store. The charm's revision, if any,
	// is ignored. If the identified resource is not in the charm store
	// then errors.NotFound is returned.
	GetResource(cURL *charm.URL, resourceName string, revision int) (io.ReadCloser, error)
}

// TestingClient expands Client with methods needed during testing.
type TestingClient interface {
	Client

	// Put makes a PUT request to the given path in the charm store. The
	// path must have a leading slash, but must not include the host
	// name or version prefix. The given value is marshaled as JSON to
	// use as the request body.
	Put(path string, val interface{}) error

	// UploadCharm uploads the given charm to the charm store with the
	// given id, which must not specify a revision. The accepted charm
	// implementations are charm.CharmDir and charm.CharmArchive.
	//
	// UploadCharm returns the id that the charm has been given in the
	// store - this will be the same as id except the revision.
	UploadCharm(id *charm.URL, ch charm.Charm) (*charm.URL, error)

	// UploadCharmWithRevision uploads the given charm to the given id
	// in the charm store, which must contain a revision. If
	// promulgatedRevision is not -1, it specifies that the charm should
	// be marked as promulgated with that revision.
	UploadCharmWithRevision(id *charm.URL, ch charm.Charm, promulgatedRevision int) error

	// UploadBundleWithRevision uploads the given bundle to the given id
	// in the charm store, which must contain a revision. If
	// promulgatedRevision is not -1, it specifies that the charm should
	// be marked as promulgated with that revision.
	UploadBundleWithRevision()
}
<commit_msg>Drop TestingClient.<commit_after>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package charmstore

import (
	"io"

	"gopkg.in/juju/charm.v6-unstable"
	charmresource "gopkg.in/juju/charm.v6-unstable/resource"
)

// Client exposes the functionality of the charm store, as provided
// by github.com/juju/charmrepo/csclient.Client.
//
// Note that the following csclient.Client methods are used as well,
// but only in tests:
//  - Put(path string, val interface{}) error
//  - UploadCharm(id *charm.URL, ch charm.Charm) (*charm.URL, error)
//  - UploadCharmWithRevision(id *charm.URL, ch charm.Charm, promulgatedRevision int) error
//  - UploadBundleWithRevision()
type Client interface {
	// TODO(ericsnow) Replace use of Get with use of more specific API methods?

	// Get makes a GET request to the given path in the charm store. The
	// path must have a leading slash, but must not include the host
	// name or version prefix. The result is parsed as JSON into the
	// given result value, which should be a pointer to the expected
	// data, but may be nil if no result is desired.
	Get(path string, result interface{}) error

	// TODO(ericsnow) Just embed resource/charmstore.BaseClient?

	// ListResources composes, for each of the identified charms, the
	// list of details for each of the charm's resources. Those details
	// are those associated with the specific charm revision. They
	// include the resource's metadata and revision.
	ListResources(charmURLs []charm.URL) ([][]charmresource.Resource, error)

	// GetResource returns a reader for the resource's data. That data
	// is streamed from the charm store. The charm's revision, if any,
	// is ignored. If the identified resource is not in the charm store
	// then errors.NotFound is returned.
	GetResource(cURL *charm.URL, resourceName string, revision int) (io.ReadCloser, error)
}
<|endoftext|>


<commit_before>package database

import (
	"fmt"
	"log"

	_ "github.com/go-sql-driver/mysql"
	"github.com/jmoiron/sqlx"
	_ "github.com/mattn/go-sqlite3"
)

var (
	// Database wrapper
	DB *sqlx.DB
)

type Databases struct {
	Type   string
	MySQL  MySQLInfo
	SQLite SQLiteInfo
}

// MySQLInfo is the details for the database connection
type MySQLInfo struct {
	Username  string
	Password  string
	Name      string
	Hostname  string
	Port      int
	Parameter string
}

// SQLiteInfo is the details for the database connection
type SQLiteInfo struct {
	Parameter string
}

// DSN returns the Data Source Name
func DSN(ci MySQLInfo) string {
	// Example: root:@tcp(localhost:3306)/test
	return ci.Username +
		":" +
		ci.Password +
		"@tcp(" +
		ci.Hostname +
		":" +
		fmt.Sprintf("%d", ci.Port) +
		")/" +
		ci.Name + ci.Parameter
}

// Connect to the database
func Connect(d Databases) {
	var err error

	switch d.Type {
	case "MySQL":
		// Connect to MySQL
		if DB, err = sqlx.Connect("mysql", DSN(d.MySQL)); err != nil {
			log.Println("SQL Driver Error", err)
		}

		// Check if is alive
		if err := DB.Ping(); err != nil {
			log.Println("Database Error", err)
		}
	case "SQLite":
		// Connect to SQLite
		if DB, err = sqlx.Connect("sqlite3", d.SQLite.Parameter); err != nil {
			log.Println("SQL Driver Error", err)
		}

		// Check if is alive
		if err := DB.Ping(); err != nil {
			log.Println("Database Error", err)
		}
	default:
		log.Println("No registered database in config")
	}
}
<commit_msg>Removed accidental extra variables<commit_after>package database

import (
	"fmt"
	"log"

	_ "github.com/go-sql-driver/mysql"
	"github.com/jmoiron/sqlx"
	_ "github.com/mattn/go-sqlite3"
)

var (
	// Database wrapper
	DB *sqlx.DB
)

type Databases struct {
	Type   string
	MySQL  MySQLInfo
	SQLite SQLiteInfo
}

// MySQLInfo is the details for the database connection
type MySQLInfo struct {
	Username  string
	Password  string
	Name      string
	Hostname  string
	Port      int
	Parameter string
}

// SQLiteInfo is the details for the database connection
type SQLiteInfo struct {
	Parameter string
}

// DSN returns the Data Source Name
func DSN(ci MySQLInfo) string {
	// Example: root:@tcp(localhost:3306)/test
	return ci.Username +
		":" +
		ci.Password +
		"@tcp(" +
		ci.Hostname +
		":" +
		fmt.Sprintf("%d", ci.Port) +
		")/" +
		ci.Name + ci.Parameter
}

// Connect to the database
func Connect(d Databases) {
	var err error

	switch d.Type {
	case "MySQL":
		// Connect to MySQL
		if DB, err = sqlx.Connect("mysql", DSN(d.MySQL)); err != nil {
			log.Println("SQL Driver Error", err)
		}

		// Check if is alive
		if err = DB.Ping(); err != nil {
			log.Println("Database Error", err)
		}
	case "SQLite":
		// Connect to SQLite
		if DB, err = sqlx.Connect("sqlite3", d.SQLite.Parameter); err != nil {
			log.Println("SQL Driver Error", err)
		}

		// Check if is alive
		if err = DB.Ping(); err != nil {
			log.Println("Database Error", err)
		}
	default:
		log.Println("No registered database in config")
	}
}
<|endoftext|>


<commit_before>package httputil

import (
	"net/http"
	"strings"
)

// IsHttps is a helper function that evaluates the http.Request
// and returns True if the Request uses HTTPS. It is able to detect,
// using the X-Forwarded-Proto, if the original request was HTTPS and
// routed through a reverse proxy with SSL termination.
func IsHttps(r *http.Request) bool {
	switch {
	case r.URL.Scheme == "https":
		return true
	case strings.HasPrefix(r.Proto, "HTTPS"):
		return true
	case r.Header.Get("X-Forwarded-Proto") == "https":
		return true
	default:
		return false
	}
}

// GetScheme is a helper function that evaluates the http.Request
// and returns the scheme, HTTP or HTTPS. It is able to detect,
// using the X-Forwarded-Proto, if the original request was HTTPS
// and routed through a reverse proxy with SSL termination.
func GetScheme(r *http.Request) string {
	switch {
	case r.URL.Scheme == "https":
		return "https"
	case strings.HasPrefix(r.Proto, "HTTPS"):
		return "https"
	case r.Header.Get("X-Forwarded-Proto") == "https":
		return "https"
	default:
		return "http"
	}
}

// GetHost is a helper function that evaluates the http.Request
// and returns the hostname. It is able to detect, using the
// X-Forarded-For header, the original hostname when routed
// through a reverse proxy.
func GetHost(r *http.Request) string {
	switch {
	case len(r.Host) != 0:
		return r.Host
	case len(r.URL.Host) != 0:
		return r.URL.Host
	case len(r.Header.Get("X-Forwarded-For")) != 0:
		return r.Header.Get("X-Forwarded-For")
	case len(r.Header.Get("X-Host")) != 0:
		return r.Header.Get("X-Host")
	case len(r.Header.Get("XFF")) != 0:
		return r.Header.Get("XFF")
	default:
		return "localhost:8080"
	}
}

// GetURL is a helper function that evaluates the http.Request
// and returns the URL as a string. Only the scheme + hostname
// are included; the path is excluded.
func GetURL(r *http.Request) string {
	return GetScheme(r) + "://" + GetHost(r)
}

// GetCookie retrieves and verifies the cookie value.
func GetCookie(r *http.Request, name string) (value string) {
	cookie, err := r.Cookie(name)
	if err != nil {
		return
	}
	value = cookie.Value
	return
}

// SetCookie writes the cookie value.
func SetCookie(w http.ResponseWriter, r *http.Request, name, value string) {
	cookie := http.Cookie{
		Name:     name,
		Value:    value,
		Path:     "/",
		Domain:   r.URL.Host,
		HttpOnly: true,
		Secure:   IsHttps(r),
	}

	http.SetCookie(w, &cookie)
}

// DelCookie deletes a cookie.
func DelCookie(w http.ResponseWriter, r *http.Request, name string) {
	cookie := http.Cookie{
		Name:   name,
		Value:  "deleted",
		Path:   "/",
		Domain: r.URL.Host,
		MaxAge: -1,
	}

	http.SetCookie(w, &cookie)
}
<commit_msg>Detect https scheme when serving ssl<commit_after>package httputil

import (
	"net/http"
	"strings"
)

// IsHttps is a helper function that evaluates the http.Request
// and returns True if the Request uses HTTPS. It is able to detect,
// using the X-Forwarded-Proto, if the original request was HTTPS and
// routed through a reverse proxy with SSL termination.
func IsHttps(r *http.Request) bool {
	switch {
	case r.URL.Scheme == "https":
		return true
	case r.TLS != nil:
		return true
	case strings.HasPrefix(r.Proto, "HTTPS"):
		return true
	case r.Header.Get("X-Forwarded-Proto") == "https":
		return true
	default:
		return false
	}
}

// GetScheme is a helper function that evaluates the http.Request
// and returns the scheme, HTTP or HTTPS. It is able to detect,
// using the X-Forwarded-Proto, if the original request was HTTPS
// and routed through a reverse proxy with SSL termination.
func GetScheme(r *http.Request) string {
	switch {
	case r.URL.Scheme == "https":
		return "https"
	case r.TLS != nil:
		return "https"
	case strings.HasPrefix(r.Proto, "HTTPS"):
		return "https"
	case r.Header.Get("X-Forwarded-Proto") == "https":
		return "https"
	default:
		return "http"
	}
}

// GetHost is a helper function that evaluates the http.Request
// and returns the hostname. It is able to detect, using the
// X-Forarded-For header, the original hostname when routed
// through a reverse proxy.
func GetHost(r *http.Request) string {
	switch {
	case len(r.Host) != 0:
		return r.Host
	case len(r.URL.Host) != 0:
		return r.URL.Host
	case len(r.Header.Get("X-Forwarded-For")) != 0:
		return r.Header.Get("X-Forwarded-For")
	case len(r.Header.Get("X-Host")) != 0:
		return r.Header.Get("X-Host")
	case len(r.Header.Get("XFF")) != 0:
		return r.Header.Get("XFF")
	default:
		return "localhost:8080"
	}
}

// GetURL is a helper function that evaluates the http.Request
// and returns the URL as a string. Only the scheme + hostname
// are included; the path is excluded.
func GetURL(r *http.Request) string {
	return GetScheme(r) + "://" + GetHost(r)
}

// GetCookie retrieves and verifies the cookie value.
func GetCookie(r *http.Request, name string) (value string) {
	cookie, err := r.Cookie(name)
	if err != nil {
		return
	}
	value = cookie.Value
	return
}

// SetCookie writes the cookie value.
func SetCookie(w http.ResponseWriter, r *http.Request, name, value string) {
	cookie := http.Cookie{
		Name:     name,
		Value:    value,
		Path:     "/",
		Domain:   r.URL.Host,
		HttpOnly: true,
		Secure:   IsHttps(r),
	}

	http.SetCookie(w, &cookie)
}

// DelCookie deletes a cookie.
func DelCookie(w http.ResponseWriter, r *http.Request, name string) {
	cookie := http.Cookie{
		Name:   name,
		Value:  "deleted",
		Path:   "/",
		Domain: r.URL.Host,
		MaxAge: -1,
	}

	http.SetCookie(w, &cookie)
}
<|endoftext|>


<commit_before>package services

import (
	"errors"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/die-net/lrucache"
	"github.com/gregjones/httpcache"
	"github.com/matrix-org/go-neb/database"
	"github.com/matrix-org/go-neb/matrix"
	"github.com/matrix-org/go-neb/polling"
	"github.com/matrix-org/go-neb/types"
	"github.com/mmcdole/gofeed"
	"github.com/prometheus/client_golang/prometheus"
	"html"
	"net/http"
	"time"
)

var cachingClient *http.Client

var (
	pollCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "goneb_rss_polls_total",
		Help: "The number of feed polls from RSS services",
	}, []string{"url", "http_status"})
)

const minPollingIntervalSeconds = 60 * 5 // 5 min (News feeds can be genuinely spammy)

type rssBotService struct {
	types.DefaultService
	id            string
	serviceUserID string
	Feeds         map[string]struct { // feed_url => { }
		PollIntervalMins         int      `json:"poll_interval_mins"`
		Rooms                    []string `json:"rooms"`
		NextPollTimestampSecs    int64    // Internal: When we should poll again
		FeedUpdatedTimestampSecs int64    // Internal: The last time the feed was updated
		RecentGUIDs              []string // Internal: The most recently seen GUIDs. Sized to the number of items in the feed.
	} `json:"feeds"`
}

func (s *rssBotService) ServiceUserID() string { return s.serviceUserID }
func (s *rssBotService) ServiceID() string     { return s.id }
func (s *rssBotService) ServiceType() string   { return "rssbot" }

// Register will check the liveness of each RSS feed given. If all feeds check out okay, no error is returned.
func (s *rssBotService) Register(oldService types.Service, client *matrix.Client) error {
	if len(s.Feeds) == 0 {
		// this is an error UNLESS the old service had some feeds in which case they are deleting us :(
		var numOldFeeds int
		oldFeedService, ok := oldService.(*rssBotService)
		if !ok {
			log.WithField("service", oldService).Error("Old service isn't a rssBotService")
		} else {
			numOldFeeds = len(oldFeedService.Feeds)
		}
		if numOldFeeds == 0 {
			return errors.New("An RSS feed must be specified.")
		}
		return nil
	}
	// Make sure we can parse the feed
	for feedURL, feedInfo := range s.Feeds {
		fp := gofeed.NewParser()
		fp.Client = cachingClient
		if _, err := fp.ParseURL(feedURL); err != nil {
			return fmt.Errorf("Failed to read URL %s: %s", feedURL, err.Error())
		}
		if len(feedInfo.Rooms) == 0 {
			return fmt.Errorf("Feed %s has no rooms to send updates to", feedURL)
		}
	}

	s.joinRooms(client)
	return nil
}

func (s *rssBotService) joinRooms(client *matrix.Client) {
	roomSet := make(map[string]bool)
	for _, feedInfo := range s.Feeds {
		for _, roomID := range feedInfo.Rooms {
			roomSet[roomID] = true
		}
	}

	for roomID := range roomSet {
		if _, err := client.JoinRoom(roomID, "", ""); err != nil {
			log.WithFields(log.Fields{
				log.ErrorKey: err,
				"room_id":    roomID,
				"user_id":    client.UserID,
			}).Error("Failed to join room")
		}
	}
}

func (s *rssBotService) PostRegister(oldService types.Service) {
	if len(s.Feeds) == 0 { // bye-bye :(
		logger := log.WithFields(log.Fields{
			"service_id":   s.ServiceID(),
			"service_type": s.ServiceType(),
		})
		logger.Info("Deleting service: No feeds remaining.")
		polling.StopPolling(s)
		if err := database.GetServiceDB().DeleteService(s.ServiceID()); err != nil {
			logger.WithError(err).Error("Failed to delete service")
		}
	}
}

func (s *rssBotService) OnPoll(cli *matrix.Client) time.Time {
	logger := log.WithFields(log.Fields{
		"service_id":   s.ServiceID(),
		"service_type": s.ServiceType(),
	})
	now := time.Now().Unix() // Second resolution

	// Work out which feeds should be polled
	var pollFeeds []string
	for u, feedInfo := range s.Feeds {
		if feedInfo.NextPollTimestampSecs == 0 || now >= feedInfo.NextPollTimestampSecs {
			// re-query this feed
			pollFeeds = append(pollFeeds, u)
		}
	}

	if len(pollFeeds) == 0 {
		return s.nextTimestamp()
	}

	// Query each feed and send new items to subscribed rooms
	for _, u := range pollFeeds {
		feed, items, err := s.queryFeed(u)
		if err != nil {
			logger.WithField("feed_url", u).WithError(err).Error("Failed to query feed")
			herr, ok := err.(gofeed.HTTPError)
			statusCode := 0 // e.g. network timeout
			if ok {
				statusCode = herr.StatusCode
			}
			pollCounter.With(prometheus.Labels{"url": u, "http_status": string(statusCode)}).Inc()
			continue
		}
		pollCounter.With(prometheus.Labels{"url": u, "http_status": "200"}).Inc() // technically 2xx but gofeed doesn't tell us which
		// Loop backwards since [0] is the most recent and we want to send in chronological order
		for i := len(items) - 1; i >= 0; i-- {
			item := items[i]
			if err := s.sendToRooms(cli, u, feed, item); err != nil {
				logger.WithFields(log.Fields{
					"feed_url":   u,
					log.ErrorKey: err,
					"item":       item,
				}).Error("Failed to send item to room")
			}
		}
	}

	// Persist the service to save the next poll times
	if _, err := database.GetServiceDB().StoreService(s); err != nil {
		logger.WithError(err).Error("Failed to persist next poll times for service")
	}

	return s.nextTimestamp()
}

func (s *rssBotService) nextTimestamp() time.Time {
	// return the earliest next poll ts
	var earliestNextTs int64
	for _, feedInfo := range s.Feeds {
		if earliestNextTs == 0 || feedInfo.NextPollTimestampSecs < earliestNextTs {
			earliestNextTs = feedInfo.NextPollTimestampSecs
		}
	}

	// Don't allow times in the past. Set a min re-poll threshold of 20s to avoid
	// tight-looping on feeds which 500.
	now := time.Now().Unix()
	if earliestNextTs <= now {
		earliestNextTs = now + 20
	}

	return time.Unix(earliestNextTs, 0)
}

// Query the given feed, update relevant timestamps and return NEW items
func (s *rssBotService) queryFeed(feedURL string) (*gofeed.Feed, []gofeed.Item, error) {
	log.WithField("feed_url", feedURL).Info("Querying feed")
	var items []gofeed.Item
	fp := gofeed.NewParser()
	fp.Client = cachingClient
	feed, err := fp.ParseURL(feedURL)
	if err != nil {
		return nil, items, err
	}

	// Work out which items are new, if any (based on the last updated TS we have)
	// If the TS is 0 then this is the first ever poll, so let's not send 10s of events
	// into the room and just do new ones from this point onwards.
	if s.Feeds[feedURL].FeedUpdatedTimestampSecs != 0 {
		items = s.newItems(feedURL, feed.Items)
	}

	now := time.Now().Unix() // Second resolution

	// Work out when this feed was last updated
	var feedLastUpdatedTs int64
	if feed.UpdatedParsed != nil {
		feedLastUpdatedTs = feed.UpdatedParsed.Unix()
	} else if len(feed.Items) > 0 {
		i := feed.Items[0]
		if i != nil && i.PublishedParsed != nil {
			feedLastUpdatedTs = i.PublishedParsed.Unix()
		}
	}

	// Work out when to next poll this feed
	nextPollTsSec := now + minPollingIntervalSeconds
	if s.Feeds[feedURL].PollIntervalMins > int(minPollingIntervalSeconds/60) {
		nextPollTsSec = now + int64(s.Feeds[feedURL].PollIntervalMins*60)
	}
	// TODO: Handle the 'sy' Syndication extension to control update interval.
	// See http://www.feedforall.com/syndication.htm and http://web.resource.org/rss/1.0/modules/syndication/

	s.updateFeedInfo(feedURL, feed.Items, nextPollTsSec, feedLastUpdatedTs)
	return feed, items, nil
}

func (s *rssBotService) newItems(feedURL string, allItems []*gofeed.Item) (items []gofeed.Item) {
	for _, i := range allItems {
		if i == nil || i.PublishedParsed == nil {
			continue
		}

		if i.PublishedParsed.Unix() > s.Feeds[feedURL].FeedUpdatedTimestampSecs {
			// if we've seen this guid before, we've sent it before (even if the timestamp is newer)
			seenBefore := false
			for _, guid := range s.Feeds[feedURL].RecentGUIDs {
				if guid == i.GUID {
					seenBefore = true
					break
				}
			}
			if seenBefore {
				continue
			}

			items = append(items, *i)
		}
	}
	return
}

func (s *rssBotService) updateFeedInfo(feedURL string, allFeedItems []*gofeed.Item, nextPollTs, feedUpdatedTs int64) {
	// map items to guid strings
	var guids []string
	for _, i := range allFeedItems {
		guids = append(guids, i.GUID)
	}

	for u := range s.Feeds {
		if u != feedURL {
			continue
		}
		f := s.Feeds[u]
		f.NextPollTimestampSecs = nextPollTs
		f.FeedUpdatedTimestampSecs = feedUpdatedTs
		f.RecentGUIDs = guids
		s.Feeds[u] = f
	}
}

func (s *rssBotService) sendToRooms(cli *matrix.Client, feedURL string, feed *gofeed.Feed, item gofeed.Item) error {
	logger := log.WithField("feed_url", feedURL).WithField("title", item.Title)
	logger.Info("New feed item")
	for _, roomID := range s.Feeds[feedURL].Rooms {
		if _, err := cli.SendMessageEvent(roomID, "m.room.message", itemToHTML(feed, item)); err != nil {
			logger.WithError(err).WithField("room_id", roomID).Error("Failed to send to room")
		}
	}
	return nil
}

// SomeOne posted a new article: Title Of The Entry ( https://someurl.com/blag )
func itemToHTML(feed *gofeed.Feed, item gofeed.Item) matrix.HTMLMessage {
	return matrix.GetHTMLMessage("m.notice", fmt.Sprintf(
		"%s posted a new article: %s ( %s )",
		html.EscapeString(feed.Title), html.EscapeString(item.Title), html.EscapeString(item.Link),
	))
}

func init() {
	lruCache := lrucache.New(1024*1024*20, 0) // 20 MB cache, no max-age
	cachingClient = httpcache.NewTransport(lruCache).Client()
	types.RegisterService(func(serviceID, serviceUserID, webhookEndpointURL string) types.Service {
		r := &rssBotService{
			id:            serviceID,
			serviceUserID: serviceUserID,
		}
		return r
	})
	prometheus.MustRegister(pollCounter)
}
<commit_msg>Only send the domain part of the RSS feed as that's more useful when doing aggregate metrics<commit_after>package services

import (
	"errors"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/die-net/lrucache"
	"github.com/gregjones/httpcache"
	"github.com/matrix-org/go-neb/database"
	"github.com/matrix-org/go-neb/matrix"
	"github.com/matrix-org/go-neb/polling"
	"github.com/matrix-org/go-neb/types"
	"github.com/mmcdole/gofeed"
	"github.com/prometheus/client_golang/prometheus"
	"html"
	"net/http"
	"net/url"
	"time"
)

var cachingClient *http.Client

var (
	pollCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "goneb_rss_polls_total",
		Help: "The number of feed polls from RSS services",
	}, []string{"url", "http_status"})
)

const minPollingIntervalSeconds = 60 * 5 // 5 min (News feeds can be genuinely spammy)

type rssBotService struct {
	types.DefaultService
	id            string
	serviceUserID string
	Feeds         map[string]struct { // feed_url => { }
		PollIntervalMins         int      `json:"poll_interval_mins"`
		Rooms                    []string `json:"rooms"`
		NextPollTimestampSecs    int64    // Internal: When we should poll again
		FeedUpdatedTimestampSecs int64    // Internal: The last time the feed was updated
		RecentGUIDs              []string // Internal: The most recently seen GUIDs. Sized to the number of items in the feed.
	} `json:"feeds"`
}

func (s *rssBotService) ServiceUserID() string { return s.serviceUserID }
func (s *rssBotService) ServiceID() string     { return s.id }
func (s *rssBotService) ServiceType() string   { return "rssbot" }

// Register will check the liveness of each RSS feed given. If all feeds check out okay, no error is returned.
func (s *rssBotService) Register(oldService types.Service, client *matrix.Client) error {
	if len(s.Feeds) == 0 {
		// this is an error UNLESS the old service had some feeds in which case they are deleting us :(
		var numOldFeeds int
		oldFeedService, ok := oldService.(*rssBotService)
		if !ok {
			log.WithField("service", oldService).Error("Old service isn't a rssBotService")
		} else {
			numOldFeeds = len(oldFeedService.Feeds)
		}
		if numOldFeeds == 0 {
			return errors.New("An RSS feed must be specified.")
		}
		return nil
	}
	// Make sure we can parse the feed
	for feedURL, feedInfo := range s.Feeds {
		fp := gofeed.NewParser()
		fp.Client = cachingClient
		if _, err := fp.ParseURL(feedURL); err != nil {
			return fmt.Errorf("Failed to read URL %s: %s", feedURL, err.Error())
		}
		if len(feedInfo.Rooms) == 0 {
			return fmt.Errorf("Feed %s has no rooms to send updates to", feedURL)
		}
	}

	s.joinRooms(client)
	return nil
}

func (s *rssBotService) joinRooms(client *matrix.Client) {
	roomSet := make(map[string]bool)
	for _, feedInfo := range s.Feeds {
		for _, roomID := range feedInfo.Rooms {
			roomSet[roomID] = true
		}
	}

	for roomID := range roomSet {
		if _, err := client.JoinRoom(roomID, "", ""); err != nil {
			log.WithFields(log.Fields{
				log.ErrorKey: err,
				"room_id":    roomID,
				"user_id":    client.UserID,
			}).Error("Failed to join room")
		}
	}
}

func (s *rssBotService) PostRegister(oldService types.Service) {
	if len(s.Feeds) == 0 { // bye-bye :(
		logger := log.WithFields(log.Fields{
			"service_id":   s.ServiceID(),
			"service_type": s.ServiceType(),
		})
		logger.Info("Deleting service: No feeds remaining.")
		polling.StopPolling(s)
		if err := database.GetServiceDB().DeleteService(s.ServiceID()); err != nil {
			logger.WithError(err).Error("Failed to delete service")
		}
	}
}

func (s *rssBotService) OnPoll(cli *matrix.Client) time.Time {
	logger := log.WithFields(log.Fields{
		"service_id":   s.ServiceID(),
		"service_type": s.ServiceType(),
	})
	now := time.Now().Unix() // Second resolution

	// Work out which feeds should be polled
	var pollFeeds []string
	for u, feedInfo := range s.Feeds {
		if feedInfo.NextPollTimestampSecs == 0 || now >= feedInfo.NextPollTimestampSecs {
			// re-query this feed
			pollFeeds = append(pollFeeds, u)
		}
	}

	if len(pollFeeds) == 0 {
		return s.nextTimestamp()
	}

	// Query each feed and send new items to subscribed rooms
	for _, u := range pollFeeds {
		feed, items, err := s.queryFeed(u)
		if err != nil {
			logger.WithField("feed_url", u).WithError(err).Error("Failed to query feed")
			sendMetric(u, err)
			continue
		}
		sendMetric(u, nil)
		// Loop backwards since [0] is the most recent and we want to send in chronological order
		for i := len(items) - 1; i >= 0; i-- {
			item := items[i]
			if err := s.sendToRooms(cli, u, feed, item); err != nil {
				logger.WithFields(log.Fields{
					"feed_url":   u,
					log.ErrorKey: err,
					"item":       item,
				}).Error("Failed to send item to room")
			}
		}
	}

	// Persist the service to save the next poll times
	if _, err := database.GetServiceDB().StoreService(s); err != nil {
		logger.WithError(err).Error("Failed to persist next poll times for service")
	}

	return s.nextTimestamp()
}

func sendMetric(urlStr string, err error) {
	// extract domain part of RSS feed URL to get coarser (more useful) statistics
	domain := urlStr
	u, urlErr := url.Parse(urlStr)
	if urlErr == nil {
		domain = u.Host
	}
	if err != nil {
		herr, ok := err.(gofeed.HTTPError)
		statusCode := 0 // e.g. network timeout
		if ok {
			statusCode = herr.StatusCode
		}
		pollCounter.With(prometheus.Labels{"url": domain, "http_status": string(statusCode)}).Inc()
	} else {
		pollCounter.With(prometheus.Labels{"url": domain, "http_status": "200"}).Inc() // technically 2xx but gofeed doesn't tell us which
	}
}

func (s *rssBotService) nextTimestamp() time.Time {
	// return the earliest next poll ts
	var earliestNextTs int64
	for _, feedInfo := range s.Feeds {
		if earliestNextTs == 0 || feedInfo.NextPollTimestampSecs < earliestNextTs {
			earliestNextTs = feedInfo.NextPollTimestampSecs
		}
	}

	// Don't allow times in the past. Set a min re-poll threshold of 20s to avoid
	// tight-looping on feeds which 500.
	now := time.Now().Unix()
	if earliestNextTs <= now {
		earliestNextTs = now + 20
	}

	return time.Unix(earliestNextTs, 0)
}

// Query the given feed, update relevant timestamps and return NEW items
func (s *rssBotService) queryFeed(feedURL string) (*gofeed.Feed, []gofeed.Item, error) {
	log.WithField("feed_url", feedURL).Info("Querying feed")
	var items []gofeed.Item
	fp := gofeed.NewParser()
	fp.Client = cachingClient
	feed, err := fp.ParseURL(feedURL)
	if err != nil {
		return nil, items, err
	}

	// Work out which items are new, if any (based on the last updated TS we have)
	// If the TS is 0 then this is the first ever poll, so let's not send 10s of events
	// into the room and just do new ones from this point onwards.
	if s.Feeds[feedURL].FeedUpdatedTimestampSecs != 0 {
		items = s.newItems(feedURL, feed.Items)
	}

	now := time.Now().Unix() // Second resolution

	// Work out when this feed was last updated
	var feedLastUpdatedTs int64
	if feed.UpdatedParsed != nil {
		feedLastUpdatedTs = feed.UpdatedParsed.Unix()
	} else if len(feed.Items) > 0 {
		i := feed.Items[0]
		if i != nil && i.PublishedParsed != nil {
			feedLastUpdatedTs = i.PublishedParsed.Unix()
		}
	}

	// Work out when to next poll this feed
	nextPollTsSec := now + minPollingIntervalSeconds
	if s.Feeds[feedURL].PollIntervalMins > int(minPollingIntervalSeconds/60) {
		nextPollTsSec = now + int64(s.Feeds[feedURL].PollIntervalMins*60)
	}
	// TODO: Handle the 'sy' Syndication extension to control update interval.
	// See http://www.feedforall.com/syndication.htm and http://web.resource.org/rss/1.0/modules/syndication/

	s.updateFeedInfo(feedURL, feed.Items, nextPollTsSec, feedLastUpdatedTs)
	return feed, items, nil
}

func (s *rssBotService) newItems(feedURL string, allItems []*gofeed.Item) (items []gofeed.Item) {
	for _, i := range allItems {
		if i == nil || i.PublishedParsed == nil {
			continue
		}

		if i.PublishedParsed.Unix() > s.Feeds[feedURL].FeedUpdatedTimestampSecs {
			// if we've seen this guid before, we've sent it before (even if the timestamp is newer)
			seenBefore := false
			for _, guid := range s.Feeds[feedURL].RecentGUIDs {
				if guid == i.GUID {
					seenBefore = true
					break
				}
			}
			if seenBefore {
				continue
			}

			items = append(items, *i)
		}
	}
	return
}

func (s *rssBotService) updateFeedInfo(feedURL string, allFeedItems []*gofeed.Item, nextPollTs, feedUpdatedTs int64) {
	// map items to guid strings
	var guids []string
	for _, i := range allFeedItems {
		guids = append(guids, i.GUID)
	}

	for u := range s.Feeds {
		if u != feedURL {
			continue
		}
		f := s.Feeds[u]
		f.NextPollTimestampSecs = nextPollTs
		f.FeedUpdatedTimestampSecs = feedUpdatedTs
		f.RecentGUIDs = guids
		s.Feeds[u] = f
	}
}

func (s *rssBotService) sendToRooms(cli *matrix.Client, feedURL string, feed *gofeed.Feed, item gofeed.Item) error {
	logger := log.WithField("feed_url", feedURL).WithField("title", item.Title)
	logger.Info("New feed item")
	for _, roomID := range s.Feeds[feedURL].Rooms {
		if _, err := cli.SendMessageEvent(roomID, "m.room.message", itemToHTML(feed, item)); err != nil {
			logger.WithError(err).WithField("room_id", roomID).Error("Failed to send to room")
		}
	}
	return nil
}

// SomeOne posted a new article: Title Of The Entry ( https://someurl.com/blag )
func itemToHTML(feed *gofeed.Feed, item gofeed.Item) matrix.HTMLMessage {
	return matrix.GetHTMLMessage("m.notice", fmt.Sprintf(
		"%s posted a new article: %s ( %s )",
		html.EscapeString(feed.Title), html.EscapeString(item.Title), html.EscapeString(item.Link),
	))
}

func init() {
	lruCache := lrucache.New(1024*1024*20, 0) // 20 MB cache, no max-age
	cachingClient = httpcache.NewTransport(lruCache).Client()
	types.RegisterService(func(serviceID, serviceUserID, webhookEndpointURL string) types.Service {
		r := &rssBotService{
			id:            serviceID,
			serviceUserID: serviceUserID,
		}
		return r
	})
	prometheus.MustRegister(pollCounter)
}
<|endoftext|>


<commit_before>//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"

	"github.com/01org/ciao/ciao-controller/api"
	"github.com/01org/ciao/ciao-controller/types"
	"github.com/01org/ciao/openstack/compute"
	"github.com/01org/ciao/payloads"
	"github.com/01org/ciao/templateutils"

	"gopkg.in/yaml.v2"
)

var workloadCommand = &command{
	SubCommands: map[string]subCommand{
		"list":   new(workloadListCommand),
		"create": new(workloadCreateCommand),
		"delete": new(workloadDeleteCommand),
		"show":   new(workloadShowCommand),
	},
}

type workloadListCommand struct {
	Flag     flag.FlagSet
	template string
}

func (cmd *workloadListCommand) usage(...string) {
	fmt.Fprintf(os.Stderr, `usage: ciao-cli [options] workload list

List all workloads

`)
	cmd.Flag.PrintDefaults()
	fmt.Fprintf(os.Stderr, "\n%s",
		templateutils.GenerateUsageDecorated("f", compute.FlavorsDetails{}.Flavors, nil))
	os.Exit(2)
}

func (cmd *workloadListCommand) parseArgs(args []string) []string {
	cmd.Flag.StringVar(&cmd.template, "f", "", "Template used to format output")
	cmd.Flag.Usage = func() { cmd.usage() }
	cmd.Flag.Parse(args)
	return cmd.Flag.Args()
}

func (cmd *workloadListCommand) run(args []string) error {
	if *tenantID == "" {
		fatalf("Missing required -tenant-id parameter")
	}

	var flavors compute.FlavorsDetails
	if *tenantID == "" {
		*tenantID = "faketenant"
	}

	url := buildComputeURL("%s/flavors/detail", *tenantID)

	resp, err := sendHTTPRequest("GET", url, nil, nil)
	if err != nil {
		fatalf(err.Error())
	}

	err = unmarshalHTTPResponse(resp, &flavors)
	if err != nil {
		fatalf(err.Error())
	}

	if cmd.template != "" {
		return templateutils.OutputToTemplate(os.Stdout, "workload-list", cmd.template,
			&flavors.Flavors, nil)
	}

	for i, flavor := range flavors.Flavors {
		fmt.Printf("Workload %d\n", i+1)
		fmt.Printf("\tName: %s\n\tUUID:%s\n\tImage UUID: %s\n\tCPUs: %d\n\tMemory: %d MB\n",
			flavor.Name, flavor.ID, flavor.Disk, flavor.Vcpus, flavor.RAM)
	}
	return nil
}

type workloadCreateCommand struct {
	Flag     flag.FlagSet
	yamlFile string
}

func (cmd *workloadCreateCommand) parseArgs(args []string) []string {
	cmd.Flag.StringVar(&cmd.yamlFile, "yaml", "", "filename for yaml which describes the workload")
	cmd.Flag.Usage = func() { cmd.usage() }
	cmd.Flag.Parse(args)
	return cmd.Flag.Args()
}

func (cmd *workloadCreateCommand) usage(...string) {
	fmt.Fprintf(os.Stderr, `usage: ciao-cli [options] workload create [flags]

Create a new workload

The create flags are:

`)
	cmd.Flag.PrintDefaults()
	os.Exit(2)
}

func getCiaoWorkloadsResource() (string, error) {
	return getCiaoResource("workloads", api.WorkloadsV1)
}

type source struct {
	Type types.SourceType `yaml:"service"`
	ID   string           `yaml:"id"`
}

type disk struct {
	ID        *string `yaml:"volume_id,omitempty"`
	Size      int     `yaml:"size"`
	Bootable  bool    `yaml:"bootable"`
	Source    *source `yaml:"source"`
	Ephemeral bool    `yaml:"ephemeral"`
}

type defaultResources struct {
	VCPUs int `yaml:"vcpus"`
	MemMB int `yaml:"mem_mb"`
}

// we currently only use the first disk due to lack of support
// in types.Workload for multiple storage resources.
type workloadOptions struct {
	Description     string           `yaml:"description"`
	VMType          string           `yaml:"vm_type"`
	FWType          string           `yaml:"fw_type,omitempty"`
	ImageName       string           `yaml:"image_name,omitempty"`
	ImageID         string           `yaml:"image_id,omitempty"`
	Defaults        defaultResources `yaml:"defaults"`
	CloudConfigFile string           `yaml:"cloud_init,omitempty"`
	Disks           []disk           `yaml:"disks,omitempty"`
}

func optToReqStorage(opt workloadOptions) ([]types.StorageResource, error) {
	storage := make([]types.StorageResource, 0)
	bootableCount := 0
	for _, disk := range opt.Disks {
		res := types.StorageResource{
			Size:      disk.Size,
			Bootable:  disk.Bootable,
			Ephemeral: disk.Ephemeral,
		}

		if disk.Source != nil && disk.Source.Type == "" {
			disk.Source.Type = types.Empty
		}

		if disk.ID != nil {
			res.ID = *disk.ID
		} else if disk.Size == 0 {
			// source had better exist.
			if disk.Source == nil {
				return nil, errors.New("Invalid workload yaml: disk source may not be nil")
			}

			res.SourceType = disk.Source.Type
			res.SourceID = disk.Source.ID

			if res.SourceID != "" && res.SourceType == types.Empty {
				return nil, errors.New("Invalid workload yaml: when specifying a source ID a type must also be specified")
			}

			if res.SourceType != types.Empty && res.SourceID == "" {
				return nil, errors.New("Invalid workload yaml: when specifying a source type other than empty an id must also be specified")
			}
		} else {
			if disk.Bootable == true {
				// you may not request a bootable drive
				// from an empty source
				return nil, errors.New("Invalid workload yaml: empty disk source may not be bootable")
			}

			res.SourceType = types.Empty
		}

		if disk.Bootable {
			bootableCount++
		}

		storage = append(storage, res)
	}

	if payloads.Hypervisor(opt.VMType) == payloads.QEMU && bootableCount == 0 {
		return nil, errors.New("Invalid workload yaml: no bootable disks specified for a VM")
	}

	return storage, nil
}

func optToReq(opt workloadOptions, req *types.Workload) error {
	b, err := ioutil.ReadFile(opt.CloudConfigFile)
	if err != nil {
		return err
	}

	config := string(b)

	// this is where you'd validate that the options make
	// sense.
	req.Description = opt.Description
	req.VMType = payloads.Hypervisor(opt.VMType)
	req.FWType = opt.FWType
	req.ImageName = opt.ImageName
	req.ImageID = opt.ImageID
	req.Config = config
	req.Storage, err = optToReqStorage(opt)

	if err != nil {
		return err
	}

	// all default resources are required.
	defaults := opt.Defaults

	r := payloads.RequestedResource{
		Type:  payloads.VCPUs,
		Value: defaults.VCPUs,
	}
	req.Defaults = append(req.Defaults, r)

	r = payloads.RequestedResource{
		Type:  payloads.MemMB,
		Value: defaults.MemMB,
	}
	req.Defaults = append(req.Defaults, r)

	return nil
}

func outputWorkload(w types.Workload) {
	var opt workloadOptions

	opt.Description = w.Description
	opt.VMType = string(w.VMType)
	opt.FWType = w.FWType
	opt.ImageName = w.ImageName
	opt.ImageID = w.ImageID
	for _, d := range w.Defaults {
		if d.Type == payloads.VCPUs {
			opt.Defaults.VCPUs = d.Value
		} else if d.Type == payloads.MemMB {
			opt.Defaults.MemMB = d.Value
		}
	}

	for _, s := range w.Storage {
		d := disk{
			Size:      s.Size,
			Bootable:  s.Bootable,
			Ephemeral: s.Ephemeral,
		}
		if s.ID != "" {
			d.ID = &s.ID
		}

		src := source{
			Type: s.SourceType,
			ID:   s.SourceID,
		}

		d.Source = &src

		opt.Disks = append(opt.Disks, d)
	}

	b, err := yaml.Marshal(opt)
	if err != nil {
		fatalf(err.Error())
	}

	fmt.Println(string(b))
	fmt.Println(w.Config)
}

func (cmd *workloadCreateCommand) run(args []string) error {
	var opt workloadOptions
	var req types.Workload

	if cmd.yamlFile == "" {
		cmd.usage()
	}

	f, err := ioutil.ReadFile(cmd.yamlFile)
	if err != nil {
		fatalf("Unable to read workload config file: %s\n", err)
	}

	err = yaml.Unmarshal(f, &opt)
	if err != nil {
		fatalf("Config file invalid: %s\n", err)
	}

	err = optToReq(opt, &req)
	if err != nil {
		fatalf(err.Error())
	}

	b, err := json.Marshal(req)
	if err != nil {
		fatalf(err.Error())
	}

	body := bytes.NewReader(b)

	url, err := getCiaoWorkloadsResource()
	if err != nil {
		fatalf(err.Error())
	}

	ver := api.WorkloadsV1

	resp, err := sendCiaoRequest("POST", url, nil, body, &ver)
	if err != nil {
		fatalf(err.Error())
	}

	if resp.StatusCode != http.StatusCreated {
		fatalf("Workload creation failed: %s", resp.Status)
	}

	var workload types.WorkloadResponse

	err = unmarshalHTTPResponse(resp, &workload)
	if err != nil {
		fatalf(err.Error())
	}

	fmt.Printf("Created new workload: %s\n", workload.Workload.ID)

	return nil
}

type workloadDeleteCommand struct {
	Flag     flag.FlagSet
	workload string
}

func (cmd *workloadDeleteCommand) usage(...string) {
	fmt.Fprintf(os.Stderr, `usage: ciao-cli [options] workload delete [flags]

Deletes a given workload

The delete flags are:

`)
	cmd.Flag.PrintDefaults()
	os.Exit(2)
}

func (cmd *workloadDeleteCommand) parseArgs(args []string) []string {
	cmd.Flag.StringVar(&cmd.workload, "workload", "", "Workload UUID")
	cmd.Flag.Usage = func() { cmd.usage() }
	cmd.Flag.Parse(args)
	return cmd.Flag.Args()
}

func (cmd *workloadDeleteCommand) run(args []string) error {
	if cmd.workload == "" {
		cmd.usage()
	}

	url, err := getCiaoWorkloadsResource()
	if err != nil {
		fatalf(err.Error())
	}

	ver := api.WorkloadsV1

	// you should do a get first and search for the workload,
	// then use the href - but not with the currently used
	// OpenStack API. Until we support GET with a ciao API,
	// just hard code the path.
	url = fmt.Sprintf("%s/%s", url, cmd.workload)

	resp, err := sendCiaoRequest("DELETE", url, nil, nil, &ver)
	if err != nil {
		fatalf(err.Error())
	}

	if resp.StatusCode != http.StatusNoContent {
		fatalf("Workload deletion failed: %s", resp.Status)
	}

	return nil
}

type workloadShowCommand struct {
	Flag     flag.FlagSet
	template string
	workload string
}

func (cmd *workloadShowCommand) usage(...string) {
	fmt.Fprintf(os.Stderr, `usage: ciao-cli [options] workload show

Show workload details

`)
	cmd.Flag.PrintDefaults()
	fmt.Fprintf(os.Stderr, "\n%s",
		templateutils.GenerateUsageDecorated("f", types.Workload{}, nil))
	os.Exit(2)
}

func (cmd *workloadShowCommand) parseArgs(args []string) []string {
	cmd.Flag.StringVar(&cmd.workload, "workload", "", "Workload UUID")
	cmd.Flag.StringVar(&cmd.template, "f", "", "Template used to format output")
	cmd.Flag.Usage = func() { cmd.usage() }
	cmd.Flag.Parse(args)
	return cmd.Flag.Args()
}

func (cmd *workloadShowCommand) run(args []string) error {
	var wl types.Workload

	if cmd.workload == "" {
		cmd.usage()
	}

	url, err := getCiaoWorkloadsResource()
	if err != nil {
		fatalf(err.Error())
	}

	ver := api.WorkloadsV1

	// you should do a get first and search for the workload,
	// then use the href - but not with the currently used
	// OpenStack API. Until we support GET with a ciao API,
	// just hard code the path.
	url = fmt.Sprintf("%s/%s", url, cmd.workload)

	resp, err := sendCiaoRequest("GET", url, nil, nil, &ver)
	if err != nil {
		fatalf(err.Error())
	}

	if resp.StatusCode != http.StatusOK {
		fatalf("Workload show failed: %s", resp.Status)
	}

	err = unmarshalHTTPResponse(resp, &wl)
	if err != nil {
		fatalf(err.Error())
	}

	if cmd.template != "" {
		return templateutils.OutputToTemplate(os.Stdout, "workload-show", cmd.template, &wl, nil)
	}

	outputWorkload(wl)
	return nil
}
<commit_msg>ciao-cli: Adjust workload validation to support resizing<commit_after>//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"

	"github.com/01org/ciao/ciao-controller/api"
	"github.com/01org/ciao/ciao-controller/types"
	"github.com/01org/ciao/openstack/compute"
	"github.com/01org/ciao/payloads"
	"github.com/01org/ciao/templateutils"

	"gopkg.in/yaml.v2"
)

var workloadCommand = &command{
	SubCommands: map[string]subCommand{
		"list":   new(workloadListCommand),
		"create": new(workloadCreateCommand),
		"delete": new(workloadDeleteCommand),
		"show":   new(workloadShowCommand),
	},
}

type workloadListCommand struct {
	Flag     flag.FlagSet
	template string
}

func (cmd *workloadListCommand) usage(...string) {
	fmt.Fprintf(os.Stderr, `usage: ciao-cli [options] workload list

List all workloads

`)
	cmd.Flag.PrintDefaults()
	fmt.Fprintf(os.Stderr, "\n%s",
		templateutils.GenerateUsageDecorated("f", compute.FlavorsDetails{}.Flavors, nil))
	os.Exit(2)
}

func (cmd *workloadListCommand) parseArgs(args []string) []string {
	cmd.Flag.StringVar(&cmd.template, "f", "", "Template used to format output")
	cmd.Flag.Usage = func() { cmd.usage() }
	cmd.Flag.Parse(args)
	return cmd.Flag.Args()
}

func (cmd *workloadListCommand) run(args []string) error {
	if *tenantID == "" {
		fatalf("Missing required -tenant-id parameter")
	}

	var flavors compute.FlavorsDetails
	if *tenantID == "" {
		*tenantID = "faketenant"
	}

	url := buildComputeURL("%s/flavors/detail", *tenantID)

	resp, err := sendHTTPRequest("GET", url, nil, nil)
	if err != nil {
		fatalf(err.Error())
	}

	err = unmarshalHTTPResponse(resp, &flavors)
	if err != nil {
		fatalf(err.Error())
	}

	if cmd.template != "" {
		return templateutils.OutputToTemplate(os.Stdout, "workload-list", cmd.template,
			&flavors.Flavors, nil)
	}

	for i, flavor := range flavors.Flavors {
		fmt.Printf("Workload %d\n", i+1)
		fmt.Printf("\tName: %s\n\tUUID:%s\n\tImage UUID: %s\n\tCPUs: %d\n\tMemory: %d MB\n",
			flavor.Name, flavor.ID, flavor.Disk, flavor.Vcpus, flavor.RAM)
	}
	return nil
}

type workloadCreateCommand struct {
	Flag     flag.FlagSet
	yamlFile string
}

func (cmd *workloadCreateCommand) parseArgs(args []string) []string {
	cmd.Flag.StringVar(&cmd.yamlFile, "yaml", "", "filename for yaml which describes the workload")
	cmd.Flag.Usage = func() { cmd.usage() }
	cmd.Flag.Parse(args)
	return cmd.Flag.Args()
}

func (cmd *workloadCreateCommand) usage(...string) {
	fmt.Fprintf(os.Stderr, `usage: ciao-cli [options] workload create [flags]

Create a new workload

The create flags are:

`)
	cmd.Flag.PrintDefaults()
	os.Exit(2)
}

func getCiaoWorkloadsResource() (string, error) {
	return getCiaoResource("workloads", api.WorkloadsV1)
}

type source struct {
	Type types.SourceType `yaml:"service"`
	ID   string           `yaml:"id"`
}

type disk struct {
	ID        *string `yaml:"volume_id,omitempty"`
	Size      int     `yaml:"size"`
	Bootable  bool    `yaml:"bootable"`
	Source    source  `yaml:"source"`
	Ephemeral bool    `yaml:"ephemeral"`
}

type defaultResources struct {
	VCPUs int `yaml:"vcpus"`
	MemMB int `yaml:"mem_mb"`
}

// we currently only use the first disk due to lack of support
// in types.Workload for multiple storage resources.
type workloadOptions struct {
	Description     string           `yaml:"description"`
	VMType          string           `yaml:"vm_type"`
	FWType          string           `yaml:"fw_type,omitempty"`
	ImageName       string           `yaml:"image_name,omitempty"`
	ImageID         string           `yaml:"image_id,omitempty"`
	Defaults        defaultResources `yaml:"defaults"`
	CloudConfigFile string           `yaml:"cloud_init,omitempty"`
	Disks           []disk           `yaml:"disks,omitempty"`
}

func optToReqStorage(opt workloadOptions) ([]types.StorageResource, error) {
	storage := make([]types.StorageResource, 0)
	bootableCount := 0
	for _, disk := range opt.Disks {
		res := types.StorageResource{
			Size:      disk.Size,
			Bootable:  disk.Bootable,
			Ephemeral: disk.Ephemeral,
		}

		// Use existing volume
		if disk.ID != nil {
			res.ID = *disk.ID
		} else {
			// Create a new one
			if disk.Source.Type == "" {
				disk.Source.Type = types.Empty
			}

			if disk.Source.Type != types.Empty {
				res.SourceType = disk.Source.Type
				res.SourceID = disk.Source.ID

				if res.SourceID == "" {
					return nil, errors.New("Invalid workload yaml: when using a source an id must also be specified")
				}
			} else {
				if disk.Bootable == true {
					// you may not request a bootable drive
					// from an empty source
					return nil, errors.New("Invalid workload yaml: empty disk source may not be bootable")
				}

				if disk.Size <= 0 {
					return nil, errors.New("Invalid workload yaml: size required when creating a volume")
				}
			}
		}

		if disk.Bootable {
			bootableCount++
		}

		storage = append(storage, res)
	}

	if payloads.Hypervisor(opt.VMType) == payloads.QEMU && bootableCount == 0 {
		return nil, errors.New("Invalid workload yaml: no bootable disks specified for a VM")
	}

	return storage, nil
}

func optToReq(opt workloadOptions, req *types.Workload) error {
	b, err := ioutil.ReadFile(opt.CloudConfigFile)
	if err != nil {
		return err
	}

	config := string(b)

	// this is where you'd validate that the options make
	// sense.
	req.Description = opt.Description
	req.VMType = payloads.Hypervisor(opt.VMType)
	req.FWType = opt.FWType
	req.ImageName = opt.ImageName
	req.ImageID = opt.ImageID
	req.Config = config
	req.Storage, err = optToReqStorage(opt)

	if err != nil {
		return err
	}

	// all default resources are required.
	defaults := opt.Defaults

	r := payloads.RequestedResource{
		Type:  payloads.VCPUs,
		Value: defaults.VCPUs,
	}
	req.Defaults = append(req.Defaults, r)

	r = payloads.RequestedResource{
		Type:  payloads.MemMB,
		Value: defaults.MemMB,
	}
	req.Defaults = append(req.Defaults, r)

	return nil
}

func outputWorkload(w types.Workload) {
	var opt workloadOptions

	opt.Description = w.Description
	opt.VMType = string(w.VMType)
	opt.FWType = w.FWType
	opt.ImageName = w.ImageName
	opt.ImageID = w.ImageID
	for _, d := range w.Defaults {
		if d.Type == payloads.VCPUs {
			opt.Defaults.VCPUs = d.Value
		} else if d.Type == payloads.MemMB {
			opt.Defaults.MemMB = d.Value
		}
	}

	for _, s := range w.Storage {
		d := disk{
			Size:      s.Size,
			Bootable:  s.Bootable,
			Ephemeral: s.Ephemeral,
		}
		if s.ID != "" {
			d.ID = &s.ID
		}

		src := source{
			Type: s.SourceType,
			ID:   s.SourceID,
		}

		d.Source = src

		opt.Disks = append(opt.Disks, d)
	}

	b, err := yaml.Marshal(opt)
	if err != nil {
		fatalf(err.Error())
	}

	fmt.Println(string(b))
	fmt.Println(w.Config)
}

func (cmd *workloadCreateCommand) run(args []string) error {
	var opt workloadOptions
	var req types.Workload

	if cmd.yamlFile == "" {
		cmd.usage()
	}

	f, err := ioutil.ReadFile(cmd.yamlFile)
	if err != nil {
		fatalf("Unable to read workload config file: %s\n", err)
	}

	err = yaml.Unmarshal(f, &opt)
	if err != nil {
		fatalf("Config file invalid: %s\n", err)
	}

	err = optToReq(opt, &req)
	if err != nil {
		fatalf(err.Error())
	}

	b, err := json.Marshal(req)
	if err != nil {
		fatalf(err.Error())
	}

	body := bytes.NewReader(b)

	url, err := getCiaoWorkloadsResource()
	if err != nil {
		fatalf(err.Error())
	}

	ver := api.WorkloadsV1

	resp, err := sendCiaoRequest("POST", url, nil, body, &ver)
	if err != nil {
		fatalf(err.Error())
	}

	if resp.StatusCode != http.StatusCreated {
		fatalf("Workload creation failed: %s", resp.Status)
	}

	var workload types.WorkloadResponse

	err = unmarshalHTTPResponse(resp, &workload)
	if err != nil {
		fatalf(err.Error())
	}

	fmt.Printf("Created new workload: %s\n", workload.Workload.ID)

	return nil
}

type workloadDeleteCommand struct {
	Flag     flag.FlagSet
	workload string
}

func (cmd *workloadDeleteCommand) usage(...string) {
	fmt.Fprintf(os.Stderr, `usage: ciao-cli [options] workload delete [flags]

Deletes a given workload

The delete flags are:

`)
	cmd.Flag.PrintDefaults()
	os.Exit(2)
}

func (cmd *workloadDeleteCommand) parseArgs(args []string) []string {
	cmd.Flag.StringVar(&cmd.workload, "workload", "", "Workload UUID")
	cmd.Flag.Usage = func() { cmd.usage() }
	cmd.Flag.Parse(args)
	return cmd.Flag.Args()
}

func (cmd *workloadDeleteCommand) run(args []string) error {
	if cmd.workload == "" {
		cmd.usage()
	}

	url, err := getCiaoWorkloadsResource()
	if err != nil {
		fatalf(err.Error())
	}

	ver := api.WorkloadsV1

	// you should do a get first and search for the workload,
	// then use the href - but not with the currently used
	// OpenStack API. Until we support GET with a ciao API,
	// just hard code the path.
	url = fmt.Sprintf("%s/%s", url, cmd.workload)

	resp, err := sendCiaoRequest("DELETE", url, nil, nil, &ver)
	if err != nil {
		fatalf(err.Error())
	}

	if resp.StatusCode != http.StatusNoContent {
		fatalf("Workload deletion failed: %s", resp.Status)
	}

	return nil
}

type workloadShowCommand struct {
	Flag     flag.FlagSet
	template string
	workload string
}

func (cmd *workloadShowCommand) usage(...string) {
	fmt.Fprintf(os.Stderr, `usage: ciao-cli [options] workload show

Show workload details

`)
	cmd.Flag.PrintDefaults()
	fmt.Fprintf(os.Stderr, "\n%s",
		templateutils.GenerateUsageDecorated("f", types.Workload{}, nil))
	os.Exit(2)
}

func (cmd *workloadShowCommand) parseArgs(args []string) []string {
	cmd.Flag.StringVar(&cmd.workload, "workload", "", "Workload UUID")
	cmd.Flag.StringVar(&cmd.template, "f", "", "Template used to format output")
	cmd.Flag.Usage = func() { cmd.usage() }
	cmd.Flag.Parse(args)
	return cmd.Flag.Args()
}

func (cmd *workloadShowCommand) run(args []string) error {
	var wl types.Workload

	if cmd.workload == "" {
		cmd.usage()
	}

	url, err := getCiaoWorkloadsResource()
	if err != nil {
		fatalf(err.Error())
	}

	ver := api.WorkloadsV1

	// you should do a get first and search for the workload,
	// then use the href - but not with the currently used
	// OpenStack API. Until we support GET with a ciao API,
	// just hard code the path.
	url = fmt.Sprintf("%s/%s", url, cmd.workload)

	resp, err := sendCiaoRequest("GET", url, nil, nil, &ver)
	if err != nil {
		fatalf(err.Error())
	}

	if resp.StatusCode != http.StatusOK {
		fatalf("Workload show failed: %s", resp.Status)
	}

	err = unmarshalHTTPResponse(resp, &wl)
	if err != nil {
		fatalf(err.Error())
	}

	if cmd.template != "" {
		return templateutils.OutputToTemplate(os.Stdout, "workload-show", cmd.template, &wl, nil)
	}

	outputWorkload(wl)
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"path/filepath"
	"strings"
	"time"

	smartlingNew "github.com/Smartling/api-sdk-go"
	"github.com/codegangsta/cli"
)

// FIXME: support conditions on list
// func ListConditionSlice(cc []string) []smartlingNew.FilesListRequest {
// 	ll := []smartlingNew.ListCondition{}
// 	for _, c := range cc {
// 		ll = append(ll, smartling.ListCondition(c))
// 	}
// 	return ll
// }

func removeEmptyStrings(ss []string) []string {
	newSs := []string{}
	for _, s := range ss {
		if s != "" {
			newSs = append(newSs, s)
		}
	}
	return newSs
}

func PrintList(uriMask string, olderThan time.Duration, long bool, conditions []string) {
	req := smartlingNew.FilesListRequest{
		URIMask: uriMask,
	}

	// FIXME: support conditions on list
	// conditions = removeEmptyStrings(conditions)
	// if len(conditions) > 0 {
	// 	req.Conditions = ListConditionSlice(conditions)
	// }
	if olderThan > 0 {
		// FIXME: check this actually works
		t := smartlingNew.UTC{Time: time.Now().Add(-olderThan)}
		req.LastUploadedBefore = t
	}

	files, err := client.List(req)
	logAndQuitIfError(err)

	// FIXME: fix this "long"
	if long {
		fmt.Println("total", files.TotalCount)
		for _, f := range files.Items {
			// t := time.Time(f.LastUploaded).Format("2 Jan 15:04")
			fmt.Println(f.FileURI)

			// fmt.Printf("%3d strings  %s  %s\n", f.StringCount, t, f.FileUri)
		}
	} else {
		for _, f := range files.Items {
			fmt.Println(f.FileURI)
		}
	}
}

var LsCommand = cli.Command{
	Name:        "ls",
	Usage:       "list remote files",
	Description: "ls [<uriMask>]",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name: "conditions",
		}, cli.StringFlag{
			Name: "older-than",
		}, cli.BoolFlag{
			Name: "long,l",
		},
	},
	Before: cmdBefore,
	Action: func(c *cli.Context) {
		if len(c.Args()) > 1 {
			log.Println("Wrong number of arguments")
			log.Fatalln("Usage: ls [<uriMask>]")
		}
		uriMask := c.Args().Get(0)

		var d time.Duration
		if len(c.String("older-than")) > 0 {
			var err error
			d, err = time.ParseDuration(c.String("older-than"))
			logAndQuitIfError(err)
		}

		conditions := strings.Split(c.String("conditions"), ",")

		PrintList(uriMask, d, c.Bool("long"), conditions)
	},
}

func PrintFileStatus(remotepath, locale string) {
	f, err := client.Status(remotepath, locale)
	logAndQuitIfError(err)

	fmt.Println("File                    ", f.FileURI)
	fmt.Println("String Count            ", f.TotalStringCount)
	fmt.Println("Word Count              ", f.TotalWordCount)
	fmt.Println("Authorized String Count ", f.AuthorizedStringCount)
	fmt.Println("Completed String Count  ", f.CompletedStringCount)
	fmt.Println("Excluded String Count   ", f.ExcludedStringCount)
	fmt.Println("Last Uploaded           ", f.LastUploaded)
	fmt.Println("File Type               ", f.FileType)
}

var StatusCommand = cli.Command{
	Name:        "stat",
	Usage:       "display the translation status of a remote file",
	Description: "stat <remote file> <locale>",
	Before:      cmdBefore,
	Action: func(c *cli.Context) {
		if len(c.Args()) != 2 {
			log.Println("Wrong number of arguments")
			log.Fatalln("Usage: stat <remote file> <locale>")
		}

		remotepath := c.Args().Get(0)
		locale := c.Args().Get(1)

		PrintFileStatus(remotepath, locale)
	},
}

var GetCommand = cli.Command{
	Name:        "get",
	Usage:       "downloads a remote file",
	Description: "get <remote file>",
	Before:      cmdBefore,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name: "locale",
		},
	},
	Action: func(c *cli.Context) {
		if len(c.Args()) != 1 {
			log.Println("Wrong number of arguments")
			log.Fatalln("Usage: get <remote file>")
		}

		remotepath := c.Args().Get(0)
		locale := c.String("locale")

		var (
			b   []byte
			err error
		)

		if locale == "" {
			b, err = client.Download(remotepath)
		} else {
			b, err = client.DownloadTranslation(locale, smartlingNew.FileDownloadRequest{
				FileURIRequest: smartlingNew.FileURIRequest{FileURI: remotepath},
			})
		}
		logAndQuitIfError(err)

		fmt.Println(string(b))
	},
}

var PutCommand = cli.Command{
	Name:        "put",
	Usage:       "uploads a local file",
	Description: "put <local file>",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name: "filetype",
		}, cli.StringFlag{
			Name: "parserconfig",
		}, cli.BoolFlag{
			Name: "approve",
		},
	},
	Before: cmdBefore,
	Action: func(c *cli.Context) {
		if len(c.Args()) != 1 {
			log.Println("Wrong number of arguments")
			log.Fatalln("Usage: put <local file>")
		}

		localpath := c.Args().Get(0)

		ft := smartlingNew.FileType(c.String("filetype"))
		if ft == "" {
			ft = smartlingNew.GetFileTypeByExtension(filepath.Ext(localpath))
		}

		parserconfig := map[string]string{}
		if c.String("parserconfig") != "" {
			parts := strings.Split(c.String("parserconfig"), ",")
			if len(parts)%2 == 1 {
				log.Fatalln("parserconfig must be in the format --parserconfig=key1,value1,key2,value2")
			}
			for i := 0; i < len(parts); i += 2 {
				parserconfig[parts[i]] = parts[i+1]
			}
		}

		f, err := ioutil.ReadFile(localpath)
		logAndQuitIfError(err)

		r, err := client.Upload(&smartlingNew.FileUploadRequest{
			File:           f,
			FileType:       ft,
			Authorize:      c.Bool("approve"),
			FileURIRequest: smartlingNew.FileURIRequest{FileURI: localpath},
		})

		logAndQuitIfError(err)

		fmt.Println("Overwritten: ", r.Overwritten)
		fmt.Println("String Count:", r.StringCount)
		fmt.Println("Word Count:  ", r.WordCount)
	},
}

var RenameCommand = cli.Command{
	Name:        "rename",
	Usage:       "renames a remote file",
	Description: "rename <remote file name> <new smartling file name>",
	Before:      cmdBefore,
	Action: func(c *cli.Context) {
		if len(c.Args()) != 2 {
			log.Println("Wrong number of arguments")
			log.Fatalln("Usage: rename <remote file> <new smartling file name>")
		}

		remotepath := c.Args().Get(0)
		newremotepath := c.Args().Get(1)

		err := client.Rename(remotepath, newremotepath)

		logAndQuitIfError(err)
	},
}

var RmCommand = cli.Command{
	Name:        "rm",
	Usage:       "removes a remote file",
	Description: "rm <remote file>...",
	Before:      cmdBefore,
	Action: func(c *cli.Context) {
		if len(c.Args()) < 1 {
			log.Println("Wrong number of arguments")
			log.Fatalln("Usage: rm <remote file>...")
		}

		for _, remotepath := range c.Args() {
			logAndQuitIfError(client.Delete(remotepath))
		}
	},
}

var LastmodifiedCommand = cli.Command{
	Name:        "lastmodified",
	Usage:       "shows when a remote file was modified last",
	Description: "lastmodified <remote file>",
	Before:      cmdBefore,
	Action: func(c *cli.Context) {
		if len(c.Args()) != 1 {
			log.Println("Wrong number of arguments")
			log.Fatalln("Usage: lastmodified <remote file>")
		}

		remotepath := c.Args().Get(0)

		locales, err := client.LastModified(smartlingNew.FileLastModifiedRequest{
			FileURIRequest: smartlingNew.FileURIRequest{FileURI: remotepath},
		})
		logAndQuitIfError(err)

		for _, i := range locales.Items {
			t := time.Time(i.LastModified.Time).Format("2 Jan 3:04")
			fmt.Printf("%s %s\n", i.LocaleID, t)
		}
	},
}

var LocalesCommand = cli.Command{
	Name:   "locales",
	Usage:  "list the locales for the project",
	Before: cmdBefore,
	Action: func(c *cli.Context) {
		if len(c.Args()) != 0 {
			log.Println("Wrong number of arguments")
			log.Fatalln("Usage: locales")
		}

		tl, err := client.Locales()
		logAndQuitIfError(err)

		for _, l := range tl {
			if l.Enabled {
				fmt.Printf("%-5s  %s\n", l.LocaleID, l.Description)
			}
		}
	},
}
<commit_msg>Remove the list conditions option<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"path/filepath"
	"strings"
	"time"

	smartlingNew "github.com/Smartling/api-sdk-go"
	"github.com/codegangsta/cli"
)

func PrintList(uriMask string, olderThan time.Duration, long bool) {
	req := smartlingNew.FilesListRequest{
		URIMask: uriMask,
	}

	if olderThan > 0 {
		// FIXME: check this actually works
		t := smartlingNew.UTC{Time: time.Now().Add(-olderThan)}
		req.LastUploadedBefore = t
	}

	files, err := client.List(req)
	logAndQuitIfError(err)

	// FIXME: fix this "long"
	if long {
		fmt.Println("total", files.TotalCount)
		for _, f := range files.Items {
			// t := time.Time(f.LastUploaded).Format("2 Jan 15:04")
			fmt.Println(f.FileURI)

			// fmt.Printf("%3d strings  %s  %s\n", f.StringCount, t, f.FileUri)
		}
	} else {
		for _, f := range files.Items {
			fmt.Println(f.FileURI)
		}
	}
}

var LsCommand = cli.Command{
	Name:        "ls",
	Usage:       "list remote files",
	Description: "ls [<uriMask>]",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name: "older-than",
		}, cli.BoolFlag{
			Name: "long,l",
		},
	},
	Before: cmdBefore,
	Action: func(c *cli.Context) {
		if len(c.Args()) > 1 {
			log.Println("Wrong number of arguments")
			log.Fatalln("Usage: ls [<uriMask>]")
		}
		uriMask := c.Args().Get(0)

		var d time.Duration
		if len(c.String("older-than")) > 0 {
			var err error
			d, err = time.ParseDuration(c.String("older-than"))
			logAndQuitIfError(err)
		}

		PrintList(uriMask, d, c.Bool("long"))
	},
}

func PrintFileStatus(remotepath, locale string) {
	f, err := client.Status(remotepath, locale)
	logAndQuitIfError(err)

	fmt.Println("File                    ", f.FileURI)
	fmt.Println("String Count            ", f.TotalStringCount)
	fmt.Println("Word Count              ", f.TotalWordCount)
	fmt.Println("Authorized String Count ", f.AuthorizedStringCount)
	fmt.Println("Completed String Count  ", f.CompletedStringCount)
	fmt.Println("Excluded String Count   ", f.ExcludedStringCount)
	fmt.Println("Last Uploaded           ", f.LastUploaded)
	fmt.Println("File Type               ", f.FileType)
}

var StatusCommand = cli.Command{
	Name:        "stat",
	Usage:       "display the translation status of a remote file",
	Description: "stat <remote file> <locale>",
	Before:      cmdBefore,
	Action: func(c *cli.Context) {
		if len(c.Args()) != 2 {
			log.Println("Wrong number of arguments")
			log.Fatalln("Usage: stat <remote file> <locale>")
		}

		remotepath := c.Args().Get(0)
		locale := c.Args().Get(1)

		PrintFileStatus(remotepath, locale)
	},
}

var GetCommand = cli.Command{
	Name:        "get",
	Usage:       "downloads a remote file",
	Description: "get <remote file>",
	Before:      cmdBefore,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name: "locale",
		},
	},
	Action: func(c *cli.Context) {
		if len(c.Args()) != 1 {
			log.Println("Wrong number of arguments")
			log.Fatalln("Usage: get <remote file>")
		}

		remotepath := c.Args().Get(0)
		locale := c.String("locale")

		var (
			b   []byte
			err error
		)

		if locale == "" {
			b, err = client.Download(remotepath)
		} else {
			b, err = client.DownloadTranslation(locale, smartlingNew.FileDownloadRequest{
				FileURIRequest: smartlingNew.FileURIRequest{FileURI: remotepath},
			})
		}
		logAndQuitIfError(err)

		fmt.Println(string(b))
	},
}

var PutCommand = cli.Command{
	Name:        "put",
	Usage:       "uploads a local file",
	Description: "put <local file>",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name: "filetype",
		}, cli.StringFlag{
			Name: "parserconfig",
		}, cli.BoolFlag{
			Name: "approve",
		},
	},
	Before: cmdBefore,
	Action: func(c *cli.Context) {
		if len(c.Args()) != 1 {
			log.Println("Wrong number of arguments")
			log.Fatalln("Usage: put <local file>")
		}

		localpath := c.Args().Get(0)

		ft := smartlingNew.FileType(c.String("filetype"))
		if ft == "" {
			ft = smartlingNew.GetFileTypeByExtension(filepath.Ext(localpath))
		}

		parserconfig := map[string]string{}
		if c.String("parserconfig") != "" {
			parts := strings.Split(c.String("parserconfig"), ",")
			if len(parts)%2 == 1 {
				log.Fatalln("parserconfig must be in the format --parserconfig=key1,value1,key2,value2")
			}
			for i := 0; i < len(parts); i += 2 {
				parserconfig[parts[i]] = parts[i+1]
			}
		}

		f, err := ioutil.ReadFile(localpath)
		logAndQuitIfError(err)

		r, err := client.Upload(&smartlingNew.FileUploadRequest{
			File:           f,
			FileType:       ft,
			Authorize:      c.Bool("approve"),
			FileURIRequest: smartlingNew.FileURIRequest{FileURI: localpath},
		})

		logAndQuitIfError(err)

		fmt.Println("Overwritten: ", r.Overwritten)
		fmt.Println("String Count:", r.StringCount)
		fmt.Println("Word Count:  ", r.WordCount)
	},
}

var RenameCommand = cli.Command{
	Name:        "rename",
	Usage:       "renames a remote file",
	Description: "rename <remote file name> <new smartling file name>",
	Before:      cmdBefore,
	Action: func(c *cli.Context) {
		if len(c.Args()) != 2 {
			log.Println("Wrong number of arguments")
			log.Fatalln("Usage: rename <remote file> <new smartling file name>")
		}

		remotepath := c.Args().Get(0)
		newremotepath := c.Args().Get(1)

		err := client.Rename(remotepath, newremotepath)

		logAndQuitIfError(err)
	},
}

var RmCommand = cli.Command{
	Name:        "rm",
	Usage:       "removes a remote file",
	Description: "rm <remote file>...",
	Before:      cmdBefore,
	Action: func(c *cli.Context) {
		if len(c.Args()) < 1 {
			log.Println("Wrong number of arguments")
			log.Fatalln("Usage: rm <remote file>...")
		}

		for _, remotepath := range c.Args() {
			logAndQuitIfError(client.Delete(remotepath))
		}
	},
}

var LastmodifiedCommand = cli.Command{
	Name:        "lastmodified",
	Usage:       "shows when a remote file was modified last",
	Description: "lastmodified <remote file>",
	Before:      cmdBefore,
	Action: func(c *cli.Context) {
		if len(c.Args()) != 1 {
			log.Println("Wrong number of arguments")
			log.Fatalln("Usage: lastmodified <remote file>")
		}

		remotepath := c.Args().Get(0)

		locales, err := client.LastModified(smartlingNew.FileLastModifiedRequest{
			FileURIRequest: smartlingNew.FileURIRequest{FileURI: remotepath},
		})
		logAndQuitIfError(err)

		for _, i := range locales.Items {
			t := time.Time(i.LastModified.Time).Format("2 Jan 3:04")
			fmt.Printf("%s %s\n", i.LocaleID, t)
		}
	},
}

var LocalesCommand = cli.Command{
	Name:   "locales",
	Usage:  "list the locales for the project",
	Before: cmdBefore,
	Action: func(c *cli.Context) {
		if len(c.Args()) != 0 {
			log.Println("Wrong number of arguments")
			log.Fatalln("Usage: locales")
		}

		tl, err := client.Locales()
		logAndQuitIfError(err)

		for _, l := range tl {
			if l.Enabled {
				fmt.Printf("%-5s  %s\n", l.LocaleID, l.Description)
			}
		}
	},
}
<|endoftext|>