<commit_before>package docker

import (
	"fmt"
	"io"
	"sync"
	"time"

	dockerclient "github.com/fsouza/go-dockerclient"
	"github.com/zenoss/glog"
)

// AllThingsDocker is a wildcard used to express interest in the Docker
// lifecycle event streams of all containers and images.
const AllThingsDocker = "*"

// Selectors for the various Docker lifecycle events.
const (
	Create  = "create"
	Delete  = "delete"
	Destroy = "destroy"
	Die     = "die"
	Export  = "export"
	Kill    = "kill"
	Restart = "restart"
	Start   = "start"
	Stop    = "stop"
	Untag   = "untag"
)

// EventMonitor implementations may be used to subscribe to Docker
// lifecycle events. This package provides such an implementation.
// Instances of it may be retreived via the client.EventMonitor() method.
type EventMonitor interface {
	// IsActive reports whether or not an EventMonitor is active, i.e., listening for Docker events.
	IsActive() bool

	// Subscribe returns a subscription to which handlers for the various Docker lifecycle events
	// for the container or image specified by ID (or all containers and images if AllThingsDocker
	// is passed) may be added.
	Subscribe(ID string) (*Subscription, error)

	// Close causes the EventMonitor to stop listening for Docker lifecycle events.
	Close() error
}

// A HandlerFunc is used to receive Docker lifecycle events.
type HandlerFunc func(e *dockerclient.APIEvents) error

type clientEventMonitor struct {
	sync.Mutex
	active        bool
	closeChannel  chan chan struct{}
	done          chan struct{}
	subscriptions map[string][]*Subscription
}

// Subscription represents a subscription to a particular container or image's Docker lifecycle
// event stream. The AllThingsDocker ID can be used to subscribe to all container and image
// event streams.
type Subscription struct {
	ID            string
	active        bool
	cancelChannel chan chan struct{}
	eventChannel  chan *dockerclient.APIEvents
	monitorDone   chan struct{}
	handlers      map[string]HandlerFunc
	monitor       *clientEventMonitor
	lock          sync.RWMutex
}

// eventMonitor is used by the client to monitor Docker lifecycle events
var eventMonitor = &clientEventMonitor{
	active:        false,
	closeChannel:  make(chan chan struct{}),
	done:          make(chan struct{}),
	subscriptions: make(map[string][]*Subscription),
}

// validEvents is a map used to check event strings for validity
var validEvents = map[string]struct{}{
	Create:  struct{}{},
	Delete:  struct{}{},
	Destroy: struct{}{},
	Die:     struct{}{},
	Export:  struct{}{},
	Kill:    struct{}{},
	Restart: struct{}{},
	Start:   struct{}{},
	Stop:    struct{}{},
	Untag:   struct{}{},
}

// MonitorEvents returns an EventMonitor that can be used to listen for and respond to
// the various events in the Docker container and image lifecycles.
func (c *Client) monitorEvents() (EventMonitor, error) {
	if err := eventMonitor.run(c); err != nil {
		return nil, err
	}
	return eventMonitor, nil
}

// IsActive reports whether or not an EventMonitor is active, i.e., listening for Docker events.
func (em *clientEventMonitor) IsActive() bool {
	em.Lock()
	defer em.Unlock()

	return em.active
}

// Close causes the EventMonitor to stop listening for Docker events.
func (em *clientEventMonitor) Close() error {
	em.Lock()
	defer em.Unlock()

	if !em.active {
		return nil
	}

	crc := make(chan struct{})
	em.closeChannel <- crc

	select {
	case <-crc:
		em.active = false
		em.subscriptions = make(map[string][]*Subscription)
		em.done = make(chan struct{})
		return nil
	}

}

// Subscribe returns a subscription to which handlers for the various Docker lifecycle events
// for the container or image specified by ID (or all containers and images if AllThingsDocker
// is passed) may be added.
func (em *clientEventMonitor) Subscribe(ID string) (*Subscription, error) {
	em.Lock()
	defer em.Unlock()

	s := &Subscription{
		ID:            ID,
		cancelChannel: make(chan chan struct{}),
		eventChannel:  make(chan *dockerclient.APIEvents),
		monitorDone:   em.done,
		handlers:      make(map[string]HandlerFunc),
		monitor:       em,
		lock:          sync.RWMutex{},
	}

	em.subscriptions[ID] = append(em.subscriptions[ID], s)
	s.run()

	return s, nil
}

// run causes the clientEventMonitor to start listening for Docker container
// and image lifecycle events
func (em *clientEventMonitor) run(c *Client) error {
	em.Lock()
	defer em.Unlock()

	if em.active {
		return nil
	}

	go func() {
		r, w := io.Pipe()

		go listenAndDispatch(c, em, r, w)

		select {
		case crc := <-em.closeChannel:
			w.Close()
			r.Close()
			close(em.done)
			crc <- struct{}{}
			return
		}
	}()

	em.active = true
	return nil
}

// dispatch sends the incoming event to the event channel of all interested subscribers.
func (em *clientEventMonitor) dispatch(evt *dockerclient.APIEvents) error {
	em.Lock()
	defer em.Unlock()

	if !em.active {
		return nil
	}

	// send the event to subscribers interested in everything
	if subs, ok := em.subscriptions[AllThingsDocker]; ok {
		for _, sub := range subs {
			select {
			case sub.eventChannel <- evt:
			case <-time.After(time.Second):
				glog.V(2).Infof("timeout sending event: %v, %v", evt, sub)
			}
		}
	}

	// send the event to subscribers interested in the particular ID
	if evt.ID != "" {
		if subs, ok := em.subscriptions[evt.ID]; ok {
			for _, sub := range subs {
				select {
				case sub.eventChannel <- evt:
				case <-time.After(time.Second):
					glog.V(2).Infof("timeout sending event: %v, %v", evt, sub)
				}
			}
		}
	}
	return nil
}

// unsubscribe removes the given Subscription from the event monitor's list of subscribers
func (em *clientEventMonitor) unsubscribe(s *Subscription) error {
	em.Lock()
	defer em.Unlock()

	ns := []*Subscription{}
	for _, sub := range em.subscriptions[s.ID] {
		if sub != s {
			ns = append(ns, sub)
		}
	}

	em.subscriptions[s.ID] = ns

	return nil
}

// listenAndDispatch reads the Docker event stream and dispatches the events
// it receives.
func listenAndDispatch(c *Client, em *clientEventMonitor, r *io.PipeReader, w *io.PipeWriter) {
	listener := make(chan *dockerclient.APIEvents)
	c.dc.AddEventListener(listener)
	for {
		evt := <-listener
		if evt != nil {
			em.dispatch(evt)
		}
	}
}

// Handle associates a HandlerFunc h with a the Docker container or image lifecycle
// event specified by es. Any HandlerFunc previously associated with es is replaced.
func (s *Subscription) Handle(es string, h HandlerFunc) error {
	if _, ok := validEvents[es]; !ok {
		return fmt.Errorf("unknown event: %s", es)
	}

	s.lock.Lock()
	s.handlers[es] = h
	s.lock.Unlock()
	return nil
}

// Cancel causes the Subscription to stop receiving and dispatching Docker container and
// image lifecycle events.
func (s *Subscription) Cancel() error {
	if !s.active {
		return nil
	}

	crc := make(chan struct{})
	s.cancelChannel <- crc

	select {
	case <-crc:
		if err := s.monitor.unsubscribe(s); err != nil {
			glog.V(2).Infof("could not unsubscribe %v (%v)", s, err)
		}
		s.active = false
		return nil
	}

}

// run causes the Subscription to start receiving and dispatching Docker container and
// image lifecycle events.
func (s *Subscription) run() error {
	if s.active {
		return nil
	}

	go func() {
		for {
			select {
			case e := <-s.eventChannel:
				if e.Status != "" {
					s.lock.RLock()
					h, ok := s.handlers[e.Status]
					s.lock.RUnlock()
					if ok {
						h(e)
					}
				}
			case crc := <-s.cancelChannel:
				crc <- struct{}{}
				return
			case <-s.monitorDone:
				s.active = false
				return
			}
		}
	}()

	s.active = true
	return nil
}
<commit_msg>fixed data race in docker event<commit_after>package docker

import (
	"fmt"
	"io"
	"sync"
	"time"

	dockerclient "github.com/fsouza/go-dockerclient"
	"github.com/zenoss/glog"
)

// AllThingsDocker is a wildcard used to express interest in the Docker
// lifecycle event streams of all containers and images.
const AllThingsDocker = "*"

// Selectors for the various Docker lifecycle events.
const (
	Create  = "create"
	Delete  = "delete"
	Destroy = "destroy"
	Die     = "die"
	Export  = "export"
	Kill    = "kill"
	Restart = "restart"
	Start   = "start"
	Stop    = "stop"
	Untag   = "untag"
)

// EventMonitor implementations may be used to subscribe to Docker
// lifecycle events. This package provides such an implementation.
// Instances of it may be retreived via the client.EventMonitor() method.
type EventMonitor interface {
	// IsActive reports whether or not an EventMonitor is active, i.e., listening for Docker events.
	IsActive() bool

	// Subscribe returns a subscription to which handlers for the various Docker lifecycle events
	// for the container or image specified by ID (or all containers and images if AllThingsDocker
	// is passed) may be added.
	Subscribe(ID string) (*Subscription, error)

	// Close causes the EventMonitor to stop listening for Docker lifecycle events.
	Close() error
}

// A HandlerFunc is used to receive Docker lifecycle events.
type HandlerFunc func(e *dockerclient.APIEvents) error

type clientEventMonitor struct {
	sync.Mutex
	active        bool
	closeChannel  chan chan struct{}
	done          chan struct{}
	subscriptions map[string][]*Subscription
}

// Subscription represents a subscription to a particular container or image's Docker lifecycle
// event stream. The AllThingsDocker ID can be used to subscribe to all container and image
// event streams.
type Subscription struct {
	ID            string
	active        bool
	cancelChannel chan chan struct{}
	eventChannel  chan *dockerclient.APIEvents
	monitorDone   chan struct{}
	handlers      map[string]HandlerFunc
	monitor       *clientEventMonitor
	lock          sync.RWMutex
}

// eventMonitor is used by the client to monitor Docker lifecycle events
var eventMonitor = &clientEventMonitor{
	active:        false,
	closeChannel:  make(chan chan struct{}),
	done:          make(chan struct{}),
	subscriptions: make(map[string][]*Subscription),
}

// validEvents is a map used to check event strings for validity
var validEvents = map[string]struct{}{
	Create:  struct{}{},
	Delete:  struct{}{},
	Destroy: struct{}{},
	Die:     struct{}{},
	Export:  struct{}{},
	Kill:    struct{}{},
	Restart: struct{}{},
	Start:   struct{}{},
	Stop:    struct{}{},
	Untag:   struct{}{},
}

// MonitorEvents returns an EventMonitor that can be used to listen for and respond to
// the various events in the Docker container and image lifecycles.
func (c *Client) monitorEvents() (EventMonitor, error) {
	if err := eventMonitor.run(c); err != nil {
		return nil, err
	}
	return eventMonitor, nil
}

// IsActive reports whether or not an EventMonitor is active, i.e., listening for Docker events.
func (em *clientEventMonitor) IsActive() bool {
	em.Lock()
	defer em.Unlock()

	return em.active
}

// Close causes the EventMonitor to stop listening for Docker events.
func (em *clientEventMonitor) Close() error {
	em.Lock()
	defer em.Unlock()

	if !em.active {
		return nil
	}

	crc := make(chan struct{})
	em.closeChannel <- crc

	select {
	case <-crc:
		em.active = false
		em.subscriptions = make(map[string][]*Subscription)
		em.done = make(chan struct{})
		return nil
	}

}

// Subscribe returns a subscription to which handlers for the various Docker lifecycle events
// for the container or image specified by ID (or all containers and images if AllThingsDocker
// is passed) may be added.
func (em *clientEventMonitor) Subscribe(ID string) (*Subscription, error) {
	em.Lock()
	defer em.Unlock()

	s := &Subscription{
		ID:            ID,
		cancelChannel: make(chan chan struct{}),
		eventChannel:  make(chan *dockerclient.APIEvents),
		monitorDone:   em.done,
		handlers:      make(map[string]HandlerFunc),
		monitor:       em,
		lock:          sync.RWMutex{},
	}

	em.subscriptions[ID] = append(em.subscriptions[ID], s)
	s.run()

	return s, nil
}

// run causes the clientEventMonitor to start listening for Docker container
// and image lifecycle events
func (em *clientEventMonitor) run(c *Client) error {
	em.Lock()
	defer em.Unlock()

	if em.active {
		return nil
	}

	go func() {
		r, w := io.Pipe()

		go listenAndDispatch(c, em, r, w)

		select {
		case crc := <-em.closeChannel:
			w.Close()
			r.Close()
			close(em.done)
			crc <- struct{}{}
			return
		}
	}()

	em.active = true
	return nil
}

// dispatch sends the incoming event to the event channel of all interested subscribers.
func (em *clientEventMonitor) dispatch(evt *dockerclient.APIEvents) error {
	em.Lock()
	defer em.Unlock()

	if !em.active {
		return nil
	}

	// send the event to subscribers interested in everything
	if subs, ok := em.subscriptions[AllThingsDocker]; ok {
		for _, sub := range subs {
			select {
			case sub.eventChannel <- evt:
			case <-time.After(time.Second):
				glog.V(2).Infof("timeout sending event: %v, %v", evt, sub)
			}
		}
	}

	// send the event to subscribers interested in the particular ID
	if evt.ID != "" {
		if subs, ok := em.subscriptions[evt.ID]; ok {
			for _, sub := range subs {
				select {
				case sub.eventChannel <- evt:
				case <-time.After(time.Second):
					glog.V(2).Infof("timeout sending event: %v, %v", evt, sub)
				}
			}
		}
	}
	return nil
}

// unsubscribe removes the given Subscription from the event monitor's list of subscribers
func (em *clientEventMonitor) unsubscribe(s *Subscription) error {
	em.Lock()
	defer em.Unlock()

	ns := []*Subscription{}
	for _, sub := range em.subscriptions[s.ID] {
		if sub != s {
			ns = append(ns, sub)
		}
	}

	em.subscriptions[s.ID] = ns

	return nil
}

// listenAndDispatch reads the Docker event stream and dispatches the events
// it receives.
func listenAndDispatch(c *Client, em *clientEventMonitor, r *io.PipeReader, w *io.PipeWriter) {
	listener := make(chan *dockerclient.APIEvents)
	c.dc.AddEventListener(listener)
	for {
		evt := <-listener
		if evt != nil {
			em.dispatch(evt)
		}
	}
}

// Handle associates a HandlerFunc h with a the Docker container or image lifecycle
// event specified by es. Any HandlerFunc previously associated with es is replaced.
func (s *Subscription) Handle(es string, h HandlerFunc) error {
	if _, ok := validEvents[es]; !ok {
		return fmt.Errorf("unknown event: %s", es)
	}

	s.lock.Lock()
	s.handlers[es] = h
	s.lock.Unlock()
	return nil
}

// Cancel causes the Subscription to stop receiving and dispatching Docker container and
// image lifecycle events.
func (s *Subscription) Cancel() error {
	if !s.active {
		return nil
	}

	crc := make(chan struct{})
	s.cancelChannel <- crc

	select {
	case <-crc:
		if err := s.monitor.unsubscribe(s); err != nil {
			glog.V(2).Infof("could not unsubscribe %v (%v)", s, err)
		}
		s.active = false
		return nil
	}

}

// run causes the Subscription to start receiving and dispatching Docker container and
// image lifecycle events.
func (s *Subscription) run() error {
	if s.active {
		return nil
	}

	go func() {
		for {
			select {
			case e := <-s.eventChannel:
				if e.Status != "" {
					s.lock.RLock()
					h, ok := s.handlers[e.Status]
					if ok {
						h(e)
					}
					s.lock.RUnlock()
				}
			case crc := <-s.cancelChannel:
				crc <- struct{}{}
				return
			case <-s.monitorDone:
				s.active = false
				return
			}
		}
	}()

	s.active = true
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"sync"
	"time"

	log "gopkg.in/inconshreveable/log15.v2"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/lxd/metrics"
	"github.com/lxc/lxd/lxd/response"
	"github.com/lxc/lxd/shared/logger"
)

type metricsCacheEntry struct {
	metrics *metrics.MetricSet
	expiry  time.Time
}

var metricsCache map[string]metricsCacheEntry
var metricsCacheLock sync.Mutex
var metricsLock sync.Mutex

var metricsCmd = APIEndpoint{
	Path: "metrics",

	Get: APIEndpointAction{Handler: metricsGet, AccessHandler: allowProjectPermission("containers", "view")},
}

// swagger:operation GET /1.0/metrics metrics metrics_get
//
// Get metrics
//
// Gets metrics of instances.
//
// ---
// produces:
//   - text/plain
// parameters:
//   - in: query
//     name: project
//     description: Project name
//     type: string
//     example: default
// responses:
//   "200":
//     description: Metrics
//     schema:
//       type: string
//       description: Instance metrics
//   "403":
//     $ref: "#/responses/Forbidden"
//   "500":
//     $ref: "#/responses/InternalServerError"
func metricsGet(d *Daemon, r *http.Request) response.Response {
	projectName := queryParam(r, "project")

	// Figure out the projects to retrieve.
	var projectNames []string

	if projectName != "" {
		projectNames = []string{projectName}
	} else {
		// Get all projects.
		err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
			projects, err := tx.GetProjects(db.ProjectFilter{})
			if err != nil {
				return err
			}

			for _, project := range projects {
				projectNames = append(projectNames, project.Name)
			}

			return nil
		})
		if err != nil {
			return response.SmartError(err)
		}
	}

	// Prepare response.
	resp := metrics.NewMetricSet(nil)

	// Review the cache.
	metricsCacheLock.Lock()
	projectMissing := []string{}
	for _, project := range projectNames {
		cache, ok := metricsCache[project]
		if !ok || cache.expiry.Before(time.Now()) {
			// If missing or expired, record it.
			projectMissing = append(projectMissing, project)
			continue
		}

		// If present and valid, merge the existing data.
		resp.Merge(cache.metrics)
	}
	metricsCacheLock.Unlock()

	// If all valid, return immediately.
	if len(projectMissing) == 0 {
		return response.SyncResponsePlain(true, resp.String())
	}

	// Acquire update lock.
	metricsLock.Lock()
	defer metricsLock.Unlock()

	// Check if any of the missing data has been filled in.
	metricsCacheLock.Lock()
	toFetch := []string{}
	for _, project := range projectMissing {
		cache, ok := metricsCache[project]
		if !ok || cache.expiry.Before(time.Now()) {
			// Still missing, queue a re-fetch.
			toFetch = append(toFetch, project)
			continue
		}

		// If present and valid, merge the existing data.
		resp.Merge(cache.metrics)
	}
	metricsCacheLock.Unlock()

	// If all valid, return immediately.
	if len(toFetch) == 0 {
		return response.SyncResponsePlain(true, resp.String())
	}

	// Prepare temporary metrics storage.
	newMetrics := map[string]*metrics.MetricSet{}
	newMetricsLock := sync.Mutex{}

	// Fetch what's missing.
	wgInstances := sync.WaitGroup{}
	for _, project := range toFetch {
		newMetrics[project] = metrics.NewMetricSet(nil)

		// Get the instances.
		instances, err := instanceLoadNodeProjectAll(d.State(), project, instancetype.Any)
		if err != nil {
			return response.SmartError(err)
		}

		for _, inst := range instances {
			// Ignore stopped instances.
			if !inst.IsRunning() {
				continue
			}

			wgInstances.Add(1)
			go func(inst instance.Instance) {
				defer wgInstances.Done()

				instanceMetrics, err := inst.Metrics()
				if err != nil {
					logger.Warn("Failed to get instance metrics", log.Ctx{"instance": inst.Name(), "project": inst.Project(), "err": err})
					return
				}

				// Add the metrics.
				newMetricsLock.Lock()
				defer newMetricsLock.Unlock()

				newMetrics[inst.Project()].Merge(instanceMetrics)
			}(inst)
		}
	}

	wgInstances.Wait()

	// Put the new data in the global cache and in response.
	metricsCacheLock.Lock()

	if metricsCache == nil {
		metricsCache = map[string]metricsCacheEntry{}
	}

	for project, entries := range newMetrics {
		metricsCache[project] = metricsCacheEntry{
			expiry:  time.Now().Add(15 * time.Second),
			metrics: entries,
		}

		resp.Merge(entries)
	}
	metricsCacheLock.Unlock()

	return response.SyncResponsePlain(true, resp.String())
}
<commit_msg>lxd/metrics: Reduce cache to 8s to accomodate 10s intervals<commit_after>package main

import (
	"net/http"
	"sync"
	"time"

	log "gopkg.in/inconshreveable/log15.v2"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/lxd/metrics"
	"github.com/lxc/lxd/lxd/response"
	"github.com/lxc/lxd/shared/logger"
)

type metricsCacheEntry struct {
	metrics *metrics.MetricSet
	expiry  time.Time
}

var metricsCache map[string]metricsCacheEntry
var metricsCacheLock sync.Mutex
var metricsLock sync.Mutex

var metricsCmd = APIEndpoint{
	Path: "metrics",

	Get: APIEndpointAction{Handler: metricsGet, AccessHandler: allowProjectPermission("containers", "view")},
}

// swagger:operation GET /1.0/metrics metrics metrics_get
//
// Get metrics
//
// Gets metrics of instances.
//
// ---
// produces:
//   - text/plain
// parameters:
//   - in: query
//     name: project
//     description: Project name
//     type: string
//     example: default
// responses:
//   "200":
//     description: Metrics
//     schema:
//       type: string
//       description: Instance metrics
//   "403":
//     $ref: "#/responses/Forbidden"
//   "500":
//     $ref: "#/responses/InternalServerError"
func metricsGet(d *Daemon, r *http.Request) response.Response {
	projectName := queryParam(r, "project")

	// Figure out the projects to retrieve.
	var projectNames []string

	if projectName != "" {
		projectNames = []string{projectName}
	} else {
		// Get all projects.
		err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
			projects, err := tx.GetProjects(db.ProjectFilter{})
			if err != nil {
				return err
			}

			for _, project := range projects {
				projectNames = append(projectNames, project.Name)
			}

			return nil
		})
		if err != nil {
			return response.SmartError(err)
		}
	}

	// Prepare response.
	resp := metrics.NewMetricSet(nil)

	// Review the cache.
	metricsCacheLock.Lock()
	projectMissing := []string{}
	for _, project := range projectNames {
		cache, ok := metricsCache[project]
		if !ok || cache.expiry.Before(time.Now()) {
			// If missing or expired, record it.
			projectMissing = append(projectMissing, project)
			continue
		}

		// If present and valid, merge the existing data.
		resp.Merge(cache.metrics)
	}
	metricsCacheLock.Unlock()

	// If all valid, return immediately.
	if len(projectMissing) == 0 {
		return response.SyncResponsePlain(true, resp.String())
	}

	// Acquire update lock.
	metricsLock.Lock()
	defer metricsLock.Unlock()

	// Check if any of the missing data has been filled in.
	metricsCacheLock.Lock()
	toFetch := []string{}
	for _, project := range projectMissing {
		cache, ok := metricsCache[project]
		if !ok || cache.expiry.Before(time.Now()) {
			// Still missing, queue a re-fetch.
			toFetch = append(toFetch, project)
			continue
		}

		// If present and valid, merge the existing data.
		resp.Merge(cache.metrics)
	}
	metricsCacheLock.Unlock()

	// If all valid, return immediately.
	if len(toFetch) == 0 {
		return response.SyncResponsePlain(true, resp.String())
	}

	// Prepare temporary metrics storage.
	newMetrics := map[string]*metrics.MetricSet{}
	newMetricsLock := sync.Mutex{}

	// Fetch what's missing.
	wgInstances := sync.WaitGroup{}
	for _, project := range toFetch {
		newMetrics[project] = metrics.NewMetricSet(nil)

		// Get the instances.
		instances, err := instanceLoadNodeProjectAll(d.State(), project, instancetype.Any)
		if err != nil {
			return response.SmartError(err)
		}

		for _, inst := range instances {
			// Ignore stopped instances.
			if !inst.IsRunning() {
				continue
			}

			wgInstances.Add(1)
			go func(inst instance.Instance) {
				defer wgInstances.Done()

				instanceMetrics, err := inst.Metrics()
				if err != nil {
					logger.Warn("Failed to get instance metrics", log.Ctx{"instance": inst.Name(), "project": inst.Project(), "err": err})
					return
				}

				// Add the metrics.
				newMetricsLock.Lock()
				defer newMetricsLock.Unlock()

				newMetrics[inst.Project()].Merge(instanceMetrics)
			}(inst)
		}
	}

	wgInstances.Wait()

	// Put the new data in the global cache and in response.
	metricsCacheLock.Lock()

	if metricsCache == nil {
		metricsCache = map[string]metricsCacheEntry{}
	}

	for project, entries := range newMetrics {
		metricsCache[project] = metricsCacheEntry{
			expiry:  time.Now().Add(8 * time.Second),
			metrics: entries,
		}

		resp.Merge(entries)
	}
	metricsCacheLock.Unlock()

	return response.SyncResponsePlain(true, resp.String())
}
<|endoftext|>


<commit_before>package cluster

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"net/http"

	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
)

// Return a TLS configuration suitable for establishing inter-node network
// connections using the cluster certificate.
func tlsClientConfig(info *shared.CertInfo) (*tls.Config, error) {
	keypair := info.KeyPair()
	ca := info.CA()
	config := shared.InitTLSConfig()
	config.Certificates = []tls.Certificate{keypair}
	config.RootCAs = x509.NewCertPool()
	if ca != nil {
		config.RootCAs.AddCert(ca)
	}
	// Since the same cluster keypair is used both as server and as client
	// cert, let's add it to the CA pool to make it trusted.
	cert, err := x509.ParseCertificate(keypair.Certificate[0])
	if err != nil {
		return nil, err
	}
	cert.IsCA = true
	cert.KeyUsage = x509.KeyUsageCertSign
	config.RootCAs.AddCert(cert)

	if cert.DNSNames != nil {
		config.ServerName = cert.DNSNames[0]
	}
	return config, nil
}

// Return true if the given request is presenting the given cluster certificate.
func tlsCheckCert(r *http.Request, info *shared.CertInfo) bool {
	cert, err := x509.ParseCertificate(info.KeyPair().Certificate[0])
	if err != nil {
		// Since we have already loaded this certificate, typically
		// using LoadX509KeyPair, an error should never happen, but
		// check for good measure.
		panic(fmt.Sprintf("invalid keypair material: %v", err))
	}
	trustedCerts := map[string]x509.Certificate{"0": *cert}

	trusted, _ := util.CheckTrustState(*r.TLS.PeerCertificates[0], trustedCerts, nil, false)

	return r.TLS != nil && trusted
}

// Return an http.Transport configured using the given configuration and a
// cleanup function to use to close all connections the transport has been
// used.
func tlsTransport(config *tls.Config) (*http.Transport, func()) {
	transport := &http.Transport{
		TLSClientConfig:   config,
		DisableKeepAlives: true,
		MaxIdleConns:      0,
	}
	return transport, transport.CloseIdleConnections
}
<commit_msg>lxd/cluster/tls: Update tlsClientConfig to accept networkCert and serverCert<commit_after>package cluster

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"net/http"

	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
)

// Return a TLS configuration suitable for establishing intra-member network connections using the server cert.
func tlsClientConfig(networkCert *shared.CertInfo, serverCert *shared.CertInfo) (*tls.Config, error) {
	if networkCert == nil {
		return nil, fmt.Errorf("Invalid networkCert")
	}

	if serverCert == nil {
		return nil, fmt.Errorf("Invalid serverCert")
	}

	keypair := serverCert.KeyPair()
	config := shared.InitTLSConfig()
	config.Certificates = []tls.Certificate{keypair}
	config.RootCAs = x509.NewCertPool()
	ca := serverCert.CA()
	if ca != nil {
		config.RootCAs.AddCert(ca)
	}

	// Since the same cluster keypair is used both as server and as client
	// cert, let's add it to the CA pool to make it trusted.
	networkKeypair := networkCert.KeyPair()
	netCert, err := x509.ParseCertificate(networkKeypair.Certificate[0])
	if err != nil {
		return nil, err
	}

	netCert.IsCA = true
	netCert.KeyUsage = x509.KeyUsageCertSign
	config.RootCAs.AddCert(netCert)

	// Always use network certificate's DNS name rather than server cert, so that it matches.
	if len(netCert.DNSNames) > 0 {
		config.ServerName = netCert.DNSNames[0]
	}

	return config, nil
}

// Return true if the given request is presenting the given cluster certificate.
func tlsCheckCert(r *http.Request, info *shared.CertInfo) bool {
	cert, err := x509.ParseCertificate(info.KeyPair().Certificate[0])
	if err != nil {
		// Since we have already loaded this certificate, typically
		// using LoadX509KeyPair, an error should never happen, but
		// check for good measure.
		panic(fmt.Sprintf("invalid keypair material: %v", err))
	}
	trustedCerts := map[string]x509.Certificate{"0": *cert}

	trusted, _ := util.CheckTrustState(*r.TLS.PeerCertificates[0], trustedCerts, nil, false)

	return r.TLS != nil && trusted
}

// Return an http.Transport configured using the given configuration and a
// cleanup function to use to close all connections the transport has been
// used.
func tlsTransport(config *tls.Config) (*http.Transport, func()) {
	transport := &http.Transport{
		TLSClientConfig:   config,
		DisableKeepAlives: true,
		MaxIdleConns:      0,
	}
	return transport, transport.CloseIdleConnections
}
<|endoftext|>


<commit_before>package information

import (
	"errors"
	"strings"

	"github.com/Necroforger/Fantasia/system"
	"github.com/Necroforger/discordgo"
	"github.com/Necroforger/dream"
)

// Module ...
type Module struct{}

// Build adds this modules commands to the system's router
func (m *Module) Build(s *system.System) {
	r := s.CommandRouter
	r.On("help", m.Help).Set("", "Displays a help menu with the available commands")
}

// Help maps a list of available commands and descends into subrouters.
func (m *Module) Help(ctx *system.Context) {

	if cmd := ctx.Args.After(); cmd != "" {
		if route, _ := ctx.System.CommandRouter.FindMatch(ctx.System.CommandRouter.Prefix + cmd); route != nil {
			ctx.ReplyEmbed(dream.NewEmbed().
				SetTitle(route.Name).
				SetDescription(route.Desc).
				SetColor(system.StatusNotify).
				MessageEmbed)
			return
		}
		ctx.ReplyError(errors.New("Command not found"))
		return
	}

	_, err := ctx.ReplyEmbed(depthcharge(ctx.System.CommandRouter, nil, 0).
		SetColor(system.StatusNotify).
		SetThumbnail(ctx.Ses.DG.State.User.AvatarURL("2048")).
		InlineAllFields().
		SetDescription("type `help [command]` for more information").
		MessageEmbed)
	if err != nil {
		ctx.ReplyError(err)
	}
}

// Depthcharge recursively generates a help embed from a CommandRouter and its subrouters
func depthcharge(r *system.CommandRouter, embed *dream.Embed, depth int) *dream.Embed {
	if embed == nil {
		embed = dream.NewEmbed()
	}

	depthString := func(text string, depth int, subrouter bool) string {
		quote := ""
		if subrouter {
			quote = "`"
		}
		return strings.Repeat("  ", depth) + quote + text + quote + "\n"
	}

	getField := func(name string) *discordgo.MessageEmbedField {
		for _, v := range embed.Fields {
			if v.Name == name {
				return v
			}
		}
		if name == "" {
			name = "undefined"
		}
		field := &discordgo.MessageEmbedField{Name: name}
		embed.Fields = append(embed.Fields, field)
		return field
	}

	removeField := func(name string) {
		for i, v := range embed.Fields {
			if v.Name == name {
				embed.Fields = append(embed.Fields[:i], embed.Fields[i+1:]...)
				return
			}
		}
	}

	for _, v := range r.Routes {
		field := getField(v.Category)

		var tag string
		if !v.Disabled {
			field.Value += depthString(tag+v.Name+tag, depth, false)
		}

		if field.Value == "" {
			removeField(v.Category)
		}
	}

	for _, v := range r.Subrouters {
		field := getField(v.Category())
		field.Value += depthString(v.Name, depth, true)
		embed = depthcharge(v.Router, embed, depth+1)
	}

	return embed
}
<commit_msg>Information no longer needs to use prefix<commit_after>package information

import (
	"errors"
	"strings"

	"github.com/Necroforger/Fantasia/system"
	"github.com/Necroforger/discordgo"
	"github.com/Necroforger/dream"
)

// Module ...
type Module struct{}

// Build adds this modules commands to the system's router
func (m *Module) Build(s *system.System) {
	r := s.CommandRouter
	r.On("help", m.Help).Set("", "Displays a help menu with the available commands")
}

// Help maps a list of available commands and descends into subrouters.
func (m *Module) Help(ctx *system.Context) {

	if cmd := ctx.Args.After(); cmd != "" {
		if route, _ := ctx.System.CommandRouter.FindMatch(cmd); route != nil {
			ctx.ReplyEmbed(dream.NewEmbed().
				SetTitle(route.Name).
				SetDescription(route.Desc).
				SetColor(system.StatusNotify).
				MessageEmbed)
			return
		}
		ctx.ReplyError(errors.New("Command not found"))
		return
	}

	_, err := ctx.ReplyEmbed(depthcharge(ctx.System.CommandRouter, nil, 0).
		SetColor(system.StatusNotify).
		SetThumbnail(ctx.Ses.DG.State.User.AvatarURL("2048")).
		InlineAllFields().
		SetDescription("type `help [command]` for more information").
		MessageEmbed)
	if err != nil {
		ctx.ReplyError(err)
	}
}

// Depthcharge recursively generates a help embed from a CommandRouter and its subrouters
func depthcharge(r *system.CommandRouter, embed *dream.Embed, depth int) *dream.Embed {
	if embed == nil {
		embed = dream.NewEmbed()
	}

	depthString := func(text string, depth int, subrouter bool) string {
		quote := ""
		if subrouter {
			quote = "`"
		}
		return strings.Repeat("  ", depth) + quote + text + quote + "\n"
	}

	getField := func(name string) *discordgo.MessageEmbedField {
		for _, v := range embed.Fields {
			if v.Name == name {
				return v
			}
		}
		if name == "" {
			name = "undefined"
		}
		field := &discordgo.MessageEmbedField{Name: name}
		embed.Fields = append(embed.Fields, field)
		return field
	}

	removeField := func(name string) {
		for i, v := range embed.Fields {
			if v.Name == name {
				embed.Fields = append(embed.Fields[:i], embed.Fields[i+1:]...)
				return
			}
		}
	}

	for _, v := range r.Routes {
		field := getField(v.Category)

		var tag string
		if !v.Disabled {
			field.Value += depthString(tag+v.Name+tag, depth, false)
		}

		if field.Value == "" {
			removeField(v.Category)
		}
	}

	for _, v := range r.Subrouters {
		field := getField(v.Category())
		field.Value += depthString(v.Name, depth, true)
		embed = depthcharge(v.Router, embed, depth+1)
	}

	return embed
}
<|endoftext|>


<commit_before>package instagram

import (
	"strings"
	"time"

	"sync"

	"net/url"

	"github.com/Seklfreak/Robyul2/cache"
	"github.com/Seklfreak/Robyul2/helpers"
	"github.com/Seklfreak/Robyul2/metrics"
	"github.com/Seklfreak/Robyul2/models"
	"github.com/globalsign/mgo/bson"
)

var (
	instagramEntryLocks = make(map[string]*sync.Mutex)
)

const (
	InstagramGraphQlWorkers = 15
)

func (m *Handler) checkInstagramPublicFeedLoop() {
	log := cache.GetLogger().WithField("module", "instagram")

	defer helpers.Recover()
	defer func() {
		go func() {
			defer helpers.Recover()
			log.Error("The checkInstagramPublicFeedLoop died." +
				"Please investigate! Will be restarted in 60 seconds")
			time.Sleep(60 * time.Second)
			m.checkInstagramPublicFeedLoop()
		}()
	}()

	var wg sync.WaitGroup
	for {
		bundledEntries, entriesCount, err := m.getBundledEntries()
		helpers.Relax(err)

		log.Infof(
			"checking graphql feed on %d accounts for %d feeds with %d workers",
			len(bundledEntries), entriesCount, InstagramGraphQlWorkers)
		start := time.Now()

		wg.Add(InstagramGraphQlWorkers)

		jobs := make(chan map[string][]models.InstagramEntry, InstagramGraphQlWorkers)

		workerEntries := make(map[int]map[string][]models.InstagramEntry, 0)
		for w := 1; w <= InstagramGraphQlWorkers; w++ {
			go m.checkInstagramPublicFeedLoopWorker(w, jobs, &wg)
			workerEntries[w] = make(map[string][]models.InstagramEntry)
		}

		lastWorker := 1
		for code, codeEntries := range bundledEntries {
			workerEntries[lastWorker][code] = codeEntries
			lastWorker++
			if lastWorker > InstagramGraphQlWorkers {
				lastWorker = 1
			}
		}

		for _, workerEntry := range workerEntries {
			jobs <- workerEntry
		}
		close(jobs)

		wg.Wait()
		elapsed := time.Since(start)
		log.Infof(
			"checked graphql feed on %d accounts for %d feeds with %d workers, took %s",
			len(bundledEntries), entriesCount, InstagramGraphQlWorkers, elapsed)
		metrics.InstagramGraphQlFeedRefreshTime.Set(elapsed.Seconds())

		if entriesCount <= 10 {
			time.Sleep(60 * time.Second)
		}
	}
}

func (m *Handler) checkInstagramPublicFeedLoopWorker(id int, jobs <-chan map[string][]models.InstagramEntry, wg *sync.WaitGroup) {
	defer helpers.Recover()
	defer func() {
		if wg != nil {
			wg.Done()
		}
	}()

	currentProxy, err := helpers.GetRandomProxy()
	helpers.Relax(err)

	for job := range jobs {
		//cache.GetLogger().WithField("module", "instagram").WithField("worker", id).Infof(
		//	"worker %d started for %d accounts", id, len(job))
	NextEntry:
		for instagramUsername, entries := range job {
			//cache.GetLogger().WithField("module", "instagram").WithField("worker", id).Infof(
			//	"checking graphql feed for %d for %d channels", instagramAccountID, len(entries))
		RetryGraphQl:
			_, receivedPosts, err := m.getInformationAndPosts(instagramUsername, currentProxy)
			if err != nil {
				if strings.Contains(err.Error(), "expected status 200; got 404") {
					// account got deleted/username got changed
					continue NextEntry
				}
				if m.retryOnError(err) {
					//cache.GetLogger().WithField("module", "instagram").Infof(
					//	"proxy error connecting to Instagram Account %s (GraphQL), "+
					//		"waiting 5 seconds, switching proxy and then trying again", instagramAccountID)
					time.Sleep(5 * time.Second)
					currentProxy, err = helpers.GetRandomProxy()
					helpers.Relax(err)
					goto RetryGraphQl
				}
				helpers.RelaxLog(err)
				continue NextEntry
			}

			postCheckTime := time.Now()

			for _, receivedPost := range receivedPosts {
				postHasBeenPostedEverywhere := true
				for _, entry := range entries {
					postAlreadyPosted := false
					if receivedPost.CreatedAt.Before(entry.LastPostCheck) {
						postAlreadyPosted = true
					}
					if !postAlreadyPosted {
						postHasBeenPostedEverywhere = false
					}
				}

				if postHasBeenPostedEverywhere {
					continue
				}

				// download specific post data
			RetryPost:
				post, err := m.getPostInformation(receivedPost.Shortcode, currentProxy)
				if err != nil {
					if m.retryOnError(err) {
						//cache.GetLogger().WithField("module", "instagram").Infof(
						//	"hit rate limit checking Instagram Account %s (GraphQL), "+
						//		"waiting 5 seconds, switching proxy and then trying again", instagramAccountID)
						time.Sleep(5 * time.Second)
						currentProxy, err = helpers.GetRandomProxy()
						helpers.Relax(err)
						goto RetryPost
					}
					if strings.Contains(err.Error(), "expected status 200; got 404") {
						// post got deleted
						continue
					}
					helpers.RelaxLog(err)
					continue NextEntry
				}

				for _, entry := range entries {
					entryID := entry.ID
					m.lockEntry(entryID)

					var entry models.InstagramEntry
					err = helpers.MdbOneWithoutLogging(
						helpers.MdbCollection(models.InstagramTable).Find(bson.M{"_id": entryID}),
						&entry,
					)

					if entry.LastPostCheck.IsZero() { // prevent spam
						entry.LastPostCheck = time.Now()
					}

					if err != nil {
						m.unlockEntry(entryID)
						helpers.RelaxLog(err)
						continue
					}

					postAlreadyPosted := false
					if receivedPost.CreatedAt.Before(entry.LastPostCheck) {
						postAlreadyPosted = true
					}

					if postAlreadyPosted == false {
						// cache.GetLogger().WithField("module", "instagram").Infof("Posting Post (GraphQL): #%s", post.ID)
						go m.postPostToChannel(entry.ChannelID, post, entry.SendPostType)
					}

					entry.LastPostCheck = postCheckTime
					err = helpers.MDbUpdateWithoutLogging(models.InstagramTable, entry.ID, entry)
					if err != nil {
						m.unlockEntry(entryID)
						helpers.RelaxLog(err)
						continue
					}

					m.unlockEntry(entryID)
				}
			}
		}
	}
}

func (m *Handler) lockEntry(entryID bson.ObjectId) {
	if _, ok := instagramEntryLocks[string(entryID)]; ok {
		instagramEntryLocks[string(entryID)].Lock()
		return
	}
	instagramEntryLocks[string(entryID)] = new(sync.Mutex)
	instagramEntryLocks[string(entryID)].Lock()
}

func (m *Handler) unlockEntry(entryID bson.ObjectId) {
	if _, ok := instagramEntryLocks[string(entryID)]; ok {
		instagramEntryLocks[string(entryID)].Unlock()
	}
}

func (m *Handler) retryOnError(err error) (retry bool) {
	if err != nil {
		if _, ok := err.(*url.Error); ok ||
			strings.Contains(err.Error(), "net/http") ||
			strings.Contains(err.Error(), "expected status 200; got 429") ||
			strings.Contains(err.Error(), "Please wait a few minutes before you try again.") ||
			strings.Contains(err.Error(), "expected status 200; got 500") ||
			strings.Contains(err.Error(), "expected status 200; got 502") ||
			strings.Contains(err.Error(), "expected status 200; got 503") ||
			strings.Contains(err.Error(), "tls: bad record MAC") ||
			strings.Contains(err.Error(), "unexpected EOF") ||
			strings.Contains(err.Error(), "page was load incorrectly") ||
			strings.Contains(err.Error(), "read: connection reset by peer") {
			return true
		}
	}
	return false
}
<commit_msg>[instagram] prevents crash 💣<commit_after>package instagram

import (
	"strings"
	"time"

	"sync"

	"net/url"

	"github.com/Seklfreak/Robyul2/cache"
	"github.com/Seklfreak/Robyul2/helpers"
	"github.com/Seklfreak/Robyul2/metrics"
	"github.com/Seklfreak/Robyul2/models"
	"github.com/globalsign/mgo/bson"
)

var (
	instagramEntryLock  sync.Mutex
	instagramEntryLocks = make(map[string]*sync.Mutex)
)

const (
	InstagramGraphQlWorkers = 15
)

func (m *Handler) checkInstagramPublicFeedLoop() {
	log := cache.GetLogger().WithField("module", "instagram")

	defer helpers.Recover()
	defer func() {
		go func() {
			defer helpers.Recover()
			log.Error("The checkInstagramPublicFeedLoop died." +
				"Please investigate! Will be restarted in 60 seconds")
			time.Sleep(60 * time.Second)
			m.checkInstagramPublicFeedLoop()
		}()
	}()

	var wg sync.WaitGroup
	for {
		bundledEntries, entriesCount, err := m.getBundledEntries()
		helpers.Relax(err)

		log.Infof(
			"checking graphql feed on %d accounts for %d feeds with %d workers",
			len(bundledEntries), entriesCount, InstagramGraphQlWorkers)
		start := time.Now()

		wg.Add(InstagramGraphQlWorkers)

		jobs := make(chan map[string][]models.InstagramEntry, InstagramGraphQlWorkers)

		workerEntries := make(map[int]map[string][]models.InstagramEntry, 0)
		for w := 1; w <= InstagramGraphQlWorkers; w++ {
			go m.checkInstagramPublicFeedLoopWorker(w, jobs, &wg)
			workerEntries[w] = make(map[string][]models.InstagramEntry)
		}

		lastWorker := 1
		for code, codeEntries := range bundledEntries {
			workerEntries[lastWorker][code] = codeEntries
			lastWorker++
			if lastWorker > InstagramGraphQlWorkers {
				lastWorker = 1
			}
		}

		for _, workerEntry := range workerEntries {
			jobs <- workerEntry
		}
		close(jobs)

		wg.Wait()
		elapsed := time.Since(start)
		log.Infof(
			"checked graphql feed on %d accounts for %d feeds with %d workers, took %s",
			len(bundledEntries), entriesCount, InstagramGraphQlWorkers, elapsed)
		metrics.InstagramGraphQlFeedRefreshTime.Set(elapsed.Seconds())

		if entriesCount <= 10 {
			time.Sleep(60 * time.Second)
		}
	}
}

func (m *Handler) checkInstagramPublicFeedLoopWorker(id int, jobs <-chan map[string][]models.InstagramEntry, wg *sync.WaitGroup) {
	defer helpers.Recover()
	defer func() {
		if wg != nil {
			wg.Done()
		}
	}()

	currentProxy, err := helpers.GetRandomProxy()
	helpers.Relax(err)

	for job := range jobs {
		//cache.GetLogger().WithField("module", "instagram").WithField("worker", id).Infof(
		//	"worker %d started for %d accounts", id, len(job))
	NextEntry:
		for instagramUsername, entries := range job {
			//cache.GetLogger().WithField("module", "instagram").WithField("worker", id).Infof(
			//	"checking graphql feed for %d for %d channels", instagramAccountID, len(entries))
		RetryGraphQl:
			_, receivedPosts, err := m.getInformationAndPosts(instagramUsername, currentProxy)
			if err != nil {
				if strings.Contains(err.Error(), "expected status 200; got 404") {
					// account got deleted/username got changed
					continue NextEntry
				}
				if m.retryOnError(err) {
					//cache.GetLogger().WithField("module", "instagram").Infof(
					//	"proxy error connecting to Instagram Account %s (GraphQL), "+
					//		"waiting 5 seconds, switching proxy and then trying again", instagramAccountID)
					time.Sleep(5 * time.Second)
					currentProxy, err = helpers.GetRandomProxy()
					helpers.Relax(err)
					goto RetryGraphQl
				}
				helpers.RelaxLog(err)
				continue NextEntry
			}

			postCheckTime := time.Now()

			for _, receivedPost := range receivedPosts {
				postHasBeenPostedEverywhere := true
				for _, entry := range entries {
					postAlreadyPosted := false
					if receivedPost.CreatedAt.Before(entry.LastPostCheck) {
						postAlreadyPosted = true
					}
					if !postAlreadyPosted {
						postHasBeenPostedEverywhere = false
					}
				}

				if postHasBeenPostedEverywhere {
					continue
				}

				// download specific post data
			RetryPost:
				post, err := m.getPostInformation(receivedPost.Shortcode, currentProxy)
				if err != nil {
					if m.retryOnError(err) {
						//cache.GetLogger().WithField("module", "instagram").Infof(
						//	"hit rate limit checking Instagram Account %s (GraphQL), "+
						//		"waiting 5 seconds, switching proxy and then trying again", instagramAccountID)
						time.Sleep(5 * time.Second)
						currentProxy, err = helpers.GetRandomProxy()
						helpers.Relax(err)
						goto RetryPost
					}
					if strings.Contains(err.Error(), "expected status 200; got 404") {
						// post got deleted
						continue
					}
					helpers.RelaxLog(err)
					continue NextEntry
				}

				for _, entry := range entries {
					entryID := entry.ID
					m.lockEntry(entryID)

					var entry models.InstagramEntry
					err = helpers.MdbOneWithoutLogging(
						helpers.MdbCollection(models.InstagramTable).Find(bson.M{"_id": entryID}),
						&entry,
					)

					if entry.LastPostCheck.IsZero() { // prevent spam
						entry.LastPostCheck = time.Now()
					}

					if err != nil {
						m.unlockEntry(entryID)
						helpers.RelaxLog(err)
						continue
					}

					postAlreadyPosted := false
					if receivedPost.CreatedAt.Before(entry.LastPostCheck) {
						postAlreadyPosted = true
					}

					if postAlreadyPosted == false {
						// cache.GetLogger().WithField("module", "instagram").Infof("Posting Post (GraphQL): #%s", post.ID)
						go m.postPostToChannel(entry.ChannelID, post, entry.SendPostType)
					}

					entry.LastPostCheck = postCheckTime
					err = helpers.MDbUpdateWithoutLogging(models.InstagramTable, entry.ID, entry)
					if err != nil {
						m.unlockEntry(entryID)
						helpers.RelaxLog(err)
						continue
					}

					m.unlockEntry(entryID)
				}
			}
		}
	}
}

func (m *Handler) lockEntry(entryID bson.ObjectId) {
	instagramEntryLock.Lock()
	defer instagramEntryLock.Unlock()

	if _, ok := instagramEntryLocks[string(entryID)]; ok {
		instagramEntryLocks[string(entryID)].Lock()
		return
	}
	instagramEntryLocks[string(entryID)] = new(sync.Mutex)
	instagramEntryLocks[string(entryID)].Lock()
}

func (m *Handler) unlockEntry(entryID bson.ObjectId) {
	instagramEntryLock.Lock()
	defer instagramEntryLock.Unlock()

	if _, ok := instagramEntryLocks[string(entryID)]; ok {
		instagramEntryLocks[string(entryID)].Unlock()
	}
}

func (m *Handler) retryOnError(err error) (retry bool) {
	if err != nil {
		if _, ok := err.(*url.Error); ok ||
			strings.Contains(err.Error(), "net/http") ||
			strings.Contains(err.Error(), "expected status 200; got 429") ||
			strings.Contains(err.Error(), "Please wait a few minutes before you try again.") ||
			strings.Contains(err.Error(), "expected status 200; got 500") ||
			strings.Contains(err.Error(), "expected status 200; got 502") ||
			strings.Contains(err.Error(), "expected status 200; got 503") ||
			strings.Contains(err.Error(), "tls: bad record MAC") ||
			strings.Contains(err.Error(), "unexpected EOF") ||
			strings.Contains(err.Error(), "page was load incorrectly") ||
			strings.Contains(err.Error(), "read: connection reset by peer") {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package auth

import (
	. "github.com/jacobsa/ogletest"
	"testing"
)

func TestSigner(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type SignerTest struct {}

func init() { RegisterTestSuite(&SignerTest{}) }

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *SignerTest) DoesFoo() {
	ExpectEq("TODO", "")
}
<commit_msg>Added test names.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package auth

import (
	. "github.com/jacobsa/ogletest"
	"testing"
)

func TestSigner(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type SignerTest struct {}

func init() { RegisterTestSuite(&SignerTest{}) }

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *SignerTest) CallsFunction() {
	ExpectEq("TODO", "")
}

func (t *SignerTest) FunctionReturnsError() {
	ExpectEq("TODO", "")
}

func (t *SignerTest) FunctionReturnsString() {
	ExpectEq("TODO", "")
}

func (t *SignerTest) GoldenTests() {
	ExpectEq("TODO", "")
}
<|endoftext|>


<commit_before>// Copyright (c) 2013 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package btcwire_test

import (
	"bytes"
	"github.com/conformal/btcwire"
	"github.com/davecgh/go-spew/spew"
	"testing"
)

// TestGenesisBlock tests the genesis block for validity by checking the encoded
// bytes and hashes.
func TestGenesisBlock(t *testing.T) {
	pver := uint32(60002)

	// Encode the genesis block to raw bytes.
	var buf bytes.Buffer
	err := btcwire.GenesisBlock.BtcEncode(&buf, pver)
	if err != nil {
		t.Errorf("TestGenesisBlock: %v", err)
		return
	}

	// Ensure the encoded block matches the expected bytes.
	if !bytes.Equal(buf.Bytes(), genesisBlockBytes) {
		t.Errorf("TestGenesisBlock: Genesis block does not appear valid - "+
			"got %v, want %v", spew.Sdump(buf.Bytes()),
			spew.Sdump(genesisBlockBytes))
		return
	}

	// Check hash of the block against expected hash.
	hash, err := btcwire.GenesisBlock.Header.BlockSha(pver)
	if err != nil {
		t.Errorf("BlockSha: %v", err)
	}
	if !btcwire.GenesisHash.IsEqual(&hash) {
		t.Errorf("TestGenesisBlock: Genesis block hash does not appear valid - "+
			"got %v, want %v", spew.Sdump(hash),
			spew.Sdump(btcwire.GenesisHash))
		return
	}
}

// genesisBlockBytes are the wire encoded bytes for the genesis block as of
// protocol version 60002.
var genesisBlockBytes = []byte{
	0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* |........| */
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* |........| */
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* |........| */
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* |........| */
	0x00, 0x00, 0x00, 0x00, 0x3b, 0xa3, 0xed, 0xfd, /* |....;...| */
	0x7a, 0x7b, 0x12, 0xb2, 0x7a, 0xc7, 0x2c, 0x3e, /* |z{..z.,>| */
	0x67, 0x76, 0x8f, 0x61, 0x7f, 0xc8, 0x1b, 0xc3, /* |gv.a....| */
	0x88, 0x8a, 0x51, 0x32, 0x3a, 0x9f, 0xb8, 0xaa, /* |..Q2:...| */
	0x4b, 0x1e, 0x5e, 0x4a, 0x29, 0xab, 0x5f, 0x49, /* |K.^J)._I| */
	0xff, 0xff, 0x00, 0x1d, 0x1d, 0xac, 0x2b, 0x7c, /* |......+|| */
	0x01, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, /* |........| */
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* |........| */
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* |........| */
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* |........| */
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, /* |........| */
	0xff, 0xff, 0x4d, 0x04, 0xff, 0xff, 0x00, 0x1d, /* |..M.....| */
	0x01, 0x04, 0x45, 0x54, 0x68, 0x65, 0x20, 0x54, /* |..EThe T| */
	0x69, 0x6d, 0x65, 0x73, 0x20, 0x30, 0x33, 0x2f, /* |imes 03/| */
	0x4a, 0x61, 0x6e, 0x2f, 0x32, 0x30, 0x30, 0x39, /* |Jan/2009| */
	0x20, 0x43, 0x68, 0x61, 0x6e, 0x63, 0x65, 0x6c, /* | Chancel| */
	0x6c, 0x6f, 0x72, 0x20, 0x6f, 0x6e, 0x20, 0x62, /* |lor on b| */
	0x72, 0x69, 0x6e, 0x6b, 0x20, 0x6f, 0x66, 0x20, /* |rink of | */
	0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x20, 0x62, /* |second b| */
	0x61, 0x69, 0x6c, 0x6f, 0x75, 0x74, 0x20, 0x66, /* |ailout f| */
	0x6f, 0x72, 0x20, 0x62, 0x61, 0x6e, 0x6b, 0x73, /* |or banks| */
	0xff, 0xff, 0xff, 0xff, 0x01, 0x00, 0xf2, 0x05, /* |........| */
	0x2a, 0x01, 0x00, 0x00, 0x00, 0x43, 0x41, 0x04, /* |*....CA.| */
	0x67, 0x8a, 0xfd, 0xb0, 0xfe, 0x55, 0x48, 0x27, /* |g....UH'| */
	0x19, 0x67, 0xf1, 0xa6, 0x71, 0x30, 0xb7, 0x10, /* |.g..q0..| */
	0x5c, 0xd6, 0xa8, 0x28, 0xe0, 0x39, 0x09, 0xa6, /* |\..(.9..| */
	0x79, 0x62, 0xe0, 0xea, 0x1f, 0x61, 0xde, 0xb6, /* |yb...a..| */
	0x49, 0xf6, 0xbc, 0x3f, 0x4c, 0xef, 0x38, 0xc4, /* |I..?L.8.| */
	0xf3, 0x55, 0x04, 0xe5, 0x1e, 0xc1, 0x12, 0xde, /* |.U......| */
	0x5c, 0x38, 0x4d, 0xf7, 0xba, 0x0b, 0x8d, 0x57, /* |\8M....W| */
	0x8a, 0x4c, 0x70, 0x2b, 0x6b, 0xf1, 0x1d, 0x5f, /* |.Lp+k.._|*/
	0xac, 0x00, 0x00, 0x00, 0x00, /* |.....|    */
}
<commit_msg>Add tests for genesis block of regression testnet.<commit_after>// Copyright (c) 2013 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package btcwire_test

import (
	"bytes"
	"github.com/conformal/btcwire"
	"github.com/davecgh/go-spew/spew"
	"testing"
)

// TestGenesisBlock tests the genesis block of the main network for validity by
// checking the encoded bytes and hashes.
func TestGenesisBlock(t *testing.T) {
	pver := uint32(60002)

	// Encode the genesis block to raw bytes.
	var buf bytes.Buffer
	err := btcwire.GenesisBlock.BtcEncode(&buf, pver)
	if err != nil {
		t.Errorf("TestGenesisBlock: %v", err)
		return
	}

	// Ensure the encoded block matches the expected bytes.
	if !bytes.Equal(buf.Bytes(), genesisBlockBytes) {
		t.Errorf("TestGenesisBlock: Genesis block does not appear valid - "+
			"got %v, want %v", spew.Sdump(buf.Bytes()),
			spew.Sdump(genesisBlockBytes))
		return
	}

	// Check hash of the block against expected hash.
	hash, err := btcwire.GenesisBlock.Header.BlockSha(pver)
	if err != nil {
		t.Errorf("BlockSha: %v", err)
	}
	if !btcwire.GenesisHash.IsEqual(&hash) {
		t.Errorf("TestGenesisBlock: Genesis block hash does not appear valid - "+
			"got %v, want %v", spew.Sdump(hash),
			spew.Sdump(btcwire.GenesisHash))
		return
	}
}

// TestTestNetGenesisBlock tests the genesis block of the regression test
// network for validity by checking the encoded bytes and hashes.
func TestTestNetGenesisBlock(t *testing.T) {
	pver := uint32(60002)

	// Encode the genesis block to raw bytes.
	var buf bytes.Buffer
	err := btcwire.TestNetGenesisBlock.BtcEncode(&buf, pver)
	if err != nil {
		t.Errorf("TestTestNetGenesisBlock: %v", err)
		return
	}

	// Ensure the encoded block matches the expected bytes.
	if !bytes.Equal(buf.Bytes(), testNetGenesisBlockBytes) {
		t.Errorf("TestTestNetGenesisBlock: Genesis block does not "+
			"appear valid - got %v, want %v",
			spew.Sdump(buf.Bytes()),
			spew.Sdump(testNetGenesisBlockBytes))
		return
	}

	// Check hash of the block against expected hash.
	hash, err := btcwire.TestNetGenesisBlock.Header.BlockSha(pver)
	if err != nil {
		t.Errorf("BlockSha: %v", err)
	}
	if !btcwire.TestNetGenesisHash.IsEqual(&hash) {
		t.Errorf("TestTestNetGenesisBlock: Genesis block hash does "+
			"not appear valid - got %v, want %v", spew.Sdump(hash),
			spew.Sdump(btcwire.GenesisHash))
		return
	}
}

// genesisBlockBytes are the wire encoded bytes for the genesis block of the
// main network as of protocol version 60002.
var genesisBlockBytes = []byte{
	0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* |........| */
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* |........| */
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* |........| */
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* |........| */
	0x00, 0x00, 0x00, 0x00, 0x3b, 0xa3, 0xed, 0xfd, /* |....;...| */
	0x7a, 0x7b, 0x12, 0xb2, 0x7a, 0xc7, 0x2c, 0x3e, /* |z{..z.,>| */
	0x67, 0x76, 0x8f, 0x61, 0x7f, 0xc8, 0x1b, 0xc3, /* |gv.a....| */
	0x88, 0x8a, 0x51, 0x32, 0x3a, 0x9f, 0xb8, 0xaa, /* |..Q2:...| */
	0x4b, 0x1e, 0x5e, 0x4a, 0x29, 0xab, 0x5f, 0x49, /* |K.^J)._I| */
	0xff, 0xff, 0x00, 0x1d, 0x1d, 0xac, 0x2b, 0x7c, /* |......+|| */
	0x01, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, /* |........| */
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* |........| */
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* |........| */
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* |........| */
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, /* |........| */
	0xff, 0xff, 0x4d, 0x04, 0xff, 0xff, 0x00, 0x1d, /* |..M.....| */
	0x01, 0x04, 0x45, 0x54, 0x68, 0x65, 0x20, 0x54, /* |..EThe T| */
	0x69, 0x6d, 0x65, 0x73, 0x20, 0x30, 0x33, 0x2f, /* |imes 03/| */
	0x4a, 0x61, 0x6e, 0x2f, 0x32, 0x30, 0x30, 0x39, /* |Jan/2009| */
	0x20, 0x43, 0x68, 0x61, 0x6e, 0x63, 0x65, 0x6c, /* | Chancel| */
	0x6c, 0x6f, 0x72, 0x20, 0x6f, 0x6e, 0x20, 0x62, /* |lor on b| */
	0x72, 0x69, 0x6e, 0x6b, 0x20, 0x6f, 0x66, 0x20, /* |rink of | */
	0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x20, 0x62, /* |second b| */
	0x61, 0x69, 0x6c, 0x6f, 0x75, 0x74, 0x20, 0x66, /* |ailout f| */
	0x6f, 0x72, 0x20, 0x62, 0x61, 0x6e, 0x6b, 0x73, /* |or banks| */
	0xff, 0xff, 0xff, 0xff, 0x01, 0x00, 0xf2, 0x05, /* |........| */
	0x2a, 0x01, 0x00, 0x00, 0x00, 0x43, 0x41, 0x04, /* |*....CA.| */
	0x67, 0x8a, 0xfd, 0xb0, 0xfe, 0x55, 0x48, 0x27, /* |g....UH'| */
	0x19, 0x67, 0xf1, 0xa6, 0x71, 0x30, 0xb7, 0x10, /* |.g..q0..| */
	0x5c, 0xd6, 0xa8, 0x28, 0xe0, 0x39, 0x09, 0xa6, /* |\..(.9..| */
	0x79, 0x62, 0xe0, 0xea, 0x1f, 0x61, 0xde, 0xb6, /* |yb...a..| */
	0x49, 0xf6, 0xbc, 0x3f, 0x4c, 0xef, 0x38, 0xc4, /* |I..?L.8.| */
	0xf3, 0x55, 0x04, 0xe5, 0x1e, 0xc1, 0x12, 0xde, /* |.U......| */
	0x5c, 0x38, 0x4d, 0xf7, 0xba, 0x0b, 0x8d, 0x57, /* |\8M....W| */
	0x8a, 0x4c, 0x70, 0x2b, 0x6b, 0xf1, 0x1d, 0x5f, /* |.Lp+k.._|*/
	0xac, 0x00, 0x00, 0x00, 0x00, /* |.....|    */
}

// testNetGenesisBlockBytes are the wire encoded bytes for the genesis block of
// the regression test network as of protocol version 60002.
var testNetGenesisBlockBytes = []byte{
	0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* |........| */
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* |........| */
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* |........| */
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* |........| */
	0x00, 0x00, 0x00, 0x00, 0x3b, 0xa3, 0xed, 0xfd, /* |....;...| */
	0x7a, 0x7b, 0x12, 0xb2, 0x7a, 0xc7, 0x2c, 0x3e, /* |z{..z.,>| */
	0x67, 0x76, 0x8f, 0x61, 0x7f, 0xc8, 0x1b, 0xc3, /* |gv.a....| */
	0x88, 0x8a, 0x51, 0x32, 0x3a, 0x9f, 0xb8, 0xaa, /* |..Q2:...| */
	0x4b, 0x1e, 0x5e, 0x4a, 0xda, 0xe5, 0x49, 0x4d, /* |K.^J)._I| */
	0xff, 0xff, 0x7f, 0x20, 0x02, 0x00, 0x00, 0x00, /* |......+|| */
	0x01, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, /* |........| */
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* |........| */
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* |........| */
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* |........| */
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, /* |........| */
	0xff, 0xff, 0x4d, 0x04, 0xff, 0xff, 0x00, 0x1d, /* |..M.....| */
	0x01, 0x04, 0x45, 0x54, 0x68, 0x65, 0x20, 0x54, /* |..EThe T| */
	0x69, 0x6d, 0x65, 0x73, 0x20, 0x30, 0x33, 0x2f, /* |imes 03/| */
	0x4a, 0x61, 0x6e, 0x2f, 0x32, 0x30, 0x30, 0x39, /* |Jan/2009| */
	0x20, 0x43, 0x68, 0x61, 0x6e, 0x63, 0x65, 0x6c, /* | Chancel| */
	0x6c, 0x6f, 0x72, 0x20, 0x6f, 0x6e, 0x20, 0x62, /* |lor on b| */
	0x72, 0x69, 0x6e, 0x6b, 0x20, 0x6f, 0x66, 0x20, /* |rink of | */
	0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x20, 0x62, /* |second b| */
	0x61, 0x69, 0x6c, 0x6f, 0x75, 0x74, 0x20, 0x66, /* |ailout f| */
	0x6f, 0x72, 0x20, 0x62, 0x61, 0x6e, 0x6b, 0x73, /* |or banks| */
	0xff, 0xff, 0xff, 0xff, 0x01, 0x00, 0xf2, 0x05, /* |........| */
	0x2a, 0x01, 0x00, 0x00, 0x00, 0x43, 0x41, 0x04, /* |*....CA.| */
	0x67, 0x8a, 0xfd, 0xb0, 0xfe, 0x55, 0x48, 0x27, /* |g....UH'| */
	0x19, 0x67, 0xf1, 0xa6, 0x71, 0x30, 0xb7, 0x10, /* |.g..q0..| */
	0x5c, 0xd6, 0xa8, 0x28, 0xe0, 0x39, 0x09, 0xa6, /* |\..(.9..| */
	0x79, 0x62, 0xe0, 0xea, 0x1f, 0x61, 0xde, 0xb6, /* |yb...a..| */
	0x49, 0xf6, 0xbc, 0x3f, 0x4c, 0xef, 0x38, 0xc4, /* |I..?L.8.| */
	0xf3, 0x55, 0x04, 0xe5, 0x1e, 0xc1, 0x12, 0xde, /* |.U......| */
	0x5c, 0x38, 0x4d, 0xf7, 0xba, 0x0b, 0x8d, 0x57, /* |\8M....W| */
	0x8a, 0x4c, 0x70, 0x2b, 0x6b, 0xf1, 0x1d, 0x5f, /* |.Lp+k.._|*/
	0xac, 0x00, 0x00, 0x00, 0x00, /* |.....|    */
}
<|endoftext|>


<commit_before>package matches

import (
	"errors"
	"fmt"
	"net/url"
	"strconv"
	"time"

	"github.com/coltiebaby/glaw"
)

// Match Filter will help you sort through match history.
// To get a list of the IDs, you'll need to look at the [constants](*url)  supports
//
// url: https://developer.games.com/game-constants.html
type MatchFilter struct {
	Client glaw.ApiClient

	AccountID string
	Start     time.Time
	End       time.Time
	Champions []int
	Queues    []int
	Seasons   []int

	maxResults int
	page       int

	totalResults int
	begin        int
	end          int
}

func (filter *MatchFilter) Debug() {
	fmt.Printf("%+v\n", filter)
}

func getIndex(max, page int) (begin int, end int) {
	end = (page * max)
	begin = (end - max) + 1

	return begin, end
}

func NewMatchFilter(c glaw.ApiClient, start, end time.Time) (filter *MatchFilter) {
	filter = &MatchFilter{Client: c}

	filter.Start = start
	filter.End = end

	filter.SetMaxResults(20)

	return filter
}

// Confirm results are between 1 - 100
// TODO: log that we changes the result to something else
func (filter *MatchFilter) SetMaxResults(results int) {
	switch {
	case results <= 0:
		results = 1
	case results > 100:
		results = 100
	}

	filter.page = 0
	filter.maxResults = results
	filter.totalResults = results
}

func (filter *MatchFilter) Next() (matches MatchStorage, err error) {
	page := filter.page
	page++

	begin, end := getIndex(filter.maxResults, page)
	fmt.Println(begin, end)

	if begin > filter.totalResults {
		return matches, MaxResultsError
	} else {
		filter.page = page
	}

	filter.begin, filter.end = begin, end

	matches, err = GetMatchlists(filter.Client, filter.AccountID, filter.CreateValues())
	filter.totalResults = matches.TotalGames
	return matches, err
}

func (filter *MatchFilter) Prev() (matches MatchStorage, err error) {
	page := filter.page

	if page <= 1 {
		return matches, MinResultsError
	}

	page--
	filter.begin, filter.end = getIndex(filter.maxResults, page)
	filter.page = page

	matches, err = GetMatchlists(filter.Client, filter.AccountID, filter.CreateValues())
	filter.totalResults = matches.TotalGames
	return matches, err
}

func (filter *MatchFilter) GoTo(page int) (matches MatchStorage, err error) {
	wanted := (page-1)*filter.maxResults + 1
	if page < 1 || wanted > filter.totalResults {
		return matches, InvalidPageError
	}

	filter.begin, filter.end = getIndex(filter.maxResults, page)
	filter.page = page

	matches, err = GetMatchlists(filter.Client, filter.AccountID, filter.CreateValues())
	filter.totalResults = matches.TotalGames
	return matches, err
}

func (filter *MatchFilter) CreateValues() url.Values {
	params := url.Values{}

	add(&params, `champion`, filter.Champions)
	add(&params, `queue`, filter.Queues)
	add(&params, `season`, filter.Seasons)

	params.Add(`startIndex`, strconv.Itoa(filter.begin))
	params.Add(`endIndex`, strconv.Itoa(filter.end))

	// TODO: Not sure why these aren't working
	// from := timeToUnixMS(filter.Start)
	// to := timeToUnixMS(filter.End)

	// params.Add(`beginTime`, strconv.FormatInt(from, 10))
	// params.Add(`endTime`, strconv.FormatInt(to, 10))

	return params
}

func (filter *MatchFilter) TotalPages() int {
	return 0
}

// time to unix millisecond
func timeToUnixMS(t time.Time) int64 {
	return t.UnixNano() / int64(1000000)
}

func add(u *url.Values, key string, values []int) {
	for _, v := range values {
		u.Add(key, strconv.Itoa(v))
	}
}

var (
	MinResultsError  error = errors.New("Reached the min results")
	MaxResultsError  error = errors.New("Reached the max results")
	InvalidPageError error = errors.New("Page is invalid")
)
<commit_msg>filtering if nothing is added (#29)<commit_after>package matches

import (
	"errors"
	"fmt"
	"net/url"
	"strconv"
	"time"

	"github.com/coltiebaby/glaw"
)

// Match Filter will help you sort through match history.
// To get a list of the IDs, you'll need to look at the [constants](*url)  supports
//
// url: https://developer.games.com/game-constants.html
type MatchFilter struct {
	Client glaw.ApiClient

	AccountID string
	Start     time.Time
	End       time.Time
	Champions []int
	Queues    []int
	Seasons   []int

	maxResults int
	page       int

	totalResults int
	begin        int
	end          int
}

func (filter *MatchFilter) Debug() {
	fmt.Printf("%+v\n", filter)
}

func getIndex(max, page int) (begin int, end int) {
	end = (page * max)
	begin = (end - max) + 1

	return begin, end
}

func NewMatchFilter(c glaw.ApiClient, start, end time.Time) (filter *MatchFilter) {
	filter = &MatchFilter{Client: c}

	filter.Start = start
	filter.End = end

	filter.SetMaxResults(20)

	return filter
}

// Confirm results are between 1 - 100
// TODO: log that we changes the result to something else
func (filter *MatchFilter) SetMaxResults(results int) {
	switch {
	case results <= 0:
		results = 1
	case results > 100:
		results = 100
	}

	filter.page = 0
	filter.maxResults = results
	filter.totalResults = results
}

func (filter *MatchFilter) Next() (matches MatchStorage, err error) {
	page := filter.page
	page++

	begin, end := getIndex(filter.maxResults, page)
	fmt.Println(begin, end)

	if begin > filter.totalResults {
		return matches, MaxResultsError
	} else {
		filter.page = page
	}

	filter.begin, filter.end = begin, end

	matches, err = GetMatchlists(filter.Client, filter.AccountID, filter.CreateValues())
	filter.totalResults = matches.TotalGames
	return matches, err
}

func (filter *MatchFilter) Prev() (matches MatchStorage, err error) {
	page := filter.page

	if page <= 1 {
		return matches, MinResultsError
	}

	page--
	filter.begin, filter.end = getIndex(filter.maxResults, page)
	filter.page = page

	matches, err = GetMatchlists(filter.Client, filter.AccountID, filter.CreateValues())
	filter.totalResults = matches.TotalGames
	return matches, err
}

func (filter *MatchFilter) GoTo(page int) (matches MatchStorage, err error) {
	wanted := (page-1)*filter.maxResults + 1
	if page < 1 || wanted > filter.totalResults {
		return matches, InvalidPageError
	}

	filter.begin, filter.end = getIndex(filter.maxResults, page)
	filter.page = page

	matches, err = GetMatchlists(filter.Client, filter.AccountID, filter.CreateValues())
	filter.totalResults = matches.TotalGames
	return matches, err
}

func (filter *MatchFilter) CreateValues() url.Values {
	params := url.Values{}

	if len(filter.Champions) > 0 {
		add(&params, `champion`, filter.Champions)
	}

	if len(filter.Queues) > 0 {
		add(&params, `queue`, filter.Queues)
	}

	if len(filter.Seasons) > 0 {
		add(&params, `season`, filter.Seasons)
	}

	if filter.begin != 0 && filter.end != 0 {
		params.Add(`startIndex`, strconv.Itoa(filter.begin))
		params.Add(`endIndex`, strconv.Itoa(filter.end))
	}

	// TODO: Not sure why these aren't working
	// from := timeToUnixMS(filter.Start)
	// to := timeToUnixMS(filter.End)

	// params.Add(`beginTime`, strconv.FormatInt(from, 10))
	// params.Add(`endTime`, strconv.FormatInt(to, 10))

	return params
}

func (filter *MatchFilter) TotalPages() int {
	return 0
}

// time to unix millisecond
func timeToUnixMS(t time.Time) int64 {
	return t.UnixNano() / int64(1000000)
}

func add(u *url.Values, key string, values []int) {
	for _, v := range values {
		u.Add(key, strconv.Itoa(v))
	}
}

var (
	MinResultsError  error = errors.New("Reached the min results")
	MaxResultsError  error = errors.New("Reached the max results")
	InvalidPageError error = errors.New("Page is invalid")
)
<|endoftext|>


<commit_before>package vespa

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"sort"
	"strconv"
	"time"

	"github.com/vespa-engine/vespa/client/go/util"
)

const (
	localTargetType  = "local"
	customTargetType = "custom"
	cloudTargetType  = "cloud"

	deployService   = "deploy"
	queryService    = "query"
	documentService = "document"

	defaultCloudAPI = "https://api.vespa-external.aws.oath.cloud:4443"

	waitRetryInterval = 2 * time.Second
)

// Service represents a Vespa service.
type Service struct {
	BaseURL    string
	Name       string
	TLSOptions TLSOptions
}

// Target represents a Vespa platform, running named Vespa services.
type Target interface {
	// Type returns this target's type, e.g. local or cloud.
	Type() string

	// Service returns the service for given name. If timeout is non-zero, wait for the service to converge.
	Service(name string, timeout time.Duration, sessionOrRunID int64) (*Service, error)
}

// TLSOptions configures the certificate to use for service requests.
type TLSOptions struct {
	KeyPair         tls.Certificate
	CertificateFile string
	PrivateKeyFile  string
}

// LogOptions configures the log output to produce when waiting for services.
type LogOptions struct {
	Writer io.Writer
	Level  int
}

type customTarget struct {
	targetType string
	baseURL    string
}

// Do sends request to this service. Any required authentication happens automatically.
func (s *Service) Do(request *http.Request, timeout time.Duration) (*http.Response, error) {
	if s.TLSOptions.KeyPair.Certificate != nil {
		util.ActiveHttpClient.UseCertificate(s.TLSOptions.KeyPair)
	}
	return util.HttpDo(request, timeout, s.Description())
}

// Wait polls the health check of this service until it succeeds or timeout passes.
func (s *Service) Wait(timeout time.Duration) (int, error) {
	url := s.BaseURL
	switch s.Name {
	case deployService:
		url += "/status.html" // because /ApplicationStatus is not publicly reachable in Vespa Cloud
	case queryService, documentService:
		url += "/ApplicationStatus"
	default:
		return 0, fmt.Errorf("invalid service: %s", s.Name)
	}
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return 0, err
	}
	okFunc := func(status int, response []byte) (bool, error) { return status/100 == 2, nil }
	return wait(okFunc, func() *http.Request { return req }, &s.TLSOptions.KeyPair, timeout)
}

func (s *Service) Description() string {
	switch s.Name {
	case queryService:
		return "Container (query API)"
	case documentService:
		return "Container (document API)"
	case deployService:
		return "Deploy API"
	}
	return fmt.Sprintf("No description of service %s", s.Name)
}

func (t *customTarget) Type() string { return t.targetType }

func (t *customTarget) Service(name string, timeout time.Duration, sessionID int64) (*Service, error) {
	if timeout > 0 && name != deployService {
		if err := t.waitForConvergence(timeout); err != nil {
			return nil, err
		}
	}
	switch name {
	case deployService, queryService, documentService:
		url, err := t.urlWithPort(name)
		if err != nil {
			return nil, err
		}
		return &Service{BaseURL: url, Name: name}, nil
	}
	return nil, fmt.Errorf("unknown service: %s", name)
}

func (t *customTarget) urlWithPort(serviceName string) (string, error) {
	u, err := url.Parse(t.baseURL)
	if err != nil {
		return "", err
	}
	port := u.Port()
	if port == "" {
		switch serviceName {
		case deployService:
			port = "19071"
		case queryService, documentService:
			port = "8080"
		default:
			return "", fmt.Errorf("unknown service: %s", serviceName)
		}
		u.Host = u.Host + ":" + port
	}
	return u.String(), nil
}

func (t *customTarget) waitForConvergence(timeout time.Duration) error {
	deployer, err := t.Service(deployService, 0, 0)
	if err != nil {
		return err
	}
	url := fmt.Sprintf("%s/application/v2/tenant/default/application/default/environment/prod/region/default/instance/default/serviceconverge", deployer.BaseURL)
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return err
	}
	converged := false
	convergedFunc := func(status int, response []byte) (bool, error) {
		if status/100 != 2 {
			return false, nil
		}
		var resp serviceConvergeResponse
		if err := json.Unmarshal(response, &resp); err != nil {
			return false, nil
		}
		converged = resp.Converged
		return converged, nil
	}
	if _, err := wait(convergedFunc, func() *http.Request { return req }, nil, timeout); err != nil {
		return err
	}
	if !converged {
		return fmt.Errorf("services have not converged")
	}
	return nil
}

type cloudTarget struct {
	cloudAPI   string
	targetType string
	deployment Deployment
	apiKey     []byte
	tlsOptions TLSOptions
	logOptions LogOptions

	queryURL    string
	documentURL string
}

func (t *cloudTarget) Type() string { return t.targetType }

func (t *cloudTarget) Service(name string, timeout time.Duration, runID int64) (*Service, error) {
	if timeout > 0 && name != deployService {
		if err := t.waitForEndpoints(timeout, runID); err != nil {
			return nil, err
		}
	}
	switch name {
	case deployService:
		return &Service{Name: name, BaseURL: t.cloudAPI}, nil
	case queryService:
		if t.queryURL == "" {
			return nil, fmt.Errorf("service %s is not discovered", name)
		}
		return &Service{Name: name, BaseURL: t.queryURL, TLSOptions: t.tlsOptions}, nil
	case documentService:
		if t.documentURL == "" {
			return nil, fmt.Errorf("service %s is not discovered", name)
		}
		return &Service{Name: name, BaseURL: t.documentURL, TLSOptions: t.tlsOptions}, nil
	}
	return nil, fmt.Errorf("unknown service: %s", name)
}

func (t *cloudTarget) waitForEndpoints(timeout time.Duration, runID int64) error {
	signer := NewRequestSigner(t.deployment.Application.SerializedForm(), t.apiKey)
	if runID > 0 {
		if err := t.waitForRun(signer, runID, timeout); err != nil {
			return err
		}
	}
	return t.discoverEndpoints(signer, timeout)
}

func (t *cloudTarget) waitForRun(signer *RequestSigner, runID int64, timeout time.Duration) error {
	runURL := fmt.Sprintf("%s/application/v4/tenant/%s/application/%s/instance/%s/job/%s-%s/run/%d",
		t.cloudAPI,
		t.deployment.Application.Tenant, t.deployment.Application.Application, t.deployment.Application.Instance,
		t.deployment.Zone.Environment, t.deployment.Zone.Region, runID)
	req, err := http.NewRequest("GET", runURL, nil)
	if err != nil {
		return err
	}
	lastID := int64(-1)
	requestFunc := func() *http.Request {
		q := req.URL.Query()
		q.Set("after", strconv.FormatInt(lastID, 10))
		req.URL.RawQuery = q.Encode()
		if err := signer.SignRequest(req); err != nil {
			panic(err)
		}
		return req
	}
	jobSuccessFunc := func(status int, response []byte) (bool, error) {
		if status/100 != 2 {
			return false, nil
		}
		var resp jobResponse
		if err := json.Unmarshal(response, &resp); err != nil {
			return false, nil
		}
		if t.logOptions.Writer != nil {
			lastID = t.printLog(resp, lastID)
		}
		if resp.Active {
			return false, nil
		}
		if resp.Status != "success" {
			return false, fmt.Errorf("run %d ended with unsuccessful status: %s", runID, resp.Status)
		}
		return true, nil
	}
	_, err = wait(jobSuccessFunc, requestFunc, &t.tlsOptions.KeyPair, timeout)
	return err
}

func (t *cloudTarget) printLog(response jobResponse, last int64) int64 {
	if response.LastID == 0 {
		return last
	}
	var msgs []logMessage
	for step, stepMsgs := range response.Log {
		for _, msg := range stepMsgs {
			if step == "copyVespaLogs" && LogLevel(msg.Type) > t.logOptions.Level {
				continue
			}
			msgs = append(msgs, msg)
		}
	}
	sort.Slice(msgs, func(i, j int) bool { return msgs[i].At < msgs[j].At })
	for _, msg := range msgs {
		tm := time.Unix(msg.At/1000, (msg.At%1000)*1000)
		fmtTime := tm.Format("15:04:05")
		fmt.Fprintf(t.logOptions.Writer, "[%s] %-7s %s\n", fmtTime, msg.Type, msg.Message)
	}
	return response.LastID
}

func (t *cloudTarget) discoverEndpoints(signer *RequestSigner, timeout time.Duration) error {
	deploymentURL := fmt.Sprintf("%s/application/v4/tenant/%s/application/%s/instance/%s/environment/%s/region/%s",
		t.cloudAPI,
		t.deployment.Application.Tenant, t.deployment.Application.Application, t.deployment.Application.Instance,
		t.deployment.Zone.Environment, t.deployment.Zone.Region)
	req, err := http.NewRequest("GET", deploymentURL, nil)
	if err != nil {
		return err
	}
	if err := signer.SignRequest(req); err != nil {
		return err
	}
	var endpointURL string
	endpointFunc := func(status int, response []byte) (bool, error) {
		if status/100 != 2 {
			return false, nil
		}
		var resp deploymentResponse
		if err := json.Unmarshal(response, &resp); err != nil {
			return false, nil
		}
		if len(resp.Endpoints) == 0 {
			return false, nil
		}
		endpointURL = resp.Endpoints[0].URL
		return true, nil
	}
	if _, err = wait(endpointFunc, func() *http.Request { return req }, &t.tlsOptions.KeyPair, timeout); err != nil {
		return err
	}
	if endpointURL == "" {
		return fmt.Errorf("no endpoint discovered")
	}
	t.queryURL = endpointURL
	t.documentURL = endpointURL
	return nil
}

// LocalTarget creates a target for a Vespa platform running locally.
func LocalTarget() Target {
	return &customTarget{targetType: localTargetType, baseURL: "http://127.0.0.1"}
}

// CustomTarget creates a Target for a Vespa platform running at baseURL.
func CustomTarget(baseURL string) Target {
	return &customTarget{targetType: customTargetType, baseURL: baseURL}
}

// CloudTarget creates a Target for the Vespa Cloud platform.
func CloudTarget(deployment Deployment, apiKey []byte, tlsOptions TLSOptions, logOptions LogOptions) Target {
	return &cloudTarget{
		cloudAPI:   defaultCloudAPI,
		targetType: cloudTargetType,
		deployment: deployment,
		apiKey:     apiKey,
		tlsOptions: tlsOptions,
		logOptions: logOptions,
	}
}

// LogLevel returns an int representing a named log level.
func LogLevel(name string) int {
	switch name {
	case "error":
		return 0
	case "warning":
		return 1
	case "info":
		return 2
	default: // everything else, e.g. debug
		return 3
	}
}

type deploymentEndpoint struct {
	URL string `json:"url"`
}

type deploymentResponse struct {
	Endpoints []deploymentEndpoint `json:"endpoints"`
}

type serviceConvergeResponse struct {
	Converged bool `json:"converged"`
}

type jobResponse struct {
	Active bool                    `json:"active"`
	Status string                  `json:"status"`
	Log    map[string][]logMessage `json:"log"`
	LastID int64                   `json:"lastId"`
}

type logMessage struct {
	At      int64  `json:"at"`
	Type    string `json:"type"`
	Message string `json:"message"`
}

type responseFunc func(status int, response []byte) (bool, error)

type requestFunc func() *http.Request

func wait(fn responseFunc, reqFn requestFunc, certificate *tls.Certificate, timeout time.Duration) (int, error) {
	if certificate != nil {
		util.ActiveHttpClient.UseCertificate(*certificate)
	}
	var (
		httpErr    error
		response   *http.Response
		statusCode int
	)
	deadline := time.Now().Add(timeout)
	loopOnce := timeout == 0
	for time.Now().Before(deadline) || loopOnce {
		response, httpErr = util.HttpDo(reqFn(), 10*time.Second, "")
		if httpErr == nil {
			statusCode = response.StatusCode
			body, err := ioutil.ReadAll(response.Body)
			if err != nil {
				return 0, err
			}
			response.Body.Close()
			ok, err := fn(statusCode, body)
			if err != nil {
				return statusCode, err
			}
			if ok {
				return statusCode, nil
			}
		}
		timeLeft := deadline.Sub(time.Now())
		if loopOnce || timeLeft < waitRetryInterval {
			break
		}
		time.Sleep(waitRetryInterval)
	}
	return statusCode, httpErr
}
<commit_msg>Do not wait on unauthorized request<commit_after>package vespa

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"sort"
	"strconv"
	"time"

	"github.com/vespa-engine/vespa/client/go/util"
)

const (
	localTargetType  = "local"
	customTargetType = "custom"
	cloudTargetType  = "cloud"

	deployService   = "deploy"
	queryService    = "query"
	documentService = "document"

	defaultCloudAPI = "https://api.vespa-external.aws.oath.cloud:4443"

	waitRetryInterval = 2 * time.Second
)

// Service represents a Vespa service.
type Service struct {
	BaseURL    string
	Name       string
	TLSOptions TLSOptions
}

// Target represents a Vespa platform, running named Vespa services.
type Target interface {
	// Type returns this target's type, e.g. local or cloud.
	Type() string

	// Service returns the service for given name. If timeout is non-zero, wait for the service to converge.
	Service(name string, timeout time.Duration, sessionOrRunID int64) (*Service, error)
}

// TLSOptions configures the certificate to use for service requests.
type TLSOptions struct {
	KeyPair         tls.Certificate
	CertificateFile string
	PrivateKeyFile  string
}

// LogOptions configures the log output to produce when waiting for services.
type LogOptions struct {
	Writer io.Writer
	Level  int
}

type customTarget struct {
	targetType string
	baseURL    string
}

// Do sends request to this service. Any required authentication happens automatically.
func (s *Service) Do(request *http.Request, timeout time.Duration) (*http.Response, error) {
	if s.TLSOptions.KeyPair.Certificate != nil {
		util.ActiveHttpClient.UseCertificate(s.TLSOptions.KeyPair)
	}
	return util.HttpDo(request, timeout, s.Description())
}

// Wait polls the health check of this service until it succeeds or timeout passes.
func (s *Service) Wait(timeout time.Duration) (int, error) {
	url := s.BaseURL
	switch s.Name {
	case deployService:
		url += "/status.html" // because /ApplicationStatus is not publicly reachable in Vespa Cloud
	case queryService, documentService:
		url += "/ApplicationStatus"
	default:
		return 0, fmt.Errorf("invalid service: %s", s.Name)
	}
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return 0, err
	}
	okFunc := func(status int, response []byte) (bool, error) { return status/100 == 2, nil }
	return wait(okFunc, func() *http.Request { return req }, &s.TLSOptions.KeyPair, timeout)
}

func (s *Service) Description() string {
	switch s.Name {
	case queryService:
		return "Container (query API)"
	case documentService:
		return "Container (document API)"
	case deployService:
		return "Deploy API"
	}
	return fmt.Sprintf("No description of service %s", s.Name)
}

func (t *customTarget) Type() string { return t.targetType }

func (t *customTarget) Service(name string, timeout time.Duration, sessionID int64) (*Service, error) {
	if timeout > 0 && name != deployService {
		if err := t.waitForConvergence(timeout); err != nil {
			return nil, err
		}
	}
	switch name {
	case deployService, queryService, documentService:
		url, err := t.urlWithPort(name)
		if err != nil {
			return nil, err
		}
		return &Service{BaseURL: url, Name: name}, nil
	}
	return nil, fmt.Errorf("unknown service: %s", name)
}

func (t *customTarget) urlWithPort(serviceName string) (string, error) {
	u, err := url.Parse(t.baseURL)
	if err != nil {
		return "", err
	}
	port := u.Port()
	if port == "" {
		switch serviceName {
		case deployService:
			port = "19071"
		case queryService, documentService:
			port = "8080"
		default:
			return "", fmt.Errorf("unknown service: %s", serviceName)
		}
		u.Host = u.Host + ":" + port
	}
	return u.String(), nil
}

func (t *customTarget) waitForConvergence(timeout time.Duration) error {
	deployer, err := t.Service(deployService, 0, 0)
	if err != nil {
		return err
	}
	url := fmt.Sprintf("%s/application/v2/tenant/default/application/default/environment/prod/region/default/instance/default/serviceconverge", deployer.BaseURL)
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return err
	}
	converged := false
	convergedFunc := func(status int, response []byte) (bool, error) {
		if status/100 != 2 {
			return false, nil
		}
		var resp serviceConvergeResponse
		if err := json.Unmarshal(response, &resp); err != nil {
			return false, nil
		}
		converged = resp.Converged
		return converged, nil
	}
	if _, err := wait(convergedFunc, func() *http.Request { return req }, nil, timeout); err != nil {
		return err
	}
	if !converged {
		return fmt.Errorf("services have not converged")
	}
	return nil
}

type cloudTarget struct {
	cloudAPI   string
	targetType string
	deployment Deployment
	apiKey     []byte
	tlsOptions TLSOptions
	logOptions LogOptions

	queryURL    string
	documentURL string
}

func (t *cloudTarget) Type() string { return t.targetType }

func (t *cloudTarget) Service(name string, timeout time.Duration, runID int64) (*Service, error) {
	if timeout > 0 && name != deployService {
		if err := t.waitForEndpoints(timeout, runID); err != nil {
			return nil, err
		}
	}
	switch name {
	case deployService:
		return &Service{Name: name, BaseURL: t.cloudAPI}, nil
	case queryService:
		if t.queryURL == "" {
			return nil, fmt.Errorf("service %s is not discovered", name)
		}
		return &Service{Name: name, BaseURL: t.queryURL, TLSOptions: t.tlsOptions}, nil
	case documentService:
		if t.documentURL == "" {
			return nil, fmt.Errorf("service %s is not discovered", name)
		}
		return &Service{Name: name, BaseURL: t.documentURL, TLSOptions: t.tlsOptions}, nil
	}
	return nil, fmt.Errorf("unknown service: %s", name)
}

func (t *cloudTarget) waitForEndpoints(timeout time.Duration, runID int64) error {
	signer := NewRequestSigner(t.deployment.Application.SerializedForm(), t.apiKey)
	if runID > 0 {
		if err := t.waitForRun(signer, runID, timeout); err != nil {
			return err
		}
	}
	return t.discoverEndpoints(signer, timeout)
}

func (t *cloudTarget) waitForRun(signer *RequestSigner, runID int64, timeout time.Duration) error {
	runURL := fmt.Sprintf("%s/application/v4/tenant/%s/application/%s/instance/%s/job/%s-%s/run/%d",
		t.cloudAPI,
		t.deployment.Application.Tenant, t.deployment.Application.Application, t.deployment.Application.Instance,
		t.deployment.Zone.Environment, t.deployment.Zone.Region, runID)
	req, err := http.NewRequest("GET", runURL, nil)
	if err != nil {
		return err
	}
	lastID := int64(-1)
	requestFunc := func() *http.Request {
		q := req.URL.Query()
		q.Set("after", strconv.FormatInt(lastID, 10))
		req.URL.RawQuery = q.Encode()
		if err := signer.SignRequest(req); err != nil {
			panic(err)
		}
		return req
	}
	jobSuccessFunc := func(status int, response []byte) (bool, error) {
		if ok, err := isOK(status); !ok {
			return ok, err
		}
		var resp jobResponse
		if err := json.Unmarshal(response, &resp); err != nil {
			return false, nil
		}
		if t.logOptions.Writer != nil {
			lastID = t.printLog(resp, lastID)
		}
		if resp.Active {
			return false, nil
		}
		if resp.Status != "success" {
			return false, fmt.Errorf("run %d ended with unsuccessful status: %s", runID, resp.Status)
		}
		return true, nil
	}
	_, err = wait(jobSuccessFunc, requestFunc, &t.tlsOptions.KeyPair, timeout)
	return err
}

func (t *cloudTarget) printLog(response jobResponse, last int64) int64 {
	if response.LastID == 0 {
		return last
	}
	var msgs []logMessage
	for step, stepMsgs := range response.Log {
		for _, msg := range stepMsgs {
			if step == "copyVespaLogs" && LogLevel(msg.Type) > t.logOptions.Level {
				continue
			}
			msgs = append(msgs, msg)
		}
	}
	sort.Slice(msgs, func(i, j int) bool { return msgs[i].At < msgs[j].At })
	for _, msg := range msgs {
		tm := time.Unix(msg.At/1000, (msg.At%1000)*1000)
		fmtTime := tm.Format("15:04:05")
		fmt.Fprintf(t.logOptions.Writer, "[%s] %-7s %s\n", fmtTime, msg.Type, msg.Message)
	}
	return response.LastID
}

func (t *cloudTarget) discoverEndpoints(signer *RequestSigner, timeout time.Duration) error {
	deploymentURL := fmt.Sprintf("%s/application/v4/tenant/%s/application/%s/instance/%s/environment/%s/region/%s",
		t.cloudAPI,
		t.deployment.Application.Tenant, t.deployment.Application.Application, t.deployment.Application.Instance,
		t.deployment.Zone.Environment, t.deployment.Zone.Region)
	req, err := http.NewRequest("GET", deploymentURL, nil)
	if err != nil {
		return err
	}
	if err := signer.SignRequest(req); err != nil {
		return err
	}
	var endpointURL string
	endpointFunc := func(status int, response []byte) (bool, error) {
		if ok, err := isOK(status); !ok {
			return ok, err
		}
		var resp deploymentResponse
		if err := json.Unmarshal(response, &resp); err != nil {
			return false, nil
		}
		if len(resp.Endpoints) == 0 {
			return false, nil
		}
		endpointURL = resp.Endpoints[0].URL
		return true, nil
	}
	if _, err = wait(endpointFunc, func() *http.Request { return req }, &t.tlsOptions.KeyPair, timeout); err != nil {
		return err
	}
	if endpointURL == "" {
		return fmt.Errorf("no endpoint discovered")
	}
	t.queryURL = endpointURL
	t.documentURL = endpointURL
	return nil
}

func isOK(status int) (bool, error) {
	if status == 401 {
		return false, fmt.Errorf("status %d: invalid api key", status)
	}
	return status/100 == 2, nil
}

// LocalTarget creates a target for a Vespa platform running locally.
func LocalTarget() Target {
	return &customTarget{targetType: localTargetType, baseURL: "http://127.0.0.1"}
}

// CustomTarget creates a Target for a Vespa platform running at baseURL.
func CustomTarget(baseURL string) Target {
	return &customTarget{targetType: customTargetType, baseURL: baseURL}
}

// CloudTarget creates a Target for the Vespa Cloud platform.
func CloudTarget(deployment Deployment, apiKey []byte, tlsOptions TLSOptions, logOptions LogOptions) Target {
	return &cloudTarget{
		cloudAPI:   defaultCloudAPI,
		targetType: cloudTargetType,
		deployment: deployment,
		apiKey:     apiKey,
		tlsOptions: tlsOptions,
		logOptions: logOptions,
	}
}

// LogLevel returns an int representing a named log level.
func LogLevel(name string) int {
	switch name {
	case "error":
		return 0
	case "warning":
		return 1
	case "info":
		return 2
	default: // everything else, e.g. debug
		return 3
	}
}

type deploymentEndpoint struct {
	URL string `json:"url"`
}

type deploymentResponse struct {
	Endpoints []deploymentEndpoint `json:"endpoints"`
}

type serviceConvergeResponse struct {
	Converged bool `json:"converged"`
}

type jobResponse struct {
	Active bool                    `json:"active"`
	Status string                  `json:"status"`
	Log    map[string][]logMessage `json:"log"`
	LastID int64                   `json:"lastId"`
}

type logMessage struct {
	At      int64  `json:"at"`
	Type    string `json:"type"`
	Message string `json:"message"`
}

type responseFunc func(status int, response []byte) (bool, error)

type requestFunc func() *http.Request

func wait(fn responseFunc, reqFn requestFunc, certificate *tls.Certificate, timeout time.Duration) (int, error) {
	if certificate != nil {
		util.ActiveHttpClient.UseCertificate(*certificate)
	}
	var (
		httpErr    error
		response   *http.Response
		statusCode int
	)
	deadline := time.Now().Add(timeout)
	loopOnce := timeout == 0
	for time.Now().Before(deadline) || loopOnce {
		response, httpErr = util.HttpDo(reqFn(), 10*time.Second, "")
		if httpErr == nil {
			statusCode = response.StatusCode
			body, err := ioutil.ReadAll(response.Body)
			if err != nil {
				return 0, err
			}
			response.Body.Close()
			ok, err := fn(statusCode, body)
			if err != nil {
				return statusCode, err
			}
			if ok {
				return statusCode, nil
			}
		}
		timeLeft := deadline.Sub(time.Now())
		if loopOnce || timeLeft < waitRetryInterval {
			break
		}
		time.Sleep(waitRetryInterval)
	}
	return statusCode, httpErr
}
<|endoftext|>


<commit_before>/*
 * Copyright 2016 Robin Engel
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package context

import (
	"time"
	"github.com/WE-Development/mosel/api"
	"sync"
	"errors"
)

type dataCache struct {
	points map[string]map[time.Time]DataPoint

	m      sync.Mutex
}

type DataPoint struct {
	Time time.Time
	Info api.NodeInfo
}

func NewDataCache() (*dataCache, error) {
	c := &dataCache{}
	c.points = make(map[string]map[time.Time]DataPoint)
	return c, nil
}

func (cache *dataCache) Add(node string, t time.Time, info api.NodeInfo) {
	var points map[time.Time]DataPoint

	t = t.Round(time.Second)
	cache.m.Lock()

	if _, ok := cache.points[node]; !ok {
		points = make(map[time.Time]DataPoint)
		cache.points[node] = points
	} else {
		points = cache.points[node]
	}

	if point, ok := points[t]; ok {
		for diag, graphs := range info {
			for graph, value := range graphs {
				if _, ok := point.Info[diag]; !ok {
					point.Info[diag] = make(map[string]string)
				}
				point.Info[diag][graph] = value
			}
		}
	} else {
		points[t] = DataPoint{
			Time: t,
			Info: info,
		}
	}

	cache.m.Unlock()
}

func (cache *dataCache) Get(node string, t time.Time) (api.NodeInfo, error) {
	points, err := cache.GetAll(node)
	t = t.Round(time.Second)

	if err != nil {
		return api.NodeInfo{}, err
	}

	for _, p := range points {
		if p.Time.Unix() == t.Unix() {
			return p.Info, nil
		}
	}

	return api.NodeInfo{}, errors.New("No datapoint found for time " + t.String())
}

func (cache *dataCache) GetSince(node string, t time.Time) ([]DataPoint, error) {

	points, err := cache.GetAll(node)

	if err != nil {
		return nil, err
	}

	result := make([]DataPoint, 0)

	for _, p := range points {
		if p.Time.Unix() > t.Unix() {
			result = append(result, p)
		}
	}

	return result, nil
}

func (cache *dataCache) GetAll(node string) ([]DataPoint, error) {
	cache.m.Lock()

	points, ok := cache.points[node]

	if !ok {
		cache.m.Unlock()
		return nil, errors.New("No node with name " + node)
	}

	cache.m.Unlock()

	res := make([]DataPoint, len(points))
	for _, point := range points {
		res = append(res, point)
	}
	return res, nil
}

func (cache *dataCache) GetNodes() []string {
	nodes := make([]string, len(cache.points))

	i := 0
	for k := range cache.points {
		nodes[i] = k
		i++
	}

	return nodes
}<commit_msg>optimize iterations in cache<commit_after>/*
 * Copyright 2016 Robin Engel
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package context

import (
	"time"
	"github.com/WE-Development/mosel/api"
	"sync"
	"errors"
)

type dataCache struct {
	points map[string]map[time.Time]DataPoint

	m      sync.Mutex
}

type DataPoint struct {
	Time time.Time
	Info api.NodeInfo
}

func NewDataCache() (*dataCache, error) {
	c := &dataCache{}
	c.points = make(map[string]map[time.Time]DataPoint)
	return c, nil
}

func (cache *dataCache) Add(node string, t time.Time, info api.NodeInfo) {
	var points map[time.Time]DataPoint

	t = t.Round(time.Second)
	cache.m.Lock()

	if _, ok := cache.points[node]; !ok {
		points = make(map[time.Time]DataPoint)
		cache.points[node] = points
	} else {
		points = cache.points[node]
	}

	if point, ok := points[t]; ok {
		for diag, graphs := range info {
			for graph, value := range graphs {
				if _, ok := point.Info[diag]; !ok {
					point.Info[diag] = make(map[string]string)
				}
				point.Info[diag][graph] = value
			}
		}
	} else {
		points[t] = DataPoint{
			Time: t,
			Info: info,
		}
	}

	cache.m.Unlock()
}

func (cache *dataCache) Get(node string, t time.Time) (api.NodeInfo, error) {
	points, err := cache.GetAll(node)
	t = t.Round(time.Second)

	if err != nil {
		return api.NodeInfo{}, err
	}

	for _, p := range points {
		if p.Time.Unix() == t.Unix() {
			return p.Info, nil
		}
	}

	return api.NodeInfo{}, errors.New("No datapoint found for time " + t.String())
}

func (cache *dataCache) GetSince(node string, t time.Time) ([]DataPoint, error) {

	points, err := cache.getAllByTime(node)

	if err != nil {
		return nil, err
	}

	result := make([]DataPoint, 0)

	for pt, p := range points {
		if pt.Unix() > t.Unix() {
			result = append(result, p)
		}
	}

	return result, nil
}

func (cache *dataCache) GetAll(node string) ([]DataPoint, error) {
	var points map[time.Time]DataPoint
	var err error

	cache.m.Lock()

	if points, err = cache.getAllByTime(node); err != nil {
		cache.m.Unlock()
		return nil, err
	}

	cache.m.Unlock()

	res := make([]DataPoint, len(points))
	for _, point := range points {
		res = append(res, point)
	}
	return res, nil
}

func (cache *dataCache) GetNodes() []string {
	nodes := make([]string, len(cache.points))

	i := 0
	for k := range cache.points {
		nodes[i] = k
		i++
	}

	return nodes
}

func (cache *dataCache) getAllByTime(node string) (map[time.Time]DataPoint, error) {
	points, ok := cache.points[node]
	if !ok {
		cache.m.Unlock()
		return nil, errors.New("No node with name " + node)
	}
	return points, nil
}<|endoftext|>


<commit_before><commit_msg>added error check for shell opening<commit_after><|endoftext|>