<commit_before>package experiment

import (
	"errors"
	log "github.com/Sirupsen/logrus"
	"os"
	"path"
	"strconv"
	"time"
)

// Experiment captures the internal data for the Experiment Driver.
type Experiment struct {
	session             session
	name                string
	workingDirectory    string
	phases              []Phase
	startingDirectory   string
	experimentDirectory string

	logFile *os.File
	logLvl  log.Level
	Log     *log.Logger
}

// NewExperiment creates a new Experiment instance,
// initialize experiment working directory and initialize logs.
// Caller have to provide slice of Phase interfaces which are going to be executed.
func NewExperiment(name string, phases []Phase,
	directory string, logLevel log.Level) (*Experiment, error) {
	if len(phases) == 0 {
		return nil, errors.New("invalid argument: no phases provided")
	}

	// TODO(mpatelcz): Check if phases names are unique!
	e := &Experiment{
		name:             name,
		session:          newSession(),
		workingDirectory: directory,
		phases:           phases,
		logLvl:           logLevel,
	}

	err := e.createExperimentDir()
	if err != nil {
		return nil, err
	}

	err = e.logInitialize()
	if err != nil {
		os.Chdir(e.startingDirectory)
		return nil, err
	}

	return e, nil
}

// Run runs experiment.
// It runs in a sequence defined phases with given repetition count.
func (e *Experiment) Run() error {
	experimentStartingTime := time.Now()

	e.Log.Info("Starting Experiment ", e.name, " with uuid ", e.session.Name)
	var err error
	for _, phase := range e.phases {
		for i := 0; i < phase.Repetitions(); i++ {
			phaseStartingTime := time.Now()

			e.Log.Info("Starting ", phase.Name(), " repetition ", i)
			err = e.createPhaseDir(phase, i)
			if err != nil {
				return err
			}

			err := phase.Run(e.Log)
			if err != nil {
				// When phase return error we stop the whole experiment.
				e.Log.Error(phase.Name(), " returned error ", err)
				return err
			}

			e.Log.Info("Ended ", phase.Name(), " repetition ", i,
				" in ", time.Since(phaseStartingTime))
		}

		// Give a chance for phase to finalize.
		// E.g to check statistical confidence of a result based on repetitions results.
		e.Log.Info("Finalizing ", phase.Name(),
			" after ", phase.Repetitions(), " repetitions")
		err := phase.Finalize()
		if err != nil {
			// When phase return error we stop the whole experiment.
			e.Log.Error(phase.Name(), " returned error ", err,
				" while finalizing.")
			return err
		}
	}
	e.Log.Info("Ended experiment ", e.name, " with uuid ", e.session.Name,
		" in ", time.Since(experimentStartingTime))
	return err
}

// createExperimentDir creates unique directory for experiment logs and results.
func (e *Experiment) createExperimentDir() error {
	e.startingDirectory, _ = os.Getwd()

	if len(e.workingDirectory) > 0 {
		e.experimentDirectory = path.Join(e.workingDirectory, e.name, e.session.Name)
	} else {
		e.experimentDirectory = path.Join(e.startingDirectory, e.name, e.session.Name)
	}

	err := os.MkdirAll(e.experimentDirectory, 0777)
	if err != nil {
		return err
	}

	err = os.Chdir(e.experimentDirectory)
	return err
}

// Finalize closes log file and returns to the previous working directory.
func (e *Experiment) Finalize() {
	e.logClose()

	// Exit experiment directory
	os.Chdir(e.startingDirectory)
}

func (e *Experiment) createPhaseDir(phase Phase, iteration int) error {
	phaseDir := path.Join(e.experimentDirectory,
		phase.Name(), strconv.FormatInt(int64(iteration), 10))

	err := os.MkdirAll(phaseDir, 0777)
	if err != nil {
		return err
	}

	err = os.Chdir(phaseDir)
	if err != nil {
		return err
	}

	return err
}

func (e *Experiment) logInitialize() error {
	var err error

	// Create master log file "master.log".
	e.logFile, err = os.OpenFile(e.experimentDirectory+"/master.log", os.O_WRONLY|os.O_CREATE, 0755)
	if err != nil {
		return err
	}

	// Create new logger.
	e.Log = &log.Logger{
		Out:       e.logFile,
		Formatter: new(log.TextFormatter),
		Hooks:     make(log.LevelHooks),
		Level:     e.logLvl,
	}

	return err
}

func (e *Experiment) logClose() {
	e.logFile.Close()
}
<commit_msg>Logger in Experiment now logs to both stdout & file. (#75)<commit_after>package experiment

import (
	"errors"
	log "github.com/Sirupsen/logrus"
	"io"
	"os"
	"path"
	"strconv"
	"time"
)

// Experiment captures the internal data for the Experiment Driver.
type Experiment struct {
	session             session
	name                string
	workingDirectory    string
	phases              []Phase
	startingDirectory   string
	experimentDirectory string

	logFile *os.File
	logLvl  log.Level
	Log     *log.Logger
}

// NewExperiment creates a new Experiment instance,
// initialize experiment working directory and initialize logs.
// Caller have to provide slice of Phase interfaces which are going to be executed.
func NewExperiment(name string, phases []Phase,
	directory string, logLevel log.Level) (*Experiment, error) {
	if len(phases) == 0 {
		return nil, errors.New("invalid argument: no phases provided")
	}

	// TODO(mpatelcz): Check if phases names are unique!
	e := &Experiment{
		name:             name,
		session:          newSession(),
		workingDirectory: directory,
		phases:           phases,
		logLvl:           logLevel,
	}

	err := e.createExperimentDir()
	if err != nil {
		return nil, err
	}

	err = e.logInitialize()
	if err != nil {
		os.Chdir(e.startingDirectory)
		return nil, err
	}

	return e, nil
}

// Run runs experiment.
// It runs in a sequence defined phases with given repetition count.
func (e *Experiment) Run() error {
	experimentStartingTime := time.Now()

	e.Log.Info("Starting Experiment ", e.name, " with uuid ", e.session.Name)
	var err error
	for _, phase := range e.phases {
		for i := 0; i < phase.Repetitions(); i++ {
			phaseStartingTime := time.Now()

			e.Log.Info("Starting ", phase.Name(), " repetition ", i)
			err = e.createPhaseDir(phase, i)
			if err != nil {
				return err
			}

			err := phase.Run(e.Log)
			if err != nil {
				// When phase return error we stop the whole experiment.
				e.Log.Error(phase.Name(), " returned error ", err)
				return err
			}

			e.Log.Info("Ended ", phase.Name(), " repetition ", i,
				" in ", time.Since(phaseStartingTime))
		}

		// Give a chance for phase to finalize.
		// E.g to check statistical confidence of a result based on repetitions results.
		e.Log.Info("Finalizing ", phase.Name(),
			" after ", phase.Repetitions(), " repetitions")
		err := phase.Finalize()
		if err != nil {
			// When phase return error we stop the whole experiment.
			e.Log.Error(phase.Name(), " returned error ", err,
				" while finalizing.")
			return err
		}
	}
	e.Log.Info("Ended experiment ", e.name, " with uuid ", e.session.Name,
		" in ", time.Since(experimentStartingTime))
	return err
}

// createExperimentDir creates unique directory for experiment logs and results.
func (e *Experiment) createExperimentDir() error {
	e.startingDirectory, _ = os.Getwd()

	if len(e.workingDirectory) > 0 {
		e.experimentDirectory = path.Join(e.workingDirectory, e.name, e.session.Name)
	} else {
		e.experimentDirectory = path.Join(e.startingDirectory, e.name, e.session.Name)
	}

	err := os.MkdirAll(e.experimentDirectory, 0777)
	if err != nil {
		return err
	}

	err = os.Chdir(e.experimentDirectory)
	return err
}

// Finalize closes log file and returns to the previous working directory.
func (e *Experiment) Finalize() {
	e.logClose()

	// Exit experiment directory
	os.Chdir(e.startingDirectory)
}

func (e *Experiment) createPhaseDir(phase Phase, iteration int) error {
	phaseDir := path.Join(e.experimentDirectory,
		phase.Name(), strconv.FormatInt(int64(iteration), 10))

	err := os.MkdirAll(phaseDir, 0777)
	if err != nil {
		return err
	}

	err = os.Chdir(phaseDir)
	if err != nil {
		return err
	}

	return err
}

func (e *Experiment) logInitialize() error {
	var err error

	// Create master log file "master.log".
	e.logFile, err = os.OpenFile(e.experimentDirectory+"/master.log", os.O_WRONLY|os.O_CREATE, 0755)
	if err != nil {
		return err
	}

	// Create new logger with logging set to both output and logFile.
	e.Log = &log.Logger{
		Out:       io.MultiWriter(e.logFile, os.Stdout),
		Formatter: new(log.TextFormatter),
		Hooks:     make(log.LevelHooks),
		Level:     e.logLvl,
	}

	return err
}

func (e *Experiment) logClose() {
	e.logFile.Close()
}
<|endoftext|>


<commit_before>package provider

import (
	"fmt"
	"net/http"
	"os"
	"strings"

	log "github.com/sirupsen/logrus"

	api "gopkg.in/ns1/ns1-go.v2/rest"
	"gopkg.in/ns1/ns1-go.v2/rest/model/dns"

	"github.com/kubernetes-incubator/external-dns/endpoint"
	"github.com/kubernetes-incubator/external-dns/plan"
)

const (
	// ns1Create is a ChangeAction enum value
	ns1Create = "CREATE"
	// ns1Delete is a ChangeAction enum value
	ns1Delete = "DELETE"
	// ns1Update is a ChangeAction enum value
	ns1Update = "UPDATE"
	// ns1DefaultTTL is the default ttl for ttls that are not set
	ns1DefaultTTL = 10
)

// NS1Config passes cli args to the NS1Provider
type NS1Config struct {
	DomainFilter DomainFilter
	ZoneIDFilter ZoneIDFilter
	DryRun       bool
}

// NS1Provider is the NS1 provider
type NS1Provider struct {
	client       *api.Client
	domainFilter DomainFilter
	zoneIDFilter ZoneIDFilter
	dryRun       bool
}

// NewNS1Provider creates a new NS1 Provider
func NewNS1Provider(config NS1Config) (*NS1Provider, error) {
	return newNS1ProviderWithHTTPClient(config, http.DefaultClient)
}

func newNS1ProviderWithHTTPClient(config NS1Config, client *http.Client) (*NS1Provider, error) {
	token, ok := os.LookupEnv("NS1_APIKEY")
	if !ok {
		return nil, fmt.Errorf("NS1_APIKEY environment variable is not set")
	}

	apiClient := api.NewClient(client, api.SetAPIKey(token))

	provider := &NS1Provider{
		client:       apiClient,
		domainFilter: config.DomainFilter,
		zoneIDFilter: config.ZoneIDFilter,
	}
	return provider, nil
}

// Records returns the endpoints this provider knows about
func (p *NS1Provider) Records() ([]*endpoint.Endpoint, error) {
	zones, err := p.zonesFiltered()
	if err != nil {
		return nil, err
	}

	var endpoints []*endpoint.Endpoint

	for _, zone := range zones {

		// TODO handle Header Codes
		zoneData, _, err := p.client.Zones.Get(zone.String())
		if err != nil {
			return nil, err
		}

		for _, record := range zoneData.Records {
			if supportedRecordType(record.Type) {
				endpoints = append(endpoints, endpoint.NewEndpointWithTTL(
					record.Domain,
					record.Type,
					endpoint.TTL(record.TTL),
					record.ShortAns...,
				),
				)
			}
		}
	}

	return endpoints, nil
}

// ns1BuildRecord returns a dns.Record for a change set
func ns1BuildRecord(zoneName string, change *ns1Change) *dns.Record {
	record := dns.NewRecord(zoneName, change.Endpoint.DNSName, change.Endpoint.RecordType)
	for _, v := range change.Endpoint.Targets {
		record.AddAnswer(dns.NewAnswer(strings.Split(v, " ")))
	}
	// set detault ttl
	var ttl = ns1DefaultTTL
	if change.Endpoint.RecordTTL.IsConfigured() {
		ttl = int(change.Endpoint.RecordTTL)
	}
	record.TTL = ttl

	return record
}

// ns1SubmitChanges takes an array of changes and sends them to NS1
func (p *NS1Provider) ns1SubmitChanges(changes []*ns1Change) error {
	// return early if there is nothing to change
	if len(changes) == 0 {
		return nil
	}

	zones, err := p.zonesFiltered()
	if err != nil {
		return err
	}

	// separate into per-zone change sets to be passed to the API.
	changesByZone := ns1ChangesByZone(zones, changes)
	for zoneName, changes := range changesByZone {
		for _, change := range changes {
			record := ns1BuildRecord(zoneName, change)
			logFields := log.Fields{
				"record": record.Domain,
				"type":   record.Type,
				"ttl":    record.TTL,
				"action": change.Action,
				"zone":   zoneName,
			}

			log.WithFields(logFields).Info("Changing record.")

			if p.dryRun {
				continue
			}

			switch change.Action {
			case ns1Create:
				_, err := p.client.Records.Create(record)
				if err != nil {
					return err
				}
			case ns1Delete:
				_, err := p.client.Records.Delete(zoneName, record.Domain, record.Type)
				if err != nil {
					return err
				}
			case ns1Update:
				_, err := p.client.Records.Update(record)
				if err != nil {
					return err
				}
			}
		}
	}
	return nil
}

// Zones returns the list of hosted zones.
func (p *NS1Provider) zonesFiltered() ([]*dns.Zone, error) {
	// TODO handle Header Codes
	zones, _, err := p.client.Zones.List()
	if err != nil {
		return nil, err
	}

	toReturn := []*dns.Zone{}

	for _, z := range zones {
		if p.domainFilter.Match(z.Zone) && p.zoneIDFilter.Match(z.ID) {
			toReturn = append(toReturn, z)
			log.Debugf("Matched %s", z.Zone)
		} else {
			log.Debugf("Filtered %s", z.Zone)
		}
	}

	return toReturn, nil
}

// ns1Change differentiates between ChangeActions
type ns1Change struct {
	Action   string
	Endpoint *endpoint.Endpoint
}

// ApplyChanges applies a given set of changes in a given zone.
func (p *NS1Provider) ApplyChanges(changes *plan.Changes) error {
	combinedChanges := make([]*ns1Change, 0, len(changes.Create)+len(changes.UpdateNew)+len(changes.Delete))

	combinedChanges = append(combinedChanges, newNS1Changes(ns1Create, changes.Create)...)
	combinedChanges = append(combinedChanges, newNS1Changes(ns1Update, changes.UpdateNew)...)
	combinedChanges = append(combinedChanges, newNS1Changes(ns1Delete, changes.Delete)...)

	return p.ns1SubmitChanges(combinedChanges)
}

// newNS1Changes returns a collection of Changes based on the given records and action.
func newNS1Changes(action string, endpoints []*endpoint.Endpoint) []*ns1Change {
	changes := make([]*ns1Change, 0, len(endpoints))

	for _, endpoint := range endpoints {
		changes = append(changes, &ns1Change{
			Action:   action,
			Endpoint: endpoint,
		},
		)
	}

	return changes
}

// ns1ChangesByZone separates a multi-zone change into a single change per zone.
func ns1ChangesByZone(zones []*dns.Zone, changeSets []*ns1Change) map[string][]*ns1Change {
	changes := make(map[string][]*ns1Change)
	zoneNameIDMapper := zoneIDName{}
	for _, z := range zones {
		zoneNameIDMapper.Add(z.Zone, z.Zone)
		changes[z.Zone] = []*ns1Change{}
	}

	for _, c := range changeSets {
		zone, _ := zoneNameIDMapper.FindZone(c.Endpoint.DNSName)
		if zone == "" {
			log.Debugf("Skipping record %s because no hosted zone matching record DNS Name was detected ", c.Endpoint.DNSName)
			continue
		}
		changes[zone] = append(changes[zone], c)
	}

	return changes
}
<commit_msg>Wrap ns1 client with interface for easier testing<commit_after>package provider

import (
	"fmt"
	"net/http"
	"os"
	"strings"

	log "github.com/sirupsen/logrus"

	api "gopkg.in/ns1/ns1-go.v2/rest"
	"gopkg.in/ns1/ns1-go.v2/rest/model/dns"

	"github.com/kubernetes-incubator/external-dns/endpoint"
	"github.com/kubernetes-incubator/external-dns/plan"
)

const (
	// ns1Create is a ChangeAction enum value
	ns1Create = "CREATE"
	// ns1Delete is a ChangeAction enum value
	ns1Delete = "DELETE"
	// ns1Update is a ChangeAction enum value
	ns1Update = "UPDATE"
	// ns1DefaultTTL is the default ttl for ttls that are not set
	ns1DefaultTTL = 10
)

// NS1DomainClient interface to ease testing
type NS1DomainClient interface {
	CreateRecord(r *dns.Record) (*http.Response, error)
	DeleteRecord(zone string, domain string, t string) (*http.Response, error)
	UpdateRecord(r *dns.Record) (*http.Response, error)
	GetZone(zone string) (*dns.Zone, *http.Response, error)
	ListZones() ([]*dns.Zone, *http.Response, error)
}

type NS1DomainService struct {
	service *api.Client
}

func (n NS1DomainService) CreateRecord(r *dns.Record) (*http.Response, error) {
	return n.service.Records.Create(r)
}

func (n NS1DomainService) DeleteRecord(zone string, domain string, t string) (*http.Response, error) {
	return n.service.Records.Delete(zone, domain, t)
}

func (n NS1DomainService) UpdateRecord(r *dns.Record) (*http.Response, error) {
	return n.service.Records.Update(r)
}

func (n NS1DomainService) GetZone(zone string) (*dns.Zone, *http.Response, error) {
	return n.service.Zones.Get(zone)
}

func (n NS1DomainService) ListZones() ([]*dns.Zone, *http.Response, error) {
	return n.service.Zones.List()
}

// NS1Config passes cli args to the NS1Provider
type NS1Config struct {
	DomainFilter DomainFilter
	ZoneIDFilter ZoneIDFilter
	DryRun       bool
}

// NS1Provider is the NS1 provider
type NS1Provider struct {
	client       NS1DomainClient
	domainFilter DomainFilter
	zoneIDFilter ZoneIDFilter
	dryRun       bool
}

// NewNS1Provider creates a new NS1 Provider
func NewNS1Provider(config NS1Config) (*NS1Provider, error) {
	return newNS1ProviderWithHTTPClient(config, http.DefaultClient)
}

func newNS1ProviderWithHTTPClient(config NS1Config, client *http.Client) (*NS1Provider, error) {
	token, ok := os.LookupEnv("NS1_APIKEY")
	if !ok {
		return nil, fmt.Errorf("NS1_APIKEY environment variable is not set")
	}

	apiClient := api.NewClient(client, api.SetAPIKey(token))

	provider := &NS1Provider{
		//client:       apiClient,
		client:       NS1DomainService{apiClient},
		domainFilter: config.DomainFilter,
		zoneIDFilter: config.ZoneIDFilter,
	}
	return provider, nil
}

// Records returns the endpoints this provider knows about
func (p *NS1Provider) Records() ([]*endpoint.Endpoint, error) {
	zones, err := p.zonesFiltered()
	if err != nil {
		return nil, err
	}

	var endpoints []*endpoint.Endpoint

	for _, zone := range zones {

		// TODO handle Header Codes
		zoneData, _, err := p.client.GetZone(zone.String())
		if err != nil {
			return nil, err
		}

		for _, record := range zoneData.Records {
			if supportedRecordType(record.Type) {
				endpoints = append(endpoints, endpoint.NewEndpointWithTTL(
					record.Domain,
					record.Type,
					endpoint.TTL(record.TTL),
					record.ShortAns...,
				),
				)
			}
		}
	}

	return endpoints, nil
}

// ns1BuildRecord returns a dns.Record for a change set
func ns1BuildRecord(zoneName string, change *ns1Change) *dns.Record {
	record := dns.NewRecord(zoneName, change.Endpoint.DNSName, change.Endpoint.RecordType)
	for _, v := range change.Endpoint.Targets {
		record.AddAnswer(dns.NewAnswer(strings.Split(v, " ")))
	}
	// set detault ttl
	var ttl = ns1DefaultTTL
	if change.Endpoint.RecordTTL.IsConfigured() {
		ttl = int(change.Endpoint.RecordTTL)
	}
	record.TTL = ttl

	return record
}

// ns1SubmitChanges takes an array of changes and sends them to NS1
func (p *NS1Provider) ns1SubmitChanges(changes []*ns1Change) error {
	// return early if there is nothing to change
	if len(changes) == 0 {
		return nil
	}

	zones, err := p.zonesFiltered()
	if err != nil {
		return err
	}

	// separate into per-zone change sets to be passed to the API.
	changesByZone := ns1ChangesByZone(zones, changes)
	for zoneName, changes := range changesByZone {
		for _, change := range changes {
			record := ns1BuildRecord(zoneName, change)
			logFields := log.Fields{
				"record": record.Domain,
				"type":   record.Type,
				"ttl":    record.TTL,
				"action": change.Action,
				"zone":   zoneName,
			}

			log.WithFields(logFields).Info("Changing record.")

			if p.dryRun {
				continue
			}

			switch change.Action {
			case ns1Create:
				_, err := p.client.CreateRecord(record)
				if err != nil {
					return err
				}
			case ns1Delete:
				_, err := p.client.DeleteRecord(zoneName, record.Domain, record.Type)
				if err != nil {
					return err
				}
			case ns1Update:
				_, err := p.client.UpdateRecord(record)
				if err != nil {
					return err
				}
			}
		}
	}
	return nil
}

// Zones returns the list of hosted zones.
func (p *NS1Provider) zonesFiltered() ([]*dns.Zone, error) {
	// TODO handle Header Codes
	zones, _, err := p.client.ListZones()
	if err != nil {
		return nil, err
	}

	toReturn := []*dns.Zone{}

	for _, z := range zones {
		if p.domainFilter.Match(z.Zone) && p.zoneIDFilter.Match(z.ID) {
			toReturn = append(toReturn, z)
			log.Debugf("Matched %s", z.Zone)
		} else {
			log.Debugf("Filtered %s", z.Zone)
		}
	}

	return toReturn, nil
}

// ns1Change differentiates between ChangeActions
type ns1Change struct {
	Action   string
	Endpoint *endpoint.Endpoint
}

// ApplyChanges applies a given set of changes in a given zone.
func (p *NS1Provider) ApplyChanges(changes *plan.Changes) error {
	combinedChanges := make([]*ns1Change, 0, len(changes.Create)+len(changes.UpdateNew)+len(changes.Delete))

	combinedChanges = append(combinedChanges, newNS1Changes(ns1Create, changes.Create)...)
	combinedChanges = append(combinedChanges, newNS1Changes(ns1Update, changes.UpdateNew)...)
	combinedChanges = append(combinedChanges, newNS1Changes(ns1Delete, changes.Delete)...)

	return p.ns1SubmitChanges(combinedChanges)
}

// newNS1Changes returns a collection of Changes based on the given records and action.
func newNS1Changes(action string, endpoints []*endpoint.Endpoint) []*ns1Change {
	changes := make([]*ns1Change, 0, len(endpoints))

	for _, endpoint := range endpoints {
		changes = append(changes, &ns1Change{
			Action:   action,
			Endpoint: endpoint,
		},
		)
	}

	return changes
}

// ns1ChangesByZone separates a multi-zone change into a single change per zone.
func ns1ChangesByZone(zones []*dns.Zone, changeSets []*ns1Change) map[string][]*ns1Change {
	changes := make(map[string][]*ns1Change)
	zoneNameIDMapper := zoneIDName{}
	for _, z := range zones {
		zoneNameIDMapper.Add(z.Zone, z.Zone)
		changes[z.Zone] = []*ns1Change{}
	}

	for _, c := range changeSets {
		zone, _ := zoneNameIDMapper.FindZone(c.Endpoint.DNSName)
		if zone == "" {
			log.Debugf("Skipping record %s because no hosted zone matching record DNS Name was detected ", c.Endpoint.DNSName)
			continue
		}
		changes[zone] = append(changes[zone], c)
	}

	return changes
}
<|endoftext|>


<commit_before>package playedplugin

import (
	"encoding/json"
	"fmt"
	"log"
	"regexp"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/iopred/bruxism"
	"github.com/iopred/discordgo"
)

type playedEntry struct {
	Name     string
	Duration time.Duration
}

type byDuration []*playedEntry

func (a byDuration) Len() int           { return len(a) }
func (a byDuration) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byDuration) Less(i, j int) bool { return a[i].Duration >= a[j].Duration }

type playedUser struct {
	Entries     map[string]*playedEntry
	Current     string
	LastChanged time.Time
	FirstSeen   time.Time
}

func (p *playedUser) Update(name string, now time.Time) {
	if p.Current != "" {
		pe := p.Entries[p.Current]
		if pe == nil {
			pe = &playedEntry{
				Name: p.Current,
			}
			p.Entries[p.Current] = pe
		}
		pe.Duration += now.Sub(p.LastChanged)
	}

	p.Current = name
	p.LastChanged = now
}

type playedPlugin struct {
	sync.RWMutex
	Users map[string]*playedUser
}

// Load will load plugin state from a byte array.
func (p *playedPlugin) Load(bot *bruxism.Bot, service bruxism.Service, data []byte) error {
	if service.Name() != bruxism.DiscordServiceName {
		panic("Carbonitex Plugin only supports Discord.")
	}

	if data != nil {
		if err := json.Unmarshal(data, p); err != nil {
			log.Println("Error loading data", err)
		}
	}

	go p.Run(bot, service)
	return nil
}

// Save will save plugin state to a byte array.
func (p *playedPlugin) Save() ([]byte, error) {
	return json.Marshal(p)
}

func (p *playedPlugin) Update(user string, entry string) {
	p.Lock()
	defer p.Unlock()

	t := time.Now()

	u := p.Users[user]
	if u == nil {
		u = &playedUser{
			Entries:     map[string]*playedEntry{},
			Current:     entry,
			LastChanged: t,
			FirstSeen:   t,
		}
		p.Users[user] = u
	}
	u.Update(entry, t)
}

// Run is the background go routine that executes for the life of the plugin.
func (p *playedPlugin) Run(bot *bruxism.Bot, service bruxism.Service) {
	discord := service.(*bruxism.Discord)

	discord.Session.AddHandler(func(s *discordgo.Session, r *discordgo.Ready) {
		for _, g := range r.Guilds {
			for _, pu := range g.Presences {
				e := ""
				if pu.Game != nil {
					e = pu.Game.Name
				}
				p.Update(pu.User.ID, e)
			}
		}
	})

	discord.Session.AddHandler(func(s *discordgo.Session, g *discordgo.GuildCreate) {
		if g.Unavailable == nil || *g.Unavailable {
			return
		}

		for _, pu := range g.Presences {
			e := ""
			if pu.Game != nil {
				e = pu.Game.Name
			}
			p.Update(pu.User.ID, e)
		}
	})

	discord.Session.AddHandler(func(s *discordgo.Session, pr *discordgo.PresencesReplace) {
		for _, pu := range *pr {
			e := ""
			if pu.Game != nil {
				e = pu.Game.Name
			}
			p.Update(pu.User.ID, e)
		}
	})

	discord.Session.AddHandler(func(s *discordgo.Session, pu *discordgo.PresenceUpdate) {
		e := ""
		if pu.Game != nil {
			e = pu.Game.Name
		}
		p.Update(pu.User.ID, e)
	})
}

// Help returns a list of help strings that are printed when the user requests them.
func (p *playedPlugin) Help(bot *bruxism.Bot, service bruxism.Service, message bruxism.Message, detailed bool) []string {
	if detailed {
		return nil
	}

	return bruxism.CommandHelp(service, "played", "[@username]", "Returns your most played games, or a users most played games if provided.")
}

var userIDRegex = regexp.MustCompile("<@!?([0-9]*)>")

func (p *playedPlugin) Message(bot *bruxism.Bot, service bruxism.Service, message bruxism.Message) {
	defer bruxism.MessageRecover()
	if service.Name() == bruxism.DiscordServiceName && !service.IsMe(message) {
		if bruxism.MatchesCommand(service, "played", message) {
			query := strings.Join(strings.Split(message.RawMessage(), " ")[1:], " ")

			id := message.UserID()
			match := userIDRegex.FindStringSubmatch(query)
			if match != nil {
				id = match[1]
			}

			p.Lock()
			defer p.Unlock()

			u := p.Users[id]
			if u == nil {
				service.SendMessage(message.Channel(), "I haven't seen that user.")
				return
			}

			if len(u.Entries) == 0 {
				service.SendMessage(message.Channel(), "I haven't seen anything played by that user.")
				return
			}

			u.Update(u.Current, time.Now())

			pes := make(byDuration, len(u.Entries))
			i := 0
			for _, pe := range u.Entries {
				pes[i] = pe
				i++
			}

			sort.Sort(pes)

			messageText := fmt.Sprintf("First seen %s.\n", humanize.Time(u.FirstSeen))
			for i = 0; i < len(pes) && i < 5; i++ {
				pe := pes[i]
				messageText += fmt.Sprintf("%s: %s\n", pe.Name, pe.Duration.String())
			}
			service.SendMessage(message.Channel(), messageText)
		}
	}
}

// Name returns the name of the plugin.
func (p *playedPlugin) Name() string {
	return "Played"
}

// New will create a played plugin.
func New() bruxism.Plugin {
	return &playedPlugin{
		Users: map[string]*playedUser{},
	}
}
<commit_msg>Improve styling.<commit_after>package playedplugin

import (
	"encoding/json"
	"fmt"
	"log"
	"regexp"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/iopred/bruxism"
	"github.com/iopred/discordgo"
)

type playedEntry struct {
	Name     string
	Duration time.Duration
}

type byDuration []*playedEntry

func (a byDuration) Len() int           { return len(a) }
func (a byDuration) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byDuration) Less(i, j int) bool { return a[i].Duration >= a[j].Duration }

type playedUser struct {
	Entries     map[string]*playedEntry
	Current     string
	LastChanged time.Time
	FirstSeen   time.Time
}

func (p *playedUser) Update(name string, now time.Time) {
	if p.Current != "" {
		pe := p.Entries[p.Current]
		if pe == nil {
			pe = &playedEntry{
				Name: p.Current,
			}
			p.Entries[p.Current] = pe
		}
		pe.Duration += now.Sub(p.LastChanged)
	}

	p.Current = name
	p.LastChanged = now
}

type playedPlugin struct {
	sync.RWMutex
	Users map[string]*playedUser
}

// Load will load plugin state from a byte array.
func (p *playedPlugin) Load(bot *bruxism.Bot, service bruxism.Service, data []byte) error {
	if service.Name() != bruxism.DiscordServiceName {
		panic("Carbonitex Plugin only supports Discord.")
	}

	if data != nil {
		if err := json.Unmarshal(data, p); err != nil {
			log.Println("Error loading data", err)
		}
	}

	go p.Run(bot, service)
	return nil
}

// Save will save plugin state to a byte array.
func (p *playedPlugin) Save() ([]byte, error) {
	return json.Marshal(p)
}

func (p *playedPlugin) Update(user string, entry string) {
	p.Lock()
	defer p.Unlock()

	t := time.Now()

	u := p.Users[user]
	if u == nil {
		u = &playedUser{
			Entries:     map[string]*playedEntry{},
			Current:     entry,
			LastChanged: t,
			FirstSeen:   t,
		}
		p.Users[user] = u
	}
	u.Update(entry, t)
}

// Run is the background go routine that executes for the life of the plugin.
func (p *playedPlugin) Run(bot *bruxism.Bot, service bruxism.Service) {
	discord := service.(*bruxism.Discord)

	discord.Session.AddHandler(func(s *discordgo.Session, r *discordgo.Ready) {
		for _, g := range r.Guilds {
			for _, pu := range g.Presences {
				e := ""
				if pu.Game != nil {
					e = pu.Game.Name
				}
				p.Update(pu.User.ID, e)
			}
		}
	})

	discord.Session.AddHandler(func(s *discordgo.Session, g *discordgo.GuildCreate) {
		if g.Unavailable == nil || *g.Unavailable {
			return
		}

		for _, pu := range g.Presences {
			e := ""
			if pu.Game != nil {
				e = pu.Game.Name
			}
			p.Update(pu.User.ID, e)
		}
	})

	discord.Session.AddHandler(func(s *discordgo.Session, pr *discordgo.PresencesReplace) {
		for _, pu := range *pr {
			e := ""
			if pu.Game != nil {
				e = pu.Game.Name
			}
			p.Update(pu.User.ID, e)
		}
	})

	discord.Session.AddHandler(func(s *discordgo.Session, pu *discordgo.PresenceUpdate) {
		e := ""
		if pu.Game != nil {
			e = pu.Game.Name
		}
		p.Update(pu.User.ID, e)
	})
}

// Help returns a list of help strings that are printed when the user requests them.
func (p *playedPlugin) Help(bot *bruxism.Bot, service bruxism.Service, message bruxism.Message, detailed bool) []string {
	if detailed {
		return nil
	}

	return bruxism.CommandHelp(service, "played", "[@username]", "Returns your most played games, or a users most played games if provided.")
}

var userIDRegex = regexp.MustCompile("<@!?([0-9]*)>")

func (p *playedPlugin) Message(bot *bruxism.Bot, service bruxism.Service, message bruxism.Message) {
	defer bruxism.MessageRecover()
	if service.Name() == bruxism.DiscordServiceName && !service.IsMe(message) {
		if bruxism.MatchesCommand(service, "played", message) {
			query := strings.Join(strings.Split(message.RawMessage(), " ")[1:], " ")

			id := message.UserID()
			match := userIDRegex.FindStringSubmatch(query)
			if match != nil {
				id = match[1]
			}

			p.Lock()
			defer p.Unlock()

			u := p.Users[id]
			if u == nil {
				service.SendMessage(message.Channel(), "I haven't seen that user.")
				return
			}

			if len(u.Entries) == 0 {
				service.SendMessage(message.Channel(), "I haven't seen anything played by that user.")
				return
			}

			lc := humanize.Time(u.LastChanged)
			u.Update(u.Current, time.Now())

			pes := make(byDuration, len(u.Entries))
			i := 0
			for _, pe := range u.Entries {
				pes[i] = pe
				i++
			}

			sort.Sort(pes)

			messageText := fmt.Sprintf("*First seen %s, last update %s*\n", humanize.Time(u.FirstSeen), lc)
			for i = 0; i < len(pes) && i < 5; i++ {
				pe := pes[i]
				messageText += fmt.Sprintf("**%s**: %s\n", pe.Name, pe.Duration.String())
			}
			service.SendMessage(message.Channel(), messageText)
		}
	}
}

// Name returns the name of the plugin.
func (p *playedPlugin) Name() string {
	return "Played"
}

// New will create a played plugin.
func New() bruxism.Plugin {
	return &playedPlugin{
		Users: map[string]*playedUser{},
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package components

import (
	"fmt"
	"strings"

	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/upup/pkg/fi/loader"
)

const DefaultBackupImage = "kopeio/etcd-backup:3.0.20190516"

// EtcdOptionsBuilder adds options for etcd to the model
type EtcdOptionsBuilder struct {
	*OptionsContext
}

var _ loader.OptionsBuilder = &EtcdOptionsBuilder{}

const (
	DefaultEtcd2Version = "2.2.1"

	//  1.11 originally recommended 3.2.18, but there was an advisory to update to 3.2.24
	DefaultEtcd3Version_1_11 = "3.2.24"

	DefaultEtcd3Version_1_13 = "3.2.24"
)

// BuildOptions is responsible for filling in the defaults for the etcd cluster model
func (b *EtcdOptionsBuilder) BuildOptions(o interface{}) error {
	spec := o.(*kops.ClusterSpec)

	for _, c := range spec.EtcdClusters {
		if c.Provider == "" {
			if b.IsKubernetesGTE("1.12") {
				c.Provider = kops.EtcdProviderTypeManager
			} else if c.Manager != nil {
				c.Provider = kops.EtcdProviderTypeManager
			} else {
				c.Provider = kops.EtcdProviderTypeLegacy
			}
		}

		// Ensure the version is set
		if c.Version == "" && c.Provider == kops.EtcdProviderTypeLegacy {
			// Even if in legacy mode, etcd version 2 is unsupported as of k8s 1.13
			if b.IsKubernetesGTE("1.13") {
				c.Version = DefaultEtcd3Version_1_13
			} else {
				c.Version = DefaultEtcd2Version
			}
		}

		if c.Version == "" && c.Provider == kops.EtcdProviderTypeManager {
			// From 1.11, we run the k8s-recommended versions of etcd when using the manager
			if b.IsKubernetesGTE("1.11") {
				c.Version = DefaultEtcd3Version_1_11
			} else {
				c.Version = DefaultEtcd2Version
			}
		}

		// From 1.12, we enable TLS if we're running EtcdManager & etcd3
		//
		// (Moving to etcd3 is a disruptive upgrade, so we
		// force TLS at the same time as we enable
		// etcd-manager by default).
		if c.Provider == kops.EtcdProviderTypeManager {
			etcdV3 := true
			version := c.Version
			version = strings.TrimPrefix(version, "v")
			if strings.HasPrefix(version, "2.") {
				etcdV3 = false
			} else if strings.HasPrefix(version, "3.") {
				etcdV3 = true
			} else {
				return fmt.Errorf("unexpected etcd version %q", c.Version)
			}

			if b.IsKubernetesGTE("1.12.0") && etcdV3 {
				c.EnableEtcdTLS = true
				c.EnableTLSAuth = true
			}
		}
	}

	// Remap the well known images
	for _, c := range spec.EtcdClusters {

		// We remap the etcd manager image when we build the manifest,
		// but we need to map the standalone images here because protokube launches them

		if c.Provider == kops.EtcdProviderTypeLegacy {

			// remap etcd image
			{
				image := c.Image
				if image == "" {
					image = fmt.Sprintf("k8s.gcr.io/etcd:%s", c.Version)
				}

				if image != "" {
					image, err := b.AssetBuilder.RemapImage(image)
					if err != nil {
						return fmt.Errorf("unable to remap container %q: %v", image, err)
					}
					c.Image = image
				}
			}

			// remap backup manager image
			if c.Backups != nil {
				image := c.Backups.Image
				if image == "" {
					image = fmt.Sprintf(DefaultBackupImage)
				}

				if image != "" {
					image, err := b.AssetBuilder.RemapImage(image)
					if err != nil {
						return fmt.Errorf("unable to remap container %q: %v", image, err)
					}
					c.Backups.Image = image
				}
			}
		}
	}

	return nil
}
<commit_msg>Default etcd-version to 3.3.10 for >= 1.14<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package components

import (
	"fmt"
	"strings"

	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/upup/pkg/fi/loader"
)

const DefaultBackupImage = "kopeio/etcd-backup:3.0.20190516"

// EtcdOptionsBuilder adds options for etcd to the model
type EtcdOptionsBuilder struct {
	*OptionsContext
}

var _ loader.OptionsBuilder = &EtcdOptionsBuilder{}

const (
	DefaultEtcd2Version = "2.2.1"

	//  1.11 originally recommended 3.2.18, but there was an advisory to update to 3.2.24
	DefaultEtcd3Version_1_11 = "3.2.24"

	DefaultEtcd3Version_1_13 = "3.2.24"

	DefaultEtcd3Version_1_14 = "3.3.10"
)

// BuildOptions is responsible for filling in the defaults for the etcd cluster model
func (b *EtcdOptionsBuilder) BuildOptions(o interface{}) error {
	spec := o.(*kops.ClusterSpec)

	for _, c := range spec.EtcdClusters {
		if c.Provider == "" {
			if b.IsKubernetesGTE("1.12") {
				c.Provider = kops.EtcdProviderTypeManager
			} else if c.Manager != nil {
				c.Provider = kops.EtcdProviderTypeManager
			} else {
				c.Provider = kops.EtcdProviderTypeLegacy
			}
		}

		// Ensure the version is set
		if c.Version == "" && c.Provider == kops.EtcdProviderTypeLegacy {
			// Even if in legacy mode, etcd version 2 is unsupported as of k8s 1.13
			if b.IsKubernetesGTE("1.14") {
				c.Version = DefaultEtcd3Version_1_14
			} else if b.IsKubernetesGTE("1.13") {
				c.Version = DefaultEtcd3Version_1_13
			} else {
				c.Version = DefaultEtcd2Version
			}
		}

		if c.Version == "" && c.Provider == kops.EtcdProviderTypeManager {
			// From 1.11, we run the k8s-recommended versions of etcd when using the manager
			if b.IsKubernetesGTE("1.14") {
				c.Version = DefaultEtcd3Version_1_14
			} else if b.IsKubernetesGTE("1.13") {
				c.Version = DefaultEtcd3Version_1_13
			} else if b.IsKubernetesGTE("1.11") {
				c.Version = DefaultEtcd3Version_1_11
			} else {
				c.Version = DefaultEtcd2Version
			}
		}

		// From 1.12, we enable TLS if we're running EtcdManager & etcd3
		//
		// (Moving to etcd3 is a disruptive upgrade, so we
		// force TLS at the same time as we enable
		// etcd-manager by default).
		if c.Provider == kops.EtcdProviderTypeManager {
			etcdV3 := true
			version := c.Version
			version = strings.TrimPrefix(version, "v")
			if strings.HasPrefix(version, "2.") {
				etcdV3 = false
			} else if strings.HasPrefix(version, "3.") {
				etcdV3 = true
			} else {
				return fmt.Errorf("unexpected etcd version %q", c.Version)
			}

			if b.IsKubernetesGTE("1.12.0") && etcdV3 {
				c.EnableEtcdTLS = true
				c.EnableTLSAuth = true
			}
		}
	}

	// Remap the well known images
	for _, c := range spec.EtcdClusters {

		// We remap the etcd manager image when we build the manifest,
		// but we need to map the standalone images here because protokube launches them

		if c.Provider == kops.EtcdProviderTypeLegacy {

			// remap etcd image
			{
				image := c.Image
				if image == "" {
					image = fmt.Sprintf("k8s.gcr.io/etcd:%s", c.Version)
				}

				if image != "" {
					image, err := b.AssetBuilder.RemapImage(image)
					if err != nil {
						return fmt.Errorf("unable to remap container %q: %v", image, err)
					}
					c.Image = image
				}
			}

			// remap backup manager image
			if c.Backups != nil {
				image := c.Backups.Image
				if image == "" {
					image = fmt.Sprintf(DefaultBackupImage)
				}

				if image != "" {
					image, err := b.AssetBuilder.RemapImage(image)
					if err != nil {
						return fmt.Errorf("unable to remap container %q: %v", image, err)
					}
					c.Backups.Image = image
				}
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package persistence

import (
	"fmt"
	"net/url"
	"time"

	"go.uber.org/zap"
)

type Database interface {
	Engine() databaseEngine
	DoesTorrentExist(infoHash []byte) (bool, error)
	AddNewTorrent(infoHash []byte, name string, files []File) error
	Close() error

	// GetNumberOfTorrents returns the number of torrents saved in the database. Might be an
	// approximation.
	GetNumberOfTorrents() (uint, error)
	// QueryTorrents returns @n torrents
	// * that are discovered before the @timePoint if @isAfter is false, else that are
	//   discovered after the @timePoint,
	// * that match the @query if it's not empty,
	// ordered by the @orderBy in ascending order if @isDescending is false, else in descending
	// order.
	QueryTorrents(query string, discoveredOnBefore int64, orderBy orderingCriteria, ascending bool, page uint, pageSize uint) ([]TorrentMetadata, error)
	// GetTorrents returns the TorrentExtMetadata for the torrent of the given InfoHash. Might return
	// nil, nil if the torrent does not exist in the database.
	GetTorrent(infoHash []byte) (*TorrentMetadata, error)
	GetFiles(infoHash []byte) ([]File, error)
	GetStatistics(n uint, granularity Granularity, to time.Time) (*Statistics, error)
}

type orderingCriteria uint8
const (
	ByRelevance orderingCriteria = iota
	BySize
	ByDiscoveredOn
	ByNFiles
)

type Granularity uint8
const (
	Yearly Granularity = iota
	Monthly
	Weekly
	Daily
	Hourly
)

type databaseEngine uint8
const (
	Sqlite3 databaseEngine = 1
)

type Statistics struct {
	N uint

	// All these slices below have the exact length equal to the Period.
	NTorrentsDiscovered []uint
	NFilesDiscovered    []uint
}

type File struct {
	Size int64
	Path string
}

type TorrentMetadata struct {
	InfoHash     []byte
	Name         string
	TotalSize    uint64
	DiscoveredOn int64
	NFiles       uint
}

func MakeDatabase(rawURL string, enableFTS bool, logger *zap.Logger) (Database, error) {
	if logger != nil {
		zap.ReplaceGlobals(logger)
	}

	url_, err := url.Parse(rawURL)
	if err != nil {
		return nil, err
	}

	switch url_.Scheme {
	case "sqlite3":
		return makeSqlite3Database(url_, enableFTS)

	case "postgresql":
		return nil, fmt.Errorf("postgresql is not yet supported!")

	case "mysql":
		return nil, fmt.Errorf("mysql is not yet supported!")

	default:
		return nil, fmt.Errorf("unknown URI scheme (database engine)!")
	}
}
<commit_msg>removed `enableFTS` arg in MakeDatabase<commit_after>package persistence

import (
	"fmt"
	"net/url"
	"time"

	"go.uber.org/zap"
)

type Database interface {
	Engine() databaseEngine
	DoesTorrentExist(infoHash []byte) (bool, error)
	AddNewTorrent(infoHash []byte, name string, files []File) error
	Close() error

	// GetNumberOfTorrents returns the number of torrents saved in the database. Might be an
	// approximation.
	GetNumberOfTorrents() (uint, error)
	// QueryTorrents returns @n torrents
	// * that are discovered before the @timePoint if @isAfter is false, else that are
	//   discovered after the @timePoint,
	// * that match the @query if it's not empty,
	// ordered by the @orderBy in ascending order if @isDescending is false, else in descending
	// order.
	QueryTorrents(query string, discoveredOnBefore int64, orderBy orderingCriteria, ascending bool, page uint, pageSize uint) ([]TorrentMetadata, error)
	// GetTorrents returns the TorrentExtMetadata for the torrent of the given InfoHash. Might return
	// nil, nil if the torrent does not exist in the database.
	GetTorrent(infoHash []byte) (*TorrentMetadata, error)
	GetFiles(infoHash []byte) ([]File, error)
	GetStatistics(n uint, granularity Granularity, to time.Time) (*Statistics, error)
}

type orderingCriteria uint8
const (
	ByRelevance orderingCriteria = iota
	BySize
	ByDiscoveredOn
	ByNFiles
)

type Granularity uint8
const (
	Yearly Granularity = iota
	Monthly
	Weekly
	Daily
	Hourly
)

type databaseEngine uint8
const (
	Sqlite3 databaseEngine = 1
)

type Statistics struct {
	N uint

	// All these slices below have the exact length equal to the Period.
	NTorrentsDiscovered []uint
	NFilesDiscovered    []uint
}

type File struct {
	Size int64
	Path string
}

type TorrentMetadata struct {
	InfoHash     []byte
	Name         string
	TotalSize    uint64
	DiscoveredOn int64
	NFiles       uint
}

func MakeDatabase(rawURL string, logger *zap.Logger) (Database, error) {
	if logger != nil {
		zap.ReplaceGlobals(logger)
	}

	url_, err := url.Parse(rawURL)
	if err != nil {
		return nil, err
	}

	switch url_.Scheme {
	case "sqlite3":
		return makeSqlite3Database(url_)

	case "postgresql":
		return nil, fmt.Errorf("postgresql is not yet supported!")

	case "mysql":
		return nil, fmt.Errorf("mysql is not yet supported!")

	default:
		return nil, fmt.Errorf("unknown URI scheme (database engine)!")
	}
}
<|endoftext|>


<commit_before>package publish

import (
	"fmt"
	"strconv"

	"github.com/drone/drone/pkg/build/buildfile"
	"github.com/drone/drone/pkg/build/repo"
)

type Docker struct {
	// The path to the dockerfile to create the image from
	Dockerfile string `yaml:"docker_file"`

	// Connection information for the docker server that will build the image
	Server string `yaml:"docker_server"`
	Port   int    `yaml:"docker_port"`
	// The Docker client version to download. This must match the docker version on the server
	DockerVersion string `yaml:"docker_version"`

	RepoBaseName string `yaml:"repo_base_name"`

	// Authentication credentials for index.docker.io
	Username string `yaml:"username"`
	Password string `yaml:"password"`
	Email    string `yaml:"email"`

	Branch string `yaml:"branch,omitempty"`
}

// Write adds commands to the buildfile to do the following:
// 1. Install the docker client in the Drone container.
// 2. Build a docker image based on the dockerfile defined in the config.
// 3. Push that docker image to index.docker.io.
// 4. Delete the docker image on the server it was build on so we conserve disk space.
func (d *Docker) Write(f *buildfile.Buildfile, r *repo.Repo) {
	if len(d.Dockerfile) == 0 || len(d.Server) == 0 || d.Port == 0 || len(d.DockerVersion) == 0 ||
		len(d.RepoBaseName) == 0 || len(d.Username) == 0 || len(d.Password) == 0 ||
		len(d.Email) == 0 {
		f.WriteCmdSilent(`echo "Docker Plugin: Missing argument(s)"`)
		return
	}

	// Install Docker on the container
	f.WriteCmd("sudo sh -c \"echo deb http://get.docker.io/ubuntu docker main\\ > " +
		"/etc/apt/sources.list.d/docker.list\"")
	f.WriteCmd("sudo apt-get update")
	f.WriteCmd("sudo apt-get --yes install lxc-docker-" + d.DockerVersion)

	dockerServerUrl := d.Server + ":" + strconv.Itoa(d.Port)
	dockerRepo := d.RepoBaseName + "/" + r.Name
	// Run the command commands to build and deploy the image. Note that the image is tagged
	// with the git hash.
	f.WriteCmd(fmt.Sprintf("docker -H %s build -t %s:$(git rev-parse --short HEAD) - < %s",
		dockerServerUrl, dockerRepo, d.Dockerfile))

	// Login and push to index.docker.io
	f.WriteCmdSilent(fmt.Sprintf("docker -H %s login -u %s -p %s -e %s",
		dockerServerUrl, d.Username, d.Password, d.Email))
	f.WriteCmd(fmt.Sprintf("docker -H %s push %s", dockerServerUrl, dockerRepo))

        // Delete the image from the docker server we built on.
	f.WriteCmd(fmt.Sprintf("docker -H %s rmi %s:$(git rev-parse --short HEAD)",
		dockerServerUrl, dockerRepo))
}
<commit_msg>Only grab the last part of the docker repo name<commit_after>package publish

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/drone/drone/pkg/build/buildfile"
	"github.com/drone/drone/pkg/build/repo"
)

type Docker struct {
	// The path to the dockerfile to create the image from
	Dockerfile string `yaml:"docker_file"`

	// Connection information for the docker server that will build the image
	Server string `yaml:"docker_server"`
	Port   int    `yaml:"docker_port"`
	// The Docker client version to download. This must match the docker version on the server
	DockerVersion string `yaml:"docker_version"`

	RepoBaseName string `yaml:"repo_base_name"`

	// Authentication credentials for index.docker.io
	Username string `yaml:"username"`
	Password string `yaml:"password"`
	Email    string `yaml:"email"`

	Branch string `yaml:"branch,omitempty"`
}

// Write adds commands to the buildfile to do the following:
// 1. Install the docker client in the Drone container.
// 2. Build a docker image based on the dockerfile defined in the config.
// 3. Push that docker image to index.docker.io.
// 4. Delete the docker image on the server it was build on so we conserve disk space.
func (d *Docker) Write(f *buildfile.Buildfile, r *repo.Repo) {
	if len(d.Dockerfile) == 0 || len(d.Server) == 0 || d.Port == 0 || len(d.DockerVersion) == 0 ||
		len(d.RepoBaseName) == 0 || len(d.Username) == 0 || len(d.Password) == 0 ||
		len(d.Email) == 0 {
		f.WriteCmdSilent(`echo "Docker Plugin: Missing argument(s)"`)
		return
	}

	// Install Docker on the container
	f.WriteCmd("sudo sh -c \"echo deb http://get.docker.io/ubuntu docker main\\ > " +
		"/etc/apt/sources.list.d/docker.list\"")
	f.WriteCmd("sudo apt-get update")
	f.WriteCmd("sudo apt-get --yes install lxc-docker-" + d.DockerVersion)

	dockerServerUrl := d.Server + ":" + strconv.Itoa(d.Port)
	splitRepoName := strings.Split(r.Name, "/")
	dockerRepo := d.RepoBaseName + "/" + splitRepoName[len(splitRepoName) - 1]
	// Run the command commands to build and deploy the image. Note that the image is tagged
	// with the git hash.
	f.WriteCmd(fmt.Sprintf("docker -H %s build -t %s:$(git rev-parse --short HEAD) - < %s",
		dockerServerUrl, dockerRepo, d.Dockerfile))

	// Login and push to index.docker.io
	f.WriteCmdSilent(fmt.Sprintf("docker -H %s login -u %s -p %s -e %s",
		dockerServerUrl, d.Username, d.Password, d.Email))
	f.WriteCmd(fmt.Sprintf("docker -H %s push %s", dockerServerUrl, dockerRepo))

        // Delete the image from the docker server we built on.
	f.WriteCmd(fmt.Sprintf("docker -H %s rmi %s:$(git rev-parse --short HEAD)",
		dockerServerUrl, dockerRepo))
}
<|endoftext|>


<commit_before>/*******************************************************************************
*
* Copyright 2018 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package plugins

import (
	"encoding/json"
	"strconv"

	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/pagination"
)

type ironicClient struct {
	*gophercloud.ServiceClient
}

func newIronicClient(provider *gophercloud.ProviderClient, eo gophercloud.EndpointOpts) (*ironicClient, error) {
	serviceType := "baremetal"
	eo.ApplyDefaults(serviceType)

	url, err := provider.EndpointLocator(eo)
	if err != nil {
		return nil, err
	}
	return &ironicClient{
		ServiceClient: &gophercloud.ServiceClient{
			ProviderClient: provider,
			Endpoint:       url,
			Type:           serviceType,
		},
	}, nil
}

////////////////////////////////////////////////////////////////////////////////
// list nodes

type ironicNode struct {
	ID                   string  `json:"uuid"`
	Name                 string  `json:"name"`
	ProvisionState       string  `json:"provision_state"`
	TargetProvisionState *string `json:"target_provision_state"`
	InstanceID           *string `json:"instance_uuid"`
	ResourceClass        *string `json:"resource_class"`
	Properties           struct {
		Cores           veryFlexibleUint64 `json:"cpus"`
		DiskGiB         veryFlexibleUint64 `json:"local_gb"`
		MemoryMiB       veryFlexibleUint64 `json:"memory_mb"`
		CPUArchitecture string             `json:"cpu_arch"`
		Capabilities    string             `json:"capabilities"` //e.g. "cpu_txt:true,cpu_aes:true"
		SerialNumber    string             `json:"serial"`
	} `json:"properties"`
}

func (n ironicNode) StableProvisionState() string {
	if n.TargetProvisionState != nil {
		return *n.TargetProvisionState
	}
	return n.ProvisionState
}

func extractNodes(page pagination.Page) (nodes []ironicNode, err error) {
	err = page.(ironicNodePage).Result.ExtractIntoSlicePtr(&nodes, "nodes")
	return
}

type ironicNodePage struct {
	pagination.MarkerPageBase
}

func (p ironicNodePage) IsEmpty() (bool, error) {
	nodes, err := extractNodes(p)
	return len(nodes) == 0, err
}

func (p ironicNodePage) LastMarker() (string, error) {
	nodes, err := extractNodes(p)
	if err != nil || len(nodes) == 0 {
		return "", err
	}
	return nodes[len(nodes)-1].ID, nil
}

func (c ironicClient) GetNodes() ([]ironicNode, error) {
	url := c.ServiceURL("nodes", "detail")
	pager := pagination.NewPager(c.ServiceClient, url, func(r pagination.PageResult) pagination.Page {
		page := ironicNodePage{pagination.MarkerPageBase{PageResult: r}}
		page.MarkerPageBase.Owner = page
		return page
	})

	// From OpenStack docs:
	// Beginning with the Kilo release, `X-OpenStack-Ironic-API-Version` header
	// SHOULD be supplied with every request. In the absence of this header,
	// each request is treated as though coming from an older pre-Kilo client.
	pager.Headers = map[string]string{
		"X-OpenStack-Ironic-API-Version": "1.58",
	}

	var result []ironicNode
	err := pager.EachPage(func(page pagination.Page) (bool, error) {
		pageNodes, err := extractNodes(page)
		if err != nil {
			return false, err
		}
		result = append(result, pageNodes...)
		return true, nil
	})
	return result, err
}

////////////////////////////////////////////////////////////////////////////////
// OpenStack is being inconsistent with itself again

//For fields that are sometimes missing, sometimes an integer, sometimes a string.
type veryFlexibleUint64 uint64

//UnmarshalJSON implements the json.Unmarshaler interface.
func (value *veryFlexibleUint64) UnmarshalJSON(buf []byte) error {
	if string(buf) == "null" {
		*value = 0
		return nil
	}

	if buf[0] == '"' {
		var str string
		err := json.Unmarshal(buf, &str)
		if err != nil {
			return err
		}
		val, err := strconv.ParseUint(str, 10, 64)
		*value = veryFlexibleUint64(val)
		return err
	}

	var val uint64
	err := json.Unmarshal(buf, &val)
	*value = veryFlexibleUint64(val)
	return err
}
<commit_msg>revert microversion increase in b329238398<commit_after>/*******************************************************************************
*
* Copyright 2018 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package plugins

import (
	"encoding/json"
	"strconv"

	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/pagination"
)

type ironicClient struct {
	*gophercloud.ServiceClient
}

func newIronicClient(provider *gophercloud.ProviderClient, eo gophercloud.EndpointOpts) (*ironicClient, error) {
	serviceType := "baremetal"
	eo.ApplyDefaults(serviceType)

	url, err := provider.EndpointLocator(eo)
	if err != nil {
		return nil, err
	}
	return &ironicClient{
		ServiceClient: &gophercloud.ServiceClient{
			ProviderClient: provider,
			Endpoint:       url,
			Type:           serviceType,
		},
	}, nil
}

////////////////////////////////////////////////////////////////////////////////
// list nodes

type ironicNode struct {
	ID                   string  `json:"uuid"`
	Name                 string  `json:"name"`
	ProvisionState       string  `json:"provision_state"`
	TargetProvisionState *string `json:"target_provision_state"`
	InstanceID           *string `json:"instance_uuid"`
	ResourceClass        *string `json:"resource_class"`
	Properties           struct {
		Cores           veryFlexibleUint64 `json:"cpus"`
		DiskGiB         veryFlexibleUint64 `json:"local_gb"`
		MemoryMiB       veryFlexibleUint64 `json:"memory_mb"`
		CPUArchitecture string             `json:"cpu_arch"`
		Capabilities    string             `json:"capabilities"` //e.g. "cpu_txt:true,cpu_aes:true"
		SerialNumber    string             `json:"serial"`
	} `json:"properties"`
}

func (n ironicNode) StableProvisionState() string {
	if n.TargetProvisionState != nil {
		return *n.TargetProvisionState
	}
	return n.ProvisionState
}

func extractNodes(page pagination.Page) (nodes []ironicNode, err error) {
	err = page.(ironicNodePage).Result.ExtractIntoSlicePtr(&nodes, "nodes")
	return
}

type ironicNodePage struct {
	pagination.MarkerPageBase
}

func (p ironicNodePage) IsEmpty() (bool, error) {
	nodes, err := extractNodes(p)
	return len(nodes) == 0, err
}

func (p ironicNodePage) LastMarker() (string, error) {
	nodes, err := extractNodes(p)
	if err != nil || len(nodes) == 0 {
		return "", err
	}
	return nodes[len(nodes)-1].ID, nil
}

func (c ironicClient) GetNodes() ([]ironicNode, error) {
	url := c.ServiceURL("nodes", "detail")
	pager := pagination.NewPager(c.ServiceClient, url, func(r pagination.PageResult) pagination.Page {
		page := ironicNodePage{pagination.MarkerPageBase{PageResult: r}}
		page.MarkerPageBase.Owner = page
		return page
	})

	// From OpenStack docs:
	// Beginning with the Kilo release, `X-OpenStack-Ironic-API-Version` header
	// SHOULD be supplied with every request. In the absence of this header,
	// each request is treated as though coming from an older pre-Kilo client.
	pager.Headers = map[string]string{
		"X-OpenStack-Ironic-API-Version": "1.22",
	}

	var result []ironicNode
	err := pager.EachPage(func(page pagination.Page) (bool, error) {
		pageNodes, err := extractNodes(page)
		if err != nil {
			return false, err
		}
		result = append(result, pageNodes...)
		return true, nil
	})
	return result, err
}

////////////////////////////////////////////////////////////////////////////////
// OpenStack is being inconsistent with itself again

//For fields that are sometimes missing, sometimes an integer, sometimes a string.
type veryFlexibleUint64 uint64

//UnmarshalJSON implements the json.Unmarshaler interface.
func (value *veryFlexibleUint64) UnmarshalJSON(buf []byte) error {
	if string(buf) == "null" {
		*value = 0
		return nil
	}

	if buf[0] == '"' {
		var str string
		err := json.Unmarshal(buf, &str)
		if err != nil {
			return err
		}
		val, err := strconv.ParseUint(str, 10, 64)
		*value = veryFlexibleUint64(val)
		return err
	}

	var val uint64
	err := json.Unmarshal(buf, &val)
	*value = veryFlexibleUint64(val)
	return err
}
<|endoftext|>


<commit_before>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package docker

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"sync"

	"github.com/GoogleCloudPlatform/skaffold/pkg/skaffold/schema/v1alpha2"
	"github.com/GoogleCloudPlatform/skaffold/pkg/skaffold/util"
	"github.com/containers/image/docker"
	"github.com/containers/image/manifest"
	"github.com/containers/image/types"
	"github.com/docker/docker/builder/dockerignore"
	"github.com/docker/docker/pkg/fileutils"
	"github.com/moby/moby/builder/dockerfile/parser"
	"github.com/moby/moby/builder/dockerfile/shell"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const (
	add    = "add"
	copy   = "copy"
	env    = "env"
	from   = "from"
	expose = "expose"
)

// For testing.
var RetrieveImage = retrieveImage

type DockerfileDepResolver struct{}

func (*DockerfileDepResolver) GetDependencies(a *v1alpha2.Artifact) ([]string, error) {
	return GetDockerfileDependencies(a.DockerArtifact.DockerfilePath, a.Workspace)
}

// GetDockerfileDependencies parses a dockerfile and returns the full paths
// of all the source files that the resulting docker image depends on.
func GetDockerfileDependencies(dockerfilePath, workspace string) ([]string, error) {
	path := filepath.Join(workspace, dockerfilePath)
	f, err := util.Fs.Open(path)
	if err != nil {
		return nil, errors.Wrapf(err, "opening dockerfile: %d", path)
	}
	defer f.Close()

	res, err := parser.Parse(f)
	if err != nil {
		return nil, errors.Wrap(err, "parsing dockerfile")
	}

	envs := map[string]string{}
	depMap := map[string]struct{}{}
	// First process onbuilds, if present.
	onbuilds := []string{}
	for _, value := range res.AST.Children {
		switch value.Value {
		case from:
			onbuilds, err = processBaseImage(value)
			if err != nil {
				logrus.Warnf("Error processing base image for onbuild triggers: %s. Dependencies may be incomplete.", err)
			}
		}
	}

	var dispatchInstructions = func(r *parser.Result) {
		for _, value := range r.AST.Children {
			switch value.Value {
			case add, copy:
				processCopy(workspace, value, depMap, envs)
			case env:
				envs[value.Next.Value] = value.Next.Next.Value
			}
		}
	}
	for _, ob := range onbuilds {
		obRes, err := parser.Parse(strings.NewReader(ob))
		if err != nil {
			return nil, err
		}
		dispatchInstructions(obRes)
	}

	dispatchInstructions(res)

	deps := []string{}
	for dep := range depMap {
		deps = append(deps, dep)
	}
	logrus.Infof("Found dependencies for dockerfile %s", deps)

	expandedDeps, err := util.ExpandPaths(workspace, deps)
	if err != nil {
		return nil, errors.Wrap(err, "expanding dockerfile paths")
	}
	logrus.Infof("deps %s", expandedDeps)

	if !util.StrSliceContains(expandedDeps, path) {
		expandedDeps = append(expandedDeps, path)
	}

	// Look for .dockerignore.
	ignorePath := filepath.Join(workspace, ".dockerignore")
	filteredDeps, err := ApplyDockerIgnore(expandedDeps, ignorePath)
	if err != nil {
		return nil, errors.Wrap(err, "applying dockerignore")
	}

	return filteredDeps, nil
}

func PortsFromDockerfile(r io.Reader) ([]string, error) {
	res, err := parser.Parse(r)
	if err != nil {
		return nil, errors.Wrap(err, "parsing dockerfile")
	}

	// Check the dockerfile and the base.
	ports := []string{}
	for _, value := range res.AST.Children {
		switch value.Value {
		case from:
			base := value.Next.Value
			if strings.ToLower(base) == "scratch" {
				logrus.Debug("Skipping port check in SCRATCH base image.")
				continue
			}
			img, err := RetrieveImage(value.Next.Value)
			if err != nil {
				logrus.Warnf("Error checking base image for ports: %s", err)
				continue
			}
			for port := range img.Config.ExposedPorts {
				logrus.Debugf("Found port %s in base image", port)
				ports = append(ports, string(port))
			}
		case expose:
			// There can be multiple ports per line.
			for {
				if value.Next == nil {
					break
				}
				port := value.Next.Value
				logrus.Debugf("Found port %s in Dockerfile", port)
				ports = append(ports, port)
				value = value.Next
			}
		}
	}
	// Sort ports for consistency in tests.
	sort.Strings(ports)
	return ports, nil
}

func processBaseImage(value *parser.Node) ([]string, error) {
	base := value.Next.Value
	logrus.Debugf("Checking base image %s for ONBUILD triggers.", base)
	if strings.ToLower(base) == "scratch" {
		logrus.Debugf("SCRATCH base image found, skipping check: %s", base)
		return nil, nil
	}
	img, err := RetrieveImage(base)
	if err != nil {
		return nil, err
	}
	logrus.Debugf("Found onbuild triggers %v in image %s", img.Config.OnBuild, base)
	return img.Config.OnBuild, nil
}

var imageCache sync.Map

func retrieveImage(image string) (*manifest.Schema2Image, error) {
	cachedCfg, present := imageCache.Load(image)
	if present {
		return cachedCfg.(*manifest.Schema2Image), nil
	}

	client, err := NewDockerAPIClient()
	if err != nil {
		return nil, err
	}

	raw, err := retrieveLocalImage(client, image)
	if err != nil {
		raw, err = retrieveRemoteImage(image)
		if err != nil {
			return nil, err
		}
	}

	cfg := &manifest.Schema2Image{}
	if err := json.Unmarshal(raw, cfg); err != nil {
		return nil, err
	}

	imageCache.Store(image, cfg)

	return cfg, nil
}

func retrieveLocalImage(client DockerAPIClient, image string) ([]byte, error) {
	_, raw, err := client.ImageInspectWithRaw(context.Background(), image)
	if err != nil {
		return nil, err
	}

	return raw, nil
}

func retrieveRemoteImage(image string) ([]byte, error) {
	context := &types.SystemContext{
		OSChoice:           "linux",
		ArchitectureChoice: "amd64",
	}

	ref, err := docker.ParseReference("//" + image)
	if err != nil {
		return nil, err
	}

	img, err := ref.NewImage(context)
	if err != nil {
		return nil, err
	}
	defer img.Close()

	return img.ConfigBlob()
}

func processCopy(workspace string, value *parser.Node, paths map[string]struct{}, envs map[string]string) error {
	slex := shell.NewLex('\\')
	for {
		// Skip last node, since it is the destination, and stop if we arrive at a comment
		if value.Next.Next == nil || strings.HasPrefix(value.Next.Next.Value, "#") {
			break
		}
		src, err := processShellWord(slex, value.Next.Value, envs)
		if err != nil {
			return errors.Wrap(err, "processing word")
		}
		// If the --from flag is provided, we are dealing with a multi-stage dockerfile
		// Adding a dependency from a different stage does not imply a source dependency
		if hasMultiStageFlag(value.Flags) {
			return nil
		}
		if !strings.HasPrefix(src, "http://") && !strings.HasPrefix(src, "https://") {
			dep := path.Join(workspace, src)
			paths[dep] = struct{}{}
		} else {
			logrus.Debugf("Skipping watch on remote dependency %s", src)
		}

		value = value.Next
	}
	return nil
}

func processShellWord(lex *shell.Lex, word string, envs map[string]string) (string, error) {
	envSlice := []string{}
	for envKey, envVal := range envs {
		envSlice = append(envSlice, fmt.Sprintf("%s=%s", envKey, envVal))
	}
	return lex.ProcessWord(word, envSlice)
}

func hasMultiStageFlag(flags []string) bool {
	for _, f := range flags {
		if strings.HasPrefix(f, "--from=") {
			return true
		}
	}
	return false
}

func ApplyDockerIgnore(paths []string, dockerIgnorePath string) ([]string, error) {
	absPaths, err := util.RelPathToAbsPath(paths)
	if err != nil {
		return nil, errors.Wrap(err, "getting absolute path of dependencies")
	}
	excludes := []string{}
	if _, err := util.Fs.Stat(dockerIgnorePath); !os.IsNotExist(err) {
		r, err := util.Fs.Open(dockerIgnorePath)
		if err != nil {
			return nil, err
		}
		defer r.Close()

		excludes, err = dockerignore.ReadAll(r)
		if err != nil {
			return nil, err
		}
		excludes = append(excludes, ".dockerignore")
	}

	absPathExcludes, err := util.RelPathToAbsPath(excludes)
	if err != nil {
		return nil, errors.Wrap(err, "getting absolute path of docker ignored paths")
	}

	filteredDeps := []string{}
	for _, d := range absPaths {
		m, err := fileutils.Matches(d, absPathExcludes)
		if err != nil {
			return nil, err
		}
		if !m {
			filteredDeps = append(filteredDeps, d)
		}
	}
	sort.Strings(filteredDeps)
	return filteredDeps, nil
}
<commit_msg>Fix error message<commit_after>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package docker

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"sync"

	"github.com/GoogleCloudPlatform/skaffold/pkg/skaffold/schema/v1alpha2"
	"github.com/GoogleCloudPlatform/skaffold/pkg/skaffold/util"
	"github.com/containers/image/docker"
	"github.com/containers/image/manifest"
	"github.com/containers/image/types"
	"github.com/docker/docker/builder/dockerignore"
	"github.com/docker/docker/pkg/fileutils"
	"github.com/moby/moby/builder/dockerfile/parser"
	"github.com/moby/moby/builder/dockerfile/shell"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const (
	add    = "add"
	copy   = "copy"
	env    = "env"
	from   = "from"
	expose = "expose"
)

// For testing.
var RetrieveImage = retrieveImage

type DockerfileDepResolver struct{}

func (*DockerfileDepResolver) GetDependencies(a *v1alpha2.Artifact) ([]string, error) {
	return GetDockerfileDependencies(a.DockerArtifact.DockerfilePath, a.Workspace)
}

// GetDockerfileDependencies parses a dockerfile and returns the full paths
// of all the source files that the resulting docker image depends on.
func GetDockerfileDependencies(dockerfilePath, workspace string) ([]string, error) {
	path := filepath.Join(workspace, dockerfilePath)
	f, err := util.Fs.Open(path)
	if err != nil {
		return nil, errors.Wrapf(err, "opening dockerfile: %s", path)
	}
	defer f.Close()

	res, err := parser.Parse(f)
	if err != nil {
		return nil, errors.Wrap(err, "parsing dockerfile")
	}

	envs := map[string]string{}
	depMap := map[string]struct{}{}
	// First process onbuilds, if present.
	onbuilds := []string{}
	for _, value := range res.AST.Children {
		switch value.Value {
		case from:
			onbuilds, err = processBaseImage(value)
			if err != nil {
				logrus.Warnf("Error processing base image for onbuild triggers: %s. Dependencies may be incomplete.", err)
			}
		}
	}

	var dispatchInstructions = func(r *parser.Result) {
		for _, value := range r.AST.Children {
			switch value.Value {
			case add, copy:
				processCopy(workspace, value, depMap, envs)
			case env:
				envs[value.Next.Value] = value.Next.Next.Value
			}
		}
	}
	for _, ob := range onbuilds {
		obRes, err := parser.Parse(strings.NewReader(ob))
		if err != nil {
			return nil, err
		}
		dispatchInstructions(obRes)
	}

	dispatchInstructions(res)

	deps := []string{}
	for dep := range depMap {
		deps = append(deps, dep)
	}
	logrus.Infof("Found dependencies for dockerfile %s", deps)

	expandedDeps, err := util.ExpandPaths(workspace, deps)
	if err != nil {
		return nil, errors.Wrap(err, "expanding dockerfile paths")
	}
	logrus.Infof("deps %s", expandedDeps)

	if !util.StrSliceContains(expandedDeps, path) {
		expandedDeps = append(expandedDeps, path)
	}

	// Look for .dockerignore.
	ignorePath := filepath.Join(workspace, ".dockerignore")
	filteredDeps, err := ApplyDockerIgnore(expandedDeps, ignorePath)
	if err != nil {
		return nil, errors.Wrap(err, "applying dockerignore")
	}

	return filteredDeps, nil
}

func PortsFromDockerfile(r io.Reader) ([]string, error) {
	res, err := parser.Parse(r)
	if err != nil {
		return nil, errors.Wrap(err, "parsing dockerfile")
	}

	// Check the dockerfile and the base.
	ports := []string{}
	for _, value := range res.AST.Children {
		switch value.Value {
		case from:
			base := value.Next.Value
			if strings.ToLower(base) == "scratch" {
				logrus.Debug("Skipping port check in SCRATCH base image.")
				continue
			}
			img, err := RetrieveImage(value.Next.Value)
			if err != nil {
				logrus.Warnf("Error checking base image for ports: %s", err)
				continue
			}
			for port := range img.Config.ExposedPorts {
				logrus.Debugf("Found port %s in base image", port)
				ports = append(ports, string(port))
			}
		case expose:
			// There can be multiple ports per line.
			for {
				if value.Next == nil {
					break
				}
				port := value.Next.Value
				logrus.Debugf("Found port %s in Dockerfile", port)
				ports = append(ports, port)
				value = value.Next
			}
		}
	}
	// Sort ports for consistency in tests.
	sort.Strings(ports)
	return ports, nil
}

func processBaseImage(value *parser.Node) ([]string, error) {
	base := value.Next.Value
	logrus.Debugf("Checking base image %s for ONBUILD triggers.", base)
	if strings.ToLower(base) == "scratch" {
		logrus.Debugf("SCRATCH base image found, skipping check: %s", base)
		return nil, nil
	}
	img, err := RetrieveImage(base)
	if err != nil {
		return nil, err
	}
	logrus.Debugf("Found onbuild triggers %v in image %s", img.Config.OnBuild, base)
	return img.Config.OnBuild, nil
}

var imageCache sync.Map

func retrieveImage(image string) (*manifest.Schema2Image, error) {
	cachedCfg, present := imageCache.Load(image)
	if present {
		return cachedCfg.(*manifest.Schema2Image), nil
	}

	client, err := NewDockerAPIClient()
	if err != nil {
		return nil, err
	}

	raw, err := retrieveLocalImage(client, image)
	if err != nil {
		raw, err = retrieveRemoteImage(image)
		if err != nil {
			return nil, err
		}
	}

	cfg := &manifest.Schema2Image{}
	if err := json.Unmarshal(raw, cfg); err != nil {
		return nil, err
	}

	imageCache.Store(image, cfg)

	return cfg, nil
}

func retrieveLocalImage(client DockerAPIClient, image string) ([]byte, error) {
	_, raw, err := client.ImageInspectWithRaw(context.Background(), image)
	if err != nil {
		return nil, err
	}

	return raw, nil
}

func retrieveRemoteImage(image string) ([]byte, error) {
	context := &types.SystemContext{
		OSChoice:           "linux",
		ArchitectureChoice: "amd64",
	}

	ref, err := docker.ParseReference("//" + image)
	if err != nil {
		return nil, err
	}

	img, err := ref.NewImage(context)
	if err != nil {
		return nil, err
	}
	defer img.Close()

	return img.ConfigBlob()
}

func processCopy(workspace string, value *parser.Node, paths map[string]struct{}, envs map[string]string) error {
	slex := shell.NewLex('\\')
	for {
		// Skip last node, since it is the destination, and stop if we arrive at a comment
		if value.Next.Next == nil || strings.HasPrefix(value.Next.Next.Value, "#") {
			break
		}
		src, err := processShellWord(slex, value.Next.Value, envs)
		if err != nil {
			return errors.Wrap(err, "processing word")
		}
		// If the --from flag is provided, we are dealing with a multi-stage dockerfile
		// Adding a dependency from a different stage does not imply a source dependency
		if hasMultiStageFlag(value.Flags) {
			return nil
		}
		if !strings.HasPrefix(src, "http://") && !strings.HasPrefix(src, "https://") {
			dep := path.Join(workspace, src)
			paths[dep] = struct{}{}
		} else {
			logrus.Debugf("Skipping watch on remote dependency %s", src)
		}

		value = value.Next
	}
	return nil
}

func processShellWord(lex *shell.Lex, word string, envs map[string]string) (string, error) {
	envSlice := []string{}
	for envKey, envVal := range envs {
		envSlice = append(envSlice, fmt.Sprintf("%s=%s", envKey, envVal))
	}
	return lex.ProcessWord(word, envSlice)
}

func hasMultiStageFlag(flags []string) bool {
	for _, f := range flags {
		if strings.HasPrefix(f, "--from=") {
			return true
		}
	}
	return false
}

func ApplyDockerIgnore(paths []string, dockerIgnorePath string) ([]string, error) {
	absPaths, err := util.RelPathToAbsPath(paths)
	if err != nil {
		return nil, errors.Wrap(err, "getting absolute path of dependencies")
	}
	excludes := []string{}
	if _, err := util.Fs.Stat(dockerIgnorePath); !os.IsNotExist(err) {
		r, err := util.Fs.Open(dockerIgnorePath)
		if err != nil {
			return nil, err
		}
		defer r.Close()

		excludes, err = dockerignore.ReadAll(r)
		if err != nil {
			return nil, err
		}
		excludes = append(excludes, ".dockerignore")
	}

	absPathExcludes, err := util.RelPathToAbsPath(excludes)
	if err != nil {
		return nil, errors.Wrap(err, "getting absolute path of docker ignored paths")
	}

	filteredDeps := []string{}
	for _, d := range absPaths {
		m, err := fileutils.Matches(d, absPathExcludes)
		if err != nil {
			return nil, err
		}
		if !m {
			filteredDeps = append(filteredDeps, d)
		}
	}
	sort.Strings(filteredDeps)
	return filteredDeps, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Radu Berinde (radu@cockroachlabs.com)

package distsqlrun

import (
	"sync"
	"time"

	"golang.org/x/net/context"

	"github.com/cockroachdb/cockroach/pkg/sql/sqlbase"
	"github.com/cockroachdb/cockroach/pkg/util/log"
)

const outboxBufRows = 16
const outboxFlushPeriod = 100 * time.Microsecond

// preferredEncoding is the encoding used for EncDatums that don't already have
// an encoding available.
const preferredEncoding = sqlbase.DatumEncoding_ASCENDING_KEY

// outbox implements an outgoing mailbox as a RowReceiver that receives rows and
// sends them to a gRPC stream. Its core logic runs in a goroutine. We send rows
// when we accumulate outboxBufRows or every outboxFlushPeriod (whichever comes
// first).
type outbox struct {
	// RowChannel implements the RowReceiver interface.
	RowChannel

	flowCtx *FlowCtx
	addr    string
	stream  DistSQL_FlowStreamClient

	// syncFlowStream is set if we are outputting to a sync flow stream; in
	// that case addr and stream will not be set.
	syncFlowStream DistSQL_RunSyncFlowServer

	encoder StreamEncoder
	// numRows is the number of rows that have been accumulated in the encoder.
	numRows int

	err error
	wg  *sync.WaitGroup
}

var _ RowReceiver = &outbox{}

func newOutbox(flowCtx *FlowCtx, addr string, flowID FlowID, streamID StreamID) *outbox {
	m := &outbox{flowCtx: flowCtx, addr: addr}
	m.encoder.setHeaderFields(flowID, streamID)
	return m
}

// newOutboxSyncFlowStream sets up an outbox for the special "sync flow"
// stream. The flow context should be provided via setFlowCtx when it is
// available.
func newOutboxSyncFlowStream(stream DistSQL_RunSyncFlowServer) *outbox {
	return &outbox{syncFlowStream: stream}
}

func (m *outbox) setFlowCtx(flowCtx *FlowCtx) {
	m.flowCtx = flowCtx
}

// addRow encodes a row into rowBuf. If enough rows were accumulated
// calls flush().
func (m *outbox) addRow(row sqlbase.EncDatumRow) error {
	err := m.encoder.AddRow(row)
	if err != nil {
		return err
	}
	m.numRows++
	if m.numRows >= outboxBufRows {
		return m.flush(false, nil)
	}
	return nil
}

// flush sends the rows accumulated so far in a StreamMessage.
func (m *outbox) flush(last bool, err error) error {
	if !last && m.numRows == 0 {
		return nil
	}
	msg := m.encoder.FormMessage(last, err)

	if log.V(3) {
		log.Infof(m.flowCtx.Context, "flushing outbox")
	}
	var sendErr error
	if m.stream != nil {
		sendErr = m.stream.Send(msg)
	} else {
		sendErr = m.syncFlowStream.Send(msg)
	}
	if sendErr != nil {
		if log.V(1) {
			log.Errorf(m.flowCtx.Context, "outbox flush error: %s", sendErr)
		}
	} else if log.V(3) {
		log.Infof(m.flowCtx.Context, "outbox flushed")
	}
	if sendErr != nil {
		return sendErr
	}

	m.numRows = 0
	return nil
}

func (m *outbox) mainLoop() error {
	if m.syncFlowStream == nil {
		conn, err := m.flowCtx.rpcCtx.GRPCDial(m.addr)
		if err != nil {
			return err
		}
		client := NewDistSQLClient(conn)
		if log.V(2) {
			log.Infof(m.flowCtx.Context, "outbox: calling FlowStream")
		}
		m.stream, err = client.FlowStream(context.TODO())
		if err != nil {
			if log.V(1) {
				log.Infof(m.flowCtx.Context, "FlowStream error: %s", err)
			}
			return err
		}
		if log.V(2) {
			log.Infof(m.flowCtx.Context, "outbox: FlowStream returned")
		}
	}

	flushTicker := time.NewTicker(outboxFlushPeriod)
	defer flushTicker.Stop()

	for {
		select {
		case d, ok := <-m.RowChannel.C:
			if !ok {
				// No more data.
				return m.flush(true, nil)
			}
			err := d.Err
			if err == nil {
				err = m.addRow(d.Row)
			}
			if err != nil {
				// Try to flush to send out the error, but ignore any
				// send error.
				_ = m.flush(true, err)
				return err
			}
		case <-flushTicker.C:
			err := m.flush(false, nil)
			if err != nil {
				return err
			}
		}
	}
}

func (m *outbox) run() {
	err := m.mainLoop()

	m.RowChannel.ConsumerDone()

	if m.stream != nil {
		resp, recvErr := m.stream.CloseAndRecv()
		if err == nil {
			if recvErr != nil {
				err = recvErr
			} else if resp.Error != nil {
				err = resp.Error.GoError()
			}
		}
	}
	m.err = err
	if m.wg != nil {
		m.wg.Done()
	}
}

func (m *outbox) start(wg *sync.WaitGroup) {
	m.wg = wg
	m.RowChannel.Init(nil)
	go m.run()
}
<commit_msg>distsql: wrap error signaled by consumer to producer<commit_after>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Radu Berinde (radu@cockroachlabs.com)

package distsqlrun

import (
	"sync"
	"time"

	"golang.org/x/net/context"

	"github.com/cockroachdb/cockroach/pkg/sql/sqlbase"
	"github.com/cockroachdb/cockroach/pkg/util/log"
	"github.com/pkg/errors"
)

const outboxBufRows = 16
const outboxFlushPeriod = 100 * time.Microsecond

// preferredEncoding is the encoding used for EncDatums that don't already have
// an encoding available.
const preferredEncoding = sqlbase.DatumEncoding_ASCENDING_KEY

// outbox implements an outgoing mailbox as a RowReceiver that receives rows and
// sends them to a gRPC stream. Its core logic runs in a goroutine. We send rows
// when we accumulate outboxBufRows or every outboxFlushPeriod (whichever comes
// first).
type outbox struct {
	// RowChannel implements the RowReceiver interface.
	RowChannel

	flowCtx *FlowCtx
	addr    string
	stream  DistSQL_FlowStreamClient

	// syncFlowStream is set if we are outputting to a sync flow stream; in
	// that case addr and stream will not be set.
	syncFlowStream DistSQL_RunSyncFlowServer

	encoder StreamEncoder
	// numRows is the number of rows that have been accumulated in the encoder.
	numRows int

	err error
	wg  *sync.WaitGroup
}

var _ RowReceiver = &outbox{}

func newOutbox(flowCtx *FlowCtx, addr string, flowID FlowID, streamID StreamID) *outbox {
	m := &outbox{flowCtx: flowCtx, addr: addr}
	m.encoder.setHeaderFields(flowID, streamID)
	return m
}

// newOutboxSyncFlowStream sets up an outbox for the special "sync flow"
// stream. The flow context should be provided via setFlowCtx when it is
// available.
func newOutboxSyncFlowStream(stream DistSQL_RunSyncFlowServer) *outbox {
	return &outbox{syncFlowStream: stream}
}

func (m *outbox) setFlowCtx(flowCtx *FlowCtx) {
	m.flowCtx = flowCtx
}

// addRow encodes a row into rowBuf. If enough rows were accumulated
// calls flush().
func (m *outbox) addRow(row sqlbase.EncDatumRow) error {
	err := m.encoder.AddRow(row)
	if err != nil {
		return err
	}
	m.numRows++
	if m.numRows >= outboxBufRows {
		return m.flush(false, nil)
	}
	return nil
}

// flush sends the rows accumulated so far in a StreamMessage.
func (m *outbox) flush(last bool, err error) error {
	if !last && m.numRows == 0 {
		return nil
	}
	msg := m.encoder.FormMessage(last, err)

	if log.V(3) {
		log.Infof(m.flowCtx.Context, "flushing outbox")
	}
	var sendErr error
	if m.stream != nil {
		sendErr = m.stream.Send(msg)
	} else {
		sendErr = m.syncFlowStream.Send(msg)
	}
	if sendErr != nil {
		if log.V(1) {
			log.Errorf(m.flowCtx.Context, "outbox flush error: %s", sendErr)
		}
	} else if log.V(3) {
		log.Infof(m.flowCtx.Context, "outbox flushed")
	}
	if sendErr != nil {
		return sendErr
	}

	m.numRows = 0
	return nil
}

func (m *outbox) mainLoop() error {
	if m.syncFlowStream == nil {
		conn, err := m.flowCtx.rpcCtx.GRPCDial(m.addr)
		if err != nil {
			return err
		}
		client := NewDistSQLClient(conn)
		if log.V(2) {
			log.Infof(m.flowCtx.Context, "outbox: calling FlowStream")
		}
		m.stream, err = client.FlowStream(context.TODO())
		if err != nil {
			if log.V(1) {
				log.Infof(m.flowCtx.Context, "FlowStream error: %s", err)
			}
			return err
		}
		if log.V(2) {
			log.Infof(m.flowCtx.Context, "outbox: FlowStream returned")
		}
	}

	flushTicker := time.NewTicker(outboxFlushPeriod)
	defer flushTicker.Stop()

	for {
		select {
		case d, ok := <-m.RowChannel.C:
			if !ok {
				// No more data.
				return m.flush(true, nil)
			}
			err := d.Err
			if err == nil {
				err = m.addRow(d.Row)
			}
			if err != nil {
				// Try to flush to send out the error, but ignore any
				// send error.
				_ = m.flush(true, err)
				return err
			}
		case <-flushTicker.C:
			err := m.flush(false, nil)
			if err != nil {
				return err
			}
		}
	}
}

func (m *outbox) run() {
	err := m.mainLoop()

	m.RowChannel.ConsumerDone()

	if m.stream != nil {
		resp, recvErr := m.stream.CloseAndRecv()
		if err == nil {
			if recvErr != nil {
				err = recvErr
			} else if resp.Error != nil {
				err = errors.Wrap(resp.Error.GoError(), "consumer signaled error")
			}
		}
	}
	m.err = err
	if m.wg != nil {
		m.wg.Done()
	}
}

func (m *outbox) start(wg *sync.WaitGroup) {
	m.wg = wg
	m.RowChannel.Init(nil)
	go m.run()
}
<|endoftext|>