<commit_before>package memprov

import (
	"errors"
	"sort"
	"sync"

	"github.com/scjalliance/resourceful/lease"
)

// leasePage is an in-memory page of lease data for a single resource.
type leasePage struct {
	mutex    sync.RWMutex
	revision uint64
	leases   lease.Set
}

// Provider provides memory-based lease management.
type Provider struct {
	mutex      sync.RWMutex
	leasePages map[string]*leasePage // The lease set for each resource
}

// New returns a new memory provider.
func New() *Provider {
	return &Provider{
		leasePages: make(map[string]*leasePage),
	}
}

// LeaseView returns the current revision and lease set for the resource.
func (p *Provider) LeaseView(resource string) (revision uint64, leases lease.Set, err error) {
	page := p.leasePage(resource)
	page.mutex.RLock()
	defer page.mutex.RUnlock()

	revision = page.revision
	leases = make(lease.Set, len(page.leases))
	copy(leases, page.leases)
	return
}

// LeaseCommit will attempt to apply the operations described in the lease
// transaction.
func (p *Provider) LeaseCommit(tx *lease.Tx) error {
	ops := tx.Ops()
	if len(ops) == 0 {
		// Nothing to commit
		return nil
	}

	page := p.leasePage(tx.Resource())
	page.mutex.Lock()
	defer page.mutex.Unlock()
	if page.revision != tx.Revision() {
		return errors.New("Unable to commit lease transaction due to opportunistic lock conflict")
	}
	page.revision++
	for _, op := range ops {
		switch op.Type {
		case lease.Create:
			page.leases = append(page.leases, op.Lease)
		case lease.Update:
			i := page.leases.Index(op.Previous.Resource, op.Previous.Consumer, op.Previous.Instance)
			if i >= 0 {
				page.leases[i] = lease.Clone(op.Lease)
			}
		case lease.Delete:
			i := page.leases.Index(op.Previous.Resource, op.Previous.Consumer, op.Previous.Instance)
			if i >= 0 {
				page.leases = append(page.leases[:i], page.leases[i+1:]...)
			}
		}
	}

	sort.Sort(page.leases)

	return nil
}

func (p *Provider) leasePage(resource string) *leasePage {
	p.mutex.RLock()
	page, ok := p.leasePages[resource]
	p.mutex.RUnlock()
	if ok {
		return page
	}

	p.mutex.Lock()
	defer p.mutex.Unlock()
	page, ok = p.leasePages[resource]
	if ok {
		return page
	}
	page = new(leasePage)
	p.leasePages[resource] = page
	return page
}

/*
// Leases will return the current set of leases for the requested resource.
//
// If the provided resource is empty all leases will be returned.
func (p *Provider) Leases(resource string) (leases lease.Set, err error) {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	p.refresh(resource)
	p.leases[]
	leases = make(lease.Set, len(p.leases))
	copy(leases, p.leases)
	return
}

// Acquire will attempt to create or renew a lease for the given resource and
// consumer.
func (p *Provider) Acquire(resource, consumer, instance string, env environment.Environment, limit uint, duration, decay time.Duration) (result lease.Lease, allocation uint, accepted bool, err error) {
	now := time.Now()

	p.mutex.Lock()
	defer p.mutex.Unlock()

	p.refresh()

	// Check to see whether this is a renewal or an existing allocation
	index := -1
	if len(p.leases) > 0 {
		index = p.leases.Index(resource, consumer, instance)
	}

	// Determine whether this is a new allocation

	// Allocate a map if this is the first lease
	if p.allocations == nil {
		p.allocations = make(map[string]uint)
	}

	// Record the lease
	result.Resource = resource
	result.Consumer = consumer
	result.Instance = instance
	result.Environment = env
	result.Status = lease.Active
	result.Renewed = now
	result.Duration = duration
	result.Decay = decay

	if index == -1 {
		// This is a new lease
		result.Started = now

		// If this is a new allocation, check whether we've already exceeded the limit
		allocation = p.allocations[resource]
		if allocation < limit {
			allocation++
			p.allocations[resource] = allocation
		}

		p.leases = append(p.leases, result)
	} else {
		// This is a renewal of a lease that may be active, expired or pending.
		result.Started = p.leases[index].Started
		result.Status = p.leases[index].Status
		p.leases[index] = result
	}

	accepted = result.Status == lease.Active

	return
}

// Update will update the environment associated with a lease. It will not
// renew the lease.
func (p *Provider) Update(resource, consumer, instance string, env environment.Environment) (result lease.Lease, err error) {
	err = errors.New("Lease updating has not been written yet")

	p.mutex.Lock()
	defer p.mutex.Unlock()

	p.refresh()

	index := p.leases.Index(resource, consumer, instance)
	if index == -1 {
		// TODO: Return error?
		return
	}

	p.leases[index].Environment = env

	return
}

// Release will remove the lease for the given resource, consumer and instance.
func (p *Provider) Release(resource, consumer, instance string) (err error) {
	p.mutex.Lock()
	defer p.mutex.Unlock()

	p.refresh()

	// Look for the lease, which might not exist after the cull
	index := -1
	if len(p.leases) > 0 {
		index = p.leases.Index(resource, consumer, instance)
	}

	// Exit if there's no lease to remove
	if index == -1 {
		return
	}

	// Remove the lease
	p.remove(index)

	return
}

// refresh will update leases statuses and remove all decayed leases from the
// provider. The caller is expected to hold a write lock for the duration of
// the call.
func (p *Provider) refresh() {
	// Remove decayed leases, update expired leases and promote pending leases
	//
	// It's safe to do this in one pass because the leases are sorted with
	// active and released coming before pending.
	for i := 0; i < len(p.leases); i++ {
		l := &p.leases[i]
		switch l.Status {
		case lease.Active, lease.Released:
			if l.Decayed() {
				p.remove(i)
				i--
			} else if l.Expired() {
				l.Status = lease.Released
			}
		case lease.Queued:
			allocation = p.allocations[l.Resource]
			if allocation < l.Limit {

			}
		}
	}

	// Pass 2: Promote pending leases to active
	for i := 0; i < len(p.leases); {
		l := &p.leases[i]
		if l.Status != lease.Queued {
			continue
		}
	}
}

// remove will remove the lease at the given index. If the index is invalid
// remove will panic.
//
// The caller is expected to hold a write lock for the duration of the call.
func (p *Provider) remove(index int) {
	// Determine the resource
	resource := p.leases[index].Resource

	// Perform some sanity checks
	if p.allocations == nil {
		panic("allocation map is nil when it shouldn't be")
	}
	allocation := p.allocations[resource]
	if allocation <= 0 {
		panic("allocation dropped below zero")
	}

	// Remove the lease
	p.leases = append(p.leases[:index], p.leases[index+1:]...)
	p.allocations[resource] = allocation - 1
}
*/
<commit_msg>memprov: Remove old code<commit_after>package memprov

import (
	"errors"
	"sort"
	"sync"

	"github.com/scjalliance/resourceful/lease"
)

// leasePage is an in-memory page of lease data for a single resource.
type leasePage struct {
	mutex    sync.RWMutex
	revision uint64
	leases   lease.Set
}

// Provider provides memory-based lease management.
type Provider struct {
	mutex      sync.RWMutex
	leasePages map[string]*leasePage // The lease set for each resource
}

// New returns a new memory provider.
func New() *Provider {
	return &Provider{
		leasePages: make(map[string]*leasePage),
	}
}

// LeaseView returns the current revision and lease set for the resource.
func (p *Provider) LeaseView(resource string) (revision uint64, leases lease.Set, err error) {
	page := p.leasePage(resource)
	page.mutex.RLock()
	defer page.mutex.RUnlock()

	revision = page.revision
	leases = make(lease.Set, len(page.leases))
	copy(leases, page.leases)
	return
}

// LeaseCommit will attempt to apply the operations described in the lease
// transaction.
func (p *Provider) LeaseCommit(tx *lease.Tx) error {
	ops := tx.Ops()
	if len(ops) == 0 {
		// Nothing to commit
		return nil
	}

	page := p.leasePage(tx.Resource())
	page.mutex.Lock()
	defer page.mutex.Unlock()
	if page.revision != tx.Revision() {
		return errors.New("Unable to commit lease transaction due to opportunistic lock conflict")
	}
	page.revision++
	for _, op := range ops {
		switch op.Type {
		case lease.Create:
			page.leases = append(page.leases, op.Lease)
		case lease.Update:
			i := page.leases.Index(op.Previous.Resource, op.Previous.Consumer, op.Previous.Instance)
			if i >= 0 {
				page.leases[i] = lease.Clone(op.Lease)
			}
		case lease.Delete:
			i := page.leases.Index(op.Previous.Resource, op.Previous.Consumer, op.Previous.Instance)
			if i >= 0 {
				page.leases = append(page.leases[:i], page.leases[i+1:]...)
			}
		}
	}

	sort.Sort(page.leases)

	return nil
}

func (p *Provider) leasePage(resource string) *leasePage {
	p.mutex.RLock()
	page, ok := p.leasePages[resource]
	p.mutex.RUnlock()
	if ok {
		return page
	}

	p.mutex.Lock()
	defer p.mutex.Unlock()
	page, ok = p.leasePages[resource]
	if ok {
		return page
	}
	page = new(leasePage)
	p.leasePages[resource] = page
	return page
}
<|endoftext|>


<commit_before>package ns1

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"
	"strings"

	"gopkg.in/ns1/ns1-go.v2/rest"
	"gopkg.in/ns1/ns1-go.v2/rest/model/dns"
	"gopkg.in/ns1/ns1-go.v2/rest/model/filter"

	"github.com/StackExchange/dnscontrol/v3/models"
	"github.com/StackExchange/dnscontrol/v3/pkg/diff"
	"github.com/StackExchange/dnscontrol/v3/providers"
)

var docNotes = providers.DocumentationNotes{
	providers.CanGetZones:            providers.Can(),
	providers.CanUseAlias:            providers.Can(),
	providers.CanUseCAA:              providers.Can(),
	providers.CanUseDS:		  providers.Can(),
	providers.CanUseDSForChildren:	  providers.Can(),
	providers.CanUseNAPTR:            providers.Can(),
	providers.CanUsePTR:              providers.Can(),
	providers.DocCreateDomains:       providers.Can(),
	providers.DocDualHost:            providers.Can(),
	providers.DocOfficiallySupported: providers.Cannot(),
}

func init() {
	fns := providers.DspFuncs{
		Initializer:   newProvider,
		RecordAuditor: AuditRecords,
	}
	providers.RegisterDomainServiceProviderType("NS1", fns, providers.CanUseSRV, docNotes)
	providers.RegisterCustomRecordType("NS1_URLFWD", "NS1", "URLFWD")
}

type nsone struct {
	*rest.Client
}

func newProvider(creds map[string]string, meta json.RawMessage) (providers.DNSServiceProvider, error) {
	if creds["api_token"] == "" {
		return nil, fmt.Errorf("api_token required for ns1")
	}
	return &nsone{rest.NewClient(http.DefaultClient, rest.SetAPIKey(creds["api_token"]))}, nil
}

func (n *nsone) EnsureDomainExists(domain string) error {
	// This enables the create-domains subcommand

	zone := dns.NewZone(domain)
	_, err := n.Zones.Create(zone)

	if err == rest.ErrZoneExists {
		// if domain exists already, just return nil, nothing to do here.
		return nil
	}

	return err
}

func (n *nsone) GetNameservers(domain string) ([]*models.Nameserver, error) {
	z, _, err := n.Zones.Get(domain)
	if err != nil {
		return nil, err
	}
	return models.ToNameservers(z.DNSServers)
}

// GetZoneRecords gets the records of a zone and returns them in RecordConfig format.
func (n *nsone) GetZoneRecords(domain string) (models.Records, error) {
	z, _, err := n.Zones.Get(domain)
	if err != nil {
		return nil, err
	}

	found := models.Records{}
	for _, r := range z.Records {
		zrs, err := convert(r, domain)
		if err != nil {
			return nil, err
		}
		found = append(found, zrs...)
	}
	return found, nil
}

func (n *nsone) GetDomainCorrections(dc *models.DomainConfig) ([]*models.Correction, error) {
	dc.Punycode()
	//dc.CombineMXs()

	domain := dc.Name

	// Get existing records
	existingRecords, err := n.GetZoneRecords(domain)
	if err != nil {
		return nil, err
	}

	existingGrouped := existingRecords.GroupedByKey()
	desiredGrouped := dc.Records.GroupedByKey()

	//  Normalize
	models.PostProcessRecords(existingRecords)

	differ := diff.New(dc)
	changedGroups, err := differ.ChangedGroups(existingRecords)
	if err != nil {
		return nil, err
	}
	corrections := []*models.Correction{}
	// each name/type is given to the api as a unit.
	for k, descs := range changedGroups {
		key := k

		desc := strings.Join(descs, "\n")
		_, current := existingGrouped[k]
		recs, wanted := desiredGrouped[k]
		if wanted && !current {
			// pure addition
			corrections = append(corrections, &models.Correction{
				Msg: desc,
				F:   func() error { return n.add(recs, dc.Name) },
			})
		} else if current && !wanted {
			// pure deletion
			corrections = append(corrections, &models.Correction{
				Msg: desc,
				F:   func() error { return n.remove(key, dc.Name) },
			})
		} else {
			// modification
			corrections = append(corrections, &models.Correction{
				Msg: desc,
				F:   func() error { return n.modify(recs, dc.Name) },
			})
		}
	}
	return corrections, nil
}

func (n *nsone) add(recs models.Records, domain string) error {
	_, err := n.Records.Create(buildRecord(recs, domain, ""))
	return err
}

func (n *nsone) remove(key models.RecordKey, domain string) error {
	_, err := n.Records.Delete(domain, key.NameFQDN, key.Type)
	return err
}

func (n *nsone) modify(recs models.Records, domain string) error {
	_, err := n.Records.Update(buildRecord(recs, domain, ""))
	return err
}

func buildRecord(recs models.Records, domain string, id string) *dns.Record {
	r := recs[0]
	rec := &dns.Record{
		Domain:  r.GetLabelFQDN(),
		Type:    r.Type,
		ID:      id,
		TTL:     int(r.TTL),
		Zone:    domain,
		Filters: []*filter.Filter{}, // Work through a bug in the NS1 API library that causes 400 Input validation failed (Value None for field '<obj>.filters' is not of type array)
	}
	for _, r := range recs {
		if r.Type == "MX" {
			rec.AddAnswer(&dns.Answer{Rdata: strings.Split(fmt.Sprintf("%d %v", r.MxPreference, r.GetTargetField()), " ")})
		} else if r.Type == "TXT" {
			rec.AddAnswer(&dns.Answer{Rdata: r.TxtStrings})
		} else if r.Type == "CAA" {
			rec.AddAnswer(&dns.Answer{
				Rdata: []string{
					fmt.Sprintf("%v", r.CaaFlag),
					r.CaaTag,
					r.GetTargetField(),
				}})
		} else if r.Type == "SRV" {
			rec.AddAnswer(&dns.Answer{Rdata: strings.Split(fmt.Sprintf("%d %d %d %v", r.SrvPriority, r.SrvWeight, r.SrvPort, r.GetTargetField()), " ")})
		} else if r.Type == "NAPTR" {
			rec.AddAnswer(&dns.Answer{Rdata: []string{
				strconv.Itoa(int(r.NaptrOrder)),
				strconv.Itoa(int(r.NaptrPreference)),
				r.NaptrFlags,
				r.NaptrService,
				r.NaptrRegexp,
				r.GetTargetField()}})
		} else if r.Type == "DS" {
			rec.AddAnswer(&dns.Answer{Rdata: []string{
				strconv.Itoa(int(r.DsKeyTag)),
				strconv.Itoa(int(r.DsAlgorithm)),
				strconv.Itoa(int(r.DsDigestType)),
				r.DsDigest}})
		} else {
			rec.AddAnswer(&dns.Answer{Rdata: strings.Split(r.GetTargetField(), " ")})
		}
	}
	return rec
}

func convert(zr *dns.ZoneRecord, domain string) ([]*models.RecordConfig, error) {
	found := []*models.RecordConfig{}
	for _, ans := range zr.ShortAns {
		rec := &models.RecordConfig{
			TTL:      uint32(zr.TTL),
			Original: zr,
		}
		rec.SetLabelFromFQDN(zr.Domain, domain)
		switch rtype := zr.Type; rtype {
		case "DNSKEY", "RRSIG":
			// if a zone is enabled for DNSSEC, NS1 autoconfigures DNSKEY & RRSIG records.
			// these entries are not modifiable via the API though, so we have to ignore them while converting.
			// 	ie. API returns "405 Operation on DNSSEC record is not allowed" on such operations
			continue
		case "ALIAS":
			rec.Type = rtype
			if err := rec.SetTarget(ans); err != nil {
				return nil, fmt.Errorf("unparsable %s record received from ns1: %w", rtype, err)
			}
		case "URLFWD":
			rec.Type = rtype
			if err := rec.SetTarget(ans); err != nil {
				return nil, fmt.Errorf("unparsable %s record received from ns1: %w", rtype, err)
			}
		case "CAA":
			//dnscontrol expects quotes around multivalue CAA entries, API doesn't add them
			xAns := strings.SplitN(ans, " ", 3)
			if err := rec.SetTargetCAAStrings(xAns[0], xAns[1], xAns[2]); err != nil {
				return nil, fmt.Errorf("unparsable %s record received from ns1: %w", rtype, err)
			}
		default:
			if err := rec.PopulateFromString(rtype, ans, domain); err != nil {
				return nil, fmt.Errorf("unparsable record received from ns1: %w", err)
			}
		}
		found = append(found, rec)
	}
	return found, nil
}
<commit_msg>ns1: enable autoDNSSEC capability (#1450)<commit_after>package ns1

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"
	"strings"

	"gopkg.in/ns1/ns1-go.v2/rest"
	"gopkg.in/ns1/ns1-go.v2/rest/model/dns"
	"gopkg.in/ns1/ns1-go.v2/rest/model/filter"

	"github.com/StackExchange/dnscontrol/v3/models"
	"github.com/StackExchange/dnscontrol/v3/pkg/diff"
	"github.com/StackExchange/dnscontrol/v3/providers"
)

var docNotes = providers.DocumentationNotes{
	providers.CanGetZones:            providers.Can(),
	providers.CanUseAlias:            providers.Can(),
	providers.CanAutoDNSSEC:	  providers.Can(),
	providers.CanUseCAA:              providers.Can(),
	providers.CanUseDS:		  providers.Can(),
	providers.CanUseDSForChildren:	  providers.Can(),
	providers.CanUseNAPTR:            providers.Can(),
	providers.CanUsePTR:              providers.Can(),
	providers.DocCreateDomains:       providers.Can(),
	providers.DocDualHost:            providers.Can(),
	providers.DocOfficiallySupported: providers.Cannot(),
}

func init() {
	fns := providers.DspFuncs{
		Initializer:   newProvider,
		RecordAuditor: AuditRecords,
	}
	providers.RegisterDomainServiceProviderType("NS1", fns, providers.CanUseSRV, docNotes)
	providers.RegisterCustomRecordType("NS1_URLFWD", "NS1", "URLFWD")
}

type nsone struct {
	*rest.Client
}

func newProvider(creds map[string]string, meta json.RawMessage) (providers.DNSServiceProvider, error) {
	if creds["api_token"] == "" {
		return nil, fmt.Errorf("api_token required for ns1")
	}
	return &nsone{rest.NewClient(http.DefaultClient, rest.SetAPIKey(creds["api_token"]))}, nil
}

func (n *nsone) EnsureDomainExists(domain string) error {
	// This enables the create-domains subcommand

	zone := dns.NewZone(domain)
	_, err := n.Zones.Create(zone)

	if err == rest.ErrZoneExists {
		// if domain exists already, just return nil, nothing to do here.
		return nil
	}

	return err
}

func (n *nsone) GetNameservers(domain string) ([]*models.Nameserver, error) {
	z, _, err := n.Zones.Get(domain)
	if err != nil {
		return nil, err
	}
	return models.ToNameservers(z.DNSServers)
}

// GetZoneRecords gets the records of a zone and returns them in RecordConfig format.
func (n *nsone) GetZoneRecords(domain string) (models.Records, error) {
	z, _, err := n.Zones.Get(domain)
	if err != nil {
		return nil, err
	}

	found := models.Records{}
	for _, r := range z.Records {
		zrs, err := convert(r, domain)
		if err != nil {
			return nil, err
		}
		found = append(found, zrs...)
	}
	return found, nil
}

// GetZoneDNSSEC gets DNSSEC status for zone. Returns true for enabled, false for disabled
// a domain in NS1 can be in 3 states:
//   1) DNSSEC is enabled  (returns true)
//   2) DNSSEC is disabled (returns false)
//   3) some error state   (return false plus the error)
func (n *nsone) GetZoneDNSSEC(domain string) (bool, error) {
	_, _, err := n.DNSSEC.Get(domain)

	// rest.ErrDNSECNotEnabled is our "disabled" state
	if err != nil && err == rest.ErrDNSECNotEnabled {
		return false, nil
	}

	// any other errors not expected, let's surface them
	if err != nil {
		return false, err
	}

	// no errors returned, we assume DNSSEC is enabled
	return true, nil
}

// getDomainCorrectionsDNSSEC creates DNSSEC zone corrections based on current state and preference
func (n *nsone) getDomainCorrectionsDNSSEC(domain, toggleDNSSEC string) (*models.Correction) {

	// get dnssec status from NS1 for domain
	// if errors are returned, we bail out without any DNSSEC corrections
	status, err := n.GetZoneDNSSEC(domain)
	if err != nil {
		return nil
	}

	if toggleDNSSEC == "on" && !status {
		// disabled, but prefer it on, let's enable DNSSEC
		return &models.Correction{
			Msg: "ENABLE DNSSEC",
			F:   func() error { return n.configureDNSSEC(domain, true) },
		}
	} else if toggleDNSSEC == "off" && status {
		// enabled, but prefer it off, let's disable DNSSEC
		return &models.Correction{
			Msg: "DISABLE DNSSEC",
			F:   func() error { return n.configureDNSSEC(domain, false) },
		}
	}
	return nil
}

func (n *nsone) GetDomainCorrections(dc *models.DomainConfig) ([]*models.Correction, error) {
	dc.Punycode()
	//dc.CombineMXs()

	domain := dc.Name

	// Get existing records
	existingRecords, err := n.GetZoneRecords(domain)
	if err != nil {
		return nil, err
	}

	existingGrouped := existingRecords.GroupedByKey()
	desiredGrouped := dc.Records.GroupedByKey()

	//  Normalize
	models.PostProcessRecords(existingRecords)

	differ := diff.New(dc)
	changedGroups, err := differ.ChangedGroups(existingRecords)
	if err != nil {
		return nil, err
	}

	corrections := []*models.Correction{}

        if dnssecCorrections := n.getDomainCorrectionsDNSSEC(domain, dc.AutoDNSSEC); dnssecCorrections != nil {
		corrections = append(corrections, dnssecCorrections)
	}

	// each name/type is given to the api as a unit.
	for k, descs := range changedGroups {
		key := k

		desc := strings.Join(descs, "\n")
		_, current := existingGrouped[k]
		recs, wanted := desiredGrouped[k]
		if wanted && !current {
			// pure addition
			corrections = append(corrections, &models.Correction{
				Msg: desc,
				F:   func() error { return n.add(recs, dc.Name) },
			})
		} else if current && !wanted {
			// pure deletion
			corrections = append(corrections, &models.Correction{
				Msg: desc,
				F:   func() error { return n.remove(key, dc.Name) },
			})
		} else {
			// modification
			corrections = append(corrections, &models.Correction{
				Msg: desc,
				F:   func() error { return n.modify(recs, dc.Name) },
			})
		}
	}
	return corrections, nil
}

func (n *nsone) add(recs models.Records, domain string) error {
	_, err := n.Records.Create(buildRecord(recs, domain, ""))
	return err
}

func (n *nsone) remove(key models.RecordKey, domain string) error {
	_, err := n.Records.Delete(domain, key.NameFQDN, key.Type)
	return err
}

func (n *nsone) modify(recs models.Records, domain string) error {
	_, err := n.Records.Update(buildRecord(recs, domain, ""))
	return err
}

// configureDNSSEC configures DNSSEC for a zone. Set 'enabled' to true to enable, false to disable.
// There's a cornercase, in which DNSSEC is globally disabled for the account.
// In that situation, enabling DNSSEC will always fail with:
//   #1: ENABLE DNSSEC
//   FAILURE! POST https://api.nsone.net/v1/zones/example.com: 400 DNSSEC support is not enabled for this account. Please contact support@ns1.com to enable it
//
// Unfortunately this is not detectable otherwise, so given that we have a nice error message, we just let this through.
//
func (n *nsone) configureDNSSEC(domain string, enabled bool) error {
	z, _, err := n.Zones.Get(domain)
	if err != nil {
		return err
	}
	z.DNSSEC = &enabled
	_, err = n.Zones.Update(z)
	return err
}

func buildRecord(recs models.Records, domain string, id string) *dns.Record {
	r := recs[0]
	rec := &dns.Record{
		Domain:  r.GetLabelFQDN(),
		Type:    r.Type,
		ID:      id,
		TTL:     int(r.TTL),
		Zone:    domain,
		Filters: []*filter.Filter{}, // Work through a bug in the NS1 API library that causes 400 Input validation failed (Value None for field '<obj>.filters' is not of type array)
	}
	for _, r := range recs {
		if r.Type == "MX" {
			rec.AddAnswer(&dns.Answer{Rdata: strings.Split(fmt.Sprintf("%d %v", r.MxPreference, r.GetTargetField()), " ")})
		} else if r.Type == "TXT" {
			rec.AddAnswer(&dns.Answer{Rdata: r.TxtStrings})
		} else if r.Type == "CAA" {
			rec.AddAnswer(&dns.Answer{
				Rdata: []string{
					fmt.Sprintf("%v", r.CaaFlag),
					r.CaaTag,
					r.GetTargetField(),
				}})
		} else if r.Type == "SRV" {
			rec.AddAnswer(&dns.Answer{Rdata: strings.Split(fmt.Sprintf("%d %d %d %v", r.SrvPriority, r.SrvWeight, r.SrvPort, r.GetTargetField()), " ")})
		} else if r.Type == "NAPTR" {
			rec.AddAnswer(&dns.Answer{Rdata: []string{
				strconv.Itoa(int(r.NaptrOrder)),
				strconv.Itoa(int(r.NaptrPreference)),
				r.NaptrFlags,
				r.NaptrService,
				r.NaptrRegexp,
				r.GetTargetField()}})
		} else if r.Type == "DS" {
			rec.AddAnswer(&dns.Answer{Rdata: []string{
				strconv.Itoa(int(r.DsKeyTag)),
				strconv.Itoa(int(r.DsAlgorithm)),
				strconv.Itoa(int(r.DsDigestType)),
				r.DsDigest}})
		} else {
			rec.AddAnswer(&dns.Answer{Rdata: strings.Split(r.GetTargetField(), " ")})
		}
	}
	return rec
}

func convert(zr *dns.ZoneRecord, domain string) ([]*models.RecordConfig, error) {
	found := []*models.RecordConfig{}
	for _, ans := range zr.ShortAns {
		rec := &models.RecordConfig{
			TTL:      uint32(zr.TTL),
			Original: zr,
		}
		rec.SetLabelFromFQDN(zr.Domain, domain)
		switch rtype := zr.Type; rtype {
		case "DNSKEY", "RRSIG":
			// if a zone is enabled for DNSSEC, NS1 autoconfigures DNSKEY & RRSIG records.
			// these entries are not modifiable via the API though, so we have to ignore them while converting.
			// 	ie. API returns "405 Operation on DNSSEC record is not allowed" on such operations
			continue
		case "ALIAS":
			rec.Type = rtype
			if err := rec.SetTarget(ans); err != nil {
				return nil, fmt.Errorf("unparsable %s record received from ns1: %w", rtype, err)
			}
		case "URLFWD":
			rec.Type = rtype
			if err := rec.SetTarget(ans); err != nil {
				return nil, fmt.Errorf("unparsable %s record received from ns1: %w", rtype, err)
			}
		case "CAA":
			//dnscontrol expects quotes around multivalue CAA entries, API doesn't add them
			xAns := strings.SplitN(ans, " ", 3)
			if err := rec.SetTargetCAAStrings(xAns[0], xAns[1], xAns[2]); err != nil {
				return nil, fmt.Errorf("unparsable %s record received from ns1: %w", rtype, err)
			}
		default:
			if err := rec.PopulateFromString(rtype, ans, domain); err != nil {
				return nil, fmt.Errorf("unparsable record received from ns1: %w", err)
			}
		}
		found = append(found, rec)
	}
	return found, nil
}
<|endoftext|>


<commit_before>// Package blackhole is an outbound handler that blocks all connections.
package blackhole

//go:generate go run $GOPATH/src/v2ray.com/core/common/errors/errorgen/main.go -pkg blackhole -path Proxy,Blackhole

import (
	"context"
	"time"

	"v2ray.com/core/common"
	"v2ray.com/core/proxy"
	"v2ray.com/core/transport/ray"
)

// Handler is an outbound connection that silently swallow the entire payload.
type Handler struct {
	response ResponseConfig
}

// New creates a new blackhole handler.
func New(ctx context.Context, config *Config) (*Handler, error) {
	response, err := config.GetInternalResponse()
	if err != nil {
		return nil, err
	}
	return &Handler{
		response: response,
	}, nil
}

// Process implements OutboundHandler.Dispatch().
func (h *Handler) Process(ctx context.Context, outboundRay ray.OutboundRay, dialer proxy.Dialer) error {
	h.response.WriteTo(outboundRay.OutboundOutput())
	// Sleep a little here to make sure the response is sent to client.
	time.Sleep(time.Second)
	outboundRay.OutboundOutput().CloseError()
	time.Sleep(time.Second * 2)
	return nil
}

func init() {
	common.Must(common.RegisterConfig((*Config)(nil), func(ctx context.Context, config interface{}) (interface{}, error) {
		return New(ctx, config.(*Config))
	}))
}
<commit_msg>fix delay in blackhole<commit_after>// Package blackhole is an outbound handler that blocks all connections.
package blackhole

//go:generate go run $GOPATH/src/v2ray.com/core/common/errors/errorgen/main.go -pkg blackhole -path Proxy,Blackhole

import (
	"context"
	"time"

	"v2ray.com/core/common"
	"v2ray.com/core/proxy"
	"v2ray.com/core/transport/ray"
)

// Handler is an outbound connection that silently swallow the entire payload.
type Handler struct {
	response ResponseConfig
}

// New creates a new blackhole handler.
func New(ctx context.Context, config *Config) (*Handler, error) {
	response, err := config.GetInternalResponse()
	if err != nil {
		return nil, err
	}
	return &Handler{
		response: response,
	}, nil
}

// Process implements OutboundHandler.Dispatch().
func (h *Handler) Process(ctx context.Context, outboundRay ray.OutboundRay, dialer proxy.Dialer) error {
	h.response.WriteTo(outboundRay.OutboundOutput())
	// Sleep a little here to make sure the response is sent to client.
	time.Sleep(time.Second)
	outboundRay.OutboundOutput().CloseError()
	return nil
}

func init() {
	common.Must(common.RegisterConfig((*Config)(nil), func(ctx context.Context, config interface{}) (interface{}, error) {
		return New(ctx, config.(*Config))
	}))
}
<|endoftext|>


<commit_before>package roger

import (
	"errors"
	"testing"

	"github.com/senseyeio/roger/constants"
)

func TestErrorPacketIsError(t *testing.T) {
	pkt := newErrorPacket(errors.New("test error"))
	if pkt.IsError() == false {
		t.Error("Test packet should return true when IsError is called")
	}
}

func TestErrorPacketResultObject(t *testing.T) {
	testError := errors.New("test error")
	pkt := newErrorPacket(testError)
	obj, err := pkt.GetResultObject()
	if err != testError {
		t.Error("An error packet should return the error when GetResultObject is called")
	}
	if obj != nil {
		t.Error("An error packet should return a nil object")
	}
}

func TestCommandFailurePacketIsError(t *testing.T) {
	failedCmdPkt := newPacket(0x01000002, []byte{})
	if failedCmdPkt.IsError() == false {
		t.Error("A command with an error flag should return true when IsError is called")
	}
}

func TestCommandFailurePacketIsOk(t *testing.T) {
	failedCmdPkt := newPacket(0x01000002, []byte{})
	if failedCmdPkt.IsOk() == true {
		t.Error("A command with an error flag should return false when IsOk is called")
	}
}

func TestCommandFailurePacketResultObject(t *testing.T) {
	failedCmdPkt := newPacket(0x01000002, []byte{})
	obj, err := failedCmdPkt.GetResultObject()
	if err.Error() != "Command error with status code: 1" {
		t.Error("A failed command packet's error message should contain the status code element of the command response")
	}
	if obj != nil {
		t.Error("A failed command packet should return a nil object")
	}
}

func TestCommandFailurePacketResultStatus(t *testing.T) {
	failedCmdPkt := newPacket(0x02000002, []byte{})
	obj, err := failedCmdPkt.GetResultObject()
	if err.Error() != "Command error with status: Invalid expression" {
		t.Error("A failed command packet's error message should contain the status message of the command response")
	}
	if obj != nil {
		t.Error("A failed command packet should return a nil object")
	}
}

func TestCommandSuccessPacketIsError(t *testing.T) {
	successfulCmdPkt := newPacket(0x01000003, []byte{})
	if successfulCmdPkt.IsError() == true {
		t.Error("A command without an error flag should return false when IsError is called")
	}
}

func TestCommandSuccessPacketIsOk(t *testing.T) {
	successfulCmdPkt := newPacket(0x01000003, []byte{})
	if successfulCmdPkt.IsOk() == false {
		t.Error("A command without an error flag should return true when IsOk is called")
	}
}

func TestEmptyResponsePacketResultObject(t *testing.T) {
	emptyPkt := newPacket(0x01000003, []byte{})
	obj, err := emptyPkt.GetResultObject()
	if err == nil {
		t.Error("An empty packet should return an error")
	}
	if obj != nil {
		t.Error("An empty packet should return a nil object")
	}
}

func TestSuccessfulResponseResultObject(t *testing.T) {
	client, _ := NewRClient("localhost", 6311)
	pkt := client.EvaluateSync("2")
	obj, err := pkt.GetResultObject()
	if err != nil {
		t.Error("A successful query should not result in an error")
	}
	if obj == nil {
		t.Error("A successful query should return a response object")
	}
}

func TestNonSEXPResponse(t *testing.T) {
	stringResp := newPacket(0x01000003, []byte{byte(constants.DtString)})
	obj, err := stringResp.GetResultObject()
	if err == nil {
		t.Error("Packets containing non SEXP content should return an error")
	}
	if obj != nil {
		t.Error("Packets containing non SEXP content should not return an object")
	}
}
<commit_msg>Added two tests to the new packet.GetError method.<commit_after>package roger

import (
	"errors"
	"testing"

	"github.com/senseyeio/roger/constants"
)

func TestErrorPacketIsError(t *testing.T) {
	pkt := newErrorPacket(errors.New("test error"))
	if pkt.IsError() == false {
		t.Error("Test packet should return true when IsError is called")
	}
}

func TestErrorPacketGetErrorNonNil(t *testing.T) {
	pkt := newErrorPacket(errors.New("test error"))
	err := pkt.GetError()
	if err == nil {
		t.Error("GetError should return a non nil error when the packet has an error")
	}
}

func TestErrorPacketGetErrorNil(t *testing.T) {
	pkt := newErrorPacket(nil)
	err := pkt.GetError()
	if err != nil {
		t.Error("GetError should return nil when the packet has no error")
	}
}

func TestErrorPacketResultObject(t *testing.T) {
	testError := errors.New("test error")
	pkt := newErrorPacket(testError)
	obj, err := pkt.GetResultObject()
	if err != testError {
		t.Error("An error packet should return the error when GetResultObject is called")
	}
	if obj != nil {
		t.Error("An error packet should return a nil object")
	}
}

func TestCommandFailurePacketIsError(t *testing.T) {
	failedCmdPkt := newPacket(0x01000002, []byte{})
	if failedCmdPkt.IsError() == false {
		t.Error("A command with an error flag should return true when IsError is called")
	}
}

func TestCommandFailurePacketIsOk(t *testing.T) {
	failedCmdPkt := newPacket(0x01000002, []byte{})
	if failedCmdPkt.IsOk() == true {
		t.Error("A command with an error flag should return false when IsOk is called")
	}
}

func TestCommandFailurePacketResultObject(t *testing.T) {
	failedCmdPkt := newPacket(0x01000002, []byte{})
	obj, err := failedCmdPkt.GetResultObject()
	if err.Error() != "Command error with status code: 1" {
		t.Error("A failed command packet's error message should contain the status code element of the command response")
	}
	if obj != nil {
		t.Error("A failed command packet should return a nil object")
	}
}

func TestCommandFailurePacketResultStatus(t *testing.T) {
	failedCmdPkt := newPacket(0x02000002, []byte{})
	obj, err := failedCmdPkt.GetResultObject()
	if err.Error() != "Command error with status: Invalid expression" {
		t.Error("A failed command packet's error message should contain the status message of the command response")
	}
	if obj != nil {
		t.Error("A failed command packet should return a nil object")
	}
}

func TestCommandSuccessPacketIsError(t *testing.T) {
	successfulCmdPkt := newPacket(0x01000003, []byte{})
	if successfulCmdPkt.IsError() == true {
		t.Error("A command without an error flag should return false when IsError is called")
	}
}

func TestCommandSuccessPacketIsOk(t *testing.T) {
	successfulCmdPkt := newPacket(0x01000003, []byte{})
	if successfulCmdPkt.IsOk() == false {
		t.Error("A command without an error flag should return true when IsOk is called")
	}
}

func TestEmptyResponsePacketResultObject(t *testing.T) {
	emptyPkt := newPacket(0x01000003, []byte{})
	obj, err := emptyPkt.GetResultObject()
	if err == nil {
		t.Error("An empty packet should return an error")
	}
	if obj != nil {
		t.Error("An empty packet should return a nil object")
	}
}

func TestSuccessfulResponseResultObject(t *testing.T) {
	client, _ := NewRClient("localhost", 6311)
	pkt := client.EvaluateSync("2")
	obj, err := pkt.GetResultObject()
	if err != nil {
		t.Error("A successful query should not result in an error")
	}
	if obj == nil {
		t.Error("A successful query should return a response object")
	}
}

func TestNonSEXPResponse(t *testing.T) {
	stringResp := newPacket(0x01000003, []byte{byte(constants.DtString)})
	obj, err := stringResp.GetResultObject()
	if err == nil {
		t.Error("Packets containing non SEXP content should return an error")
	}
	if obj != nil {
		t.Error("Packets containing non SEXP content should not return an object")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"testing"
)

func TestPrepareSDQuery(t *testing.T) {
	original := "What is artificial intelligence?"
	expected := "#weight( 0.85 #combine( what is artificial intelligence ) 0.10 #combine( #1( what is ) #1( is artificial ) #1( artificial intelligence ) ) 0.05 #combine ( #uw8( what is ) #uw8( is artificial ) #uw8( artificial intelligence ) ) )"

	if sd := PrepareSDQuery(GetQueryTerms(original)); sd != expected {
		t.Errorf("Expected payload `%s` but got `%s`", expected, sd)
	}
}
<commit_msg>Update indriproducer_test.go<commit_after>package main

import (
	"testing"
)

func TestPrepareSDQuery(t *testing.T) {
	original := "What is artificial intelligence?"
	expected := "#weight( 0.85 #combine( what is artificial intelligence ) 0.10 #combine( #1( artificial intelligence ) ) 0.05 #combine ( #uw8( artificial intelligence ) ) )"

	if sd := PrepareSDQuery(GetQueryTerms(original)); sd != expected {
		t.Errorf("Expected payload `%s` but got `%s`", expected, sd)
	}
}
<|endoftext|>


<commit_before>package integration_tests

import (
	"fmt"
	"io/ioutil"
	"path/filepath"
	"testing"

	"github.com/foomo/htpasswd"
	"github.com/stretchr/testify/assert"

	"github.com/reviewboard/rb-gateway/config"
	"github.com/reviewboard/rb-gateway/helpers"
	"github.com/reviewboard/rb-gateway/repositories"
	"github.com/reviewboard/rb-gateway/repositories/events"
	"github.com/reviewboard/rb-gateway/repositories/hooks"
)

// Set up the configuration and write it to disk.
func setupConfig(t *testing.T, upstream repositories.Repository) (string, config.Config) {
	t.Helper()
	assert := assert.New(t)

	cfgDir, err := ioutil.TempDir("", "rb-gateway-config-")
	assert.Nil(err)
	cfg := helpers.CreateTestConfig(t, upstream)

	hookStorePath := filepath.Join(cfgDir, "webhooks.json")
	cfg.WebhookStorePath = hookStorePath
	cfg.TokenStorePath = filepath.Join(cfgDir, "tokens.dat")
	cfg.HtpasswdPath = filepath.Join(cfgDir, "htpasswd")

	assert.Nil(htpasswd.SetPassword(cfg.HtpasswdPath, "username", "password", htpasswd.HashBCrypt))

	helpers.WriteConfig(t, filepath.Join(cfgDir, "config.json"), &cfg)

	return cfgDir, cfg
}

// Set up the webhook store and write it to disk.
func setupStore(t *testing.T, serverUrl string, cfg *config.Config) *hooks.Webhook {
	assert := assert.New(t)
	t.Helper()

	var repoName string
	for repoName, _ = range cfg.Repositories {
		break
	}

	assert.NotEqual(0, len(repoName))
	hook := &hooks.Webhook{
		Id:      "test-hook",
		Enabled: true,
		Url:     fmt.Sprintf("%s/test-hook", serverUrl),
		Secret:  "top-secret-123",
		Events:  []string{events.PushEvent},
		Repos:   []string{repoName},
	}

	store := hooks.WebhookStore{
		hook.Id: hook,
	}

    assert.Nil(store.Save(cfg.WebhookStorePath))

	return hook
}

type testCase struct {
	recorded *helpers.RecordedRequest
	message  string
	commitId string
	target   events.PushPayloadCommitTarget
}

func runTests(t *testing.T, cases []testCase, upstream repositories.Repository, hook *hooks.Webhook) {
	assert := assert.New(t)

	for i, testCase := range cases {
		request := testCase.recorded.Request
		body := testCase.recorded.Body

		assert.Equalf("/test-hook", request.URL.Path, "URL for request %d does not match", i)
		assert.Equalf(events.PushEvent, request.Header.Get("X-RBG-Event"), "X-RBG-Event header for request %d does not match", i)
		assert.Equalf(hook.SignPayload(body), request.Header.Get("X-RBG-Signature"), "Signature for request %d does not match", i)
		payload := events.PushPayload{
			Repository: upstream.GetName(),
			Commits: []events.PushPayloadCommit{
				{
					Id:      testCase.commitId,
					Message: testCase.message,
					Target:  testCase.target,
				},
			},
		}

		rawJson, err := events.MarshalPayload(payload)
		assert.Nil(err)

		assert.Equalf(string(rawJson), string(body), "Body for request %d does not match", i)
	}
}
<commit_msg>Ran go fmt<commit_after>package integration_tests

import (
	"fmt"
	"io/ioutil"
	"path/filepath"
	"testing"

	"github.com/foomo/htpasswd"
	"github.com/stretchr/testify/assert"

	"github.com/reviewboard/rb-gateway/config"
	"github.com/reviewboard/rb-gateway/helpers"
	"github.com/reviewboard/rb-gateway/repositories"
	"github.com/reviewboard/rb-gateway/repositories/events"
	"github.com/reviewboard/rb-gateway/repositories/hooks"
)

// Set up the configuration and write it to disk.
func setupConfig(t *testing.T, upstream repositories.Repository) (string, config.Config) {
	t.Helper()
	assert := assert.New(t)

	cfgDir, err := ioutil.TempDir("", "rb-gateway-config-")
	assert.Nil(err)
	cfg := helpers.CreateTestConfig(t, upstream)

	hookStorePath := filepath.Join(cfgDir, "webhooks.json")
	cfg.WebhookStorePath = hookStorePath
	cfg.TokenStorePath = filepath.Join(cfgDir, "tokens.dat")
	cfg.HtpasswdPath = filepath.Join(cfgDir, "htpasswd")

	assert.Nil(htpasswd.SetPassword(cfg.HtpasswdPath, "username", "password", htpasswd.HashBCrypt))

	helpers.WriteConfig(t, filepath.Join(cfgDir, "config.json"), &cfg)

	return cfgDir, cfg
}

// Set up the webhook store and write it to disk.
func setupStore(t *testing.T, serverUrl string, cfg *config.Config) *hooks.Webhook {
	assert := assert.New(t)
	t.Helper()

	var repoName string
	for repoName, _ = range cfg.Repositories {
		break
	}

	assert.NotEqual(0, len(repoName))
	hook := &hooks.Webhook{
		Id:      "test-hook",
		Enabled: true,
		Url:     fmt.Sprintf("%s/test-hook", serverUrl),
		Secret:  "top-secret-123",
		Events:  []string{events.PushEvent},
		Repos:   []string{repoName},
	}

	store := hooks.WebhookStore{
		hook.Id: hook,
	}

	assert.Nil(store.Save(cfg.WebhookStorePath))

	return hook
}

type testCase struct {
	recorded *helpers.RecordedRequest
	message  string
	commitId string
	target   events.PushPayloadCommitTarget
}

func runTests(t *testing.T, cases []testCase, upstream repositories.Repository, hook *hooks.Webhook) {
	assert := assert.New(t)

	for i, testCase := range cases {
		request := testCase.recorded.Request
		body := testCase.recorded.Body

		assert.Equalf("/test-hook", request.URL.Path, "URL for request %d does not match", i)
		assert.Equalf(events.PushEvent, request.Header.Get("X-RBG-Event"), "X-RBG-Event header for request %d does not match", i)
		assert.Equalf(hook.SignPayload(body), request.Header.Get("X-RBG-Signature"), "Signature for request %d does not match", i)
		payload := events.PushPayload{
			Repository: upstream.GetName(),
			Commits: []events.PushPayloadCommit{
				{
					Id:      testCase.commitId,
					Message: testCase.message,
					Target:  testCase.target,
				},
			},
		}

		rawJson, err := events.MarshalPayload(payload)
		assert.Nil(err)

		assert.Equalf(string(rawJson), string(body), "Body for request %d does not match", i)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Ceph-CSI Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cephfs

import (
	"context"
	"strings"

	"github.com/ceph/ceph-csi/internal/util"

	"github.com/golang/protobuf/ptypes/timestamp"
)

// autoProtect points to the snapshot auto-protect feature of
// the subvolume.
const (
	autoProtect = "snapshot-autoprotect"
)

// cephfsSnapshot represents a CSI snapshot and its cluster information.
type cephfsSnapshot struct {
	NamePrefix string
	Monitors   string
	// MetadataPool & Pool fields are not used atm. But its definitely good to have it in this struct
	// so keeping it here
	MetadataPool string
	Pool         string
	ClusterID    string
	RequestName  string
	// ReservedID represents the ID reserved for a snapshot
	ReservedID string
}

func (vo *volumeOptions) createSnapshot(ctx context.Context, cr *util.Credentials, snapID, volID volumeID) error {
	args := []string{
		"fs",
		"subvolume",
		"snapshot",
		"create",
		vo.FsName,
		string(volID),
		string(snapID),
		"--group_name",
		vo.SubvolumeGroup,
		"-m", vo.Monitors,
		"-c", util.CephConfigPath,
		"-n", cephEntityClientPrefix + cr.ID,
		"--keyfile=" + cr.KeyFile,
	}

	err := execCommandErr(
		ctx,
		"ceph",
		args[:]...)
	if err != nil {
		util.ErrorLog(ctx, "failed to create subvolume snapshot %s %s(%s) in fs %s", string(snapID), string(volID), err, vo.FsName)
		return err
	}
	return nil
}

func deleteSnapshot(ctx context.Context, volOptions *volumeOptions, cr *util.Credentials, snapID, volID volumeID) error {
	args := []string{
		"fs",
		"subvolume",
		"snapshot",
		"rm",
		volOptions.FsName,
		string(volID),
		string(snapID),
		"--group_name",
		volOptions.SubvolumeGroup,
		"-m", volOptions.Monitors,
		"-c", util.CephConfigPath,
		"-n", cephEntityClientPrefix + cr.ID,
		"--keyfile=" + cr.KeyFile,
		"--force",
	}

	err := execCommandErr(
		ctx,
		"ceph",
		args[:]...)
	if err != nil {
		util.ErrorLog(ctx, "failed to delete subvolume snapshot %s %s(%s) in fs %s", string(snapID), string(volID), err, volOptions.FsName)
		return err
	}
	return nil
}

type snapshotInfo struct {
	CreatedAt        string `json:"created_at"`
	CreationTime     *timestamp.Timestamp
	DataPool         string `json:"data_pool"`
	HasPendingClones string `json:"has_pending_clones"`
	Protected        string `json:"protected"`
	Size             int    `json:"size"`
}

func getSnapshotInfo(ctx context.Context, volOptions *volumeOptions, cr *util.Credentials, snapID, volID volumeID) (snapshotInfo, error) {
	snap := snapshotInfo{}
	args := []string{
		"fs",
		"subvolume",
		"snapshot",
		"info",
		volOptions.FsName,
		string(volID),
		string(snapID),
		"--group_name",
		volOptions.SubvolumeGroup,
		"-m", volOptions.Monitors,
		"-c", util.CephConfigPath,
		"-n", cephEntityClientPrefix + cr.ID,
		"--keyfile=" + cr.KeyFile,
		"--format=json",
	}
	err := execCommandJSON(
		ctx,
		&snap,
		"ceph",
		args[:]...)
	if err != nil {
		if strings.Contains(err.Error(), snapNotFound) {
			return snapshotInfo{}, ErrSnapNotFound
		}
		util.ErrorLog(ctx, "failed to get subvolume snapshot info %s %s(%s) in fs %s", string(snapID), string(volID), err, volOptions.FsName)
		return snapshotInfo{}, err
	}
	return snap, nil
}

func protectSnapshot(ctx context.Context, volOptions *volumeOptions, cr *util.Credentials, snapID, volID volumeID) error {
	// If "snapshot-autoprotect" feature is present, The ProtectSnapshot
	// call should be treated as a no-op.
	if checkSubvolumeHasFeature(autoProtect, volOptions.Features) {
		return nil
	}
	args := []string{
		"fs",
		"subvolume",
		"snapshot",
		"protect",
		volOptions.FsName,
		string(volID),
		string(snapID),
		"--group_name",
		volOptions.SubvolumeGroup,
		"-m", volOptions.Monitors,
		"-c", util.CephConfigPath,
		"-n", cephEntityClientPrefix + cr.ID,
		"--keyfile=" + cr.KeyFile,
	}

	err := execCommandErr(
		ctx,
		"ceph",
		args[:]...)
	if err != nil {
		if strings.Contains(err.Error(), snapProtectionExist) {
			return nil
		}
		util.ErrorLog(ctx, "failed to protect subvolume snapshot %s %s(%s) in fs %s", string(snapID), string(volID), err, volOptions.FsName)
		return err
	}
	return nil
}

func unprotectSnapshot(ctx context.Context, volOptions *volumeOptions, cr *util.Credentials, snapID, volID volumeID) error {
	// If "snapshot-autoprotect" feature is present, The UnprotectSnapshot
	// call should be treated as a no-op.
	if checkSubvolumeHasFeature(autoProtect, volOptions.Features) {
		return nil
	}
	args := []string{
		"fs",
		"subvolume",
		"snapshot",
		"unprotect",
		volOptions.FsName,
		string(volID),
		string(snapID),
		"--group_name",
		volOptions.SubvolumeGroup,
		"-m", volOptions.Monitors,
		"-c", util.CephConfigPath,
		"-n", cephEntityClientPrefix + cr.ID,
		"--keyfile=" + cr.KeyFile,
	}

	err := execCommandErr(
		ctx,
		"ceph",
		args[:]...)
	if err != nil {
		// In case the snap is already unprotected we get ErrSnapProtectionExist error code
		// in that case we are safe and we could discard this error.
		if strings.Contains(err.Error(), snapProtectionExist) {
			return nil
		}
		util.ErrorLog(ctx, "failed to unprotect subvolume snapshot %s %s(%s) in fs %s", string(snapID), string(volID), err, volOptions.FsName)
		return err
	}
	return nil
}

func cloneSnapshot(ctx context.Context, parentVolOptions *volumeOptions, cr *util.Credentials, volID, snapID, cloneID volumeID, cloneVolOptions *volumeOptions) error {
	args := []string{
		"fs",
		"subvolume",
		"snapshot",
		"clone",
		parentVolOptions.FsName,
		string(volID),
		string(snapID),
		string(cloneID),
		"--group_name",
		parentVolOptions.SubvolumeGroup,
		"--target_group_name",
		cloneVolOptions.SubvolumeGroup,
		"-m", parentVolOptions.Monitors,
		"-c", util.CephConfigPath,
		"-n", cephEntityClientPrefix + cr.ID,
		"--keyfile=" + cr.KeyFile,
	}
	if cloneVolOptions.Pool != "" {
		args = append(args, "--pool_layout", cloneVolOptions.Pool)
	}

	err := execCommandErr(
		ctx,
		"ceph",
		args[:]...)

	if err != nil {
		util.ErrorLog(ctx, "failed to clone subvolume snapshot %s %s(%s) in fs %s", string(cloneID), string(volID), err, parentVolOptions.FsName)
		if strings.HasPrefix(err.Error(), volumeNotFound) {
			return ErrVolumeNotFound
		}
		return err
	}
	return nil
}
<commit_msg>cephfs: implement createSnapshot() with go-ceph<commit_after>/*
Copyright 2020 The Ceph-CSI Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cephfs

import (
	"context"
	"strings"

	"github.com/ceph/ceph-csi/internal/util"

	"github.com/golang/protobuf/ptypes/timestamp"
)

// autoProtect points to the snapshot auto-protect feature of
// the subvolume.
const (
	autoProtect = "snapshot-autoprotect"
)

// cephfsSnapshot represents a CSI snapshot and its cluster information.
type cephfsSnapshot struct {
	NamePrefix string
	Monitors   string
	// MetadataPool & Pool fields are not used atm. But its definitely good to have it in this struct
	// so keeping it here
	MetadataPool string
	Pool         string
	ClusterID    string
	RequestName  string
	// ReservedID represents the ID reserved for a snapshot
	ReservedID string
}

func (vo *volumeOptions) createSnapshot(ctx context.Context, cr *util.Credentials, snapID, volID volumeID) error {
	fsa, err := vo.conn.GetFSAdmin()
	if err != nil {
		util.ErrorLog(ctx, "could not get FSAdmin: %s", err)
		return err
	}

	err = fsa.CreateSubVolumeSnapshot(vo.FsName, vo.SubvolumeGroup, string(volID), string(snapID))
	if err != nil {
		util.ErrorLog(ctx, "failed to create subvolume snapshot %s %s(%s) in fs %s", string(snapID), string(volID), err, vo.FsName)
		return err
	}
	return nil
}

func deleteSnapshot(ctx context.Context, volOptions *volumeOptions, cr *util.Credentials, snapID, volID volumeID) error {
	args := []string{
		"fs",
		"subvolume",
		"snapshot",
		"rm",
		volOptions.FsName,
		string(volID),
		string(snapID),
		"--group_name",
		volOptions.SubvolumeGroup,
		"-m", volOptions.Monitors,
		"-c", util.CephConfigPath,
		"-n", cephEntityClientPrefix + cr.ID,
		"--keyfile=" + cr.KeyFile,
		"--force",
	}

	err := execCommandErr(
		ctx,
		"ceph",
		args[:]...)
	if err != nil {
		util.ErrorLog(ctx, "failed to delete subvolume snapshot %s %s(%s) in fs %s", string(snapID), string(volID), err, volOptions.FsName)
		return err
	}
	return nil
}

type snapshotInfo struct {
	CreatedAt        string `json:"created_at"`
	CreationTime     *timestamp.Timestamp
	DataPool         string `json:"data_pool"`
	HasPendingClones string `json:"has_pending_clones"`
	Protected        string `json:"protected"`
	Size             int    `json:"size"`
}

func getSnapshotInfo(ctx context.Context, volOptions *volumeOptions, cr *util.Credentials, snapID, volID volumeID) (snapshotInfo, error) {
	snap := snapshotInfo{}
	args := []string{
		"fs",
		"subvolume",
		"snapshot",
		"info",
		volOptions.FsName,
		string(volID),
		string(snapID),
		"--group_name",
		volOptions.SubvolumeGroup,
		"-m", volOptions.Monitors,
		"-c", util.CephConfigPath,
		"-n", cephEntityClientPrefix + cr.ID,
		"--keyfile=" + cr.KeyFile,
		"--format=json",
	}
	err := execCommandJSON(
		ctx,
		&snap,
		"ceph",
		args[:]...)
	if err != nil {
		if strings.Contains(err.Error(), snapNotFound) {
			return snapshotInfo{}, ErrSnapNotFound
		}
		util.ErrorLog(ctx, "failed to get subvolume snapshot info %s %s(%s) in fs %s", string(snapID), string(volID), err, volOptions.FsName)
		return snapshotInfo{}, err
	}
	return snap, nil
}

func protectSnapshot(ctx context.Context, volOptions *volumeOptions, cr *util.Credentials, snapID, volID volumeID) error {
	// If "snapshot-autoprotect" feature is present, The ProtectSnapshot
	// call should be treated as a no-op.
	if checkSubvolumeHasFeature(autoProtect, volOptions.Features) {
		return nil
	}
	args := []string{
		"fs",
		"subvolume",
		"snapshot",
		"protect",
		volOptions.FsName,
		string(volID),
		string(snapID),
		"--group_name",
		volOptions.SubvolumeGroup,
		"-m", volOptions.Monitors,
		"-c", util.CephConfigPath,
		"-n", cephEntityClientPrefix + cr.ID,
		"--keyfile=" + cr.KeyFile,
	}

	err := execCommandErr(
		ctx,
		"ceph",
		args[:]...)
	if err != nil {
		if strings.Contains(err.Error(), snapProtectionExist) {
			return nil
		}
		util.ErrorLog(ctx, "failed to protect subvolume snapshot %s %s(%s) in fs %s", string(snapID), string(volID), err, volOptions.FsName)
		return err
	}
	return nil
}

func unprotectSnapshot(ctx context.Context, volOptions *volumeOptions, cr *util.Credentials, snapID, volID volumeID) error {
	// If "snapshot-autoprotect" feature is present, The UnprotectSnapshot
	// call should be treated as a no-op.
	if checkSubvolumeHasFeature(autoProtect, volOptions.Features) {
		return nil
	}
	args := []string{
		"fs",
		"subvolume",
		"snapshot",
		"unprotect",
		volOptions.FsName,
		string(volID),
		string(snapID),
		"--group_name",
		volOptions.SubvolumeGroup,
		"-m", volOptions.Monitors,
		"-c", util.CephConfigPath,
		"-n", cephEntityClientPrefix + cr.ID,
		"--keyfile=" + cr.KeyFile,
	}

	err := execCommandErr(
		ctx,
		"ceph",
		args[:]...)
	if err != nil {
		// In case the snap is already unprotected we get ErrSnapProtectionExist error code
		// in that case we are safe and we could discard this error.
		if strings.Contains(err.Error(), snapProtectionExist) {
			return nil
		}
		util.ErrorLog(ctx, "failed to unprotect subvolume snapshot %s %s(%s) in fs %s", string(snapID), string(volID), err, volOptions.FsName)
		return err
	}
	return nil
}

func cloneSnapshot(ctx context.Context, parentVolOptions *volumeOptions, cr *util.Credentials, volID, snapID, cloneID volumeID, cloneVolOptions *volumeOptions) error {
	args := []string{
		"fs",
		"subvolume",
		"snapshot",
		"clone",
		parentVolOptions.FsName,
		string(volID),
		string(snapID),
		string(cloneID),
		"--group_name",
		parentVolOptions.SubvolumeGroup,
		"--target_group_name",
		cloneVolOptions.SubvolumeGroup,
		"-m", parentVolOptions.Monitors,
		"-c", util.CephConfigPath,
		"-n", cephEntityClientPrefix + cr.ID,
		"--keyfile=" + cr.KeyFile,
	}
	if cloneVolOptions.Pool != "" {
		args = append(args, "--pool_layout", cloneVolOptions.Pool)
	}

	err := execCommandErr(
		ctx,
		"ceph",
		args[:]...)

	if err != nil {
		util.ErrorLog(ctx, "failed to clone subvolume snapshot %s %s(%s) in fs %s", string(cloneID), string(volID), err, parentVolOptions.FsName)
		if strings.HasPrefix(err.Error(), volumeNotFound) {
			return ErrVolumeNotFound
		}
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cache

import (
	"bytes"
	"context"
	"fmt"
	"go/ast"
	"go/scanner"
	"go/types"
	"sort"
	"sync"

	"golang.org/x/tools/go/packages"
	"golang.org/x/tools/internal/lsp/source"
	"golang.org/x/tools/internal/lsp/telemetry"
	"golang.org/x/tools/internal/memoize"
	"golang.org/x/tools/internal/telemetry/log"
	"golang.org/x/tools/internal/telemetry/trace"
	errors "golang.org/x/xerrors"
)

type importer struct {
	snapshot *snapshot
	ctx      context.Context

	// seen maintains the set of previously imported packages.
	// If we have seen a package that is already in this map, we have a circular import.
	seen map[packageID]struct{}

	// topLevelPackageID is the ID of the package from which type-checking began.
	topLevelPackageID packageID

	// parentPkg is the package that imports the current package.
	parentPkg *pkg

	// parentCheckPackageHandle is the check package handle that imports the current package.
	parentCheckPackageHandle *checkPackageHandle
}

// checkPackageHandle implements source.CheckPackageHandle.
type checkPackageHandle struct {
	handle *memoize.Handle

	// files are the ParseGoHandles that compose the package.
	files []source.ParseGoHandle

	// mode is the mode the the files were parsed in.
	mode source.ParseMode

	// imports is the map of the package's imports.
	imports map[packagePath]packageID

	// m is the metadata associated with the package.
	m *metadata

	// key is the hashed key for the package.
	key []byte
}

func (cph *checkPackageHandle) packageKey() packageKey {
	return packageKey{
		id:   cph.m.id,
		mode: cph.mode,
	}
}

// checkPackageData contains the data produced by type-checking a package.
type checkPackageData struct {
	memoize.NoCopy

	pkg *pkg
	err error
}

// checkPackageHandle returns a source.CheckPackageHandle for a given package and config.
func (imp *importer) checkPackageHandle(ctx context.Context, id packageID) (*checkPackageHandle, error) {
	// Determine the mode that the files should be parsed in.
	mode := imp.mode(id)

	// Check if we already have this CheckPackageHandle cached.
	if cph := imp.snapshot.getPackage(id, mode); cph != nil {
		return cph, nil
	}

	// Build the CheckPackageHandle for this ID and its dependencies.
	cph, err := imp.buildKey(ctx, id, mode)
	if err != nil {
		return nil, err
	}

	h := imp.snapshot.view.session.cache.store.Bind(string(cph.key), func(ctx context.Context) interface{} {
		data := &checkPackageData{}
		data.pkg, data.err = imp.typeCheck(ctx, cph)
		return data
	})
	cph.handle = h

	return cph, nil
}

// buildKey computes the checkPackageKey for a given checkPackageHandle.
func (imp *importer) buildKey(ctx context.Context, id packageID, mode source.ParseMode) (*checkPackageHandle, error) {
	m := imp.snapshot.getMetadata(id)
	if m == nil {
		return nil, errors.Errorf("no metadata for %s", id)
	}
	phs, err := imp.parseGoHandles(ctx, m, mode)
	if err != nil {
		return nil, err
	}
	cph := &checkPackageHandle{
		m:       m,
		files:   phs,
		imports: make(map[packagePath]packageID),
		mode:    mode,
	}

	// Make sure all of the deps are sorted.
	deps := append([]packageID{}, m.deps...)
	sort.Slice(deps, func(i, j int) bool {
		return deps[i] < deps[j]
	})

	// Create the dep importer for use on the dependency handles.
	depImporter := &importer{
		snapshot:          imp.snapshot,
		topLevelPackageID: imp.topLevelPackageID,
	}
	// Begin computing the key by getting the depKeys for all dependencies.
	var depKeys [][]byte
	for _, dep := range deps {
		depHandle, err := depImporter.checkPackageHandle(ctx, dep)
		if err != nil {
			log.Error(ctx, "no dep handle", err, telemetry.Package.Of(dep))

			// One bad dependency should not prevent us from checking the entire package.
			// Add a special key to mark a bad dependency.
			depKeys = append(depKeys, []byte(fmt.Sprintf("%s import not found", id)))
			continue
		}
		cph.imports[depHandle.m.pkgPath] = depHandle.m.id
		depKeys = append(depKeys, depHandle.key)
	}
	cph.key = checkPackageKey(cph.m.id, cph.files, m.config, depKeys)

	// Cache the CheckPackageHandle in the snapshot.
	imp.snapshot.addPackage(cph)

	return cph, nil
}

func checkPackageKey(id packageID, phs []source.ParseGoHandle, cfg *packages.Config, deps [][]byte) []byte {
	return []byte(hashContents([]byte(fmt.Sprintf("%s%s%s%s", id, hashParseKeys(phs), hashConfig(cfg), hashContents(bytes.Join(deps, nil))))))
}

// hashConfig returns the hash for the *packages.Config.
func hashConfig(config *packages.Config) string {
	b := bytes.NewBuffer(nil)

	// Dir, Mode, Env, BuildFlags are the parts of the config that can change.
	b.WriteString(config.Dir)
	b.WriteString(string(config.Mode))

	for _, e := range config.Env {
		b.WriteString(e)
	}
	for _, f := range config.BuildFlags {
		b.WriteString(f)
	}
	return hashContents(b.Bytes())
}

func (cph *checkPackageHandle) Check(ctx context.Context) (source.Package, error) {
	return cph.check(ctx)
}

func (cph *checkPackageHandle) check(ctx context.Context) (*pkg, error) {
	v := cph.handle.Get(ctx)
	if v == nil {
		return nil, errors.Errorf("no package for %s", cph.m.id)
	}
	data := v.(*checkPackageData)
	return data.pkg, data.err
}

func (cph *checkPackageHandle) Files() []source.ParseGoHandle {
	return cph.files
}

func (cph *checkPackageHandle) ID() string {
	return string(cph.m.id)
}

func (cph *checkPackageHandle) MissingDependencies() []string {
	var md []string
	for i := range cph.m.missingDeps {
		md = append(md, string(i))
	}
	return md
}

func (cph *checkPackageHandle) Cached(ctx context.Context) (source.Package, error) {
	return cph.cached(ctx)
}

func (cph *checkPackageHandle) cached(ctx context.Context) (*pkg, error) {
	v := cph.handle.Cached()
	if v == nil {
		return nil, errors.Errorf("no cached type information for %s", cph.m.pkgPath)
	}
	data := v.(*checkPackageData)
	return data.pkg, data.err
}

func (imp *importer) parseGoHandles(ctx context.Context, m *metadata, mode source.ParseMode) ([]source.ParseGoHandle, error) {
	phs := make([]source.ParseGoHandle, 0, len(m.files))
	for _, uri := range m.files {
		f, err := imp.snapshot.view.GetFile(ctx, uri)
		if err != nil {
			return nil, err
		}
		fh := imp.snapshot.Handle(ctx, f)
		phs = append(phs, imp.snapshot.view.session.cache.ParseGoHandle(fh, mode))
	}
	return phs, nil
}

func (imp *importer) mode(id packageID) source.ParseMode {
	if imp.topLevelPackageID == id {
		return source.ParseFull
	}
	return source.ParseExported
}

func (imp *importer) Import(pkgPath string) (*types.Package, error) {
	ctx, done := trace.StartSpan(imp.ctx, "cache.importer.Import", telemetry.PackagePath.Of(pkgPath))
	defer done()

	// We need to set the parent package's imports, so there should always be one.
	if imp.parentPkg == nil {
		return nil, errors.Errorf("no parent package for import %s", pkgPath)
	}
	// Get the CheckPackageHandle from the importing package.
	id, ok := imp.parentCheckPackageHandle.imports[packagePath(pkgPath)]
	if !ok {
		return nil, errors.Errorf("no package data for import path %s", pkgPath)
	}
	cph := imp.snapshot.getPackage(id, source.ParseExported)
	if cph == nil {
		return nil, errors.Errorf("no package for %s", id)
	}
	pkg, err := cph.check(ctx)
	if err != nil {
		return nil, err
	}
	imp.parentPkg.imports[packagePath(pkgPath)] = pkg
	return pkg.GetTypes(), nil
}

func (imp *importer) typeCheck(ctx context.Context, cph *checkPackageHandle) (*pkg, error) {
	ctx, done := trace.StartSpan(ctx, "cache.importer.typeCheck", telemetry.Package.Of(cph.m.id))
	defer done()

	pkg := &pkg{
		snapshot:   imp.snapshot,
		id:         cph.m.id,
		mode:       cph.mode,
		pkgPath:    cph.m.pkgPath,
		files:      cph.Files(),
		imports:    make(map[packagePath]*pkg),
		typesSizes: cph.m.typesSizes,
		typesInfo: &types.Info{
			Types:      make(map[ast.Expr]types.TypeAndValue),
			Defs:       make(map[*ast.Ident]types.Object),
			Uses:       make(map[*ast.Ident]types.Object),
			Implicits:  make(map[ast.Node]types.Object),
			Selections: make(map[*ast.SelectorExpr]*types.Selection),
			Scopes:     make(map[ast.Node]*types.Scope),
		},
	}
	// If the package comes back with errors from `go list`,
	// don't bother type-checking it.
	for _, err := range cph.m.errors {
		pkg.errors = append(cph.m.errors, err)
	}
	var (
		files       = make([]*ast.File, len(pkg.files))
		parseErrors = make([]error, len(pkg.files))
		wg          sync.WaitGroup
	)
	for i, ph := range pkg.files {
		wg.Add(1)
		go func(i int, ph source.ParseGoHandle) {
			defer wg.Done()

			files[i], _, parseErrors[i], _ = ph.Parse(ctx)
		}(i, ph)
	}
	wg.Wait()

	for _, err := range parseErrors {
		if err != nil {
			imp.snapshot.view.session.cache.appendPkgError(pkg, err)
		}
	}

	var i int
	for _, f := range files {
		if f != nil {
			files[i] = f
			i++
		}
	}
	files = files[:i]

	// Use the default type information for the unsafe package.
	if cph.m.pkgPath == "unsafe" {
		pkg.types = types.Unsafe
	} else if len(files) == 0 { // not the unsafe package, no parsed files
		return nil, errors.Errorf("no parsed files for package %s", pkg.pkgPath)
	} else {
		pkg.types = types.NewPackage(string(cph.m.pkgPath), cph.m.name)
	}

	cfg := &types.Config{
		Error: func(err error) {
			imp.snapshot.view.session.cache.appendPkgError(pkg, err)
		},
		Importer: imp.depImporter(ctx, cph, pkg),
	}
	check := types.NewChecker(cfg, imp.snapshot.view.session.cache.FileSet(), pkg.types, pkg.typesInfo)

	// Type checking errors are handled via the config, so ignore them here.
	_ = check.Files(files)

	return pkg, nil
}

func (imp *importer) depImporter(ctx context.Context, cph *checkPackageHandle, pkg *pkg) *importer {
	// Handle circular imports by copying previously seen imports.
	seen := make(map[packageID]struct{})
	for k, v := range imp.seen {
		seen[k] = v
	}
	seen[pkg.id] = struct{}{}
	return &importer{
		snapshot:                 imp.snapshot,
		topLevelPackageID:        imp.topLevelPackageID,
		parentPkg:                pkg,
		parentCheckPackageHandle: cph,
		seen:                     seen,
		ctx:                      ctx,
	}
}

func (c *cache) appendPkgError(pkg *pkg, err error) {
	if err == nil {
		return
	}
	var errs []packages.Error
	switch err := err.(type) {
	case *scanner.Error:
		errs = append(errs, packages.Error{
			Pos:  err.Pos.String(),
			Msg:  err.Msg,
			Kind: packages.ParseError,
		})
	case scanner.ErrorList:
		// The first parser error is likely the root cause of the problem.
		if err.Len() > 0 {
			errs = append(errs, packages.Error{
				Pos:  err[0].Pos.String(),
				Msg:  err[0].Msg,
				Kind: packages.ParseError,
			})
		}
	case types.Error:
		errs = append(errs, packages.Error{
			Pos:  c.FileSet().Position(err.Pos).String(),
			Msg:  err.Msg,
			Kind: packages.TypeError,
		})
	}
	pkg.errors = append(pkg.errors, errs...)
}
<commit_msg>internal/lsp: fix race in checkPackageHandles<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cache

import (
	"bytes"
	"context"
	"fmt"
	"go/ast"
	"go/scanner"
	"go/types"
	"sort"
	"sync"

	"golang.org/x/tools/go/packages"
	"golang.org/x/tools/internal/lsp/source"
	"golang.org/x/tools/internal/lsp/telemetry"
	"golang.org/x/tools/internal/memoize"
	"golang.org/x/tools/internal/telemetry/log"
	"golang.org/x/tools/internal/telemetry/trace"
	errors "golang.org/x/xerrors"
)

type importer struct {
	snapshot *snapshot
	ctx      context.Context

	// seen maintains the set of previously imported packages.
	// If we have seen a package that is already in this map, we have a circular import.
	seen map[packageID]struct{}

	// topLevelPackageID is the ID of the package from which type-checking began.
	topLevelPackageID packageID

	// parentPkg is the package that imports the current package.
	parentPkg *pkg

	// parentCheckPackageHandle is the check package handle that imports the current package.
	parentCheckPackageHandle *checkPackageHandle
}

// checkPackageHandle implements source.CheckPackageHandle.
type checkPackageHandle struct {
	handle *memoize.Handle

	// files are the ParseGoHandles that compose the package.
	files []source.ParseGoHandle

	// mode is the mode the the files were parsed in.
	mode source.ParseMode

	// imports is the map of the package's imports.
	imports map[packagePath]packageID

	// m is the metadata associated with the package.
	m *metadata

	// key is the hashed key for the package.
	key []byte
}

func (cph *checkPackageHandle) packageKey() packageKey {
	return packageKey{
		id:   cph.m.id,
		mode: cph.mode,
	}
}

// checkPackageData contains the data produced by type-checking a package.
type checkPackageData struct {
	memoize.NoCopy

	pkg *pkg
	err error
}

// checkPackageHandle returns a source.CheckPackageHandle for a given package and config.
func (imp *importer) checkPackageHandle(ctx context.Context, id packageID) (*checkPackageHandle, error) {
	// Determine the mode that the files should be parsed in.
	mode := imp.mode(id)

	// Check if we already have this CheckPackageHandle cached.
	if cph := imp.snapshot.getPackage(id, mode); cph != nil {
		return cph, nil
	}

	// Build the CheckPackageHandle for this ID and its dependencies.
	cph, err := imp.buildKey(ctx, id, mode)
	if err != nil {
		return nil, err
	}

	h := imp.snapshot.view.session.cache.store.Bind(string(cph.key), func(ctx context.Context) interface{} {
		data := &checkPackageData{}
		data.pkg, data.err = imp.typeCheck(ctx, cph)
		return data
	})
	cph.handle = h

	// Cache the CheckPackageHandle in the snapshot.
	imp.snapshot.addPackage(cph)

	return cph, nil
}

// buildKey computes the checkPackageKey for a given checkPackageHandle.
func (imp *importer) buildKey(ctx context.Context, id packageID, mode source.ParseMode) (*checkPackageHandle, error) {
	m := imp.snapshot.getMetadata(id)
	if m == nil {
		return nil, errors.Errorf("no metadata for %s", id)
	}
	phs, err := imp.parseGoHandles(ctx, m, mode)
	if err != nil {
		return nil, err
	}
	cph := &checkPackageHandle{
		m:       m,
		files:   phs,
		imports: make(map[packagePath]packageID),
		mode:    mode,
	}

	// Make sure all of the deps are sorted.
	deps := append([]packageID{}, m.deps...)
	sort.Slice(deps, func(i, j int) bool {
		return deps[i] < deps[j]
	})

	// Create the dep importer for use on the dependency handles.
	depImporter := &importer{
		snapshot:          imp.snapshot,
		topLevelPackageID: imp.topLevelPackageID,
	}
	// Begin computing the key by getting the depKeys for all dependencies.
	var depKeys [][]byte
	for _, dep := range deps {
		depHandle, err := depImporter.checkPackageHandle(ctx, dep)
		if err != nil {
			log.Error(ctx, "no dep handle", err, telemetry.Package.Of(dep))

			// One bad dependency should not prevent us from checking the entire package.
			// Add a special key to mark a bad dependency.
			depKeys = append(depKeys, []byte(fmt.Sprintf("%s import not found", id)))
			continue
		}
		cph.imports[depHandle.m.pkgPath] = depHandle.m.id
		depKeys = append(depKeys, depHandle.key)
	}
	cph.key = checkPackageKey(cph.m.id, cph.files, m.config, depKeys)

	return cph, nil
}

func checkPackageKey(id packageID, phs []source.ParseGoHandle, cfg *packages.Config, deps [][]byte) []byte {
	return []byte(hashContents([]byte(fmt.Sprintf("%s%s%s%s", id, hashParseKeys(phs), hashConfig(cfg), hashContents(bytes.Join(deps, nil))))))
}

// hashConfig returns the hash for the *packages.Config.
func hashConfig(config *packages.Config) string {
	b := bytes.NewBuffer(nil)

	// Dir, Mode, Env, BuildFlags are the parts of the config that can change.
	b.WriteString(config.Dir)
	b.WriteString(string(config.Mode))

	for _, e := range config.Env {
		b.WriteString(e)
	}
	for _, f := range config.BuildFlags {
		b.WriteString(f)
	}
	return hashContents(b.Bytes())
}

func (cph *checkPackageHandle) Check(ctx context.Context) (source.Package, error) {
	return cph.check(ctx)
}

func (cph *checkPackageHandle) check(ctx context.Context) (*pkg, error) {
	v := cph.handle.Get(ctx)
	if v == nil {
		return nil, errors.Errorf("no package for %s", cph.m.id)
	}
	data := v.(*checkPackageData)
	return data.pkg, data.err
}

func (cph *checkPackageHandle) Files() []source.ParseGoHandle {
	return cph.files
}

func (cph *checkPackageHandle) ID() string {
	return string(cph.m.id)
}

func (cph *checkPackageHandle) MissingDependencies() []string {
	var md []string
	for i := range cph.m.missingDeps {
		md = append(md, string(i))
	}
	return md
}

func (cph *checkPackageHandle) Cached(ctx context.Context) (source.Package, error) {
	return cph.cached(ctx)
}

func (cph *checkPackageHandle) cached(ctx context.Context) (*pkg, error) {
	v := cph.handle.Cached()
	if v == nil {
		return nil, errors.Errorf("no cached type information for %s", cph.m.pkgPath)
	}
	data := v.(*checkPackageData)
	return data.pkg, data.err
}

func (imp *importer) parseGoHandles(ctx context.Context, m *metadata, mode source.ParseMode) ([]source.ParseGoHandle, error) {
	phs := make([]source.ParseGoHandle, 0, len(m.files))
	for _, uri := range m.files {
		f, err := imp.snapshot.view.GetFile(ctx, uri)
		if err != nil {
			return nil, err
		}
		fh := imp.snapshot.Handle(ctx, f)
		phs = append(phs, imp.snapshot.view.session.cache.ParseGoHandle(fh, mode))
	}
	return phs, nil
}

func (imp *importer) mode(id packageID) source.ParseMode {
	if imp.topLevelPackageID == id {
		return source.ParseFull
	}
	return source.ParseExported
}

func (imp *importer) Import(pkgPath string) (*types.Package, error) {
	ctx, done := trace.StartSpan(imp.ctx, "cache.importer.Import", telemetry.PackagePath.Of(pkgPath))
	defer done()

	// We need to set the parent package's imports, so there should always be one.
	if imp.parentPkg == nil {
		return nil, errors.Errorf("no parent package for import %s", pkgPath)
	}
	// Get the CheckPackageHandle from the importing package.
	id, ok := imp.parentCheckPackageHandle.imports[packagePath(pkgPath)]
	if !ok {
		return nil, errors.Errorf("no package data for import path %s", pkgPath)
	}
	cph := imp.snapshot.getPackage(id, source.ParseExported)
	if cph == nil {
		return nil, errors.Errorf("no package for %s", id)
	}
	pkg, err := cph.check(ctx)
	if err != nil {
		return nil, err
	}
	imp.parentPkg.imports[packagePath(pkgPath)] = pkg
	return pkg.GetTypes(), nil
}

func (imp *importer) typeCheck(ctx context.Context, cph *checkPackageHandle) (*pkg, error) {
	ctx, done := trace.StartSpan(ctx, "cache.importer.typeCheck", telemetry.Package.Of(cph.m.id))
	defer done()

	pkg := &pkg{
		snapshot:   imp.snapshot,
		id:         cph.m.id,
		mode:       cph.mode,
		pkgPath:    cph.m.pkgPath,
		files:      cph.Files(),
		imports:    make(map[packagePath]*pkg),
		typesSizes: cph.m.typesSizes,
		typesInfo: &types.Info{
			Types:      make(map[ast.Expr]types.TypeAndValue),
			Defs:       make(map[*ast.Ident]types.Object),
			Uses:       make(map[*ast.Ident]types.Object),
			Implicits:  make(map[ast.Node]types.Object),
			Selections: make(map[*ast.SelectorExpr]*types.Selection),
			Scopes:     make(map[ast.Node]*types.Scope),
		},
	}
	// If the package comes back with errors from `go list`,
	// don't bother type-checking it.
	for _, err := range cph.m.errors {
		pkg.errors = append(cph.m.errors, err)
	}
	var (
		files       = make([]*ast.File, len(pkg.files))
		parseErrors = make([]error, len(pkg.files))
		wg          sync.WaitGroup
	)
	for i, ph := range pkg.files {
		wg.Add(1)
		go func(i int, ph source.ParseGoHandle) {
			defer wg.Done()

			files[i], _, parseErrors[i], _ = ph.Parse(ctx)
		}(i, ph)
	}
	wg.Wait()

	for _, err := range parseErrors {
		if err != nil {
			imp.snapshot.view.session.cache.appendPkgError(pkg, err)
		}
	}

	var i int
	for _, f := range files {
		if f != nil {
			files[i] = f
			i++
		}
	}
	files = files[:i]

	// Use the default type information for the unsafe package.
	if cph.m.pkgPath == "unsafe" {
		pkg.types = types.Unsafe
	} else if len(files) == 0 { // not the unsafe package, no parsed files
		return nil, errors.Errorf("no parsed files for package %s", pkg.pkgPath)
	} else {
		pkg.types = types.NewPackage(string(cph.m.pkgPath), cph.m.name)
	}

	cfg := &types.Config{
		Error: func(err error) {
			imp.snapshot.view.session.cache.appendPkgError(pkg, err)
		},
		Importer: imp.depImporter(ctx, cph, pkg),
	}
	check := types.NewChecker(cfg, imp.snapshot.view.session.cache.FileSet(), pkg.types, pkg.typesInfo)

	// Type checking errors are handled via the config, so ignore them here.
	_ = check.Files(files)

	return pkg, nil
}

func (imp *importer) depImporter(ctx context.Context, cph *checkPackageHandle, pkg *pkg) *importer {
	// Handle circular imports by copying previously seen imports.
	seen := make(map[packageID]struct{})
	for k, v := range imp.seen {
		seen[k] = v
	}
	seen[pkg.id] = struct{}{}
	return &importer{
		snapshot:                 imp.snapshot,
		topLevelPackageID:        imp.topLevelPackageID,
		parentPkg:                pkg,
		parentCheckPackageHandle: cph,
		seen:                     seen,
		ctx:                      ctx,
	}
}

func (c *cache) appendPkgError(pkg *pkg, err error) {
	if err == nil {
		return
	}
	var errs []packages.Error
	switch err := err.(type) {
	case *scanner.Error:
		errs = append(errs, packages.Error{
			Pos:  err.Pos.String(),
			Msg:  err.Msg,
			Kind: packages.ParseError,
		})
	case scanner.ErrorList:
		// The first parser error is likely the root cause of the problem.
		if err.Len() > 0 {
			errs = append(errs, packages.Error{
				Pos:  err[0].Pos.String(),
				Msg:  err[0].Msg,
				Kind: packages.ParseError,
			})
		}
	case types.Error:
		errs = append(errs, packages.Error{
			Pos:  c.FileSet().Position(err.Pos).String(),
			Msg:  err.Msg,
			Kind: packages.TypeError,
		})
	}
	pkg.errors = append(pkg.errors, errs...)
}
<|endoftext|>


<commit_before>package midilib

import (
	"github.com/gomidi/midi"
	"io"
)

func hasBitU8(n uint8, pos uint8) bool {
	val := n & (1 << pos)
	return (val > 0)
}

func IsChannelMessage(b uint8) bool {
	return !hasBitU8(b, 6)
}

func IsStatusByte(b uint8) bool {
	return hasBitU8(b, 7)
}

func ReadNBytes(n int, rd io.Reader) ([]byte, error) {
	var b []byte = make([]byte, n)
	num, err := rd.Read(b)

	if err != nil {
		return nil, err
	}
	if num != n {
		return nil, midi.ErrUnexpectedEOF
	}

	return b, nil
}

func ReadByte(rd io.Reader) (byte, error) {
	b, err := ReadNBytes(1, rd)

	if err != nil {
		return 0, err
	}

	return b[0], nil
}

// ReadUint16 reads a 2-byte 16 bit integer from a ReadSeeker.
// It returns the 16-bit value and an error.
func ReadUint16(rd io.Reader) (uint16, error) {
	b, err := ReadNBytes(2, rd)

	if err != nil {
		return 0, err
	}

	var val uint16 = 0x00
	val |= uint16(b[1]) << 0
	val |= uint16(b[0]) << 8

	return val, nil
}

// ReadUint24 parse a 3-byte 24 bit integer from a ReadSeeker.
// It returns the 32-bit value and an error.
// TODO TEST
func ReadUint24(rd io.Reader) (uint32, error) {
	b, err := ReadNBytes(3, rd)

	if err != nil {
		return 0, err
	}

	var val uint32 = 0x00
	val |= uint32(b[2]) << 0
	val |= uint32(b[1]) << 8
	val |= uint32(b[0]) << 16

	return val, nil
}

// ReadUint32 parse a 4-byte 32 bit integer from a ReadSeeker.
// It returns the 32-bit value and an error.
func ReadUint32(rd io.Reader) (uint32, error) {
	b, err := ReadNBytes(4, rd)

	if err != nil {
		return 0, err
	}

	var val uint32 = 0x00
	val |= uint32(b[3]) << 0
	val |= uint32(b[2]) << 8
	val |= uint32(b[1]) << 16
	val |= uint32(b[0]) << 24

	return val, nil
}

// readVarLength reads a variable length value from a ReadSeeker.
// It returns the [up to] 32-bit value and an error.
func ReadVarLength(reader io.Reader) (uint32, error) {

	// Single byte buffer to read byte by byte.
	var buffer []byte = make([]uint8, 1)

	// The number of bytes returned.
	// Should always be 1 unless we reach the EOF
	var num int = 1

	// Result value
	var result uint32 = 0x00

	// RTFM.
	var first = true
	for (first || (buffer[0]&0x80 == 0x80)) && (num > 0) {
		result = result << 7

		num, _ = reader.Read(buffer)
		result |= (uint32(buffer[0]) & 0x7f)
		first = false
	}

	if num == 0 && !first {
		return result, midi.ErrUnexpectedEOF
	}

	return result, nil
}

// readVarLengthData reads data that is prefixed by a varLength that tells the length of the data
func ReadVarLengthData(reader io.Reader) ([]byte, error) {
	length, err := ReadVarLength(reader)

	if err != nil {
		return []byte{}, err
	}

	var buffer []byte = make([]byte, length)

	num, err := reader.Read(buffer)

	// If we couldn't read the entire expected-length buffer, that's a problem.
	if num != int(length) {
		return []byte{}, midi.ErrUnexpectedEOF
	}

	// If there was some other problem, that's also a problem.
	if err != nil {
		return []byte{}, err
	}

	return buffer, nil
}
<commit_msg>fix ReadNBytes: don't return error, if num bytes read is correct<commit_after>package midilib

import (
	"github.com/gomidi/midi"
	"io"
)

func hasBitU8(n uint8, pos uint8) bool {
	val := n & (1 << pos)
	return (val > 0)
}

func IsChannelMessage(b uint8) bool {
	return !hasBitU8(b, 6)
}

func IsStatusByte(b uint8) bool {
	return hasBitU8(b, 7)
}

func ReadNBytes(n int, rd io.Reader) ([]byte, error) {
	var b []byte = make([]byte, n)
	num, err := rd.Read(b)

	// if num is correct, we are not interested in io.EOF errors
	if num == n {
		err = nil
	}

	return b, err
}

func ReadByte(rd io.Reader) (byte, error) {
	b, err := ReadNBytes(1, rd)

	if err != nil {
		return 0, err
	}

	return b[0], nil
}

// ReadUint16 reads a 2-byte 16 bit integer from a ReadSeeker.
// It returns the 16-bit value and an error.
func ReadUint16(rd io.Reader) (uint16, error) {
	b, err := ReadNBytes(2, rd)

	if err != nil {
		return 0, err
	}

	var val uint16 = 0x00
	val |= uint16(b[1]) << 0
	val |= uint16(b[0]) << 8

	return val, nil
}

// ReadUint24 parse a 3-byte 24 bit integer from a ReadSeeker.
// It returns the 32-bit value and an error.
// TODO TEST
func ReadUint24(rd io.Reader) (uint32, error) {
	b, err := ReadNBytes(3, rd)

	if err != nil {
		return 0, err
	}

	var val uint32 = 0x00
	val |= uint32(b[2]) << 0
	val |= uint32(b[1]) << 8
	val |= uint32(b[0]) << 16

	return val, nil
}

// ReadUint32 parse a 4-byte 32 bit integer from a ReadSeeker.
// It returns the 32-bit value and an error.
func ReadUint32(rd io.Reader) (uint32, error) {
	b, err := ReadNBytes(4, rd)

	if err != nil {
		return 0, err
	}

	var val uint32 = 0x00
	val |= uint32(b[3]) << 0
	val |= uint32(b[2]) << 8
	val |= uint32(b[1]) << 16
	val |= uint32(b[0]) << 24

	return val, nil
}

// readVarLength reads a variable length value from a ReadSeeker.
// It returns the [up to] 32-bit value and an error.
func ReadVarLength(reader io.Reader) (uint32, error) {

	// Single byte buffer to read byte by byte.
	var buffer []byte = make([]uint8, 1)

	// The number of bytes returned.
	// Should always be 1 unless we reach the EOF
	var num int = 1

	// Result value
	var result uint32 = 0x00

	// RTFM.
	var first = true
	for (first || (buffer[0]&0x80 == 0x80)) && (num > 0) {
		result = result << 7

		num, _ = reader.Read(buffer)
		result |= (uint32(buffer[0]) & 0x7f)
		first = false
	}

	if num == 0 && !first {
		return result, midi.ErrUnexpectedEOF
	}

	return result, nil
}

// readVarLengthData reads data that is prefixed by a varLength that tells the length of the data
func ReadVarLengthData(reader io.Reader) ([]byte, error) {
	length, err := ReadVarLength(reader)

	if err != nil {
		return []byte{}, err
	}

	var buffer []byte = make([]byte, length)

	num, err := reader.Read(buffer)

	// If we couldn't read the entire expected-length buffer, that's a problem.
	if num != int(length) {
		return []byte{}, midi.ErrUnexpectedEOF
	}

	// If there was some other problem, that's also a problem.
	if err != nil {
		return []byte{}, err
	}

	return buffer, nil
}
<|endoftext|>


<commit_before>// Copyright © 2017 Circonus, Inc. <support@circonus.com>
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//

package plugins

import (
	"context"
	"encoding/json"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/circonus-labs/circonus-agent/api"
	"github.com/circonus-labs/circonus-agent/internal/config"
	"github.com/circonus-labs/circonus-agent/internal/config/defaults"
	cgm "github.com/circonus-labs/circonus-gometrics/v3"
	"github.com/maier/go-appstats"
	"github.com/pkg/errors"
	"github.com/rs/zerolog"
	"github.com/rs/zerolog/log"
	"github.com/spf13/viper"
)

// Plugins defines plugin manager
type Plugins struct {
	active        map[string]*plugin
	plugList      []string
	ctx           context.Context
	logger        zerolog.Logger
	pluginDir     string
	reservedNames map[string]bool
	running       bool
	sync.RWMutex
}

// Plugin defines a specific plugin
type plugin struct {
	cmd             *exec.Cmd
	command         string
	ctx             context.Context
	id              string
	instanceArgs    []string
	instanceID      string
	lastError       error
	lastRunDuration time.Duration
	currStart       time.Time
	lastStart       time.Time
	lastEnd         time.Time
	logger          zerolog.Logger
	metrics         *cgm.Metrics
	name            string
	prevMetrics     *cgm.Metrics
	runDir          string
	running         bool
	runTTL          time.Duration
	baseTags        []string
	sync.Mutex
}

const (
	fieldDelimiter  = "\t"
	nullMetricValue = "[[null]]"
)

// New returns a new instance of the plugins manager
func New(ctx context.Context, defaultPluginPath string) (*Plugins, error) {
	p := Plugins{
		ctx:           ctx,
		running:       false,
		logger:        log.With().Str("pkg", "plugins").Logger(),
		reservedNames: map[string]bool{"prom": true, "write": true, "statsd": true},
		active:        make(map[string]*plugin),
	}

	pluginDir := viper.GetString(config.KeyPluginDir)
	pluginList := viper.GetStringSlice(config.KeyPluginList)

	// if neither specified, use default plugin directory
	if pluginDir == "" && len(pluginList) == 0 {
		pluginDir = defaultPluginPath
	}

	if pluginDir != "" && len(pluginList) > 0 {
		return nil, errors.New("invalid configuration cannot specifiy plugin-dir AND plugin-list")
	}

	if pluginDir == "" {
		for _, cmdSpec := range pluginList {
			if _, err := os.Stat(cmdSpec); err != nil {
				p.logger.Warn().Err(err).Str("cmd", cmdSpec).Msg("skipping")
			}
		}
		return &p, nil
	}

	errMsg := "Invalid plugin directory"
	absDir, err := filepath.Abs(pluginDir)
	if err != nil {
		return nil, errors.Wrap(err, errMsg)
	}

	pluginDir = absDir

	fi, err := os.Stat(pluginDir)
	if err != nil {
		if os.IsNotExist(err) {
			p.logger.Warn().Err(err).Str("path", pluginDir).Msg("not found, ignoring")
			p.pluginDir = ""
			return &p, nil
		}
		return nil, errors.Wrap(err, errMsg)
	}

	if !fi.Mode().IsDir() {
		return nil, errors.Errorf(errMsg+" (%s) not a directory", pluginDir)
	}

	// also try opening, to verify permissions
	// if last dir on path is not accessible to user, stat doesn't return EPERM
	f, err := os.Open(pluginDir)
	if err != nil {
		return nil, errors.Wrap(err, errMsg)
	}
	f.Close()

	p.pluginDir = pluginDir

	return &p, nil
}

// Flush plugin metrics
func (p *Plugins) Flush(pluginName string) *cgm.Metrics {
	p.RLock()
	defer p.RUnlock()

	appstats.SetString("plugins.last_flush", time.Now().String())
	// appstats.MapSet("plugins", "last_flush", time.Now())

	metrics := cgm.Metrics{}

	for pluginID, plug := range p.active {
		if pluginName == "" || // all plugins
			pluginID == pluginName || // specific plugin
			strings.HasPrefix(pluginID, pluginName+defaults.MetricNameSeparator) { // specific plugin with instances

			m := plug.drain()
			for mn, mv := range *m {
				metrics[mn] = mv
			}
		}
	}

	return &metrics
}

// Stop any long running plugins
func (p *Plugins) Stop() error {
	p.logger.Info().Msg("stopping")
	return nil
}

// Run one or all plugins
func (p *Plugins) Run(pluginName string) error {
	p.Lock()

	if p.running {
		msg := "plugin run already in progress"
		p.logger.Info().Msg(msg)
		p.Unlock()
		return errors.Errorf(msg)
	}

	if len(p.active) == 0 {
		p.logger.Debug().Msg("no active plugins, skipping run")
		p.Unlock()
		return nil
	}

	if len(p.plugList) == 0 {
		p.plugList = make([]string, len(p.active))
		i := 0
		for name := range p.active {
			p.plugList[i] = name
			i++
		}
	}

	start := time.Now()
	appstats.SetString("plugins.last_run_start", start.String())
	// appstats.MapSet("plugins", "last_run_start", start)

	p.running = true
	p.Unlock()

	var wg sync.WaitGroup

	if pluginName != "" {
		numFound := 0
		for pluginID, pluginRef := range p.active {
			if pluginID == pluginName || // specific plugin
				strings.HasPrefix(pluginID, pluginName+"`") { // specific plugin with instances
				numFound++
				wg.Add(1)
				p.logger.Debug().Str("id", pluginID).Msg("running")
				go func(id string, plug *plugin) {
					plug.exec()
					plug.logger.Debug().Str("id", id).Str("duration", time.Since(start).String()).Msg("done")
					wg.Done()
				}(pluginID, pluginRef)
			}
		}
		if numFound == 0 {
			p.logger.Error().Str("id", pluginName).Msg("invalid/unknown")
			p.running = false
			return errors.Errorf("invalid plugin (%s)", pluginName)
		}
	} else {
		p.logger.Debug().Str("plugin(s)", strings.Join(p.plugList, ",")).Msg("running")
		for pluginID, pluginRef := range p.active {
			wg.Add(1)
			go func(id string, plug *plugin) {
				plug.exec()
				plug.logger.Debug().Str("id", id).Str("duration", time.Since(start).String()).Msg("done")
				wg.Done()
			}(pluginID, pluginRef)
		}
	}

	wg.Wait()

	appstats.SetString("plugins.last_run_end", time.Now().String())
	appstats.SetString("plugins.last_run_duration", time.Since(start).String())
	// appstats.MapSet("plugins", "last_run_end", time.Now())
	// appstats.MapSet("plugins", "last_run_duration", time.Since(start))

	p.Lock()
	p.running = false
	p.logger.Debug().Str("duration", time.Since(start).String()).Msg("all plugins done")
	p.Unlock()

	return nil
}

// IsValid determines if a specific plugin is valid
func (p *Plugins) IsValid(pluginName string) bool {
	if pluginName == "" {
		return false
	}

	p.RLock()
	defer p.RUnlock()

	for pluginID := range p.active {
		// specific plugin       plugin with instances
		if pluginID == pluginName || strings.HasPrefix(pluginID, pluginName+"`") {
			return true
		}
	}

	return false
}

// IsInternal checks to see if the plugin is one of the internal plugins (write|statsd)
func (p *Plugins) IsInternal(pluginName string) bool {
	if pluginName == "" {
		return false
	}
	_, reserved := p.reservedNames[pluginName]

	return reserved
}

// Inventory returns list of active plugins
func (p *Plugins) Inventory() []byte {
	p.Lock()
	defer p.Unlock()
	inventory := api.Inventory{}
	for id, plug := range p.active {
		plug.Lock()
		pinfo := api.Plugin{
			ID:              id,
			Name:            plug.id,
			Instance:        plug.instanceID,
			Command:         plug.command,
			Args:            plug.instanceArgs,
			LastRunStart:    plug.lastStart.Format(time.RFC3339Nano),
			LastRunEnd:      plug.lastEnd.Format(time.RFC3339Nano),
			LastRunDuration: plug.lastRunDuration.String(),
		}
		if plug.lastError != nil {
			pinfo.LastError = plug.lastError.Error()
		}
		plug.Unlock()
		inventory = append(inventory, pinfo)
	}
	data, err := json.Marshal(inventory)
	if err != nil {
		p.logger.Fatal().Err(err).Msg("inventory -> json")
	}
	return data
}
<commit_msg>upd: lint, print exec errors<commit_after>// Copyright © 2017 Circonus, Inc. <support@circonus.com>
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//

package plugins

import (
	"context"
	"encoding/json"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/circonus-labs/circonus-agent/api"
	"github.com/circonus-labs/circonus-agent/internal/config"
	"github.com/circonus-labs/circonus-agent/internal/config/defaults"
	cgm "github.com/circonus-labs/circonus-gometrics/v3"
	"github.com/maier/go-appstats"
	"github.com/pkg/errors"
	"github.com/rs/zerolog"
	"github.com/rs/zerolog/log"
	"github.com/spf13/viper"
)

// Plugins defines plugin manager
type Plugins struct {
	active        map[string]*plugin
	plugList      []string
	ctx           context.Context
	logger        zerolog.Logger
	pluginDir     string
	reservedNames map[string]bool
	running       bool
	sync.RWMutex
}

// Plugin defines a specific plugin
type plugin struct {
	cmd             *exec.Cmd
	command         string
	ctx             context.Context
	id              string
	instanceArgs    []string
	instanceID      string
	lastError       error
	lastRunDuration time.Duration
	currStart       time.Time
	lastStart       time.Time
	lastEnd         time.Time
	logger          zerolog.Logger
	metrics         *cgm.Metrics
	name            string
	prevMetrics     *cgm.Metrics
	runDir          string
	running         bool
	runTTL          time.Duration
	baseTags        []string
	sync.Mutex
}

const (
	fieldDelimiter  = "\t"
	nullMetricValue = "[[null]]"
)

// New returns a new instance of the plugins manager
func New(ctx context.Context, defaultPluginPath string) (*Plugins, error) {
	p := Plugins{
		ctx:           ctx,
		running:       false,
		logger:        log.With().Str("pkg", "plugins").Logger(),
		reservedNames: map[string]bool{"prom": true, "write": true, "statsd": true},
		active:        make(map[string]*plugin),
	}

	pluginDir := viper.GetString(config.KeyPluginDir)
	pluginList := viper.GetStringSlice(config.KeyPluginList)

	// if neither specified, use default plugin directory
	if pluginDir == "" && len(pluginList) == 0 {
		pluginDir = defaultPluginPath
	}

	if pluginDir != "" && len(pluginList) > 0 {
		return nil, errors.New("invalid configuration cannot specifiy plugin-dir AND plugin-list")
	}

	if pluginDir == "" {
		for _, cmdSpec := range pluginList {
			if _, err := os.Stat(cmdSpec); err != nil {
				p.logger.Warn().Err(err).Str("cmd", cmdSpec).Msg("skipping")
			}
		}
		return &p, nil
	}

	errMsg := "Invalid plugin directory"
	absDir, err := filepath.Abs(pluginDir)
	if err != nil {
		return nil, errors.Wrap(err, errMsg)
	}

	pluginDir = absDir

	fi, err := os.Stat(pluginDir)
	if err != nil {
		if os.IsNotExist(err) {
			p.logger.Warn().Err(err).Str("path", pluginDir).Msg("not found, ignoring")
			p.pluginDir = ""
			return &p, nil
		}
		return nil, errors.Wrap(err, errMsg)
	}

	if !fi.Mode().IsDir() {
		return nil, errors.Errorf(errMsg+" (%s) not a directory", pluginDir)
	}

	// also try opening, to verify permissions
	// if last dir on path is not accessible to user, stat doesn't return EPERM
	f, err := os.Open(pluginDir)
	if err != nil {
		return nil, errors.Wrap(err, errMsg)
	}
	f.Close()

	p.pluginDir = pluginDir

	return &p, nil
}

// Flush plugin metrics
func (p *Plugins) Flush(pluginName string) *cgm.Metrics {
	p.RLock()
	defer p.RUnlock()

	_ = appstats.SetString("plugins.last_flush", time.Now().String())
	// appstats.MapSet("plugins", "last_flush", time.Now())

	metrics := cgm.Metrics{}

	for pluginID, plug := range p.active {
		if pluginName == "" || // all plugins
			pluginID == pluginName || // specific plugin
			strings.HasPrefix(pluginID, pluginName+defaults.MetricNameSeparator) { // specific plugin with instances

			m := plug.drain()
			for mn, mv := range *m {
				metrics[mn] = mv
			}
		}
	}

	return &metrics
}

// Stop any long running plugins
func (p *Plugins) Stop() error {
	p.logger.Info().Msg("stopping")
	return nil
}

// Run one or all plugins
func (p *Plugins) Run(pluginName string) error {
	p.Lock()

	if p.running {
		msg := "plugin run already in progress"
		p.logger.Info().Msg(msg)
		p.Unlock()
		return errors.Errorf(msg)
	}

	if len(p.active) == 0 {
		p.logger.Debug().Msg("no active plugins, skipping run")
		p.Unlock()
		return nil
	}

	if len(p.plugList) == 0 {
		p.plugList = make([]string, len(p.active))
		i := 0
		for name := range p.active {
			p.plugList[i] = name
			i++
		}
	}

	start := time.Now()
	_ = appstats.SetString("plugins.last_run_start", start.String())
	// appstats.MapSet("plugins", "last_run_start", start)

	p.running = true
	p.Unlock()

	var wg sync.WaitGroup

	if pluginName != "" {
		numFound := 0
		for pluginID, pluginRef := range p.active {
			if pluginID == pluginName || // specific plugin
				strings.HasPrefix(pluginID, pluginName+"`") { // specific plugin with instances
				numFound++
				wg.Add(1)
				p.logger.Debug().Str("id", pluginID).Msg("running")
				go func(id string, plug *plugin) {
					if err := plug.exec(); err != nil {
						plug.logger.Error().Err(err).Msg("executing")
					}
					plug.logger.Debug().Str("id", id).Str("duration", time.Since(start).String()).Msg("done")
					wg.Done()
				}(pluginID, pluginRef)
			}
		}
		if numFound == 0 {
			p.logger.Error().Str("id", pluginName).Msg("invalid/unknown")
			p.running = false
			return errors.Errorf("invalid plugin (%s)", pluginName)
		}
	} else {
		p.logger.Debug().Str("plugin(s)", strings.Join(p.plugList, ",")).Msg("running")
		for pluginID, pluginRef := range p.active {
			wg.Add(1)
			go func(id string, plug *plugin) {
				if err := plug.exec(); err != nil {
					plug.logger.Error().Err(err).Msg("executing")
				}
				plug.logger.Debug().Str("id", id).Str("duration", time.Since(start).String()).Msg("done")
				wg.Done()
			}(pluginID, pluginRef)
		}
	}

	wg.Wait()

	_ = appstats.SetString("plugins.last_run_end", time.Now().String())
	_ = appstats.SetString("plugins.last_run_duration", time.Since(start).String())
	// appstats.MapSet("plugins", "last_run_end", time.Now())
	// appstats.MapSet("plugins", "last_run_duration", time.Since(start))

	p.Lock()
	p.running = false
	p.logger.Debug().Str("duration", time.Since(start).String()).Msg("all plugins done")
	p.Unlock()

	return nil
}

// IsValid determines if a specific plugin is valid
func (p *Plugins) IsValid(pluginName string) bool {
	if pluginName == "" {
		return false
	}

	p.RLock()
	defer p.RUnlock()

	for pluginID := range p.active {
		// specific plugin       plugin with instances
		if pluginID == pluginName || strings.HasPrefix(pluginID, pluginName+"`") {
			return true
		}
	}

	return false
}

// IsInternal checks to see if the plugin is one of the internal plugins (write|statsd)
func (p *Plugins) IsInternal(pluginName string) bool {
	if pluginName == "" {
		return false
	}
	_, reserved := p.reservedNames[pluginName]

	return reserved
}

// Inventory returns list of active plugins
func (p *Plugins) Inventory() []byte {
	p.Lock()
	defer p.Unlock()
	inventory := api.Inventory{}
	for id, plug := range p.active {
		plug.Lock()
		pinfo := api.Plugin{
			ID:              id,
			Name:            plug.id,
			Instance:        plug.instanceID,
			Command:         plug.command,
			Args:            plug.instanceArgs,
			LastRunStart:    plug.lastStart.Format(time.RFC3339Nano),
			LastRunEnd:      plug.lastEnd.Format(time.RFC3339Nano),
			LastRunDuration: plug.lastRunDuration.String(),
		}
		if plug.lastError != nil {
			pinfo.LastError = plug.lastError.Error()
		}
		plug.Unlock()
		inventory = append(inventory, pinfo)
	}
	data, err := json.Marshal(inventory)
	if err != nil {
		p.logger.Fatal().Err(err).Msg("inventory -> json")
	}
	return data
}
<|endoftext|>