<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/couchbaselabs/go-couchbase"
	"log"
	"math/rand"
	"os"
	"text/tabwriter"
	"time"
)

var poolName = flag.String("pool", "default", "Pool to connect to")
var bucketName = flag.String("bucket", "default", "Bucket to connect to")

const myfmt = "2006-02-01-15:04:05.000000000"

var names = []string{
	"Jan Lehnardt",
	"John Christopher Anderson",
	"Noah Slater",
	"Filipe David Borba Manana",
	"Adam Kocoloski",
	"Paul Joseph Davis",
	"Christopher Lenz",
	"Damien F. Katz",
	"Robert Newson",
	"Benoit Chesneau",
	"Jason David Davies",
	"Mark Hammond",
	"Randall Leeds",
	"Bin Cui",
	"Benjamin Young",
	"Dustin Sallings",
	"Steve Yen",
	"Joe Schaefer",
}

var actions = []string{
	"submitted", "aborted", "approved", "declined",
}

var projects = []string{
	"ep-engine", "couchdb", "ns_server", "moxi", "libcouchbase",
}

type Record struct {
	Author   string `json:"author"`
	Reviewer string `json:"reviewer"`
	Action   string `json:"action"`
	Project  string `json:"project"`
	Score    int    `json:"score"`
}

func report(b couchbase.Bucket) {
	fmt.Printf("-----------------------------------------------------\n")
	tr := tabwriter.NewWriter(os.Stdout, 8, 8, 1, ' ', 0)
	defer tr.Flush()
	params := map[string]interface{}{
		"group_level": 1,
	}
	vres, err := b.View("test", "test", params)
	if err != nil {
		log.Fatalf("Error executing view:  %v", err)
	}

	for _, r := range vres.Rows {
		fmt.Fprintf(tr, "%v:\t%v\n", r.Key, r.Value)
	}
}

func harass(b couchbase.Bucket) {
	fmt.Printf("Doing stuff\n")

	go func() {
		for {
			time.Sleep(5 * time.Second)
			report(b)
		}
	}()

	for {
		r := Record{
			Author:   names[rand.Intn(len(names))],
			Reviewer: names[rand.Intn(len(names))],
			Action:   actions[rand.Intn(len(actions))],
			Project:  projects[rand.Intn(len(projects))],
			Score:    rand.Intn(4) - 2,
		}

		k := time.Now().Format(myfmt)

		if err := b.Set(k, r); err != nil {
			log.Fatalf("Oops, failed a store of %s:  %v", k, err)
		}
	}
}

func main() {
	flag.Parse()
	c, err := couchbase.Connect(flag.Arg(0))
	if err != nil {
		log.Fatalf("Error connecting:  %v", err)
	}

	pool, err := c.GetPool(*poolName)
	if err != nil {
		log.Fatalf("Error getting pool:  %v", err)
	}

	bucket, err := pool.GetBucket(*bucketName)
	if err != nil {
		log.Fatalf("Error getting bucket:  %v", err)
	}

	harass(bucket)
}
<commit_msg>Minor enhancements on the populate test.<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/couchbaselabs/go-couchbase"
	"log"
	"math/rand"
	"os"
	"sync/atomic"
	"text/tabwriter"
	"time"
)

var poolName = flag.String("pool", "default", "Pool to connect to")
var bucketName = flag.String("bucket", "default", "Bucket to connect to")

const myfmt = "2006-02-01-15:04:05.000000000"

var names = []string{
	"Jan Lehnardt",
	"John Christopher Anderson",
	"Noah Slater",
	"Filipe David Borba Manana",
	"Adam Kocoloski",
	"Paul Joseph Davis",
	"Christopher Lenz",
	"Damien F. Katz",
	"Robert Newson",
	"Benoit Chesneau",
	"Jason David Davies",
	"Mark Hammond",
	"Randall Leeds",
	"Bin Cui",
	"Benjamin Young",
	"Dustin Sallings",
	"Steve Yen",
	"Joe Schaefer",
}

var actions = []string{
	"submitted", "aborted", "approved", "declined",
}

var projects = []string{
	"ep-engine", "couchdb", "ns_server", "moxi", "libcouchbase",
}

type Record struct {
	Author   string `json:"author"`
	Reviewer string `json:"reviewer"`
	Action   string `json:"action"`
	Project  string `json:"project"`
	Score    int    `json:"score"`
}

func report(b couchbase.Bucket, msgs *uint64) {
	fmt.Printf("-----------------------------------------------------\n")
	fmt.Printf("Sent %d messages\n", *msgs)
	fmt.Printf("-----------------------------------------------------\n")
	tr := tabwriter.NewWriter(os.Stdout, 8, 8, 1, ' ', 0)
	defer tr.Flush()
	params := map[string]interface{}{
		"group_level":        1,
		"stale":              "update_after",
		"connection_timeout": 60000,
	}
	vres, err := b.View("test", "test", params)
	if err != nil {
		log.Printf("Error executing view:  %v", err)
	}

	for _, r := range vres.Rows {
		fmt.Fprintf(tr, "%v:\t%v\n", r.Key, r.Value)
	}
}

func harass(b couchbase.Bucket) {
	fmt.Printf("Doing stuff\n")
	msgs := uint64(0)

	go func() {
		for {
			time.Sleep(5 * time.Second)
			report(b, &msgs)
		}
	}()

	for {
		r := Record{
			Author:   names[rand.Intn(len(names))],
			Reviewer: names[rand.Intn(len(names))],
			Action:   actions[rand.Intn(len(actions))],
			Project:  projects[rand.Intn(len(projects))],
			Score:    rand.Intn(4) - 2,
		}

		k := time.Now().Format(myfmt)

		if err := b.Set(k, r); err != nil {
			log.Fatalf("Oops, failed a store of %s:  %v", k, err)
		}
		atomic.AddUint64(&msgs, 1)
	}
}

func main() {
	flag.Parse()
	c, err := couchbase.Connect(flag.Arg(0))
	if err != nil {
		log.Fatalf("Error connecting:  %v", err)
	}

	pool, err := c.GetPool(*poolName)
	if err != nil {
		log.Fatalf("Error getting pool:  %v", err)
	}

	bucket, err := pool.GetBucket(*bucketName)
	if err != nil {
		log.Fatalf("Error getting bucket:  %v", err)
	}

	harass(bucket)
}
<|endoftext|>


<commit_before>package certs

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"

	deis "github.com/deis/controller-sdk-go"
	"github.com/deis/controller-sdk-go/api"
)

// List certs registered with the controller.
func List(c *deis.Client, results int) ([]api.Cert, int, error) {
	body, count, err := c.LimitedRequest("/v2/certs/", results)

	if err != nil {
		return []api.Cert{}, -1, err
	}

	var res []api.Cert
	if err = json.Unmarshal([]byte(body), &res); err != nil {
		return []api.Cert{}, -1, err
	}

	return res, count, nil
}

// New creates a cert.
func New(c *deis.Client, cert string, key string, name string) (api.Cert, error) {
	req := api.CertCreateRequest{Certificate: cert, Key: key, Name: name}
	reqBody, err := json.Marshal(req)
	if err != nil {
		return api.Cert{}, err
	}

	res, reqErr := c.Request("POST", "/v2/certs/", reqBody)
	if reqErr != nil && !deis.IsErrAPIMismatch(reqErr) {
		return api.Cert{}, reqErr
	}
	// Fix json.Decoder bug in <go1.7
	defer func() {
		io.Copy(ioutil.Discard, res.Body)
		res.Body.Close()
	}()

	resCert := api.Cert{}
	if err = json.NewDecoder(res.Body).Decode(&resCert); err != nil {
		return api.Cert{}, err
	}

	return resCert, reqErr
}

// Get information for a certificate
func Get(c *deis.Client, name string) (api.Cert, error) {
	url := fmt.Sprintf("/v2/certs/%s", name)
	res, reqErr := c.Request("GET", url, nil)
	if reqErr != nil {
		return api.Cert{}, reqErr
	}
	// Fix json.Decoder bug in <go1.7
	defer func() {
		io.Copy(ioutil.Discard, res.Body)
		res.Body.Close()
	}()

	resCert := api.Cert{}
	if err := json.NewDecoder(res.Body).Decode(&resCert); err != nil {
		return api.Cert{}, err
	}

	return resCert, reqErr
}

// Delete removes a cert.
func Delete(c *deis.Client, name string) error {
	url := fmt.Sprintf("/v2/certs/%s", name)
	res, err := c.Request("DELETE", url, nil)
	if err == nil {
		res.Body.Close()
	}
	return err
}

// Attach a certificate to a domain
func Attach(c *deis.Client, name string, domain string) error {
	req := api.CertAttachRequest{Domain: domain}
	reqBody, err := json.Marshal(req)
	if err != nil {
		return err
	}

	url := fmt.Sprintf("/v2/certs/%s/domain/", name)
	res, err := c.Request("POST", url, reqBody)
	if err == nil {
		res.Body.Close()
	}
	return err
}

// Detach a certificate from a domain
func Detach(c *deis.Client, name string, domain string) error {
	url := fmt.Sprintf("/v2/certs/%s/domain/%s", name, domain)
	res, err := c.Request("DELETE", url, nil)
	if err == nil {
		res.Body.Close()
	}
	return err
}
<commit_msg>docs(certs): added godoc for certs package (#34)<commit_after>// Package certs manages SSL keys and certificates on the deis platform
package certs

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"

	deis "github.com/deis/controller-sdk-go"
	"github.com/deis/controller-sdk-go/api"
)

// List lists certificates added to deis.
func List(c *deis.Client, results int) ([]api.Cert, int, error) {
	body, count, err := c.LimitedRequest("/v2/certs/", results)

	if err != nil {
		return []api.Cert{}, -1, err
	}

	var res []api.Cert
	if err = json.Unmarshal([]byte(body), &res); err != nil {
		return []api.Cert{}, -1, err
	}

	return res, count, nil
}

// New creates a new certificate.
// Certificates are created independently from apps and are applied on a per domain basis.
// So to enable SSL for an app with the domain test.com, you would first create the certificate,
// then use the attach method to attach test.com to the certificate.
func New(c *deis.Client, cert string, key string, name string) (api.Cert, error) {
	req := api.CertCreateRequest{Certificate: cert, Key: key, Name: name}
	reqBody, err := json.Marshal(req)
	if err != nil {
		return api.Cert{}, err
	}

	res, reqErr := c.Request("POST", "/v2/certs/", reqBody)
	if reqErr != nil && !deis.IsErrAPIMismatch(reqErr) {
		return api.Cert{}, reqErr
	}
	// Fix json.Decoder bug in <go1.7
	defer func() {
		io.Copy(ioutil.Discard, res.Body)
		res.Body.Close()
	}()

	resCert := api.Cert{}
	if err = json.NewDecoder(res.Body).Decode(&resCert); err != nil {
		return api.Cert{}, err
	}

	return resCert, reqErr
}

// Get retrieves information about a certificate
func Get(c *deis.Client, name string) (api.Cert, error) {
	url := fmt.Sprintf("/v2/certs/%s", name)
	res, reqErr := c.Request("GET", url, nil)
	if reqErr != nil {
		return api.Cert{}, reqErr
	}
	// Fix json.Decoder bug in <go1.7
	defer func() {
		io.Copy(ioutil.Discard, res.Body)
		res.Body.Close()
	}()

	resCert := api.Cert{}
	if err := json.NewDecoder(res.Body).Decode(&resCert); err != nil {
		return api.Cert{}, err
	}

	return resCert, reqErr
}

// Delete removes a certificate.
func Delete(c *deis.Client, name string) error {
	url := fmt.Sprintf("/v2/certs/%s", name)
	res, err := c.Request("DELETE", url, nil)
	if err == nil {
		res.Body.Close()
	}
	return err
}

// Attach adds a domain to a certificate.
func Attach(c *deis.Client, name string, domain string) error {
	req := api.CertAttachRequest{Domain: domain}
	reqBody, err := json.Marshal(req)
	if err != nil {
		return err
	}

	url := fmt.Sprintf("/v2/certs/%s/domain/", name)
	res, err := c.Request("POST", url, reqBody)
	if err == nil {
		res.Body.Close()
	}
	return err
}

// Detach removes a domain from a certificate.
func Detach(c *deis.Client, name string, domain string) error {
	url := fmt.Sprintf("/v2/certs/%s/domain/%s", name, domain)
	res, err := c.Request("DELETE", url, nil)
	if err == nil {
		res.Body.Close()
	}
	return err
}
<|endoftext|>


<commit_before>package crawler

import (
	"os"
	"path/filepath"
	"time"

	cfg "github.com/elastic/filebeat/config"
	"github.com/elastic/filebeat/harvester"
	"github.com/elastic/filebeat/input"
	"github.com/elastic/libbeat/logp"
)

type Prospector struct {
	FileConfig     cfg.FileConfig
	prospectorList map[string]ProspectorFileStat
	iteration      uint32
	lastscan       time.Time
	crawler        *Crawler
	missingFiles   map[string]os.FileInfo
}

// Contains statistic about file when it was last seend by the prospector
type ProspectorFileStat struct {
	Fileinfo      os.FileInfo /* the file info */
	Harvester     chan int64  /* the harvester will send an event with its offset when it closes */
	LastIteration uint32      /* int number of the last iterations in which we saw this file */
}

// Init sets up default config for prospector
func (p *Prospector) Init() error {

	if p.FileConfig.IgnoreOlder != "" {

		var err error
		// Default ignore time time
		p.FileConfig.IgnoreOlderDuration, err = time.ParseDuration(p.FileConfig.IgnoreOlder)

		if err != nil {
			logp.Warn("Failed to parse dead time duration '%s'. Error was: %s\n", p.FileConfig.IgnoreOlder, err)
			return err
		}
	} else {
		logp.Debug("propsector", "Set ignoreOlderDuration to %s", cfg.DefaultIgnoreOlderDuration)
		// Set it to default
		p.FileConfig.IgnoreOlderDuration = cfg.DefaultIgnoreOlderDuration
	}

	if p.FileConfig.ScanFrequency != "" {

		var err error
		// Default ignore time time
		p.FileConfig.ScanFrequencyDuration, err = time.ParseDuration(p.FileConfig.ScanFrequency)

		if err != nil {
			logp.Warn("Failed to parse dead time duration '%s'. Error was: %s\n", p.FileConfig.IgnoreOlder, err)
			return err
		}
	} else {
		logp.Debug("propsector", "Set ignoreOlderDuration to %s", cfg.DefaultIgnoreOlderDuration)
		// Set it to default
		p.FileConfig.ScanFrequencyDuration = cfg.DefaultScanFrequency
	}

	// Init list
	p.prospectorList = make(map[string]ProspectorFileStat)

	return nil

}

// Starts scanning through all the file paths and fetch the related files. Start a harvester for each file
func (p *Prospector) Run(spoolChan chan *input.FileEvent) {

	// Handle any "-" (stdin) paths
	for i, path := range p.FileConfig.Paths {

		logp.Debug("prospector", "Harvest path: %s", path)

		if path == "-" {
			// Offset and Initial never get used when path is "-"
			h := harvester.Harvester{
				Path:       path,
				FileConfig: p.FileConfig,
				// TODO: SpoolerChan is passed around, but could be part of prospector (init)
				SpoolerChan: spoolChan,
			}

			h.Start()

			// Remove it from the file list
			p.FileConfig.Paths = append(p.FileConfig.Paths[:i], p.FileConfig.Paths[i+1:]...)
		}
	}

	// Seed last scan time
	p.lastscan = time.Now()

	// Now let's do one quick scan to pick up new files
	for _, path := range p.FileConfig.Paths {
		p.scan(path, spoolChan)
	}

	// This signals we finished considering the previous state
	event := &input.FileState{
		Source: nil,
	}
	p.crawler.Persist <- event

	for {
		newlastscan := time.Now()

		for _, path := range p.FileConfig.Paths {
			// Scan - flag false so new files always start at beginning TODO: is this still working as expected?
			p.scan(path, spoolChan)
		}

		p.lastscan = newlastscan

		// Defer next scan for a bit.
		// TODO: Implement config variable or find algorithm when to rescan https://github.com/elastic/filebeat/issues/19
		time.Sleep(10 * time.Second) // Make this tunable
		logp.Debug("prospector", "Start next scan")

		// Clear out files that disappeared and we've stopped harvesting
		for file, lastinfo := range p.prospectorList {
			if len(lastinfo.Harvester) != 0 && lastinfo.LastIteration < p.iteration {
				delete(p.prospectorList, file)
			}
		}

		p.iteration++ // Overflow is allowed
	}
}

// Scans the specific path which can be a glob (/**/**/*.log)
// For all found files it is checked if a harvester should be started
func (p *Prospector) scan(path string, output chan *input.FileEvent) {

	logp.Debug("prospector", "scan path %s", path)
	// Evaluate the path as a wildcards/shell glob
	matches, err := filepath.Glob(path)
	if err != nil {
		logp.Debug("prospector", "glob(%s) failed: %v", path, err)
		return
	}

	p.missingFiles = map[string]os.FileInfo{}

	// Check any matched files to see if we need to start a harvester
	for _, file := range matches {
		logp.Debug("prospector", "Check file for harvesting: %s", file)

		// Stat the file, following any symlinks.
		fileinfo, err := os.Stat(file)

		// TODO(sissel): check err
		if err != nil {
			logp.Debug("prospector", "stat(%s) failed: %s", file, err)
			continue
		}

		newFile := input.File{
			FileInfo: fileinfo,
		}

		if newFile.FileInfo.IsDir() {
			logp.Debug("prospector", "Skipping directory: %s", file)
			continue
		}

		// Check the current info against p.prospectorinfo[file]
		lastinfo, isKnown := p.prospectorList[file]

		oldFile := input.File{
			FileInfo: lastinfo.Fileinfo,
		}

		// Create a new prospector info with the stat info for comparison
		newInfo := ProspectorFileStat{
			Fileinfo:      newFile.FileInfo,
			Harvester:     make(chan int64, 1),
			LastIteration: p.iteration,
		}

		// Conditions for starting a new harvester:
		// - file path hasn't been seen before
		// - the file's inode or device changed
		if !isKnown {
			p.checkNewFile(&newInfo, file, output)
		} else {
			p.checkExistingFile(&newInfo, &newFile, &oldFile, file, output)
		}

		// Track the stat data for this file for later comparison to check for
		// rotation/etc
		p.prospectorList[file] = newInfo
	} // for each file matched by the glob
}

// Check if harvester for new file hast to be started
// For a new file the following options exist:
func (p *Prospector) checkNewFile(newinfo *ProspectorFileStat, file string, output chan *input.FileEvent) {

	logp.Debug("prospector", "Start harvesting unkown file:", file)

	// Init harvester with info
	h := &harvester.Harvester{
		Path:        file,
		FileConfig:  p.FileConfig,
		FinishChan:  newinfo.Harvester,
		SpoolerChan: output,
	}

	// Check for unmodified time, but only if the file modification time is before the last scan started
	// This ensures we don't skip genuine creations with dead times less than 10s
	if newinfo.Fileinfo.ModTime().Before(p.lastscan) &&
		time.Since(newinfo.Fileinfo.ModTime()) > p.FileConfig.IgnoreOlderDuration {

		// Call crawler if there if there exists a state for the given file
		offset, resuming := p.crawler.fetchState(file, newinfo.Fileinfo)

		// Are we resuming a dead file? We have to resume even if dead so we catch any old updates to the file
		// This is safe as the harvester, once it hits the EOF and a timeout, will stop harvesting
		// Once we detect changes again we can resume another harvester again - this keeps number of go routines to a minimum
		if resuming {
			logp.Debug("prospector", "Resuming harvester on a previously harvested file: %s", file)

			h.Offset = offset
			h.Start()
		} else {
			// Old file, skip it, but push offset of file size so we start from the end if this file changes and needs picking up
			logp.Debug("prospector", "Skipping file (older than ignore older of %v): %s", p.FileConfig.IgnoreOlderDuration, file)
			newinfo.Harvester <- newinfo.Fileinfo.Size()
		}
	} else if previousFile := p.isFileRenamed(file, newinfo.Fileinfo); previousFile != "" {
		// This file was simply renamed (known inode+dev) - link the same harvester channel as the old file
		logp.Debug("prospector", "File rename was detected: %s -> %s", previousFile, file)
		newinfo.Harvester = p.prospectorList[previousFile].Harvester

	} else {

		// Call crawler if there if there exists a state for the given file
		offset, resuming := p.crawler.fetchState(file, newinfo.Fileinfo)

		// Are we resuming a file or is this a completely new file?
		if resuming {
			logp.Debug("prospector", "Resuming harvester on a previously harvested file: %s", file)
		} else {
			logp.Debug("prospector", "Launching harvester on new file: %s", file)
		}

		// Launch the harvester
		h.Offset = offset
		h.Start()
	}
}

// checkExistingFile checks if a harvester has to be started for a already known file
// For existing files the following options exist:
// * Last reading position is 0, no harvester has to be started as old harvester probably still busy
// * The old known modification time is older then the current one. Start at last known position
// * The new file is not the same as the old file, means file was renamed
// ** New file is actually really a new file, start a new harvester
// ** Renamed file has a state, continue there
func (p *Prospector) checkExistingFile(newinfo *ProspectorFileStat, newFile *input.File, oldFile *input.File, file string, output chan *input.FileEvent) {

	logp.Debug("prospector", "Update existing file for harvesting:", file)

	h := &harvester.Harvester{
		Path:        file,
		FileConfig:  p.FileConfig,
		FinishChan:  newinfo.Harvester,
		SpoolerChan: output,
	}

	if !oldFile.IsSameFile(newFile) {

		if previousFile := p.isFileRenamed(file, newinfo.Fileinfo); previousFile != "" {
			// This file was renamed from another file we know - link the same harvester channel as the old file
			logp.Debug("prospector", "File rename was detected: %s -> %s", previousFile, file)
			logp.Debug("prospector", "Launching harvester on renamed file: %s", file)

			newinfo.Harvester = p.prospectorList[previousFile].Harvester
		} else {
			// File is not the same file we saw previously, it must have rotated and is a new file
			logp.Debug("prospector", "Launching harvester on rotated file: %s", file)

			// Forget about the previous harvester and let it continue on the old file - so start a new channel to use with the new harvester
			newinfo.Harvester = make(chan int64, 1)

			// Start a new harvester on the path
			h.Start()
		}

		// Keep the old file in missingFiles so we don't rescan it if it was renamed and we've not yet reached the new filename
		// We only need to keep it for the remainder of this iteration then we can assume it was deleted and forget about it
		p.missingFiles[file] = oldFile.FileInfo

	} else if len(newinfo.Harvester) != 0 && oldFile.FileInfo.ModTime() != newinfo.Fileinfo.ModTime() {
		// Resume harvesting of an old file we've stopped harvesting from
		logp.Debug("prospector", "Resuming harvester on an old file that was just modified: %s", file)

		// Start a harvester on the path; an old file was just modified and it doesn't have a harvester
		// The offset to continue from will be stored in the harvester channel - so take that to use and also clear the channel
		h.Offset = <-newinfo.Harvester
		h.Start()
	} else {
		logp.Debug("prospector", "Not harvesting, file didn't change: ", file)
	}
}

func (p *Prospector) Stop() {

}
<commit_msg>Introduce config in code<commit_after>package crawler

import (
	"os"
	"path/filepath"
	"time"

	cfg "github.com/elastic/filebeat/config"
	"github.com/elastic/filebeat/harvester"
	"github.com/elastic/filebeat/input"
	"github.com/elastic/libbeat/logp"
)

type Prospector struct {
	FileConfig     cfg.FileConfig
	prospectorList map[string]ProspectorFileStat
	iteration      uint32
	lastscan       time.Time
	crawler        *Crawler
	missingFiles   map[string]os.FileInfo
}

// Contains statistic about file when it was last seend by the prospector
type ProspectorFileStat struct {
	Fileinfo      os.FileInfo /* the file info */
	Harvester     chan int64  /* the harvester will send an event with its offset when it closes */
	LastIteration uint32      /* int number of the last iterations in which we saw this file */
}

// Init sets up default config for prospector
func (p *Prospector) Init() error {

	if p.FileConfig.IgnoreOlder != "" {

		var err error
		// Default ignore time time
		p.FileConfig.IgnoreOlderDuration, err = time.ParseDuration(p.FileConfig.IgnoreOlder)

		if err != nil {
			logp.Warn("Failed to parse dead time duration '%s'. Error was: %s\n", p.FileConfig.IgnoreOlder, err)
			return err
		}
	} else {
		logp.Debug("propsector", "Set ignoreOlderDuration to %s", cfg.DefaultIgnoreOlderDuration)
		// Set it to default
		p.FileConfig.IgnoreOlderDuration = cfg.DefaultIgnoreOlderDuration
	}

	if p.FileConfig.ScanFrequency != "" {

		var err error
		// Default ignore time time
		p.FileConfig.ScanFrequencyDuration, err = time.ParseDuration(p.FileConfig.ScanFrequency)

		if err != nil {
			logp.Warn("Failed to parse dead time duration '%s'. Error was: %s\n", p.FileConfig.IgnoreOlder, err)
			return err
		}
	} else {
		logp.Debug("propsector", "Set ignoreOlderDuration to %s", cfg.DefaultIgnoreOlderDuration)
		// Set it to default
		p.FileConfig.ScanFrequencyDuration = cfg.DefaultScanFrequency
	}

	// Init list
	p.prospectorList = make(map[string]ProspectorFileStat)

	return nil

}

// Starts scanning through all the file paths and fetch the related files. Start a harvester for each file
func (p *Prospector) Run(spoolChan chan *input.FileEvent) {

	// Handle any "-" (stdin) paths
	for i, path := range p.FileConfig.Paths {

		logp.Debug("prospector", "Harvest path: %s", path)

		if path == "-" {
			// Offset and Initial never get used when path is "-"
			h := harvester.Harvester{
				Path:       path,
				FileConfig: p.FileConfig,
				// TODO: SpoolerChan is passed around, but could be part of prospector (init)
				SpoolerChan: spoolChan,
			}

			h.Start()

			// Remove it from the file list
			p.FileConfig.Paths = append(p.FileConfig.Paths[:i], p.FileConfig.Paths[i+1:]...)
		}
	}

	// Seed last scan time
	p.lastscan = time.Now()

	// Now let's do one quick scan to pick up new files
	for _, path := range p.FileConfig.Paths {
		p.scan(path, spoolChan)
	}

	// This signals we finished considering the previous state
	event := &input.FileState{
		Source: nil,
	}
	p.crawler.Persist <- event

	for {
		newlastscan := time.Now()

		for _, path := range p.FileConfig.Paths {
			// Scan - flag false so new files always start at beginning TODO: is this still working as expected?
			p.scan(path, spoolChan)
		}

		p.lastscan = newlastscan

		// Defer next scan for the defined scanFrequency
		time.Sleep(p.FileConfig.ScanFrequencyDuration)
		logp.Debug("prospector", "Start next scan")

		// Clear out files that disappeared and we've stopped harvesting
		for file, lastinfo := range p.prospectorList {
			if len(lastinfo.Harvester) != 0 && lastinfo.LastIteration < p.iteration {
				delete(p.prospectorList, file)
			}
		}

		p.iteration++ // Overflow is allowed
	}
}

// Scans the specific path which can be a glob (/**/**/*.log)
// For all found files it is checked if a harvester should be started
func (p *Prospector) scan(path string, output chan *input.FileEvent) {

	logp.Debug("prospector", "scan path %s", path)
	// Evaluate the path as a wildcards/shell glob
	matches, err := filepath.Glob(path)
	if err != nil {
		logp.Debug("prospector", "glob(%s) failed: %v", path, err)
		return
	}

	p.missingFiles = map[string]os.FileInfo{}

	// Check any matched files to see if we need to start a harvester
	for _, file := range matches {
		logp.Debug("prospector", "Check file for harvesting: %s", file)

		// Stat the file, following any symlinks.
		fileinfo, err := os.Stat(file)

		// TODO(sissel): check err
		if err != nil {
			logp.Debug("prospector", "stat(%s) failed: %s", file, err)
			continue
		}

		newFile := input.File{
			FileInfo: fileinfo,
		}

		if newFile.FileInfo.IsDir() {
			logp.Debug("prospector", "Skipping directory: %s", file)
			continue
		}

		// Check the current info against p.prospectorinfo[file]
		lastinfo, isKnown := p.prospectorList[file]

		oldFile := input.File{
			FileInfo: lastinfo.Fileinfo,
		}

		// Create a new prospector info with the stat info for comparison
		newInfo := ProspectorFileStat{
			Fileinfo:      newFile.FileInfo,
			Harvester:     make(chan int64, 1),
			LastIteration: p.iteration,
		}

		// Conditions for starting a new harvester:
		// - file path hasn't been seen before
		// - the file's inode or device changed
		if !isKnown {
			p.checkNewFile(&newInfo, file, output)
		} else {
			p.checkExistingFile(&newInfo, &newFile, &oldFile, file, output)
		}

		// Track the stat data for this file for later comparison to check for
		// rotation/etc
		p.prospectorList[file] = newInfo
	} // for each file matched by the glob
}

// Check if harvester for new file hast to be started
// For a new file the following options exist:
func (p *Prospector) checkNewFile(newinfo *ProspectorFileStat, file string, output chan *input.FileEvent) {

	logp.Debug("prospector", "Start harvesting unkown file:", file)

	// Init harvester with info
	h := &harvester.Harvester{
		Path:        file,
		FileConfig:  p.FileConfig,
		FinishChan:  newinfo.Harvester,
		SpoolerChan: output,
	}

	// Check for unmodified time, but only if the file modification time is before the last scan started
	// This ensures we don't skip genuine creations with dead times less than 10s
	if newinfo.Fileinfo.ModTime().Before(p.lastscan) &&
		time.Since(newinfo.Fileinfo.ModTime()) > p.FileConfig.IgnoreOlderDuration {

		// Call crawler if there if there exists a state for the given file
		offset, resuming := p.crawler.fetchState(file, newinfo.Fileinfo)

		// Are we resuming a dead file? We have to resume even if dead so we catch any old updates to the file
		// This is safe as the harvester, once it hits the EOF and a timeout, will stop harvesting
		// Once we detect changes again we can resume another harvester again - this keeps number of go routines to a minimum
		if resuming {
			logp.Debug("prospector", "Resuming harvester on a previously harvested file: %s", file)

			h.Offset = offset
			h.Start()
		} else {
			// Old file, skip it, but push offset of file size so we start from the end if this file changes and needs picking up
			logp.Debug("prospector", "Skipping file (older than ignore older of %v): %s", p.FileConfig.IgnoreOlderDuration, file)
			newinfo.Harvester <- newinfo.Fileinfo.Size()
		}
	} else if previousFile := p.isFileRenamed(file, newinfo.Fileinfo); previousFile != "" {
		// This file was simply renamed (known inode+dev) - link the same harvester channel as the old file
		logp.Debug("prospector", "File rename was detected: %s -> %s", previousFile, file)
		newinfo.Harvester = p.prospectorList[previousFile].Harvester

	} else {

		// Call crawler if there if there exists a state for the given file
		offset, resuming := p.crawler.fetchState(file, newinfo.Fileinfo)

		// Are we resuming a file or is this a completely new file?
		if resuming {
			logp.Debug("prospector", "Resuming harvester on a previously harvested file: %s", file)
		} else {
			logp.Debug("prospector", "Launching harvester on new file: %s", file)
		}

		// Launch the harvester
		h.Offset = offset
		h.Start()
	}
}

// checkExistingFile checks if a harvester has to be started for a already known file
// For existing files the following options exist:
// * Last reading position is 0, no harvester has to be started as old harvester probably still busy
// * The old known modification time is older then the current one. Start at last known position
// * The new file is not the same as the old file, means file was renamed
// ** New file is actually really a new file, start a new harvester
// ** Renamed file has a state, continue there
func (p *Prospector) checkExistingFile(newinfo *ProspectorFileStat, newFile *input.File, oldFile *input.File, file string, output chan *input.FileEvent) {

	logp.Debug("prospector", "Update existing file for harvesting:", file)

	h := &harvester.Harvester{
		Path:        file,
		FileConfig:  p.FileConfig,
		FinishChan:  newinfo.Harvester,
		SpoolerChan: output,
	}

	if !oldFile.IsSameFile(newFile) {

		if previousFile := p.isFileRenamed(file, newinfo.Fileinfo); previousFile != "" {
			// This file was renamed from another file we know - link the same harvester channel as the old file
			logp.Debug("prospector", "File rename was detected: %s -> %s", previousFile, file)
			logp.Debug("prospector", "Launching harvester on renamed file: %s", file)

			newinfo.Harvester = p.prospectorList[previousFile].Harvester
		} else {
			// File is not the same file we saw previously, it must have rotated and is a new file
			logp.Debug("prospector", "Launching harvester on rotated file: %s", file)

			// Forget about the previous harvester and let it continue on the old file - so start a new channel to use with the new harvester
			newinfo.Harvester = make(chan int64, 1)

			// Start a new harvester on the path
			h.Start()
		}

		// Keep the old file in missingFiles so we don't rescan it if it was renamed and we've not yet reached the new filename
		// We only need to keep it for the remainder of this iteration then we can assume it was deleted and forget about it
		p.missingFiles[file] = oldFile.FileInfo

	} else if len(newinfo.Harvester) != 0 && oldFile.FileInfo.ModTime() != newinfo.Fileinfo.ModTime() {
		// Resume harvesting of an old file we've stopped harvesting from
		logp.Debug("prospector", "Resuming harvester on an old file that was just modified: %s", file)

		// Start a harvester on the path; an old file was just modified and it doesn't have a harvester
		// The offset to continue from will be stored in the harvester channel - so take that to use and also clear the channel
		h.Offset = <-newinfo.Harvester
		h.Start()
	} else {
		logp.Debug("prospector", "Not harvesting, file didn't change: ", file)
	}
}

func (p *Prospector) Stop() {

}
<|endoftext|>


<commit_before>package credentials

import (
	"bytes"
	"encoding/binary"
	"errors"
	"github.com/jcmturner/asn1"
	"github.com/jcmturner/gokrb5/types"
	"io/ioutil"
	"strings"
	"time"
	"unsafe"
)

const (
	headerFieldTagKDCOffset = 1
)

// The first byte of the file always has the value 5.
// The value of the second byte contains the version number (1 through 4)
// Versions 1 and 2 of the file format use native byte order for integer representations.
// Versions 3 and 4 always use big-endian byte order
// After the two-byte version indicator, the file has three parts:
//   1) the header (in version 4 only)
//   2) the default principal name
//   3) a sequence of credentials

// CCache is the file credentials cache as define here: https://web.mit.edu/kerberos/krb5-latest/doc/formats/ccache_file_format.html
type CCache struct {
	Version          uint8
	Header           header
	DefaultPrincipal principal
	Credentials      []credential
	Path             string
}

type header struct {
	length uint16
	fields []headerField
}

type headerField struct {
	tag    uint16
	length uint16
	value  []byte
}

// Credential cache entry principal struct.
type principal struct {
	Realm         string
	PrincipalName types.PrincipalName
}

type credential struct {
	Client       principal
	Server       principal
	Key          types.EncryptionKey
	AuthTime     time.Time
	StartTime    time.Time
	EndTime      time.Time
	RenewTill    time.Time
	IsSKey       bool
	TicketFlags  asn1.BitString
	Addresses    []types.HostAddress
	AuthData     []types.AuthorizationDataEntry
	Ticket       []byte
	SecondTicket []byte
}

// LoadCCache loads a credential cache file into a CCache type.
func LoadCCache(cpath string) (CCache, error) {
	k, err := ioutil.ReadFile(cpath)
	if err != nil {
		return CCache{}, err
	}
	c, err := ParseCCache(k)
	c.Path = cpath
	return c, err
}

// ParseCCache byte slice of credential cache data into CCache type.
func ParseCCache(b []byte) (c CCache, err error) {
	p := 0
	//The first byte of the file always has the value 5
	if int8(b[p]) != 5 {
		err = errors.New("Invalid credential cache data. First byte does not equal 5")
		return
	}
	p++
	//Get credential cache version
	//The second byte contains the version number (1 to 4)
	c.Version = uint8(b[p])
	if c.Version < 1 || c.Version > 4 {
		err = errors.New("Invalid credential cache data. Keytab version is not within 1 to 4")
		if err != nil {
			return
		}
	}
	p++
	//Version 1 or 2 of the file format uses native byte order for integer representations. Versions 3 & 4 always uses big-endian byte order
	var endian binary.ByteOrder
	endian = binary.BigEndian
	if (c.Version == 1 || c.Version == 2) && isNativeEndianLittle() {
		endian = binary.LittleEndian
	}
	if c.Version == 4 {
		err = parse_header(b, &p, &c, &endian)
		if err != nil {
			return
		}
	}
	c.DefaultPrincipal = parse_principal(b, &p, &c, &endian)
	for p < len(b) {
		cred, e := parse_credential(b, &p, &c, &endian)
		if e != nil {
			err = e
			return
		}
		c.Credentials = append(c.Credentials, cred)
	}
	return
}

func parse_header(b []byte, p *int, c *CCache, e *binary.ByteOrder) error {
	if c.Version != 4 {
		return errors.New("Credentials cache version is not 4 so there is no header to parse.")
	}
	h := header{}
	h.length = uint16(read_int16(b, p, e))
	for *p <= int(h.length) {
		f := headerField{}
		f.tag = uint16(read_int16(b, p, e))
		f.length = uint16(read_int16(b, p, e))
		f.value = b[*p : *p+int(f.length)]
		*p += int(f.length)
		if !f.valid() {
			return errors.New("Invalid credential cache header found")
		}
		h.fields = append(h.fields, f)
	}
	c.Header = h
	return nil
}

// Parse the Keytab bytes of a principal into a Keytab entry's principal.
func parse_principal(b []byte, p *int, c *CCache, e *binary.ByteOrder) (princ principal) {
	if c.Version != 1 {
		//Name Type is omitted in version 1
		princ.PrincipalName.NameType = int(read_int32(b, p, e))
	}
	nc := int(read_int32(b, p, e))
	if c.Version == 1 {
		//In version 1 the number of components includes the realm. Minus 1 to make consistent with version 2
		nc--
	}
	len_realm := read_int32(b, p, e)
	princ.Realm = string(read_Bytes(b, p, int(len_realm), e))
	for i := 0; i < int(nc); i++ {
		l := read_int32(b, p, e)
		princ.PrincipalName.NameString = append(princ.PrincipalName.NameString, string(read_Bytes(b, p, int(l), e)))
	}
	return princ
}

func parse_credential(b []byte, p *int, c *CCache, e *binary.ByteOrder) (cred credential, err error) {
	cred.Client = parse_principal(b, p, c, e)
	cred.Server = parse_principal(b, p, c, e)
	key := types.EncryptionKey{}
	key.KeyType = int(read_int16(b, p, e))
	if c.Version == 3 {
		//repeated twice in version 3
		key.KeyType = int(read_int16(b, p, e))
	}
	key.KeyValue = read_data(b, p, e)
	cred.Key = key
	cred.AuthTime = read_timestamp(b, p, e)
	cred.StartTime = read_timestamp(b, p, e)
	cred.EndTime = read_timestamp(b, p, e)
	cred.RenewTill = read_timestamp(b, p, e)
	if ik := read_int8(b, p, e); ik == 0 {
		cred.IsSKey = false
	} else {
		cred.IsSKey = true
	}
	cred.TicketFlags = types.NewKrbFlags()
	cred.TicketFlags.Bytes = read_Bytes(b, p, 4, e)
	l := int(read_int32(b, p, e))
	cred.Addresses = make([]types.HostAddress, l, l)
	for i := range cred.Addresses {
		cred.Addresses[i] = read_address(b, p, e)
	}
	l = int(read_int32(b, p, e))
	cred.AuthData = make([]types.AuthorizationDataEntry, l, l)
	for i := range cred.AuthData {
		cred.AuthData[i] = read_authDataEntry(b, p, e)
	}
	cred.Ticket = read_data(b, p, e)
	cred.SecondTicket = read_data(b, p, e)
	return
}

// GetClientPrincipalName returns a PrincipalName type for the client the credentials cache is for.
func (c *CCache) GetClientPrincipalName() types.PrincipalName {
	return c.DefaultPrincipal.PrincipalName
}

// GetClientRealm returns the reals of teh client the credentials cache is for.
func (c *CCache) GetClientRealm() string {
	return c.DefaultPrincipal.Realm
}

// GetClientCredentials returns a Credentials object representing the client of the credentials cache.
func (c *CCache) GetClientCredentials() *Credentials {
	return &Credentials{
		Username: c.DefaultPrincipal.PrincipalName.GetPrincipalNameString(),
		Realm:    c.GetClientRealm(),
		CName:    c.DefaultPrincipal.PrincipalName,
	}
}

// Contains tests if the cache contains a credential for the provided server PrincipalName
func (c *CCache) Contains(p types.PrincipalName) bool {
	for _, cred := range c.Credentials {
		if cred.Server.PrincipalName.Equal(p) {
			return true
		}
	}
	return false
}

// GetEntry returns a specific credential for the PrincipalName provided.
func (c *CCache) GetEntry(p types.PrincipalName) (credential, bool) {
	var cred credential
	var found bool
	for i := range c.Credentials {
		if c.Credentials[i].Server.PrincipalName.Equal(p) {
			cred = c.Credentials[i]
			found = true
			break
		}
	}
	if !found {
		return cred, false
	}
	return cred, true
}

// GetEntries filters out configuration entries an returns a slice of credentials.
func (c *CCache) GetEntries() []credential {
	var creds []credential
	for _, cred := range c.Credentials {
		// Filter out configuration entries
		if strings.HasPrefix(cred.Server.Realm, "X-CACHECONF") {
			continue
		}
		creds = append(creds, cred)
	}
	return creds
}

func (h *headerField) valid() bool {
	// At this time there is only one defined header field.
	// Its tag value is 1, its length is always 8.
	// Its contents are two 32-bit integers giving the seconds and microseconds
	// of the time offset of the KDC relative to the client.
	// Adding this offset to the current time on the client should give the current time on the KDC, if that offset has not changed since the initial authentication.

	// Done as a switch in case other tag values are added in the future.
	switch h.tag {
	case headerFieldTagKDCOffset:
		if h.length != 8 || len(h.value) != 8 {
			return false
		}
		return true
	}
	return false
}

func read_data(b []byte, p *int, e *binary.ByteOrder) []byte {
	l := read_int32(b, p, e)
	return read_Bytes(b, p, int(l), e)
}

func read_address(b []byte, p *int, e *binary.ByteOrder) types.HostAddress {
	a := types.HostAddress{}
	a.AddrType = int(read_int16(b, p, e))
	a.Address = read_data(b, p, e)
	return a
}

func read_authDataEntry(b []byte, p *int, e *binary.ByteOrder) types.AuthorizationDataEntry {
	a := types.AuthorizationDataEntry{}
	a.ADType = int(read_int16(b, p, e))
	a.ADData = read_data(b, p, e)
	return a
}

// Read bytes representing a timestamp.
func read_timestamp(b []byte, p *int, e *binary.ByteOrder) time.Time {
	return time.Unix(int64(read_int32(b, p, e)), 0)
}

// Read bytes representing an eight bit integer.
func read_int8(b []byte, p *int, e *binary.ByteOrder) (i int8) {
	buf := bytes.NewBuffer(b[*p : *p+1])
	binary.Read(buf, *e, &i)
	*p++
	return
}

// Read bytes representing a sixteen bit integer.
func read_int16(b []byte, p *int, e *binary.ByteOrder) (i int16) {
	buf := bytes.NewBuffer(b[*p : *p+2])
	binary.Read(buf, *e, &i)
	*p += 2
	return
}

// Read bytes representing a thirty two bit integer.
func read_int32(b []byte, p *int, e *binary.ByteOrder) (i int32) {
	buf := bytes.NewBuffer(b[*p : *p+4])
	binary.Read(buf, *e, &i)
	*p += 4
	return
}

func read_Bytes(b []byte, p *int, s int, e *binary.ByteOrder) []byte {
	buf := bytes.NewBuffer(b[*p : *p+s])
	r := make([]byte, s)
	binary.Read(buf, *e, &r)
	*p += s
	return r
}

func isNativeEndianLittle() bool {
	var x = 0x012345678
	var p = unsafe.Pointer(&x)
	var bp = (*[4]byte)(p)

	var endian bool
	if 0x01 == bp[0] {
		endian = false
	} else if (0x78 & 0xff) == (bp[0] & 0xff) {
		endian = true
	} else {
		// Default to big endian
		endian = false
	}
	return endian
}
<commit_msg>spelling<commit_after>package credentials

import (
	"bytes"
	"encoding/binary"
	"errors"
	"github.com/jcmturner/asn1"
	"github.com/jcmturner/gokrb5/types"
	"io/ioutil"
	"strings"
	"time"
	"unsafe"
)

const (
	headerFieldTagKDCOffset = 1
)

// The first byte of the file always has the value 5.
// The value of the second byte contains the version number (1 through 4)
// Versions 1 and 2 of the file format use native byte order for integer representations.
// Versions 3 and 4 always use big-endian byte order
// After the two-byte version indicator, the file has three parts:
//   1) the header (in version 4 only)
//   2) the default principal name
//   3) a sequence of credentials

// CCache is the file credentials cache as define here: https://web.mit.edu/kerberos/krb5-latest/doc/formats/ccache_file_format.html
type CCache struct {
	Version          uint8
	Header           header
	DefaultPrincipal principal
	Credentials      []credential
	Path             string
}

type header struct {
	length uint16
	fields []headerField
}

type headerField struct {
	tag    uint16
	length uint16
	value  []byte
}

// Credential cache entry principal struct.
type principal struct {
	Realm         string
	PrincipalName types.PrincipalName
}

type credential struct {
	Client       principal
	Server       principal
	Key          types.EncryptionKey
	AuthTime     time.Time
	StartTime    time.Time
	EndTime      time.Time
	RenewTill    time.Time
	IsSKey       bool
	TicketFlags  asn1.BitString
	Addresses    []types.HostAddress
	AuthData     []types.AuthorizationDataEntry
	Ticket       []byte
	SecondTicket []byte
}

// LoadCCache loads a credential cache file into a CCache type.
func LoadCCache(cpath string) (CCache, error) {
	k, err := ioutil.ReadFile(cpath)
	if err != nil {
		return CCache{}, err
	}
	c, err := ParseCCache(k)
	c.Path = cpath
	return c, err
}

// ParseCCache byte slice of credential cache data into CCache type.
func ParseCCache(b []byte) (c CCache, err error) {
	p := 0
	//The first byte of the file always has the value 5
	if int8(b[p]) != 5 {
		err = errors.New("Invalid credential cache data. First byte does not equal 5")
		return
	}
	p++
	//Get credential cache version
	//The second byte contains the version number (1 to 4)
	c.Version = uint8(b[p])
	if c.Version < 1 || c.Version > 4 {
		err = errors.New("Invalid credential cache data. Keytab version is not within 1 to 4")
		if err != nil {
			return
		}
	}
	p++
	//Version 1 or 2 of the file format uses native byte order for integer representations. Versions 3 & 4 always uses big-endian byte order
	var endian binary.ByteOrder
	endian = binary.BigEndian
	if (c.Version == 1 || c.Version == 2) && isNativeEndianLittle() {
		endian = binary.LittleEndian
	}
	if c.Version == 4 {
		err = parse_header(b, &p, &c, &endian)
		if err != nil {
			return
		}
	}
	c.DefaultPrincipal = parse_principal(b, &p, &c, &endian)
	for p < len(b) {
		cred, e := parse_credential(b, &p, &c, &endian)
		if e != nil {
			err = e
			return
		}
		c.Credentials = append(c.Credentials, cred)
	}
	return
}

func parse_header(b []byte, p *int, c *CCache, e *binary.ByteOrder) error {
	if c.Version != 4 {
		return errors.New("Credentials cache version is not 4 so there is no header to parse.")
	}
	h := header{}
	h.length = uint16(read_int16(b, p, e))
	for *p <= int(h.length) {
		f := headerField{}
		f.tag = uint16(read_int16(b, p, e))
		f.length = uint16(read_int16(b, p, e))
		f.value = b[*p : *p+int(f.length)]
		*p += int(f.length)
		if !f.valid() {
			return errors.New("Invalid credential cache header found")
		}
		h.fields = append(h.fields, f)
	}
	c.Header = h
	return nil
}

// Parse the Keytab bytes of a principal into a Keytab entry's principal.
func parse_principal(b []byte, p *int, c *CCache, e *binary.ByteOrder) (princ principal) {
	if c.Version != 1 {
		//Name Type is omitted in version 1
		princ.PrincipalName.NameType = int(read_int32(b, p, e))
	}
	nc := int(read_int32(b, p, e))
	if c.Version == 1 {
		//In version 1 the number of components includes the realm. Minus 1 to make consistent with version 2
		nc--
	}
	len_realm := read_int32(b, p, e)
	princ.Realm = string(read_Bytes(b, p, int(len_realm), e))
	for i := 0; i < int(nc); i++ {
		l := read_int32(b, p, e)
		princ.PrincipalName.NameString = append(princ.PrincipalName.NameString, string(read_Bytes(b, p, int(l), e)))
	}
	return princ
}

func parse_credential(b []byte, p *int, c *CCache, e *binary.ByteOrder) (cred credential, err error) {
	cred.Client = parse_principal(b, p, c, e)
	cred.Server = parse_principal(b, p, c, e)
	key := types.EncryptionKey{}
	key.KeyType = int(read_int16(b, p, e))
	if c.Version == 3 {
		//repeated twice in version 3
		key.KeyType = int(read_int16(b, p, e))
	}
	key.KeyValue = read_data(b, p, e)
	cred.Key = key
	cred.AuthTime = read_timestamp(b, p, e)
	cred.StartTime = read_timestamp(b, p, e)
	cred.EndTime = read_timestamp(b, p, e)
	cred.RenewTill = read_timestamp(b, p, e)
	if ik := read_int8(b, p, e); ik == 0 {
		cred.IsSKey = false
	} else {
		cred.IsSKey = true
	}
	cred.TicketFlags = types.NewKrbFlags()
	cred.TicketFlags.Bytes = read_Bytes(b, p, 4, e)
	l := int(read_int32(b, p, e))
	cred.Addresses = make([]types.HostAddress, l, l)
	for i := range cred.Addresses {
		cred.Addresses[i] = read_address(b, p, e)
	}
	l = int(read_int32(b, p, e))
	cred.AuthData = make([]types.AuthorizationDataEntry, l, l)
	for i := range cred.AuthData {
		cred.AuthData[i] = read_authDataEntry(b, p, e)
	}
	cred.Ticket = read_data(b, p, e)
	cred.SecondTicket = read_data(b, p, e)
	return
}

// GetClientPrincipalName returns a PrincipalName type for the client the credentials cache is for.
func (c *CCache) GetClientPrincipalName() types.PrincipalName {
	return c.DefaultPrincipal.PrincipalName
}

// GetClientRealm returns the reals of the client the credentials cache is for.
func (c *CCache) GetClientRealm() string {
	return c.DefaultPrincipal.Realm
}

// GetClientCredentials returns a Credentials object representing the client of the credentials cache.
func (c *CCache) GetClientCredentials() *Credentials {
	return &Credentials{
		Username: c.DefaultPrincipal.PrincipalName.GetPrincipalNameString(),
		Realm:    c.GetClientRealm(),
		CName:    c.DefaultPrincipal.PrincipalName,
	}
}

// Contains tests if the cache contains a credential for the provided server PrincipalName
func (c *CCache) Contains(p types.PrincipalName) bool {
	for _, cred := range c.Credentials {
		if cred.Server.PrincipalName.Equal(p) {
			return true
		}
	}
	return false
}

// GetEntry returns a specific credential for the PrincipalName provided.
func (c *CCache) GetEntry(p types.PrincipalName) (credential, bool) {
	var cred credential
	var found bool
	for i := range c.Credentials {
		if c.Credentials[i].Server.PrincipalName.Equal(p) {
			cred = c.Credentials[i]
			found = true
			break
		}
	}
	if !found {
		return cred, false
	}
	return cred, true
}

// GetEntries filters out configuration entries an returns a slice of credentials.
func (c *CCache) GetEntries() []credential {
	var creds []credential
	for _, cred := range c.Credentials {
		// Filter out configuration entries
		if strings.HasPrefix(cred.Server.Realm, "X-CACHECONF") {
			continue
		}
		creds = append(creds, cred)
	}
	return creds
}

func (h *headerField) valid() bool {
	// At this time there is only one defined header field.
	// Its tag value is 1, its length is always 8.
	// Its contents are two 32-bit integers giving the seconds and microseconds
	// of the time offset of the KDC relative to the client.
	// Adding this offset to the current time on the client should give the current time on the KDC, if that offset has not changed since the initial authentication.

	// Done as a switch in case other tag values are added in the future.
	switch h.tag {
	case headerFieldTagKDCOffset:
		if h.length != 8 || len(h.value) != 8 {
			return false
		}
		return true
	}
	return false
}

func read_data(b []byte, p *int, e *binary.ByteOrder) []byte {
	l := read_int32(b, p, e)
	return read_Bytes(b, p, int(l), e)
}

func read_address(b []byte, p *int, e *binary.ByteOrder) types.HostAddress {
	a := types.HostAddress{}
	a.AddrType = int(read_int16(b, p, e))
	a.Address = read_data(b, p, e)
	return a
}

func read_authDataEntry(b []byte, p *int, e *binary.ByteOrder) types.AuthorizationDataEntry {
	a := types.AuthorizationDataEntry{}
	a.ADType = int(read_int16(b, p, e))
	a.ADData = read_data(b, p, e)
	return a
}

// Read bytes representing a timestamp.
func read_timestamp(b []byte, p *int, e *binary.ByteOrder) time.Time {
	return time.Unix(int64(read_int32(b, p, e)), 0)
}

// Read bytes representing an eight bit integer.
func read_int8(b []byte, p *int, e *binary.ByteOrder) (i int8) {
	buf := bytes.NewBuffer(b[*p : *p+1])
	binary.Read(buf, *e, &i)
	*p++
	return
}

// Read bytes representing a sixteen bit integer.
func read_int16(b []byte, p *int, e *binary.ByteOrder) (i int16) {
	buf := bytes.NewBuffer(b[*p : *p+2])
	binary.Read(buf, *e, &i)
	*p += 2
	return
}

// Read bytes representing a thirty two bit integer.
func read_int32(b []byte, p *int, e *binary.ByteOrder) (i int32) {
	buf := bytes.NewBuffer(b[*p : *p+4])
	binary.Read(buf, *e, &i)
	*p += 4
	return
}

func read_Bytes(b []byte, p *int, s int, e *binary.ByteOrder) []byte {
	buf := bytes.NewBuffer(b[*p : *p+s])
	r := make([]byte, s)
	binary.Read(buf, *e, &r)
	*p += s
	return r
}

func isNativeEndianLittle() bool {
	var x = 0x012345678
	var p = unsafe.Pointer(&x)
	var bp = (*[4]byte)(p)

	var endian bool
	if 0x01 == bp[0] {
		endian = false
	} else if (0x78 & 0xff) == (bp[0] & 0xff) {
		endian = true
	} else {
		// Default to big endian
		endian = false
	}
	return endian
}
<|endoftext|>


<commit_before>// Copyright (c) 2013-2015 The btcsuite developers
// Copyright (c) 2015-2016 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package chain

import (
	"errors"
	"fmt"
	"sync"
	"time"

	"github.com/decred/dcrd/chaincfg"
	"github.com/decred/dcrd/chaincfg/chainhash"
	"github.com/decred/dcrrpcclient"
)

var requiredChainServerAPI = semver{major: 2, minor: 0, patch: 0}

// RPCClient represents a persistent client connection to a decred RPC server
// for information regarding the current best block chain.
type RPCClient struct {
	*dcrrpcclient.Client
	connConfig        *dcrrpcclient.ConnConfig // Work around unexported field
	chainParams       *chaincfg.Params
	reconnectAttempts int

	enqueueNotification       chan interface{}
	dequeueNotification       chan interface{}
	enqueueVotingNotification chan interface{}
	dequeueVotingNotification chan interface{}

	quit    chan struct{}
	wg      sync.WaitGroup
	started bool
	quitMtx sync.Mutex
}

// NewRPCClient creates a client connection to the server described by the
// connect string.  If disableTLS is false, the remote RPC certificate must be
// provided in the certs slice.  The connection is not established immediately,
// but must be done using the Start method.  If the remote server does not
// operate on the same bitcoin network as described by the passed chain
// parameters, the connection will be disconnected.
func NewRPCClient(chainParams *chaincfg.Params, connect, user, pass string, certs []byte,
	disableTLS bool, reconnectAttempts int) (*RPCClient, error) {

	if reconnectAttempts < 0 {
		return nil, errors.New("reconnectAttempts must be positive")
	}

	client := &RPCClient{
		connConfig: &dcrrpcclient.ConnConfig{
			Host:                 connect,
			Endpoint:             "ws",
			User:                 user,
			Pass:                 pass,
			Certificates:         certs,
			DisableAutoReconnect: true,
			DisableConnectOnNew:  true,
			DisableTLS:           disableTLS,
		},
		chainParams:               chainParams,
		reconnectAttempts:         reconnectAttempts,
		enqueueNotification:       make(chan interface{}),
		dequeueNotification:       make(chan interface{}),
		enqueueVotingNotification: make(chan interface{}),
		dequeueVotingNotification: make(chan interface{}),
		quit: make(chan struct{}),
	}
	ntfnCallbacks := &dcrrpcclient.NotificationHandlers{
		OnClientConnected:       client.onClientConnect,
		OnBlockConnected:        client.onBlockConnected,
		OnBlockDisconnected:     client.onBlockDisconnected,
		OnRelevantTxAccepted:    client.onRelevantTxAccepted,
		OnReorganization:        client.onReorganization,
		OnWinningTickets:        client.onWinningTickets,
		OnSpentAndMissedTickets: client.onSpentAndMissedTickets,
		OnStakeDifficulty:       client.onStakeDifficulty,
	}
	rpcClient, err := dcrrpcclient.New(client.connConfig, ntfnCallbacks)
	if err != nil {
		return nil, err
	}
	client.Client = rpcClient
	return client, nil
}

// Start attempts to establish a client connection with the remote server.
// If successful, handler goroutines are started to process notifications
// sent by the server.  After a limited number of connection attempts, this
// function gives up, and therefore will not block forever waiting for the
// connection to be established to a server that may not exist.
func (c *RPCClient) Start() (err error) {
	err = c.Connect(c.reconnectAttempts)
	if err != nil {
		return err
	}

	defer func() {
		if err != nil {
			c.Disconnect()
		}
	}()

	// Verify that the server is running on the expected network.
	net, err := c.GetCurrentNet()
	if err != nil {
		return err
	}
	if net != c.chainParams.Net {
		return errors.New("mismatched networks")
	}

	// Ensure the RPC server has a compatible API version.
	var serverAPI semver
	versions, err := c.Version()
	if err == nil {
		versionResult := versions["dcrdjsonrpcapi"]
		serverAPI = semver{
			major: versionResult.Major,
			minor: versionResult.Minor,
			patch: versionResult.Patch,
		}
	}
	if !semverCompatible(requiredChainServerAPI, serverAPI) {
		return fmt.Errorf("consensus JSON-RPC server does not have a "+
			"compatible API version: advertises %v but require %v",
			serverAPI, requiredChainServerAPI)
	}

	c.quitMtx.Lock()
	c.started = true
	c.quitMtx.Unlock()

	c.wg.Add(2)
	go c.handler()
	go c.handlerVoting()
	return nil
}

// Stop disconnects the client and signals the shutdown of all goroutines
// started by Start.
func (c *RPCClient) Stop() {
	c.quitMtx.Lock()
	select {
	case <-c.quit:
	default:
		close(c.quit)
		c.Client.Shutdown()

		if !c.started {
			close(c.dequeueNotification)
			close(c.dequeueVotingNotification)
		}
	}
	c.quitMtx.Unlock()
}

// WaitForShutdown blocks until both the client has finished disconnecting
// and all handlers have exited.
func (c *RPCClient) WaitForShutdown() {
	c.Client.WaitForShutdown()
	c.wg.Wait()
}

// Notification types.  These are defined here and processed from from reading
// a notificationChan to avoid handling these notifications directly in
// dcrrpcclient callbacks, which isn't very Go-like and doesn't allow
// blocking client calls.
type (
	// ClientConnected is a notification for when a client connection is
	// opened or reestablished to the chain server.
	ClientConnected struct{}

	// BlockConnected is a notification for a newly-attached block to the
	// best chain.
	BlockConnected struct {
		BlockHeader  []byte
		Transactions [][]byte
	}

	// BlockDisconnected is a notifcation that the block described by the
	// BlockStamp was reorganized out of the best chain.
	BlockDisconnected struct {
		BlockHeader []byte
	}

	// RelevantTxAccepted is a notification that a transaction accepted by
	// mempool passed the client's transaction filter.
	RelevantTxAccepted struct {
		Transaction []byte
	}

	// Reorganization is a notification that a reorg has happen with the new
	// old and new tip included.
	Reorganization struct {
		OldHash   *chainhash.Hash
		OldHeight int64
		NewHash   *chainhash.Hash
		NewHeight int64
	}

	// WinningTickets is a notification with the winning tickets (and the
	// block they are in.
	WinningTickets struct {
		BlockHash   *chainhash.Hash
		BlockHeight int64
		Tickets     []*chainhash.Hash
	}

	// MissedTickets is a notifcation for tickets that have been missed.
	MissedTickets struct {
		BlockHash   *chainhash.Hash
		BlockHeight int64
		Tickets     []*chainhash.Hash
	}

	// StakeDifficulty is a notification for the current stake difficulty.
	StakeDifficulty struct {
		BlockHash   *chainhash.Hash
		BlockHeight int64
		StakeDiff   int64
	}
)

// Notifications returns a channel of parsed notifications sent by the remote
// decred RPC server.  This channel must be continually read or the process
// may abort for running out memory, as unread notifications are queued for
// later reads.
func (c *RPCClient) Notifications() <-chan interface{} {
	return c.dequeueNotification
}

// NotificationsVoting returns a channel of parsed voting notifications sent
// by the remote RPC server.  This channel must be continually read or the
// process may abort for running out memory, as unread notifications are
// queued for later reads.
func (c *RPCClient) NotificationsVoting() <-chan interface{} {
	return c.dequeueVotingNotification
}

func (c *RPCClient) onClientConnect() {
	select {
	case c.enqueueNotification <- ClientConnected{}:
	case <-c.quit:
	}
}

func (c *RPCClient) onBlockConnected(header []byte, transactions [][]byte) {
	select {
	case c.enqueueNotification <- BlockConnected{
		BlockHeader:  header,
		Transactions: transactions,
	}:
	case <-c.quit:
	}
}

func (c *RPCClient) onBlockDisconnected(header []byte) {
	select {
	case c.enqueueNotification <- BlockDisconnected{
		BlockHeader: header,
	}:
	case <-c.quit:
	}
}

func (c *RPCClient) onRelevantTxAccepted(transaction []byte) {
	select {
	case c.enqueueNotification <- RelevantTxAccepted{
		Transaction: transaction,
	}:
	case <-c.quit:
	}
}

// onReorganization handles reorganization notifications and passes them
// downstream to the notifications queue.
func (c *RPCClient) onReorganization(oldHash *chainhash.Hash, oldHeight int32,
	newHash *chainhash.Hash, newHeight int32) {
	select {
	case c.enqueueNotification <- Reorganization{
		oldHash,
		int64(oldHeight),
		newHash,
		int64(newHeight),
	}:
	case <-c.quit:
	}
}

// onWinningTickets handles winning tickets notifications data and passes it
// downstream to the notifications queue.
func (c *RPCClient) onWinningTickets(hash *chainhash.Hash, height int64,
	tickets []*chainhash.Hash) {
	select {
	case c.enqueueVotingNotification <- WinningTickets{
		hash,
		height,
		tickets,
	}:
	case <-c.quit:
	}
}

// onSpentAndMissedTickets handles missed tickets notifications data and passes
// it downstream to the notifications queue.
func (c *RPCClient) onSpentAndMissedTickets(hash *chainhash.Hash,
	height int64,
	stakeDiff int64,
	tickets map[chainhash.Hash]bool) {

	var missedTickets []*chainhash.Hash

	// Copy the missing ticket hashes to a slice.
	for ticket, isSpent := range tickets {
		newTicket := ticket
		if !isSpent { // if missed
			missedTickets = append(missedTickets, &newTicket)
		}
	}

	select {
	case c.enqueueVotingNotification <- MissedTickets{
		hash,
		height,
		missedTickets,
	}:
	case <-c.quit:
	}
}

// onStakeDifficulty handles stake difficulty notifications data and passes it
// downstream to the notification queue.
func (c *RPCClient) onStakeDifficulty(hash *chainhash.Hash,
	height int64,
	stakeDiff int64) {

	select {
	case c.enqueueNotification <- StakeDifficulty{
		hash,
		height,
		stakeDiff,
	}:
	case <-c.quit:
	}
}

// handler maintains a queue of notifications and the current state (best
// block) of the chain.
func (c *RPCClient) handler() {
	// TODO: Rather than leaving this as an unbounded queue for all types of
	// notifications, try dropping ones where a later enqueued notification
	// can fully invalidate one waiting to be processed.  For example,
	// blockconnected notifications for greater block heights can remove the
	// need to process earlier blockconnected notifications still waiting
	// here.

	var notifications []interface{}
	enqueue := c.enqueueNotification
	var dequeue chan interface{}
	var next interface{}
	pingChan := time.After(time.Minute)
out:
	for {
		select {
		case n, ok := <-enqueue:
			if !ok {
				// If no notifications are queued for handling,
				// the queue is finished.
				if len(notifications) == 0 {
					break out
				}
				// nil channel so no more reads can occur.
				enqueue = nil
				continue
			}
			if len(notifications) == 0 {
				next = n
				dequeue = c.dequeueNotification
			}
			notifications = append(notifications, n)
			pingChan = time.After(time.Minute)

		case dequeue <- next:
			notifications[0] = nil
			notifications = notifications[1:]
			if len(notifications) != 0 {
				next = notifications[0]
			} else {
				// If no more notifications can be enqueued, the
				// queue is finished.
				if enqueue == nil {
					break out
				}
				dequeue = nil
			}

		case <-pingChan:
			// No notifications were received in the last 60s.
			// Ensure the connection is still active by making a new
			// request to the server.
			// TODO: A minute timeout is used to prevent the handler
			// loop from blocking here forever, but this is much larger
			// than it needs to be due to dcrd processing websocket
			// requests synchronously (see
			// https://github.com/btcsuite/btcd/issues/504).  Decrease
			// this to something saner like 3s when the above issue is
			// fixed.
			type sessionResult struct {
				err error
			}
			sessionResponse := make(chan sessionResult, 1)
			go func() {
				_, err := c.Session()
				sessionResponse <- sessionResult{err}
			}()

			select {
			case resp := <-sessionResponse:
				if resp.err != nil {
					log.Errorf("Failed to receive session "+
						"result: %v", resp.err)
					c.Stop()
					break out
				}
				pingChan = time.After(time.Minute)

			case <-time.After(time.Minute):
				log.Errorf("Timeout waiting for session RPC")
				c.Stop()
				break out
			}

		case <-c.quit:
			break out
		}
	}

	c.Stop()
	close(c.dequeueNotification)
	c.wg.Done()
}

// handler maintains a queue of notifications and the current state (best
// block) of the chain.
func (c *RPCClient) handlerVoting() {
	var notifications []interface{}
	enqueue := c.enqueueVotingNotification
	var dequeue chan interface{}
	var next interface{}
out:
	for {
		select {
		case n, ok := <-enqueue:
			if !ok {
				// If no notifications are queued for handling,
				// the queue is finished.
				if len(notifications) == 0 {
					break out
				}
				// nil channel so no more reads can occur.
				enqueue = nil
				continue
			}
			if len(notifications) == 0 {
				next = n
				dequeue = c.dequeueVotingNotification
			}
			notifications = append(notifications, n)

		case dequeue <- next:
			notifications[0] = nil
			notifications = notifications[1:]
			if len(notifications) != 0 {
				next = notifications[0]
			} else {
				// If no more notifications can be enqueued, the
				// queue is finished.
				if enqueue == nil {
					break out
				}
				dequeue = nil
			}

		case <-c.quit:
			break out
		}
	}
	close(c.dequeueVotingNotification)
	c.wg.Done()
}

// POSTClient creates the equivalent HTTP POST dcrrpcclient.Client.
func (c *RPCClient) POSTClient() (*dcrrpcclient.Client, error) {
	configCopy := *c.connConfig
	configCopy.HTTPPostMode = true
	return dcrrpcclient.New(&configCopy, nil)
}
<commit_msg>Fix RPC ping/pong deadlock and timeout issue. (#612)<commit_after>// Copyright (c) 2013-2015 The btcsuite developers
// Copyright (c) 2015-2016 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package chain

import (
	"errors"
	"fmt"
	"sync"
	"time"

	"github.com/decred/dcrd/chaincfg"
	"github.com/decred/dcrd/chaincfg/chainhash"
	"github.com/decred/dcrrpcclient"
)

var requiredChainServerAPI = semver{major: 2, minor: 0, patch: 0}

// RPCClient represents a persistent client connection to a decred RPC server
// for information regarding the current best block chain.
type RPCClient struct {
	*dcrrpcclient.Client
	connConfig        *dcrrpcclient.ConnConfig // Work around unexported field
	chainParams       *chaincfg.Params
	reconnectAttempts int

	enqueueNotification       chan interface{}
	dequeueNotification       chan interface{}
	enqueueVotingNotification chan interface{}
	dequeueVotingNotification chan interface{}

	quit    chan struct{}
	wg      sync.WaitGroup
	started bool
	quitMtx sync.Mutex
}

// NewRPCClient creates a client connection to the server described by the
// connect string.  If disableTLS is false, the remote RPC certificate must be
// provided in the certs slice.  The connection is not established immediately,
// but must be done using the Start method.  If the remote server does not
// operate on the same bitcoin network as described by the passed chain
// parameters, the connection will be disconnected.
func NewRPCClient(chainParams *chaincfg.Params, connect, user, pass string, certs []byte,
	disableTLS bool, reconnectAttempts int) (*RPCClient, error) {

	if reconnectAttempts < 0 {
		return nil, errors.New("reconnectAttempts must be positive")
	}

	client := &RPCClient{
		connConfig: &dcrrpcclient.ConnConfig{
			Host:                 connect,
			Endpoint:             "ws",
			User:                 user,
			Pass:                 pass,
			Certificates:         certs,
			DisableAutoReconnect: true,
			DisableConnectOnNew:  true,
			DisableTLS:           disableTLS,
		},
		chainParams:               chainParams,
		reconnectAttempts:         reconnectAttempts,
		enqueueNotification:       make(chan interface{}),
		dequeueNotification:       make(chan interface{}),
		enqueueVotingNotification: make(chan interface{}),
		dequeueVotingNotification: make(chan interface{}),
		quit: make(chan struct{}),
	}
	ntfnCallbacks := &dcrrpcclient.NotificationHandlers{
		OnClientConnected:       client.onClientConnect,
		OnBlockConnected:        client.onBlockConnected,
		OnBlockDisconnected:     client.onBlockDisconnected,
		OnRelevantTxAccepted:    client.onRelevantTxAccepted,
		OnReorganization:        client.onReorganization,
		OnWinningTickets:        client.onWinningTickets,
		OnSpentAndMissedTickets: client.onSpentAndMissedTickets,
		OnStakeDifficulty:       client.onStakeDifficulty,
	}
	rpcClient, err := dcrrpcclient.New(client.connConfig, ntfnCallbacks)
	if err != nil {
		return nil, err
	}
	client.Client = rpcClient
	return client, nil
}

// Start attempts to establish a client connection with the remote server.
// If successful, handler goroutines are started to process notifications
// sent by the server.  After a limited number of connection attempts, this
// function gives up, and therefore will not block forever waiting for the
// connection to be established to a server that may not exist.
func (c *RPCClient) Start() (err error) {
	err = c.Connect(c.reconnectAttempts)
	if err != nil {
		return err
	}

	defer func() {
		if err != nil {
			c.Disconnect()
		}
	}()

	// Verify that the server is running on the expected network.
	net, err := c.GetCurrentNet()
	if err != nil {
		return err
	}
	if net != c.chainParams.Net {
		return errors.New("mismatched networks")
	}

	// Ensure the RPC server has a compatible API version.
	var serverAPI semver
	versions, err := c.Version()
	if err == nil {
		versionResult := versions["dcrdjsonrpcapi"]
		serverAPI = semver{
			major: versionResult.Major,
			minor: versionResult.Minor,
			patch: versionResult.Patch,
		}
	}
	if !semverCompatible(requiredChainServerAPI, serverAPI) {
		return fmt.Errorf("consensus JSON-RPC server does not have a "+
			"compatible API version: advertises %v but require %v",
			serverAPI, requiredChainServerAPI)
	}

	c.quitMtx.Lock()
	c.started = true
	c.quitMtx.Unlock()

	c.wg.Add(2)
	go c.handler()
	go c.handlerVoting()
	return nil
}

// Stop disconnects the client and signals the shutdown of all goroutines
// started by Start.
func (c *RPCClient) Stop() {
	c.quitMtx.Lock()
	select {
	case <-c.quit:
	default:
		close(c.quit)
		c.Client.Shutdown()

		if !c.started {
			close(c.dequeueNotification)
			close(c.dequeueVotingNotification)
		}
	}
	c.quitMtx.Unlock()
}

// WaitForShutdown blocks until both the client has finished disconnecting
// and all handlers have exited.
func (c *RPCClient) WaitForShutdown() {
	c.Client.WaitForShutdown()
	c.wg.Wait()
}

// Notification types.  These are defined here and processed from from reading
// a notificationChan to avoid handling these notifications directly in
// dcrrpcclient callbacks, which isn't very Go-like and doesn't allow
// blocking client calls.
type (
	// ClientConnected is a notification for when a client connection is
	// opened or reestablished to the chain server.
	ClientConnected struct{}

	// BlockConnected is a notification for a newly-attached block to the
	// best chain.
	BlockConnected struct {
		BlockHeader  []byte
		Transactions [][]byte
	}

	// BlockDisconnected is a notifcation that the block described by the
	// BlockStamp was reorganized out of the best chain.
	BlockDisconnected struct {
		BlockHeader []byte
	}

	// RelevantTxAccepted is a notification that a transaction accepted by
	// mempool passed the client's transaction filter.
	RelevantTxAccepted struct {
		Transaction []byte
	}

	// Reorganization is a notification that a reorg has happen with the new
	// old and new tip included.
	Reorganization struct {
		OldHash   *chainhash.Hash
		OldHeight int64
		NewHash   *chainhash.Hash
		NewHeight int64
	}

	// WinningTickets is a notification with the winning tickets (and the
	// block they are in.
	WinningTickets struct {
		BlockHash   *chainhash.Hash
		BlockHeight int64
		Tickets     []*chainhash.Hash
	}

	// MissedTickets is a notifcation for tickets that have been missed.
	MissedTickets struct {
		BlockHash   *chainhash.Hash
		BlockHeight int64
		Tickets     []*chainhash.Hash
	}

	// StakeDifficulty is a notification for the current stake difficulty.
	StakeDifficulty struct {
		BlockHash   *chainhash.Hash
		BlockHeight int64
		StakeDiff   int64
	}
)

// Notifications returns a channel of parsed notifications sent by the remote
// decred RPC server.  This channel must be continually read or the process
// may abort for running out memory, as unread notifications are queued for
// later reads.
func (c *RPCClient) Notifications() <-chan interface{} {
	return c.dequeueNotification
}

// NotificationsVoting returns a channel of parsed voting notifications sent
// by the remote RPC server.  This channel must be continually read or the
// process may abort for running out memory, as unread notifications are
// queued for later reads.
func (c *RPCClient) NotificationsVoting() <-chan interface{} {
	return c.dequeueVotingNotification
}

func (c *RPCClient) onClientConnect() {
	select {
	case c.enqueueNotification <- ClientConnected{}:
	case <-c.quit:
	}
}

func (c *RPCClient) onBlockConnected(header []byte, transactions [][]byte) {
	select {
	case c.enqueueNotification <- BlockConnected{
		BlockHeader:  header,
		Transactions: transactions,
	}:
	case <-c.quit:
	}
}

func (c *RPCClient) onBlockDisconnected(header []byte) {
	select {
	case c.enqueueNotification <- BlockDisconnected{
		BlockHeader: header,
	}:
	case <-c.quit:
	}
}

func (c *RPCClient) onRelevantTxAccepted(transaction []byte) {
	select {
	case c.enqueueNotification <- RelevantTxAccepted{
		Transaction: transaction,
	}:
	case <-c.quit:
	}
}

// onReorganization handles reorganization notifications and passes them
// downstream to the notifications queue.
func (c *RPCClient) onReorganization(oldHash *chainhash.Hash, oldHeight int32,
	newHash *chainhash.Hash, newHeight int32) {
	select {
	case c.enqueueNotification <- Reorganization{
		oldHash,
		int64(oldHeight),
		newHash,
		int64(newHeight),
	}:
	case <-c.quit:
	}
}

// onWinningTickets handles winning tickets notifications data and passes it
// downstream to the notifications queue.
func (c *RPCClient) onWinningTickets(hash *chainhash.Hash, height int64,
	tickets []*chainhash.Hash) {
	select {
	case c.enqueueVotingNotification <- WinningTickets{
		hash,
		height,
		tickets,
	}:
	case <-c.quit:
	}
}

// onSpentAndMissedTickets handles missed tickets notifications data and passes
// it downstream to the notifications queue.
func (c *RPCClient) onSpentAndMissedTickets(hash *chainhash.Hash,
	height int64,
	stakeDiff int64,
	tickets map[chainhash.Hash]bool) {

	var missedTickets []*chainhash.Hash

	// Copy the missing ticket hashes to a slice.
	for ticket, isSpent := range tickets {
		newTicket := ticket
		if !isSpent { // if missed
			missedTickets = append(missedTickets, &newTicket)
		}
	}

	select {
	case c.enqueueVotingNotification <- MissedTickets{
		hash,
		height,
		missedTickets,
	}:
	case <-c.quit:
	}
}

// onStakeDifficulty handles stake difficulty notifications data and passes it
// downstream to the notification queue.
func (c *RPCClient) onStakeDifficulty(hash *chainhash.Hash,
	height int64,
	stakeDiff int64) {

	select {
	case c.enqueueNotification <- StakeDifficulty{
		hash,
		height,
		stakeDiff,
	}:
	case <-c.quit:
	}
}

// handler maintains a queue of notifications and the current state (best
// block) of the chain.
func (c *RPCClient) handler() {
	// TODO: Rather than leaving this as an unbounded queue for all types of
	// notifications, try dropping ones where a later enqueued notification
	// can fully invalidate one waiting to be processed.  For example,
	// blockconnected notifications for greater block heights can remove the
	// need to process earlier blockconnected notifications still waiting
	// here.

	var notifications []interface{}
	enqueue := c.enqueueNotification
	var dequeue chan interface{}
	var next interface{}
	pingChan := time.After(time.Minute)
	pingChanReset := make(chan (<-chan time.Time))
out:
	for {
		select {
		case n, ok := <-enqueue:
			if !ok {
				// If no notifications are queued for handling,
				// the queue is finished.
				if len(notifications) == 0 {
					break out
				}
				// nil channel so no more reads can occur.
				enqueue = nil
				continue
			}
			if len(notifications) == 0 {
				next = n
				dequeue = c.dequeueNotification
			}
			notifications = append(notifications, n)
			pingChan = time.After(time.Minute)

		case dequeue <- next:
			notifications[0] = nil
			notifications = notifications[1:]
			if len(notifications) != 0 {
				next = notifications[0]
			} else {
				// If no more notifications can be enqueued, the
				// queue is finished.
				if enqueue == nil {
					break out
				}
				dequeue = nil
			}

		case <-pingChan:
			// No notifications were received in the last 60s. Ensure the
			// connection is still active by making a new request to the server.
			//
			// This MUST wait for the response in a new goroutine so as to not
			// block channel sends enqueueing more notifications.  Doing so
			// would cause a deadlock and after the timeout expires, the client
			// would be shut down.
			//
			// TODO: A minute timeout is used to prevent the handler loop from
			// blocking here forever, but this is much larger than it needs to
			// be due to dcrd processing websocket requests synchronously (see
			// https://github.com/btcsuite/btcd/issues/504).  Decrease this to
			// something saner like 3s when the above issue is fixed.
			type sessionResult struct {
				err error
			}
			sessionResponse := make(chan sessionResult, 1)
			go func() {
				_, err := c.Session()
				sessionResponse <- sessionResult{err}
			}()
			go func() {
				select {
				case resp := <-sessionResponse:
					if resp.err != nil {
						log.Errorf("Failed to receive session "+
							"result: %v", resp.err)
						c.Stop()
					}
					pingChanReset <- time.After(time.Minute)

				case <-time.After(time.Minute):
					log.Errorf("Timeout waiting for session RPC")
					c.Stop()
				}
			}()

		case ch := <-pingChanReset:
			pingChan = ch

		case <-c.quit:
			break out
		}
	}

	c.Stop()
	close(c.dequeueNotification)
	c.wg.Done()
}

// handler maintains a queue of notifications and the current state (best
// block) of the chain.
func (c *RPCClient) handlerVoting() {
	var notifications []interface{}
	enqueue := c.enqueueVotingNotification
	var dequeue chan interface{}
	var next interface{}
out:
	for {
		select {
		case n, ok := <-enqueue:
			if !ok {
				// If no notifications are queued for handling,
				// the queue is finished.
				if len(notifications) == 0 {
					break out
				}
				// nil channel so no more reads can occur.
				enqueue = nil
				continue
			}
			if len(notifications) == 0 {
				next = n
				dequeue = c.dequeueVotingNotification
			}
			notifications = append(notifications, n)

		case dequeue <- next:
			notifications[0] = nil
			notifications = notifications[1:]
			if len(notifications) != 0 {
				next = notifications[0]
			} else {
				// If no more notifications can be enqueued, the
				// queue is finished.
				if enqueue == nil {
					break out
				}
				dequeue = nil
			}

		case <-c.quit:
			break out
		}
	}
	close(c.dequeueVotingNotification)
	c.wg.Done()
}

// POSTClient creates the equivalent HTTP POST dcrrpcclient.Client.
func (c *RPCClient) POSTClient() (*dcrrpcclient.Client, error) {
	configCopy := *c.connConfig
	configCopy.HTTPPostMode = true
	return dcrrpcclient.New(&configCopy, nil)
}
<|endoftext|>


<commit_before>// Copyright 2017-2021 Jeff Foley. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package requests

import (
	"context"
	"errors"
	"net"
	"strings"
	"time"

	"github.com/OWASP/Amass/v3/config"
	amassdns "github.com/OWASP/Amass/v3/net/dns"
	"github.com/caffix/eventbus"
	"github.com/caffix/pipeline"
	"github.com/caffix/stringset"
	"github.com/miekg/dns"
)

// Request tag types.
const (
	NONE     = "none"
	ALT      = "alt"
	GUESS    = "guess"
	ARCHIVE  = "archive"
	API      = "api"
	AXFR     = "axfr"
	BRUTE    = "brute"
	CERT     = "cert"
	CRAWL    = "crawl"
	DNS      = "dns"
	RIR      = "rir"
	EXTERNAL = "ext"
	SCRAPE   = "scrape"
)

// ContextKey is the type used for context value keys.
type ContextKey int

// The key used when values are obtained during service requests.
const (
	ContextConfig ContextKey = iota
	ContextEventBus
)

// Request Pub/Sub topics used across Amass.
const (
	NewNameTopic       = "amass:newname"
	NewAddrTopic       = "amass:newaddr"
	SubDiscoveredTopic = "amass:newsub"
	ASNRequestTopic    = "amass:asnreq"
	NewASNTopic        = "amass:newasn"
	WhoisRequestTopic  = "amass:whoisreq"
	NewWhoisTopic      = "amass:whoisinfo"
	LogTopic           = "amass:log"
	OutputTopic        = "amass:output"
)

// ContextConfigBus extracts the Config and EventBus references from the Context argument.
func ContextConfigBus(ctx context.Context) (*config.Config, *eventbus.EventBus, error) {
	var ok bool
	var cfg *config.Config

	if c := ctx.Value(ContextConfig); c != nil {
		cfg, ok = c.(*config.Config)
		if !ok {
			return nil, nil, errors.New("Failed to extract the configuration from the context")
		}
	} else {
		return nil, nil, errors.New("Failed to extract the configuration from the context")
	}

	var bus *eventbus.EventBus
	if b := ctx.Value(ContextEventBus); b != nil {
		bus, ok = b.(*eventbus.EventBus)
		if !ok {
			return nil, nil, errors.New("Failed to extract the event bus from the context")
		}
	} else {
		return nil, nil, errors.New("Failed to extract the event bus from the context")
	}

	return cfg, bus, nil
}

// DNSAnswer is the type used by Amass to represent a DNS record.
type DNSAnswer struct {
	Name string `json:"name"`
	Type int    `json:"type"`
	TTL  int    `json:"TTL"`
	Data string `json:"data"`
}

// DNSRequest handles data needed throughout Service processing of a DNS name.
type DNSRequest struct {
	Name    string
	Domain  string
	Records []DNSAnswer
	Tag     string
	Source  string
}

// Clone implements pipeline Data.
func (d *DNSRequest) Clone() pipeline.Data {
	return &DNSRequest{
		Name:    d.Name,
		Domain:  d.Domain,
		Records: append([]DNSAnswer(nil), d.Records...),
		Tag:     d.Tag,
		Source:  d.Source,
	}
}

// MarkAsProcessed implements pipeline Data.
func (d *DNSRequest) MarkAsProcessed() {}

// Valid performs input validation of the receiver.
func (d *DNSRequest) Valid() bool {
	if _, ok := dns.IsDomainName(d.Name); !ok {
		return false
	}
	if _, ok := dns.IsDomainName(d.Domain); !ok {
		return false
	}
	if !dns.IsSubDomain(d.Domain, d.Name) {
		return false
	}
	return true
}

// ResolvedRequest allows services to identify DNS names that have been resolved.

type ResolvedRequest struct {
	Name    string
	Domain  string
	Records []DNSAnswer
	Tag     string
	Source  string
}

// Clone implements pipeline Data.
func (r *ResolvedRequest) Clone() pipeline.Data {
	return &ResolvedRequest{
		Name:    r.Name,
		Domain:  r.Domain,
		Records: append([]DNSAnswer(nil), r.Records...),
		Tag:     r.Tag,
		Source:  r.Source,
	}
}

// MarkAsProcessed implements pipeline Data.
func (r *ResolvedRequest) MarkAsProcessed() {}

// Valid performs input validation of the receiver.
func (r *ResolvedRequest) Valid() bool {
	if _, ok := dns.IsDomainName(r.Name); !ok {
		return false
	}
	if _, ok := dns.IsDomainName(r.Domain); !ok {
		return false
	}
	if !dns.IsSubDomain(r.Domain, r.Name) {
		return false
	}
	return true
}

// SubdomainRequest handles subdomain data processed by enumeration.
type SubdomainRequest struct {
	Name    string
	Domain  string
	Records []DNSAnswer
	Tag     string
	Source  string
	Times   int
}

// Clone implements pipeline Data.
func (s *SubdomainRequest) Clone() pipeline.Data {
	return &SubdomainRequest{
		Name:    s.Name,
		Domain:  s.Domain,
		Records: append([]DNSAnswer(nil), s.Records...),
		Tag:     s.Tag,
		Source:  s.Source,
	}
}

// MarkAsProcessed implements pipeline Data.
func (s *SubdomainRequest) MarkAsProcessed() {}

// Valid performs input validation of the receiver.
func (s *SubdomainRequest) Valid() bool {
	if _, ok := dns.IsDomainName(s.Name); !ok {
		return false
	}
	if _, ok := dns.IsDomainName(s.Domain); !ok {
		return false
	}
	if !dns.IsSubDomain(s.Domain, s.Name) {
		return false
	}
	if s.Times == 0 {
		return false
	}
	return true
}

// ZoneXFRRequest handles zone transfer requests.
type ZoneXFRRequest struct {
	Name   string
	Domain string
	Server string
	Tag    string
	Source string
}

// Clone implements pipeline Data.
func (z *ZoneXFRRequest) Clone() pipeline.Data {
	return &ZoneXFRRequest{
		Name:   z.Name,
		Domain: z.Domain,
		Server: z.Server,
		Tag:    z.Tag,
		Source: z.Source,
	}
}

// MarkAsProcessed implements pipeline Data.
func (z *ZoneXFRRequest) MarkAsProcessed() {}

// AddrRequest handles data needed throughout Service processing of a network address.
type AddrRequest struct {
	Address string
	InScope bool
	Domain  string
	Tag     string
	Source  string
}

// Clone implements pipeline Data.
func (a *AddrRequest) Clone() pipeline.Data {
	return &AddrRequest{
		Address: a.Address,
		InScope: a.InScope,
		Domain:  a.Domain,
		Tag:     a.Tag,
		Source:  a.Source,
	}
}

// MarkAsProcessed implements pipeline Data.
func (a *AddrRequest) MarkAsProcessed() {}

// Valid performs input validation of the receiver.
func (a *AddrRequest) Valid() bool {
	if ip := net.ParseIP(a.Address); ip == nil {
		return false
	}
	if a.Domain != "" {
		if _, ok := dns.IsDomainName(a.Domain); !ok {
			return false
		}
	}
	return true
}

// ASNRequest handles all autonomous system information needed by Amass.
type ASNRequest struct {
	Address        string
	ASN            int
	Prefix         string
	CC             string
	Registry       string
	AllocationDate time.Time
	Description    string
	Netblocks      stringset.Set
	Tag            string
	Source         string
}

// Clone implements pipeline Data.
func (a *ASNRequest) Clone() pipeline.Data {
	return &ASNRequest{
		Address:        a.Address,
		ASN:            a.ASN,
		Prefix:         a.Prefix,
		CC:             a.CC,
		Registry:       a.Registry,
		AllocationDate: a.AllocationDate,
		Description:    a.Description,
		Netblocks:      stringset.New(a.Netblocks.Slice()...),
		Tag:            a.Tag,
		Source:         a.Source,
	}
}

// MarkAsProcessed implements pipeline Data.
func (a *ASNRequest) MarkAsProcessed() {}

// Valid performs input validation of the receiver.
func (a *ASNRequest) Valid() bool {
	if ip := net.ParseIP(a.Address); ip == nil {
		return false
	}
	if _, _, err := net.ParseCIDR(a.Prefix); err != nil {
		return false
	}
	for _, netblock := range a.Netblocks.Slice() {
		if _, _, err := net.ParseCIDR(netblock); err != nil {
			return false
		}
	}
	return true
}

// WhoisRequest handles data needed throughout Service processing of reverse whois.
type WhoisRequest struct {
	Domain     string
	Company    string
	Email      string
	NewDomains []string
	Tag        string
	Source     string
}

// Output contains all the output data for an enumerated DNS name.
type Output struct {
	Name      string        `json:"name"`
	Domain    string        `json:"domain"`
	Addresses []AddressInfo `json:"addresses"`
	Tag       string        `json:"tag"`
	Sources   []string      `json:"sources"`
}

// Clone implements pipeline Data.
func (o *Output) Clone() pipeline.Data {
	return &Output{
		Name:      o.Name,
		Domain:    o.Domain,
		Addresses: append([]AddressInfo(nil), o.Addresses...),
		Tag:       o.Tag,
		Sources:   append([]string(nil), o.Sources...),
	}
}

// MarkAsProcessed implements pipeline Data.
func (o *Output) MarkAsProcessed() {}

// AddressInfo stores all network addressing info for the Output type.
type AddressInfo struct {
	Address     net.IP     `json:"ip"`
	Netblock    *net.IPNet `json:"-"`
	CIDRStr     string     `json:"cidr"`
	ASN         int        `json:"asn"`
	Description string     `json:"desc"`
}

// TrustedTag returns true when the tag parameter is of a type that should be trusted even
// facing DNS wildcards.
func TrustedTag(tag string) bool {
	if tag == ARCHIVE || tag == AXFR || tag == CERT || tag == CRAWL || tag == DNS {
		return true
	}
	return false
}

// SanitizeDNSRequest cleans the Name and Domain elements of the receiver.
func SanitizeDNSRequest(req *DNSRequest) {
	req.Name = strings.ToLower(req.Name)
	req.Name = strings.TrimSpace(req.Name)
	req.Name = amassdns.RemoveAsteriskLabel(req.Name)
	req.Name = strings.Trim(req.Name, ".")

	req.Domain = strings.ToLower(req.Domain)
	req.Domain = strings.TrimSpace(req.Domain)
	req.Domain = strings.Trim(req.Domain, ".")
}
<commit_msg>Added checking for output to help fix #566<commit_after>// Copyright 2017-2021 Jeff Foley. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package requests

import (
	"context"
	"errors"
	"net"
	"strings"
	"time"

	"github.com/OWASP/Amass/v3/config"
	amassdns "github.com/OWASP/Amass/v3/net/dns"
	"github.com/caffix/eventbus"
	"github.com/caffix/pipeline"
	"github.com/miekg/dns"
)

// Request tag types.
const (
	NONE     = "none"
	ALT      = "alt"
	GUESS    = "guess"
	ARCHIVE  = "archive"
	API      = "api"
	AXFR     = "axfr"
	BRUTE    = "brute"
	CERT     = "cert"
	CRAWL    = "crawl"
	DNS      = "dns"
	RIR      = "rir"
	EXTERNAL = "ext"
	SCRAPE   = "scrape"
)

// ContextKey is the type used for context value keys.
type ContextKey int

// The key used when values are obtained during service requests.
const (
	ContextConfig ContextKey = iota
	ContextEventBus
)

// Request Pub/Sub topics used across Amass.
const (
	NewNameTopic       = "amass:newname"
	NewAddrTopic       = "amass:newaddr"
	SubDiscoveredTopic = "amass:newsub"
	ASNRequestTopic    = "amass:asnreq"
	NewASNTopic        = "amass:newasn"
	WhoisRequestTopic  = "amass:whoisreq"
	NewWhoisTopic      = "amass:whoisinfo"
	LogTopic           = "amass:log"
	OutputTopic        = "amass:output"
)

// ContextConfigBus extracts the Config and EventBus references from the Context argument.
func ContextConfigBus(ctx context.Context) (*config.Config, *eventbus.EventBus, error) {
	var ok bool
	var cfg *config.Config

	if c := ctx.Value(ContextConfig); c != nil {
		cfg, ok = c.(*config.Config)
		if !ok {
			return nil, nil, errors.New("Failed to extract the configuration from the context")
		}
	} else {
		return nil, nil, errors.New("Failed to extract the configuration from the context")
	}

	var bus *eventbus.EventBus
	if b := ctx.Value(ContextEventBus); b != nil {
		bus, ok = b.(*eventbus.EventBus)
		if !ok {
			return nil, nil, errors.New("Failed to extract the event bus from the context")
		}
	} else {
		return nil, nil, errors.New("Failed to extract the event bus from the context")
	}

	return cfg, bus, nil
}

// DNSAnswer is the type used by Amass to represent a DNS record.
type DNSAnswer struct {
	Name string `json:"name"`
	Type int    `json:"type"`
	TTL  int    `json:"TTL"`
	Data string `json:"data"`
}

// DNSRequest handles data needed throughout Service processing of a DNS name.
type DNSRequest struct {
	Name    string
	Domain  string
	Records []DNSAnswer
	Tag     string
	Source  string
}

// Clone implements pipeline Data.
func (d *DNSRequest) Clone() pipeline.Data {
	return &DNSRequest{
		Name:    d.Name,
		Domain:  d.Domain,
		Records: append([]DNSAnswer(nil), d.Records...),
		Tag:     d.Tag,
		Source:  d.Source,
	}
}

// MarkAsProcessed implements pipeline Data.
func (d *DNSRequest) MarkAsProcessed() {}

// Valid performs input validation of the receiver.
func (d *DNSRequest) Valid() bool {
	if _, ok := dns.IsDomainName(d.Name); !ok {
		return false
	}
	if _, ok := dns.IsDomainName(d.Domain); !ok {
		return false
	}
	if !dns.IsSubDomain(d.Domain, d.Name) {
		return false
	}
	return true
}

// ResolvedRequest allows services to identify DNS names that have been resolved.

type ResolvedRequest struct {
	Name    string
	Domain  string
	Records []DNSAnswer
	Tag     string
	Source  string
}

// Clone implements pipeline Data.
func (r *ResolvedRequest) Clone() pipeline.Data {
	return &ResolvedRequest{
		Name:    r.Name,
		Domain:  r.Domain,
		Records: append([]DNSAnswer(nil), r.Records...),
		Tag:     r.Tag,
		Source:  r.Source,
	}
}

// MarkAsProcessed implements pipeline Data.
func (r *ResolvedRequest) MarkAsProcessed() {}

// Valid performs input validation of the receiver.
func (r *ResolvedRequest) Valid() bool {
	if _, ok := dns.IsDomainName(r.Name); !ok {
		return false
	}
	if _, ok := dns.IsDomainName(r.Domain); !ok {
		return false
	}
	if !dns.IsSubDomain(r.Domain, r.Name) {
		return false
	}
	return true
}

// SubdomainRequest handles subdomain data processed by enumeration.
type SubdomainRequest struct {
	Name    string
	Domain  string
	Records []DNSAnswer
	Tag     string
	Source  string
	Times   int
}

// Clone implements pipeline Data.
func (s *SubdomainRequest) Clone() pipeline.Data {
	return &SubdomainRequest{
		Name:    s.Name,
		Domain:  s.Domain,
		Records: append([]DNSAnswer(nil), s.Records...),
		Tag:     s.Tag,
		Source:  s.Source,
	}
}

// MarkAsProcessed implements pipeline Data.
func (s *SubdomainRequest) MarkAsProcessed() {}

// Valid performs input validation of the receiver.
func (s *SubdomainRequest) Valid() bool {
	if _, ok := dns.IsDomainName(s.Name); !ok {
		return false
	}
	if _, ok := dns.IsDomainName(s.Domain); !ok {
		return false
	}
	if !dns.IsSubDomain(s.Domain, s.Name) {
		return false
	}
	if s.Times == 0 {
		return false
	}
	return true
}

// ZoneXFRRequest handles zone transfer requests.
type ZoneXFRRequest struct {
	Name   string
	Domain string
	Server string
	Tag    string
	Source string
}

// Clone implements pipeline Data.
func (z *ZoneXFRRequest) Clone() pipeline.Data {
	return &ZoneXFRRequest{
		Name:   z.Name,
		Domain: z.Domain,
		Server: z.Server,
		Tag:    z.Tag,
		Source: z.Source,
	}
}

// MarkAsProcessed implements pipeline Data.
func (z *ZoneXFRRequest) MarkAsProcessed() {}

// AddrRequest handles data needed throughout Service processing of a network address.
type AddrRequest struct {
	Address string
	InScope bool
	Domain  string
	Tag     string
	Source  string
}

// Clone implements pipeline Data.
func (a *AddrRequest) Clone() pipeline.Data {
	return &AddrRequest{
		Address: a.Address,
		InScope: a.InScope,
		Domain:  a.Domain,
		Tag:     a.Tag,
		Source:  a.Source,
	}
}

// MarkAsProcessed implements pipeline Data.
func (a *AddrRequest) MarkAsProcessed() {}

// Valid performs input validation of the receiver.
func (a *AddrRequest) Valid() bool {
	if ip := net.ParseIP(a.Address); ip == nil {
		return false
	}
	if a.Domain != "" {
		if _, ok := dns.IsDomainName(a.Domain); !ok {
			return false
		}
	}
	return true
}

// ASNRequest handles all autonomous system information needed by Amass.
type ASNRequest struct {
	Address        string
	ASN            int
	Prefix         string
	CC             string
	Registry       string
	AllocationDate time.Time
	Description    string
	Netblocks      []string
	Tag            string
	Source         string
}

// Clone implements pipeline Data.
func (a *ASNRequest) Clone() pipeline.Data {
	return &ASNRequest{
		Address:        a.Address,
		ASN:            a.ASN,
		Prefix:         a.Prefix,
		CC:             a.CC,
		Registry:       a.Registry,
		AllocationDate: a.AllocationDate,
		Description:    a.Description,
		Netblocks:      a.Netblocks,
		Tag:            a.Tag,
		Source:         a.Source,
	}
}

// MarkAsProcessed implements pipeline Data.
func (a *ASNRequest) MarkAsProcessed() {}

// Valid performs input validation of the receiver.
func (a *ASNRequest) Valid() bool {
	if ip := net.ParseIP(a.Address); ip == nil {
		return false
	}
	if _, _, err := net.ParseCIDR(a.Prefix); err != nil {
		return false
	}
	for _, netblock := range a.Netblocks {
		if _, _, err := net.ParseCIDR(netblock); err != nil {
			return false
		}
	}
	return true
}

// WhoisRequest handles data needed throughout Service processing of reverse whois.
type WhoisRequest struct {
	Domain     string
	Company    string
	Email      string
	NewDomains []string
	Tag        string
	Source     string
}

// Output contains all the output data for an enumerated DNS name.
type Output struct {
	Name      string        `json:"name"`
	Domain    string        `json:"domain"`
	Addresses []AddressInfo `json:"addresses"`
	Tag       string        `json:"tag"`
	Sources   []string      `json:"sources"`
}

// Clone implements pipeline Data.
func (o *Output) Clone() pipeline.Data {
	return &Output{
		Name:      o.Name,
		Domain:    o.Domain,
		Addresses: append([]AddressInfo(nil), o.Addresses...),
		Tag:       o.Tag,
		Sources:   append([]string(nil), o.Sources...),
	}
}

// MarkAsProcessed implements pipeline Data.
func (o *Output) MarkAsProcessed() {}

// Complete checks that all the required fields have been populated.
func (o *Output) Complete(passive bool) bool {
	if o.Name == "" || o.Domain == "" || o.Tag == "" || len(o.Sources) == 0 {
		return false
	}

	for _, src := range o.Sources {
		if src == "" {
			return false
		}
	}

	if !passive {
		for _, a := range o.Addresses {
			if a.Address == nil || a.Netblock == nil || a.CIDRStr == "" || a.Description == "" {
				return false
			}
		}
	}

	return true
}

// AddressInfo stores all network addressing info for the Output type.
type AddressInfo struct {
	Address     net.IP     `json:"ip"`
	Netblock    *net.IPNet `json:"-"`
	CIDRStr     string     `json:"cidr"`
	ASN         int        `json:"asn"`
	Description string     `json:"desc"`
}

// TrustedTag returns true when the tag parameter is of a type that should be trusted even
// facing DNS wildcards.
func TrustedTag(tag string) bool {
	if tag == ARCHIVE || tag == AXFR || tag == CERT || tag == CRAWL || tag == DNS {
		return true
	}
	return false
}

// SanitizeDNSRequest cleans the Name and Domain elements of the receiver.
func SanitizeDNSRequest(req *DNSRequest) {
	req.Name = strings.ToLower(req.Name)
	req.Name = strings.TrimSpace(req.Name)
	req.Name = amassdns.RemoveAsteriskLabel(req.Name)
	req.Name = strings.Trim(req.Name, ".")

	req.Domain = strings.ToLower(req.Domain)
	req.Domain = strings.TrimSpace(req.Domain)
	req.Domain = strings.Trim(req.Domain, ".")
}
<|endoftext|>


<commit_before>package view

// Page packages up any data and/or message that is
// needed to be rendered to the screen for the user
type Page struct {
	Error   string
	Success string
	Warning string
	Content interface{}
}
<commit_msg>toss the view package<commit_after><|endoftext|>