<commit_before>package misc

import (
	"context"
	"errors"
	"math/rand"
	"time"

	"github.com/HouzuoGuo/laitos/lalog"
)

// Periodic invokes a function continuously with a regular interval in between.
type Periodic struct {
	// LogActorName is a string used in log messages. These messages are rare.
	LogActorName string
	// Interval between each invocation.
	Interval time.Duration
	// MaxInt determines the upper bound range of the integer value given
	// to the invoked function.
	// If the function does not use this integer, then set this field to 1.
	MaxInt int
	// Func is the function to be invoked at regular interval. With each
	// invocation, the function will receive:
	// - A context, which may be cancelled.
	// - Current round number starting with 0. A round is completed after the
	//   function has been invoked with all of [0,MaxInt).
	// - An integer in the range of [0,MaxInt).
	// If the function returns a non-nil error, then the periodic invocation
	// will stop entirely. The function's error can be retrieved by calling
	// WaitForErr function.
	Func func(context.Context, int, int) error
	// RandomInt determines whether each invocation of the function will receive
	// a randomly chosen integer within the range of [0,MaxInt).
	// Otherwise, each consecutive invocation will receive the next integer in
	// sequence, which wraps around after reaching MaxInt-1.
	// Caller of Start function should seed the default PRNG.
	RandomOrder bool
	// StableInterval lines up the start of each invocation of the function to
	// the interval.
	// Otherwise, the interval between consecutive invocations of the function
	// will always be the fixed duration.
	StableInterval bool
	// RapidFirstRound causes the first round of invocations (that is Func
	// invoked with all of [0,MaxInt)) to be invoked in rapid succession, without
	// having to wait for the interval in between.
	RapidFirstRound bool

	cancelFunc  func()
	funcErrChan chan error
	funcErr     error
}

// Start invoking the periodic function continuously at regular interval.
// The function does not block caller.
// Optionally, use WaitForErr to block-wait for the periodic function to return
// an error, in which case the periodic invocations will stop entirely.
func (p *Periodic) Start(ctx context.Context) error {
	if p.Interval == 0 {
		return errors.New("Interval must be greater than 0")
	}
	if p.MaxInt < 1 {
		return errors.New("MaxInt must be greater than 0")
	}
	funcInputInts := make([]int, p.MaxInt)
	for i := 0; i < p.MaxInt; i++ {
		funcInputInts[i] = i
	}
	if p.RandomOrder {
		rand.Shuffle(len(funcInputInts), func(a, b int) {
			funcInputInts[a], funcInputInts[b] = funcInputInts[b], funcInputInts[a]
		})
	}
	// Log a message every ~ 3 minutes
	logIntervalSec := 3 * time.Minute / p.Interval
	if logIntervalSec < 1 {
		// If the interval is greater than 3 minutes, then log a message with
		// every periodic invocation of the function
		logIntervalSec = 1
	}
	ctx, cancelFunc := context.WithCancel(ctx)
	p.cancelFunc = cancelFunc
	p.funcErrChan = make(chan error, 1)
	p.funcErr = nil
	go func() {
		for roundNum := 0; ; roundNum++ {
			for _, anInt := range funcInputInts {
				if EmergencyLockDown {
					lalog.DefaultLogger.Warning("Periodic.Start", p.LogActorName, ErrEmergencyLockDown, "stop immediately")
					p.funcErrChan <- ErrEmergencyLockDown
					return
				}
				if ((roundNum+1)*len(funcInputInts))%int(logIntervalSec) == 0 {
					lalog.DefaultLogger.Info("Periodic.Start", p.LogActorName, nil, "is still alive in round %d, integer input %d", roundNum, anInt)
				}
				startTime := time.Now()
				if err := p.Func(ctx, roundNum, anInt); err != nil {
					p.funcErrChan <- err
					return
				}
				runDuration := time.Now().Sub(startTime)
				// Calculate the interval to wait
				waitInterval := p.Interval
				if p.StableInterval {
					// Handle overrun (the previous invocation took longer than
					// the regular interval) by waiting for the next interval
					// to start.
					if runDuration > waitInterval {
						waitInterval -= runDuration % waitInterval
					} else {
						waitInterval -= runDuration
					}
				}
				if p.RapidFirstRound && roundNum == 0 {
					waitInterval = 0
				}
				select {
				case <-time.After(waitInterval):
				case <-ctx.Done():
					p.funcErrChan <- ctx.Err()
					return
				}
			}
		}
	}()
	return nil
}

// Wait for the periodically invoked function or its context to return an error,
// and then return the error to the caller. The function blocks caller.
func (p *Periodic) WaitForErr() error {
	if p.funcErr == nil {
		err := <-p.funcErrChan
		p.funcErr = err
	}
	return p.funcErr
}

// Stop the periodic invocation of the function. The result of the final
// invocation can be discovered from the return value of Wait function.
func (p *Periodic) Stop() {
	p.cancelFunc()
}
<commit_msg>print a log message at the start of periodic<commit_after>package misc

import (
	"context"
	"errors"
	"math/rand"
	"time"

	"github.com/HouzuoGuo/laitos/lalog"
)

// Periodic invokes a function continuously with a regular interval in between.
type Periodic struct {
	// LogActorName is a string used in log messages. These messages are rare.
	LogActorName string
	// Interval between each invocation.
	Interval time.Duration
	// MaxInt determines the upper bound range of the integer value given
	// to the invoked function.
	// If the function does not use this integer, then set this field to 1.
	MaxInt int
	// Func is the function to be invoked at regular interval. With each
	// invocation, the function will receive:
	// - A context, which may be cancelled.
	// - Current round number starting with 0. A round is completed after the
	//   function has been invoked with all of [0,MaxInt).
	// - An integer in the range of [0,MaxInt).
	// If the function returns a non-nil error, then the periodic invocation
	// will stop entirely. The function's error can be retrieved by calling
	// WaitForErr function.
	Func func(context.Context, int, int) error
	// RandomInt determines whether each invocation of the function will receive
	// a randomly chosen integer within the range of [0,MaxInt).
	// Otherwise, each consecutive invocation will receive the next integer in
	// sequence, which wraps around after reaching MaxInt-1.
	// Caller of Start function should seed the default PRNG.
	RandomOrder bool
	// StableInterval lines up the start of each invocation of the function to
	// the interval.
	// Otherwise, the interval between consecutive invocations of the function
	// will always be the fixed duration.
	StableInterval bool
	// RapidFirstRound causes the first round of invocations (that is Func
	// invoked with all of [0,MaxInt)) to be invoked in rapid succession, without
	// having to wait for the interval in between.
	RapidFirstRound bool

	cancelFunc  func()
	funcErrChan chan error
	funcErr     error
}

// Start invoking the periodic function continuously at regular interval.
// The function does not block caller.
// Optionally, use WaitForErr to block-wait for the periodic function to return
// an error, in which case the periodic invocations will stop entirely.
func (p *Periodic) Start(ctx context.Context) error {
	if p.Interval == 0 {
		return errors.New("Interval must be greater than 0")
	}
	if p.MaxInt < 1 {
		return errors.New("MaxInt must be greater than 0")
	}
	funcInputInts := make([]int, p.MaxInt)
	for i := 0; i < p.MaxInt; i++ {
		funcInputInts[i] = i
	}
	if p.RandomOrder {
		rand.Shuffle(len(funcInputInts), func(a, b int) {
			funcInputInts[a], funcInputInts[b] = funcInputInts[b], funcInputInts[a]
		})
	}
	// Log a message every ~ 3 minutes
	logIntervalSec := 3 * time.Minute / p.Interval
	if logIntervalSec < 1 {
		// If the interval is greater than 3 minutes, then log a message with
		// every periodic invocation of the function
		logIntervalSec = 1
	}
	ctx, cancelFunc := context.WithCancel(ctx)
	p.cancelFunc = cancelFunc
	p.funcErrChan = make(chan error, 1)
	p.funcErr = nil
	go func() {
		lalog.DefaultLogger.Info("Periodic.Start", p.LogActorName, nil, "starting now using interval %v, max.int input %v, random order? %v, stable interval? %v, rapid first round? %v",
			p.Interval, p.MaxInt, p.RandomOrder, p.StableInterval, p.RapidFirstRound)
		for roundNum := 0; ; roundNum++ {
			for _, anInt := range funcInputInts {
				if EmergencyLockDown {
					lalog.DefaultLogger.Warning("Periodic.Start", p.LogActorName, ErrEmergencyLockDown, "stop immediately")
					p.funcErrChan <- ErrEmergencyLockDown
					return
				}
				if ((roundNum+1)*len(funcInputInts))%int(logIntervalSec) == 1 {
					lalog.DefaultLogger.Info("Periodic.Start", p.LogActorName, nil, "is still alive in round %d, integer input %d", roundNum, anInt)
				}
				startTime := time.Now()
				if err := p.Func(ctx, roundNum, anInt); err != nil {
					p.funcErrChan <- err
					return
				}
				runDuration := time.Now().Sub(startTime)
				// Calculate the interval to wait
				waitInterval := p.Interval
				if p.StableInterval {
					// Handle overrun (the previous invocation took longer than
					// the regular interval) by waiting for the next interval
					// to start.
					if runDuration > waitInterval {
						waitInterval -= runDuration % waitInterval
					} else {
						waitInterval -= runDuration
					}
				}
				if p.RapidFirstRound && roundNum == 0 {
					waitInterval = 0
				}
				select {
				case <-time.After(waitInterval):
				case <-ctx.Done():
					p.funcErrChan <- ctx.Err()
					return
				}
			}
		}
	}()
	return nil
}

// Wait for the periodically invoked function or its context to return an error,
// and then return the error to the caller. The function blocks caller.
func (p *Periodic) WaitForErr() error {
	if p.funcErr == nil {
		err := <-p.funcErrChan
		p.funcErr = err
	}
	return p.funcErr
}

// Stop the periodic invocation of the function. The result of the final
// invocation can be discovered from the return value of Wait function.
func (p *Periodic) Stop() {
	p.cancelFunc()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/BurntSushi/toml"
	"github.com/pkg/errors"
)

type BuildpackTOML struct {
	API       string `toml:"api"`
	Buildpack struct {
		ID      string `toml:"id"`
		Name    string `toml:"name"`
		Version string `toml:"version"`
	} `toml:"buildpack"`
	Metadata Metadata `toml:"metadata"`
	Orders   Orders   `toml:"order"`
	Stacks   Stacks   `toml:"stacks"`
}

type Metadata struct {
	IncludeFiles               []string         `toml:"include_files"`
	Dependencies               Dependencies     `toml:"dependencies"`
	DependencyDeprecationDates DeprecationDates `toml:"dependency_deprecation_dates"`
}

func (buildpackTOML BuildpackTOML) WriteToFile(filepath string) error {
	buildpackTOMLFile, err := os.OpenFile(filepath, os.O_RDWR|os.O_CREATE, 0666)
	if err != nil {
		return errors.Wrap(err, fmt.Sprintf("failed to open buildpack.toml at: %s", filepath))
	}
	defer buildpackTOMLFile.Close()

	if err := toml.NewEncoder(buildpackTOMLFile).Encode(buildpackTOML); err != nil {
		return errors.Wrap(err, "failed to save updated buildpack.toml")
	}
	return nil
}
<commit_msg>Add pre_package key into update-cnb-dependency buildpackTOML struct<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/BurntSushi/toml"
	"github.com/pkg/errors"
)

type BuildpackTOML struct {
	API       string `toml:"api"`
	Buildpack struct {
		ID      string `toml:"id"`
		Name    string `toml:"name"`
		Version string `toml:"version"`
	} `toml:"buildpack"`
	Metadata Metadata `toml:"metadata"`
	Orders   Orders   `toml:"order"`
	Stacks   Stacks   `toml:"stacks"`
}

type Metadata struct {
	IncludeFiles               []string         `toml:"include_files"`
	PrePackage                 string           `toml:"pre_package"`
	Dependencies               Dependencies     `toml:"dependencies"`
	DependencyDeprecationDates DeprecationDates `toml:"dependency_deprecation_dates"`
}

func (buildpackTOML BuildpackTOML) WriteToFile(filepath string) error {
	buildpackTOMLFile, err := os.OpenFile(filepath, os.O_RDWR|os.O_CREATE, 0666)
	if err != nil {
		return errors.Wrap(err, fmt.Sprintf("failed to open buildpack.toml at: %s", filepath))
	}
	defer buildpackTOMLFile.Close()

	if err := toml.NewEncoder(buildpackTOMLFile).Encode(buildpackTOML); err != nil {
		return errors.Wrap(err, "failed to save updated buildpack.toml")
	}
	return nil
}
<|endoftext|>


<commit_before><commit_msg>Release 0.11.3<commit_after><|endoftext|>


<commit_before><commit_msg>Revert "project.go"<commit_after><|endoftext|>


<commit_before>package models

import (
	"fmt"
	"time"

	"github.com/jinzhu/gorm"
)

// CveDetails is for sorting
type CveDetails []CveDetail

func (c CveDetails) Len() int {
	return len(c)
}

func (c CveDetails) Swap(i, j int) {
	c[i], c[j] = c[j], c[i]
}

func (c CveDetails) Less(i, j int) bool {
	return c[i].CveID < c[j].CveID
}

// CveDetail is a parent of Jnv/Nvd model
type CveDetail struct {
	gorm.Model `json:"-"`
	CveInfoID  uint `json:"-"`

	CveID string
	Nvd   Nvd
	Jvn   Jvn
}

// CvssScore returns CVSS Score of the CVE
func (c CveDetail) CvssScore(lang string) float64 {
	switch lang {
	case "en":
		if 0 < c.Nvd.CvssScore() {
			return c.Nvd.CvssScore()
		} else if 0 < c.Jvn.CvssScore() {
			return c.Jvn.CvssScore()
		}
		return -1
	case "ja":
		if 0 < c.Jvn.CvssScore() {
			return c.Jvn.CvssScore()
		} else if 0 < c.Nvd.CvssScore() {
			return c.Nvd.CvssScore()
		}
		return -1
	default:
		return c.CvssScore("en")
	}
}

// CvssV2CalculatorLink returns cvss Caluculate site URL.
// https://nvd.nist.gov/cvss/v2-calculator?name=CVE-2015-5477&vector=(AV:N/AC:L/Au:N/C:N/I:N/A:C)
func (c CveDetail) CvssV2CalculatorLink(lang string) string {
	var vector string
	switch lang {
	case "en":
		vector = c.Nvd.CvssVector()
	case "ja":
		vector = c.Jvn.CvssVector()
	default:
		return ""
	}
	return fmt.Sprintf("https://nvd.nist.gov/cvss/v2-calculator?name=%s&vector=%s",
		c.CveID,
		vector)
}

// CweID returns CWE ID of NVD struct
func (c CveDetail) CweID() string {
	return c.Nvd.CweID
}

// CveDictionary is interface of JVN, NVD
type CveDictionary interface {
	GetID() uint
	GetCveDetailID() uint
	SetCveDetailID(id uint)
	CveTitle() string
	CveSummary() string
	CvssScore() float64
	CvssVector() string
	CvssSeverity() string
	Link() string
	VulnSiteReferences() []Reference
	SetVulnSiteReferences([]Reference)
	SetCpes([]Cpe)
	LastModified() time.Time
}

// Nvd is a model of NVD
type Nvd struct {
	gorm.Model  `json:"-"`
	CveDetailID uint `json:"-"`

	// Increased size of summary to ensure we can fit arbitrary CVE content.
	Summary               string `gorm:"size:4096"`

	Score                 float64 // 1 to 10
	AccessVector          string
	AccessComplexity      string
	Authentication        string
	ConfidentialityImpact string
	IntegrityImpact       string
	AvailabilityImpact    string

	Cpes       []Cpe
	CweID      string
	References []Reference

	PublishedDate    time.Time
	LastModifiedDate time.Time
}

// GetID return title
func (c Nvd) GetID() uint {
	return c.ID
}

// GetCveDetailID return title
func (c Nvd) GetCveDetailID() uint {
	return c.CveDetailID
}

// SetCveDetailID return title
func (c *Nvd) SetCveDetailID(id uint) {
	c.CveDetailID = id
}

//CveTitle return title
func (c Nvd) CveTitle() string {
	return ""
}

//CveSummary return summary
func (c Nvd) CveSummary() string {
	return c.Summary
}

//CvssScore return title
func (c Nvd) CvssScore() float64 {
	return c.Score
}

// CvssSeverity return severity ranking that NVD povided.
// https://nvd.nist.gov/cvss.cfm
func (c Nvd) CvssSeverity() string {
	switch {
	case 7 <= c.CvssScore():
		return "High"
	case 4 <= c.CvssScore() && c.CvssScore() < 7:
		return "Medium"
	case 0 < c.CvssScore() && c.CvssScore() < 4:
		return "Low"
	default:
		return "?"
	}
}

// CvssVector make CVSS Vector string
// https://nvd.nist.gov/cvss.cfm?vectorinfo&version=2
//  Example 1: (AV:L/AC:H/Au:N/C:N/I:P/A:C)
//  Example 2: (AV:A/AC:L/Au:M/C:C/I:N/A:P)
func (c Nvd) CvssVector() string {
	return fmt.Sprintf("(AV:%s/AC:%s/Au:%s/C:%s/I:%s/A:%s)",
		firstChar(c.AccessVector),
		firstChar(c.AccessComplexity),
		firstChar(c.Authentication),
		firstChar(c.ConfidentialityImpact),
		firstChar(c.IntegrityImpact),
		firstChar(c.AvailabilityImpact),
	)
}

func firstChar(str string) string {
	if len(str) == 0 {
		return "?"
	}
	return string(str[0])
}

// Link return empty string
func (c Nvd) Link() string {
	return ""
}

// VulnSiteReferences return References
func (c Nvd) VulnSiteReferences() []Reference {
	return c.References
}

// SetVulnSiteReferences set References
func (c Nvd) SetVulnSiteReferences(r []Reference) {
	c.References = r
}

// SetCpes set cpes
func (c *Nvd) SetCpes(r []Cpe) {
	c.Cpes = r
}

// LastModified get LastModifiedDate
func (c Nvd) LastModified() time.Time {
	return c.LastModifiedDate
}

// Jvn is a model of JVN
type Jvn struct {
	gorm.Model  `json:"-"`
	CveDetailID uint `json:"-"`

	Title   string
	Summary string    `gorm:"size:8192"`
	JvnLink string
	JvnID   string

	Score    float64 // 1 to 10
	Severity string  // Low|Medium|High
	Vector   string  // (AV:N/AC:M/Au:N/C:N/I:P/A:N)
	//  Version   string  // 2.0

	References []Reference
	Cpes       []Cpe

	PublishedDate    time.Time
	LastModifiedDate time.Time
}

// GetID return title
func (c Jvn) GetID() uint {
	return c.ID
}

// SetCveDetailID return title
func (c *Jvn) SetCveDetailID(id uint) {
	c.CveDetailID = id
}

// GetCveDetailID return title
func (c Jvn) GetCveDetailID() uint {
	return c.CveDetailID
}

//CveTitle return title
func (c Jvn) CveTitle() string {
	return c.Title
}

// CveSummary return summary
func (c Jvn) CveSummary() string {
	return c.Summary
}

//CvssScore return CvssScore
func (c Jvn) CvssScore() float64 {
	return c.Score
}

//CvssVector return severity
func (c Jvn) CvssVector() string {
	return c.Vector
}

// Link return summary
func (c Jvn) Link() string {
	return c.JvnLink
}

//CvssSeverity return severity ranking
func (c Jvn) CvssSeverity() string {
	return c.Severity
}

// VulnSiteReferences return summary
func (c Jvn) VulnSiteReferences() []Reference {
	return c.References
}

// SetVulnSiteReferences set References
func (c *Jvn) SetVulnSiteReferences(r []Reference) {
	c.References = r
}

// SetCpes set cpes
func (c *Jvn) SetCpes(r []Cpe) {
	c.Cpes = r
}

// LastModified set cpes
func (c Jvn) LastModified() time.Time {
	return c.LastModifiedDate
}

// Cpe is Child model of Jvn/Nvd.
// see https://www.ipa.go.jp/security/vuln/CPE.html
type Cpe struct {
	gorm.Model `json:"-"`
	JvnID      uint `json:"-"`
	NvdID      uint `json:"-"`

	// CPE Name (URL sytle)
	// JVN ... cpe:/a:oracle:mysql
	// NVD ... cpe:/a:cisco:unified_contact_center_express:11.0%281%29
	CpeName string

	// each items
	Part     string
	Vendor   string
	Product  string
	Version  string
	Update   string
	Edition  string
	Language string
}

// Reference is Child model of Jvn/Nvd.
// It holds reference information about the CVE.
type Reference struct {
	gorm.Model `json:"-"`
	JvnID      uint `json:"-"`
	NvdID      uint `json:"-"`

	Source string
	Link   string   `gorm:"size:512"`
}
<commit_msg>add struct tag for xml<commit_after>package models

import (
	"fmt"
	"time"

	"github.com/jinzhu/gorm"
)

// CveDetails is for sorting
type CveDetails []CveDetail

func (c CveDetails) Len() int {
	return len(c)
}

func (c CveDetails) Swap(i, j int) {
	c[i], c[j] = c[j], c[i]
}

func (c CveDetails) Less(i, j int) bool {
	return c[i].CveID < c[j].CveID
}

// CveDetail is a parent of Jnv/Nvd model
type CveDetail struct {
	gorm.Model `json:"-" xml:"-"`
	CveInfoID  uint `json:"-" xml:"-"`

	CveID string
	Nvd   Nvd
	Jvn   Jvn
}

// CvssScore returns CVSS Score of the CVE
func (c CveDetail) CvssScore(lang string) float64 {
	switch lang {
	case "en":
		if 0 < c.Nvd.CvssScore() {
			return c.Nvd.CvssScore()
		} else if 0 < c.Jvn.CvssScore() {
			return c.Jvn.CvssScore()
		}
		return -1
	case "ja":
		if 0 < c.Jvn.CvssScore() {
			return c.Jvn.CvssScore()
		} else if 0 < c.Nvd.CvssScore() {
			return c.Nvd.CvssScore()
		}
		return -1
	default:
		return c.CvssScore("en")
	}
}

// CvssV2CalculatorLink returns cvss Caluculate site URL.
// https://nvd.nist.gov/cvss/v2-calculator?name=CVE-2015-5477&vector=(AV:N/AC:L/Au:N/C:N/I:N/A:C)
func (c CveDetail) CvssV2CalculatorLink(lang string) string {
	var vector string
	switch lang {
	case "en":
		vector = c.Nvd.CvssVector()
	case "ja":
		vector = c.Jvn.CvssVector()
	default:
		return ""
	}
	return fmt.Sprintf("https://nvd.nist.gov/cvss/v2-calculator?name=%s&vector=%s",
		c.CveID,
		vector)
}

// CweID returns CWE ID of NVD struct
func (c CveDetail) CweID() string {
	return c.Nvd.CweID
}

// CveDictionary is interface of JVN, NVD
type CveDictionary interface {
	GetID() uint
	GetCveDetailID() uint
	SetCveDetailID(id uint)
	CveTitle() string
	CveSummary() string
	CvssScore() float64
	CvssVector() string
	CvssSeverity() string
	Link() string
	VulnSiteReferences() []Reference
	SetVulnSiteReferences([]Reference)
	SetCpes([]Cpe)
	LastModified() time.Time
}

// Nvd is a model of NVD
type Nvd struct {
	gorm.Model  `json:"-" xml:"-"`
	CveDetailID uint `json:"-" xml:"-"`

	// Increased size of summary to ensure we can fit arbitrary CVE content.
	Summary               string `gorm:"size:4096"`

	Score                 float64 // 1 to 10
	AccessVector          string
	AccessComplexity      string
	Authentication        string
	ConfidentialityImpact string
	IntegrityImpact       string
	AvailabilityImpact    string

	Cpes       []Cpe
	CweID      string
	References []Reference

	PublishedDate    time.Time
	LastModifiedDate time.Time
}

// GetID return title
func (c Nvd) GetID() uint {
	return c.ID
}

// GetCveDetailID return title
func (c Nvd) GetCveDetailID() uint {
	return c.CveDetailID
}

// SetCveDetailID return title
func (c *Nvd) SetCveDetailID(id uint) {
	c.CveDetailID = id
}

//CveTitle return title
func (c Nvd) CveTitle() string {
	return ""
}

//CveSummary return summary
func (c Nvd) CveSummary() string {
	return c.Summary
}

//CvssScore return title
func (c Nvd) CvssScore() float64 {
	return c.Score
}

// CvssSeverity return severity ranking that NVD povided.
// https://nvd.nist.gov/cvss.cfm
func (c Nvd) CvssSeverity() string {
	switch {
	case 7 <= c.CvssScore():
		return "High"
	case 4 <= c.CvssScore() && c.CvssScore() < 7:
		return "Medium"
	case 0 < c.CvssScore() && c.CvssScore() < 4:
		return "Low"
	default:
		return "?"
	}
}

// CvssVector make CVSS Vector string
// https://nvd.nist.gov/cvss.cfm?vectorinfo&version=2
//  Example 1: (AV:L/AC:H/Au:N/C:N/I:P/A:C)
//  Example 2: (AV:A/AC:L/Au:M/C:C/I:N/A:P)
func (c Nvd) CvssVector() string {
	return fmt.Sprintf("(AV:%s/AC:%s/Au:%s/C:%s/I:%s/A:%s)",
		firstChar(c.AccessVector),
		firstChar(c.AccessComplexity),
		firstChar(c.Authentication),
		firstChar(c.ConfidentialityImpact),
		firstChar(c.IntegrityImpact),
		firstChar(c.AvailabilityImpact),
	)
}

func firstChar(str string) string {
	if len(str) == 0 {
		return "?"
	}
	return string(str[0])
}

// Link return empty string
func (c Nvd) Link() string {
	return ""
}

// VulnSiteReferences return References
func (c Nvd) VulnSiteReferences() []Reference {
	return c.References
}

// SetVulnSiteReferences set References
func (c Nvd) SetVulnSiteReferences(r []Reference) {
	c.References = r
}

// SetCpes set cpes
func (c *Nvd) SetCpes(r []Cpe) {
	c.Cpes = r
}

// LastModified get LastModifiedDate
func (c Nvd) LastModified() time.Time {
	return c.LastModifiedDate
}

// Jvn is a model of JVN
type Jvn struct {
	gorm.Model  `json:"-" xml:"-"`
	CveDetailID uint `json:"-" xml:"-"`

	Title   string
	Summary string    `gorm:"size:8192"`
	JvnLink string
	JvnID   string

	Score    float64 // 1 to 10
	Severity string  // Low|Medium|High
	Vector   string  // (AV:N/AC:M/Au:N/C:N/I:P/A:N)
	//  Version   string  // 2.0

	References []Reference
	Cpes       []Cpe

	PublishedDate    time.Time
	LastModifiedDate time.Time
}

// GetID return title
func (c Jvn) GetID() uint {
	return c.ID
}

// SetCveDetailID return title
func (c *Jvn) SetCveDetailID(id uint) {
	c.CveDetailID = id
}

// GetCveDetailID return title
func (c Jvn) GetCveDetailID() uint {
	return c.CveDetailID
}

//CveTitle return title
func (c Jvn) CveTitle() string {
	return c.Title
}

// CveSummary return summary
func (c Jvn) CveSummary() string {
	return c.Summary
}

//CvssScore return CvssScore
func (c Jvn) CvssScore() float64 {
	return c.Score
}

//CvssVector return severity
func (c Jvn) CvssVector() string {
	return c.Vector
}

// Link return summary
func (c Jvn) Link() string {
	return c.JvnLink
}

//CvssSeverity return severity ranking
func (c Jvn) CvssSeverity() string {
	return c.Severity
}

// VulnSiteReferences return summary
func (c Jvn) VulnSiteReferences() []Reference {
	return c.References
}

// SetVulnSiteReferences set References
func (c *Jvn) SetVulnSiteReferences(r []Reference) {
	c.References = r
}

// SetCpes set cpes
func (c *Jvn) SetCpes(r []Cpe) {
	c.Cpes = r
}

// LastModified set cpes
func (c Jvn) LastModified() time.Time {
	return c.LastModifiedDate
}

// Cpe is Child model of Jvn/Nvd.
// see https://www.ipa.go.jp/security/vuln/CPE.html
type Cpe struct {
	gorm.Model `json:"-" xml:"-"`
	JvnID      uint `json:"-" xml:"-"`
	NvdID      uint `json:"-" xml:"-"`

	// CPE Name (URL sytle)
	// JVN ... cpe:/a:oracle:mysql
	// NVD ... cpe:/a:cisco:unified_contact_center_express:11.0%281%29
	CpeName string

	// each items
	Part     string
	Vendor   string
	Product  string
	Version  string
	Update   string
	Edition  string
	Language string
}

// Reference is Child model of Jvn/Nvd.
// It holds reference information about the CVE.
type Reference struct {
	gorm.Model `json:"-" xml:"-"`
	JvnID      uint `json:"-" xml:"-"`
	NvdID      uint `json:"-" xml:"-"`

	Source string
	Link   string   `gorm:"size:512"`
}
<|endoftext|>


<commit_before>package models

import (
	"fmt"
	"time"
)

// PullRequest interface
type PullRequest interface {
	ToString() string
}

// PullRequestList struct
type PullRequestList struct {
	PageLen int               `json:"pagelen"`
	Page    int               `json:"page"`
	Size    int               `json:"size"`
	Items   []PullRequestInfo `json:"values"`
}

// PullRequestInfo struct
type PullRequestInfo struct {
	ID          int       `json:"id"`
	Title       string    `json:"title"`
	CreatedOn   time.Time `json:"created_on"`
	UpdatedOn   time.Time `json:"updated_on"`
	Author      User      `json:"author"`
	Destination Commit    `json:"destination"`
	Source      Commit    `json:"source"`
	Description string    `json:"description"`
	Links       Links     `json:"links"`
}

// PullRequestDetail struct
type PullRequestDetail struct {
	PullRequestInfo
	Participants []Reviewer `json:"participants"`
}

// Links struct
type Links struct {
	Html Link `json:"html"`
}

// Link struct
type Link struct {
	Href string `json:"href"`
}

// Reviewer struct
type Reviewer struct {
	User     User `json:"user"`
	Approved bool `json:"approved"`
}

// User struct
type User struct {
	Username    string `json:"username"`
	DisplayName string `json:"display_name"`
}

// Commit struct
type Commit struct {
	Branch Branch     `json:"branch"`
	Commit CommitInfo `json:"commit"`
}

// CommitInfo struct
type CommitInfo struct {
	Hash string `json:"hash"`
}

// Branch struct
type Branch struct {
	Name string `json:"name"`
}

// PullRequestActivity struct
type PullRequestActivity struct {
	Update  Update  `json:"update,omitempty"`
	Comment Comment `json:"comment,omitempty"`
}

// Update struct
type Update struct {
	Date   time.Time `json:"date"`
	Source Commit    `json:"source"`
	Author User      `json:"author"`
}

// Comment struct
type Comment struct {
	Content   CommentContent `json:"content"`
	CreatedOn time.Time      `json:"created_on"`
	UpdatedOn time.Time      `json:"updated_on"`
	User      User           `json:"user"`
}

// CommentContent struct
type CommentContent struct {
	Raw string `json:"raw"`
}

// Repository struct
type Repository struct {
	Org  string
	Name string
}

// UserCredential struct
type UserCredential struct {
	Username string
	Password string
}

// IsApproved checks the pull request has been approved by
// user with the specified username
func (pr PullRequestDetail) IsApproved(username string) bool {
	for _, reviewer := range pr.Participants {
		if reviewer.Approved && reviewer.User.Username == username {
			return true
		}
	}
	return false
}

// ToString returns the description of a pull request
func (pr PullRequestInfo) ToString() string {
	return fmt.Sprintf("%d %s %s\n\t%s->%s\n\t%s\n",
		pr.ID,
		formatLocalTime(pr.UpdatedOn),
		pr.Author.DisplayName,
		pr.Source.Branch.Name,
		pr.Destination.Branch.Name,
		pr.Title)
}

// ToString returns the description of a pull request
func (pr PullRequestDetail) ToString() string {
	return fmt.Sprintf("%d %s %s\n\t%s->%s\n\t%s\n%s\n",
		pr.ID,
		formatLocalTime(pr.UpdatedOn),
		pr.Author.DisplayName,
		pr.Source.Branch.Name,
		pr.Destination.Branch.Name,
		pr.Title,
		pr.Description)
}

// ToString returns the description of a comment
func (c Comment) ToString() string {
	return fmt.Sprintf(
		"Comment by %s (%s): %s",
		c.User.DisplayName,
		formatLocalTime(c.UpdatedOn),
		c.Content.Raw)
}

// ToString returns the description of an update
func (c Update) ToString() string {
	return fmt.Sprintf(
		"Commit made by %s (%s): %s",
		c.Author.DisplayName,
		formatLocalTime(c.Date),
		c.Source.Commit.Hash)
}

func formatLocalTime(t time.Time) string {
	loc, _ := time.LoadLocation("Local")
	lt := t.In(loc)
	return lt.Format("2006-01-02 15:04")
}
<commit_msg>Cosmetic changes<commit_after>package models

import (
	"fmt"
	"time"
)

// PullRequest interface
type PullRequest interface {
	ToString() string
}

// PullRequestList struct
type PullRequestList struct {
	PageLen int               `json:"pagelen"`
	Page    int               `json:"page"`
	Size    int               `json:"size"`
	Items   []PullRequestInfo `json:"values"`
}

// PullRequestInfo struct
type PullRequestInfo struct {
	ID          int       `json:"id"`
	Title       string    `json:"title"`
	CreatedOn   time.Time `json:"created_on"`
	UpdatedOn   time.Time `json:"updated_on"`
	Author      User      `json:"author"`
	Destination Commit    `json:"destination"`
	Source      Commit    `json:"source"`
	Description string    `json:"description"`
	Links       Links     `json:"links"`
}

// PullRequestDetail struct
type PullRequestDetail struct {
	PullRequestInfo
	Participants []Reviewer `json:"participants"`
}

// Links struct
type Links struct {
	Html Link `json:"html"`
}

// Link struct
type Link struct {
	Href string `json:"href"`
}

// Reviewer struct
type Reviewer struct {
	User     User `json:"user"`
	Approved bool `json:"approved"`
}

// User struct
type User struct {
	Username    string `json:"username"`
	DisplayName string `json:"display_name"`
}

// Commit struct
type Commit struct {
	Branch Branch     `json:"branch"`
	Commit CommitInfo `json:"commit"`
}

// CommitInfo struct
type CommitInfo struct {
	Hash string `json:"hash"`
}

// Branch struct
type Branch struct {
	Name string `json:"name"`
}

// PullRequestActivity struct
type PullRequestActivity struct {
	Update  Update  `json:"update,omitempty"`
	Comment Comment `json:"comment,omitempty"`
}

// Update struct
type Update struct {
	Date   time.Time `json:"date"`
	Source Commit    `json:"source"`
	Author User      `json:"author"`
}

// Comment struct
type Comment struct {
	Content   CommentContent `json:"content"`
	CreatedOn time.Time      `json:"created_on"`
	UpdatedOn time.Time      `json:"updated_on"`
	User      User           `json:"user"`
}

// CommentContent struct
type CommentContent struct {
	Raw string `json:"raw"`
}

// Repository struct
type Repository struct {
	Org  string
	Name string
}

// UserCredential struct
type UserCredential struct {
	Username string
	Password string
}

// IsApproved checks the pull request has been approved by
// user with the specified username
func (pr PullRequestDetail) IsApproved(username string) bool {
	for _, reviewer := range pr.Participants {
		if reviewer.Approved && reviewer.User.Username == username {
			return true
		}
	}
	return false
}

// ToString returns the description of a pull request
func (pr PullRequestInfo) ToString() string {
	return fmt.Sprintf("%d %s %s\n\t%s -> %s\n\t%s\n",
		pr.ID,
		formatLocalTime(pr.UpdatedOn),
		pr.Author.DisplayName,
		pr.Source.Branch.Name,
		pr.Destination.Branch.Name,
		pr.Title)
}

// ToString returns the description of a pull request
func (pr PullRequestDetail) ToString() string {
	return fmt.Sprintf("%d %s %s\n\t%s -> %s\n\t%s\n%s\n",
		pr.ID,
		formatLocalTime(pr.UpdatedOn),
		pr.Author.DisplayName,
		pr.Source.Branch.Name,
		pr.Destination.Branch.Name,
		pr.Title,
		pr.Description)
}

// ToString returns the description of a comment
func (c Comment) ToString() string {
	return fmt.Sprintf(
		"Comment by %s (%s): %s",
		c.User.DisplayName,
		formatLocalTime(c.UpdatedOn),
		c.Content.Raw)
}

// ToString returns the description of an update
func (c Update) ToString() string {
	return fmt.Sprintf(
		"Commit made by %s (%s): %s",
		c.Author.DisplayName,
		formatLocalTime(c.Date),
		c.Source.Commit.Hash)
}

func formatLocalTime(t time.Time) string {
	loc, _ := time.LoadLocation("Local")
	lt := t.In(loc)
	return lt.Format("2006-01-02 15:04")
}
<|endoftext|>


<commit_before><commit_msg>solution(contains-duplicate-iii): Time O(n)<commit_after><|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.
// +build !windows

package rsyslog_test

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/juju/names"
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/api"
	"github.com/juju/juju/cert"
	"github.com/juju/juju/network"
	"github.com/juju/juju/state"
	coretesting "github.com/juju/juju/testing"
	"github.com/juju/juju/utils/syslog"
	"github.com/juju/juju/worker/rsyslog"
)

func waitForRestart(c *gc.C, restarted chan struct{}) {
	timeout := time.After(coretesting.LongWait)
	for {
		select {
		case <-timeout:
			c.Fatalf("timed out waiting for rsyslog to be restarted")
		case <-restarted:
			return
		}
	}
}

func assertPathExists(c *gc.C, path string) {
	_, err := os.Stat(path)
	c.Assert(err, gc.IsNil)
}

func (s *RsyslogSuite) TestStartStop(c *gc.C) {
	st, m := s.OpenAPIAsNewMachine(c, state.JobHostUnits)
	worker, err := rsyslog.NewRsyslogConfigWorker(st.Rsyslog(), rsyslog.RsyslogModeForwarding, m.Tag(), "", []string{"0.1.2.3"})
	c.Assert(err, gc.IsNil)
	worker.Kill()
	c.Assert(worker.Wait(), gc.IsNil)
}

func (s *RsyslogSuite) TestTearDown(c *gc.C) {
	st, m := s.st, s.machine
	worker, err := rsyslog.NewRsyslogConfigWorker(st.Rsyslog(), rsyslog.RsyslogModeAccumulate, m.Tag(), "", []string{"0.1.2.3"})
	c.Assert(err, gc.IsNil)
	confFile := filepath.Join(*rsyslog.RsyslogConfDir, "25-juju.conf")
	// On worker teardown, the rsyslog config file should be removed.
	defer func() {
		_, err := os.Stat(confFile)
		c.Assert(err, jc.Satisfies, os.IsNotExist)
	}()
	defer func() { c.Assert(worker.Wait(), gc.IsNil) }()
	defer worker.Kill()
	waitForFile(c, confFile)
}

func (s *RsyslogSuite) TestModeAccumulate(c *gc.C) {
	st, m := s.st, s.machine
	worker, err := rsyslog.NewRsyslogConfigWorker(st.Rsyslog(), rsyslog.RsyslogModeAccumulate, m.Tag(), "", nil)
	c.Assert(err, gc.IsNil)
	defer func() { c.Assert(worker.Wait(), gc.IsNil) }()
	defer worker.Kill()
	waitForFile(c, filepath.Join(*rsyslog.LogDir, "ca-cert.pem"))

	// We should have ca-cert.pem, rsyslog-cert.pem, and rsyslog-key.pem.
	caCertPEM, err := ioutil.ReadFile(filepath.Join(*rsyslog.LogDir, "ca-cert.pem"))
	c.Assert(err, gc.IsNil)
	rsyslogCertPEM, err := ioutil.ReadFile(filepath.Join(*rsyslog.LogDir, "rsyslog-cert.pem"))
	c.Assert(err, gc.IsNil)
	rsyslogKeyPEM, err := ioutil.ReadFile(filepath.Join(*rsyslog.LogDir, "rsyslog-key.pem"))
	c.Assert(err, gc.IsNil)

	_, _, err = cert.ParseCertAndKey(string(rsyslogCertPEM), string(rsyslogKeyPEM))
	c.Assert(err, gc.IsNil)
	err = cert.Verify(string(rsyslogCertPEM), string(caCertPEM), time.Now().UTC())
	c.Assert(err, gc.IsNil)

	// Verify rsyslog configuration.
	waitForFile(c, filepath.Join(*rsyslog.RsyslogConfDir, "25-juju.conf"))
	rsyslogConf, err := ioutil.ReadFile(filepath.Join(*rsyslog.RsyslogConfDir, "25-juju.conf"))
	c.Assert(err, gc.IsNil)

	syslogPort := s.Environ.Config().SyslogPort()
	syslogConfig := syslog.NewAccumulateConfig(m.Tag().String(), *rsyslog.LogDir, syslogPort, "", []string{})
	syslogConfig.ConfigDir = *rsyslog.RsyslogConfDir
	rendered, err := syslogConfig.Render()
	c.Assert(err, gc.IsNil)

	c.Assert(string(rsyslogConf), gc.DeepEquals, string(rendered))

	// Verify logrotate files
	assertPathExists(c, filepath.Join(*rsyslog.LogDir, "logrotate.conf"))
	assertPathExists(c, filepath.Join(*rsyslog.LogDir, "logrotate.run"))

}

func (s *RsyslogSuite) TestAccumulateHA(c *gc.C) {
	m := s.machine
	syslogConfig := syslog.NewAccumulateConfig(m.Tag().String(), *rsyslog.LogDir, 6541, "", []string{"192.168.1", "127.0.0.1"})
	rendered, err := syslogConfig.Render()
	c.Assert(err, gc.IsNil)

	stateServer1Config := ":syslogtag, startswith, \"juju-\" @@192.168.1:6541;LongTagForwardFormat"
	stateServer2Config := ":syslogtag, startswith, \"juju-\" @@127.0.0.1:6541;LongTagForwardFormat"

	c.Assert(strings.Contains(string(rendered), stateServer1Config), gc.Equals, true)
	c.Assert(strings.Contains(string(rendered), stateServer2Config), gc.Equals, true)
}

func (s *RsyslogSuite) TestNamespace(c *gc.C) {
	st := s.st
	// set the rsyslog cert
	err := s.APIState.Client().EnvironmentSet(map[string]interface{}{"rsyslog-ca-cert": coretesting.CACert})
	c.Assert(err, gc.IsNil)

	// namespace only takes effect in filenames
	// for machine-0; all others assume isolation.
	s.testNamespace(c, st, names.NewMachineTag("0"), "", "25-juju.conf", *rsyslog.LogDir)
	s.testNamespace(c, st, names.NewMachineTag("0"), "mynamespace", "25-juju-mynamespace.conf", *rsyslog.LogDir+"-mynamespace")
	s.testNamespace(c, st, names.NewMachineTag("1"), "", "25-juju.conf", *rsyslog.LogDir)
	s.testNamespace(c, st, names.NewMachineTag("1"), "mynamespace", "25-juju.conf", *rsyslog.LogDir)
	s.testNamespace(c, st, names.NewUnitTag("myservice/0"), "", "26-juju-unit-myservice-0.conf", *rsyslog.LogDir)
	s.testNamespace(c, st, names.NewUnitTag("myservice/0"), "mynamespace", "26-juju-unit-myservice-0.conf", *rsyslog.LogDir)
}

// testNamespace starts a worker and ensures that
// the rsyslog config file has the expected filename,
// and the appropriate log dir is used.
func (s *RsyslogSuite) testNamespace(c *gc.C, st *api.State, tag names.Tag, namespace, expectedFilename, expectedLogDir string) {
	restarted := make(chan struct{}, 2) // once for create, once for teardown
	s.PatchValue(rsyslog.RestartRsyslog, func() error {
		restarted <- struct{}{}
		return nil
	})

	err := os.MkdirAll(expectedLogDir, 0755)
	c.Assert(err, gc.IsNil)
	worker, err := rsyslog.NewRsyslogConfigWorker(st.Rsyslog(), rsyslog.RsyslogModeAccumulate, tag, namespace, []string{"0.1.2.3"})
	c.Assert(err, gc.IsNil)
	defer func() { c.Assert(worker.Wait(), gc.IsNil) }()
	defer worker.Kill()

	// change the API HostPorts to trigger an rsyslog restart
	newHostPorts := network.AddressesWithPort(network.NewAddresses("127.0.0.1"), 6541)
	err = s.State.SetAPIHostPorts([][]network.HostPort{newHostPorts})
	c.Assert(err, gc.IsNil)

	// Wait for rsyslog to be restarted, so we can check to see
	// what the name of the config file is.
	waitForRestart(c, restarted)

	// Ensure that ca-cert.pem gets written to the expected log dir.
	waitForFile(c, filepath.Join(expectedLogDir, "ca-cert.pem"))

	dir, err := os.Open(*rsyslog.RsyslogConfDir)
	c.Assert(err, gc.IsNil)
	names, err := dir.Readdirnames(-1)
	dir.Close()
	c.Assert(err, gc.IsNil)
	c.Assert(names, gc.HasLen, 1)
	c.Assert(names[0], gc.Equals, expectedFilename)
}
<commit_msg>rsyslog: test certificate fields<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.
// +build !windows

package rsyslog_test

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/juju/names"
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/api"
	"github.com/juju/juju/cert"
	"github.com/juju/juju/network"
	"github.com/juju/juju/state"
	coretesting "github.com/juju/juju/testing"
	"github.com/juju/juju/utils/syslog"
	"github.com/juju/juju/worker/rsyslog"
)

func waitForRestart(c *gc.C, restarted chan struct{}) {
	timeout := time.After(coretesting.LongWait)
	for {
		select {
		case <-timeout:
			c.Fatalf("timed out waiting for rsyslog to be restarted")
		case <-restarted:
			return
		}
	}
}

func assertPathExists(c *gc.C, path string) {
	_, err := os.Stat(path)
	c.Assert(err, gc.IsNil)
}

func (s *RsyslogSuite) TestStartStop(c *gc.C) {
	st, m := s.OpenAPIAsNewMachine(c, state.JobHostUnits)
	worker, err := rsyslog.NewRsyslogConfigWorker(st.Rsyslog(), rsyslog.RsyslogModeForwarding, m.Tag(), "", []string{"0.1.2.3"})
	c.Assert(err, gc.IsNil)
	worker.Kill()
	c.Assert(worker.Wait(), gc.IsNil)
}

func (s *RsyslogSuite) TestTearDown(c *gc.C) {
	st, m := s.st, s.machine
	worker, err := rsyslog.NewRsyslogConfigWorker(st.Rsyslog(), rsyslog.RsyslogModeAccumulate, m.Tag(), "", []string{"0.1.2.3"})
	c.Assert(err, gc.IsNil)
	confFile := filepath.Join(*rsyslog.RsyslogConfDir, "25-juju.conf")
	// On worker teardown, the rsyslog config file should be removed.
	defer func() {
		_, err := os.Stat(confFile)
		c.Assert(err, jc.Satisfies, os.IsNotExist)
	}()
	defer func() { c.Assert(worker.Wait(), gc.IsNil) }()
	defer worker.Kill()
	waitForFile(c, confFile)
}

func (s *RsyslogSuite) TestRsyslogCert(c *gc.C) {
	st, m := s.st, s.machine
	worker, err := rsyslog.NewRsyslogConfigWorker(st.Rsyslog(), rsyslog.RsyslogModeAccumulate, m.Tag(), "", nil)
	c.Assert(err, gc.IsNil)
	defer func() { c.Assert(worker.Wait(), gc.IsNil) }()
	defer worker.Kill()
	waitForFile(c, filepath.Join(*rsyslog.LogDir, "ca-cert.pem"))

	rsyslogCertPEM, err := ioutil.ReadFile(filepath.Join(*rsyslog.LogDir, "rsyslog-cert.pem"))
	c.Assert(err, gc.IsNil)

	cert, err := cert.ParseCert(string(rsyslogCertPEM))
	c.Assert(err, gc.IsNil)

	c.Assert(cert.DNSNames, gc.DeepEquals, []string{"*"})

	subject := cert.Subject
	c.Assert(subject.CommonName, gc.Equals, "*")
	c.Assert(subject.Organization, gc.DeepEquals, []string{"juju"})

	issuer := cert.Issuer
	c.Assert(issuer.CommonName, gc.Equals, "juju-generated CA for environment \"rsyslog\"")
	c.Assert(issuer.Organization, gc.DeepEquals, []string{"juju"})
}

func (s *RsyslogSuite) TestModeAccumulate(c *gc.C) {
	st, m := s.st, s.machine
	worker, err := rsyslog.NewRsyslogConfigWorker(st.Rsyslog(), rsyslog.RsyslogModeAccumulate, m.Tag(), "", nil)
	c.Assert(err, gc.IsNil)
	defer func() { c.Assert(worker.Wait(), gc.IsNil) }()
	defer worker.Kill()
	waitForFile(c, filepath.Join(*rsyslog.LogDir, "ca-cert.pem"))

	// We should have ca-cert.pem, rsyslog-cert.pem, and rsyslog-key.pem.
	caCertPEM, err := ioutil.ReadFile(filepath.Join(*rsyslog.LogDir, "ca-cert.pem"))
	c.Assert(err, gc.IsNil)
	rsyslogCertPEM, err := ioutil.ReadFile(filepath.Join(*rsyslog.LogDir, "rsyslog-cert.pem"))
	c.Assert(err, gc.IsNil)
	rsyslogKeyPEM, err := ioutil.ReadFile(filepath.Join(*rsyslog.LogDir, "rsyslog-key.pem"))
	c.Assert(err, gc.IsNil)

	_, _, err = cert.ParseCertAndKey(string(rsyslogCertPEM), string(rsyslogKeyPEM))
	c.Assert(err, gc.IsNil)
	err = cert.Verify(string(rsyslogCertPEM), string(caCertPEM), time.Now().UTC())
	c.Assert(err, gc.IsNil)

	// Verify rsyslog configuration.
	waitForFile(c, filepath.Join(*rsyslog.RsyslogConfDir, "25-juju.conf"))
	rsyslogConf, err := ioutil.ReadFile(filepath.Join(*rsyslog.RsyslogConfDir, "25-juju.conf"))
	c.Assert(err, gc.IsNil)

	syslogPort := s.Environ.Config().SyslogPort()
	syslogConfig := syslog.NewAccumulateConfig(m.Tag().String(), *rsyslog.LogDir, syslogPort, "", []string{})
	syslogConfig.ConfigDir = *rsyslog.RsyslogConfDir
	rendered, err := syslogConfig.Render()
	c.Assert(err, gc.IsNil)

	c.Assert(string(rsyslogConf), gc.DeepEquals, string(rendered))

	// Verify logrotate files
	assertPathExists(c, filepath.Join(*rsyslog.LogDir, "logrotate.conf"))
	assertPathExists(c, filepath.Join(*rsyslog.LogDir, "logrotate.run"))

}

func (s *RsyslogSuite) TestAccumulateHA(c *gc.C) {
	m := s.machine
	syslogConfig := syslog.NewAccumulateConfig(m.Tag().String(), *rsyslog.LogDir, 6541, "", []string{"192.168.1", "127.0.0.1"})
	rendered, err := syslogConfig.Render()
	c.Assert(err, gc.IsNil)

	stateServer1Config := ":syslogtag, startswith, \"juju-\" @@192.168.1:6541;LongTagForwardFormat"
	stateServer2Config := ":syslogtag, startswith, \"juju-\" @@127.0.0.1:6541;LongTagForwardFormat"

	c.Assert(strings.Contains(string(rendered), stateServer1Config), gc.Equals, true)
	c.Assert(strings.Contains(string(rendered), stateServer2Config), gc.Equals, true)
}

func (s *RsyslogSuite) TestNamespace(c *gc.C) {
	st := s.st
	// set the rsyslog cert
	err := s.APIState.Client().EnvironmentSet(map[string]interface{}{"rsyslog-ca-cert": coretesting.CACert})
	c.Assert(err, gc.IsNil)

	// namespace only takes effect in filenames
	// for machine-0; all others assume isolation.
	s.testNamespace(c, st, names.NewMachineTag("0"), "", "25-juju.conf", *rsyslog.LogDir)
	s.testNamespace(c, st, names.NewMachineTag("0"), "mynamespace", "25-juju-mynamespace.conf", *rsyslog.LogDir+"-mynamespace")
	s.testNamespace(c, st, names.NewMachineTag("1"), "", "25-juju.conf", *rsyslog.LogDir)
	s.testNamespace(c, st, names.NewMachineTag("1"), "mynamespace", "25-juju.conf", *rsyslog.LogDir)
	s.testNamespace(c, st, names.NewUnitTag("myservice/0"), "", "26-juju-unit-myservice-0.conf", *rsyslog.LogDir)
	s.testNamespace(c, st, names.NewUnitTag("myservice/0"), "mynamespace", "26-juju-unit-myservice-0.conf", *rsyslog.LogDir)
}

// testNamespace starts a worker and ensures that
// the rsyslog config file has the expected filename,
// and the appropriate log dir is used.
func (s *RsyslogSuite) testNamespace(c *gc.C, st *api.State, tag names.Tag, namespace, expectedFilename, expectedLogDir string) {
	restarted := make(chan struct{}, 2) // once for create, once for teardown
	s.PatchValue(rsyslog.RestartRsyslog, func() error {
		restarted <- struct{}{}
		return nil
	})

	err := os.MkdirAll(expectedLogDir, 0755)
	c.Assert(err, gc.IsNil)
	worker, err := rsyslog.NewRsyslogConfigWorker(st.Rsyslog(), rsyslog.RsyslogModeAccumulate, tag, namespace, []string{"0.1.2.3"})
	c.Assert(err, gc.IsNil)
	defer func() { c.Assert(worker.Wait(), gc.IsNil) }()
	defer worker.Kill()

	// change the API HostPorts to trigger an rsyslog restart
	newHostPorts := network.AddressesWithPort(network.NewAddresses("127.0.0.1"), 6541)
	err = s.State.SetAPIHostPorts([][]network.HostPort{newHostPorts})
	c.Assert(err, gc.IsNil)

	// Wait for rsyslog to be restarted, so we can check to see
	// what the name of the config file is.
	waitForRestart(c, restarted)

	// Ensure that ca-cert.pem gets written to the expected log dir.
	waitForFile(c, filepath.Join(expectedLogDir, "ca-cert.pem"))

	dir, err := os.Open(*rsyslog.RsyslogConfDir)
	c.Assert(err, gc.IsNil)
	names, err := dir.Readdirnames(-1)
	dir.Close()
	c.Assert(err, gc.IsNil)
	c.Assert(names, gc.HasLen, 1)
	c.Assert(names[0], gc.Equals, expectedFilename)
}
<|endoftext|>


<commit_before>package workspace

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"
)

const metadataFilename = "metadata.json"
const legacyMetadataFilename = ".solution.json"
const ignoreSubdir = ".exercism"

var metadataFilepath = filepath.Join(ignoreSubdir, metadataFilename)

// ExerciseMetadata contains metadata about a user's exercise.
type ExerciseMetadata struct {
	Track        string     `json:"track"`
	ExerciseSlug string     `json:"exercise"`
	ID           string     `json:"id"`
	Team         string     `json:"team,omitempty"`
	URL          string     `json:"url"`
	Handle       string     `json:"handle"`
	IsRequester  bool       `json:"is_requester"`
	SubmittedAt  *time.Time `json:"submitted_at,omitempty"`
	Dir          string     `json:"-"`
	AutoApprove  bool       `json:"auto_approve"`
}

// NewExerciseMetadata reads exercise metadata from a file in the given directory.
func NewExerciseMetadata(dir string) (*ExerciseMetadata, error) {
	b, err := ioutil.ReadFile(filepath.Join(dir, metadataFilepath))
	if err != nil {
		return nil, err
	}
	var metadata ExerciseMetadata
	if err := json.Unmarshal(b, &metadata); err != nil {
		return nil, err
	}
	metadata.Dir = dir
	return &metadata, nil
}

// Suffix is the serial numeric value appended to an exercise directory.
// This is appended to avoid name conflicts, and does not indicate a particular
// iteration.
func (em *ExerciseMetadata) Suffix() string {
	return strings.Trim(strings.Replace(filepath.Base(em.Dir), em.ExerciseSlug, "", 1), "-.")
}

func (em *ExerciseMetadata) String() string {
	str := fmt.Sprintf("%s/%s", em.Track, em.ExerciseSlug)
	if em.Suffix() != "" {
		str = fmt.Sprintf("%s (%s)", str, em.Suffix())
	}
	if !em.IsRequester && em.Handle != "" {
		str = fmt.Sprintf("%s by @%s", str, em.Handle)
	}
	return str
}

// Write stores exercise metadata to a file.
func (em *ExerciseMetadata) Write(dir string) error {
	b, err := json.Marshal(em)
	if err != nil {
		return err
	}
	metadataAbsoluteFilepath := filepath.Join(dir, metadataFilepath)
	if err = os.MkdirAll(filepath.Dir(metadataAbsoluteFilepath), os.FileMode(0755)); err != nil {
		return err
	}
	if err = ioutil.WriteFile(metadataAbsoluteFilepath, b, os.FileMode(0600)); err != nil {
		return err
	}
	em.Dir = dir
	return nil
}

// PathToParent is the relative path from the workspace to the parent dir.
func (em *ExerciseMetadata) PathToParent() string {
	var dir string
	if !em.IsRequester {
		dir = filepath.Join("users")
	}
	return filepath.Join(dir, em.Track)
}
<commit_msg>Create workspace exercise from metadata<commit_after>package workspace

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"
)

const metadataFilename = "metadata.json"
const legacyMetadataFilename = ".solution.json"
const ignoreSubdir = ".exercism"

var metadataFilepath = filepath.Join(ignoreSubdir, metadataFilename)

// ExerciseMetadata contains metadata about a user's exercise.
type ExerciseMetadata struct {
	Track        string     `json:"track"`
	ExerciseSlug string     `json:"exercise"`
	ID           string     `json:"id"`
	Team         string     `json:"team,omitempty"`
	URL          string     `json:"url"`
	Handle       string     `json:"handle"`
	IsRequester  bool       `json:"is_requester"`
	SubmittedAt  *time.Time `json:"submitted_at,omitempty"`
	Dir          string     `json:"-"`
	AutoApprove  bool       `json:"auto_approve"`
}

// NewExerciseMetadata reads exercise metadata from a file in the given directory.
func NewExerciseMetadata(dir string) (*ExerciseMetadata, error) {
	b, err := ioutil.ReadFile(filepath.Join(dir, metadataFilepath))
	if err != nil {
		return nil, err
	}
	var metadata ExerciseMetadata
	if err := json.Unmarshal(b, &metadata); err != nil {
		return nil, err
	}
	metadata.Dir = dir
	return &metadata, nil
}

// Suffix is the serial numeric value appended to an exercise directory.
// This is appended to avoid name conflicts, and does not indicate a particular
// iteration.
func (em *ExerciseMetadata) Suffix() string {
	return strings.Trim(strings.Replace(filepath.Base(em.Dir), em.ExerciseSlug, "", 1), "-.")
}

func (em *ExerciseMetadata) String() string {
	str := fmt.Sprintf("%s/%s", em.Track, em.ExerciseSlug)
	if em.Suffix() != "" {
		str = fmt.Sprintf("%s (%s)", str, em.Suffix())
	}
	if !em.IsRequester && em.Handle != "" {
		str = fmt.Sprintf("%s by @%s", str, em.Handle)
	}
	return str
}

// Write stores exercise metadata to a file.
func (em *ExerciseMetadata) Write(dir string) error {
	b, err := json.Marshal(em)
	if err != nil {
		return err
	}
	metadataAbsoluteFilepath := filepath.Join(dir, metadataFilepath)
	if err = os.MkdirAll(filepath.Dir(metadataAbsoluteFilepath), os.FileMode(0755)); err != nil {
		return err
	}
	if err = ioutil.WriteFile(metadataAbsoluteFilepath, b, os.FileMode(0600)); err != nil {
		return err
	}
	em.Dir = dir
	return nil
}

// PathToParent is the relative path from the workspace to the parent dir.
func (em *ExerciseMetadata) PathToParent() string {
	var dir string
	if !em.IsRequester {
		dir = filepath.Join("users")
	}
	return filepath.Join(dir, em.Track)
}

// Exercise is an implementation of a problem on disk.
func (em *ExerciseMetadata) Exercise(workspace string) Exercise {
	return Exercise{
		Root:  em.root(workspace),
		Track: em.Track,
		Slug:  em.ExerciseSlug,
	}
}

// root represents the root of the exercise.
func (em *ExerciseMetadata) root(workspace string) string {
	if em.Team != "" {
		return filepath.Join(workspace, "teams", em.Team)
	}
	if !em.IsRequester {
		return filepath.Join(workspace, "users", em.Handle)
	}
	return workspace
}
<|endoftext|>


<commit_before>package generators

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/stretchr/testify/require"
)

func TestAppendRoute(t *testing.T) {
	r := require.New(t)

	tmpDir := os.TempDir()
	packagePath := filepath.Join(tmpDir, "src", "sample")
	os.MkdirAll(packagePath, 0755)
	os.Chdir(packagePath)

	const shortAppFileExample = `package actions

import (
	"os"

	"github.com/gobuffalo/buffalo"
	"github.com/gobuffalo/buffalo/examples/json-crud/models"
	"github.com/gobuffalo/buffalo/middleware"
	"github.com/markbates/going/defaults"
)

var ENV = defaults.String(os.Getenv("GO_ENV"), "development")
var app *buffalo.App
func App() *buffalo.App {
	if app == nil {
		app = buffalo.Automatic(buffalo.Options{
			Env: ENV,
		})

		app.Use(middleware.SetContentType("application/json"))
		app.Use(middleware.PopTransaction(models.DB))

		app.Use(findUserMW)
		app.GET("/users", UsersList)
	}

	return app
}`

	ioutil.WriteFile(filepath.Join(packagePath, "actions", "app.go"), []byte(shortAppFileExample), 0755)

	AddRoute("GET", "/new/route", "UserCoolHandler")

	contentAfter, err := ioutil.ReadFile(filepath.Join(packagePath, "actions", "app.go"))
	r.Equal(`package actions

import (
	"os"

	"github.com/gobuffalo/buffalo"
	"github.com/gobuffalo/buffalo/examples/json-crud/models"
	"github.com/gobuffalo/buffalo/middleware"
	"github.com/markbates/going/defaults"
)

var ENV = defaults.String(os.Getenv("GO_ENV"), "development")
var app *buffalo.App
func App() *buffalo.App {
	if app == nil {
		app = buffalo.Automatic(buffalo.Options{
			Env: ENV,
		})

		app.Use(middleware.SetContentType("application/json"))
		app.Use(middleware.PopTransaction(models.DB))

		app.Use(findUserMW)
		app.GET("/users", UsersList)
		app.GET("/new/route", UserCoolHandler)
	}

	return app
}`, string(contentAfter), err.Error())

}
<commit_msg>changing permission for the helper test<commit_after>package generators

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/stretchr/testify/require"
)

func TestAppendRoute(t *testing.T) {
	r := require.New(t)

	tmpDir := os.TempDir()
	packagePath := filepath.Join(tmpDir, "src", "sample")
	os.MkdirAll(packagePath, 0755)
	os.Chdir(packagePath)

	const shortAppFileExample = `package actions

import (
	"os"

	"github.com/gobuffalo/buffalo"
	"github.com/gobuffalo/buffalo/examples/json-crud/models"
	"github.com/gobuffalo/buffalo/middleware"
	"github.com/markbates/going/defaults"
)

var ENV = defaults.String(os.Getenv("GO_ENV"), "development")
var app *buffalo.App
func App() *buffalo.App {
	if app == nil {
		app = buffalo.Automatic(buffalo.Options{
			Env: ENV,
		})

		app.Use(middleware.SetContentType("application/json"))
		app.Use(middleware.PopTransaction(models.DB))

		app.Use(findUserMW)
		app.GET("/users", UsersList)
	}

	return app
}`

	ioutil.WriteFile(filepath.Join(packagePath, "actions", "app.go"), []byte(shortAppFileExample), 0644)

	AddRoute("GET", "/new/route", "UserCoolHandler")

	contentAfter, _ := ioutil.ReadFile(filepath.Join(packagePath, "actions", "app.go"))
	r.Equal(`package actions

import (
	"os"

	"github.com/gobuffalo/buffalo"
	"github.com/gobuffalo/buffalo/examples/json-crud/models"
	"github.com/gobuffalo/buffalo/middleware"
	"github.com/markbates/going/defaults"
)

var ENV = defaults.String(os.Getenv("GO_ENV"), "development")
var app *buffalo.App
func App() *buffalo.App {
	if app == nil {
		app = buffalo.Automatic(buffalo.Options{
			Env: ENV,
		})

		app.Use(middleware.SetContentType("application/json"))
		app.Use(middleware.PopTransaction(models.DB))

		app.Use(findUserMW)
		app.GET("/users", UsersList)
		app.GET("/new/route", UserCoolHandler)
	}

	return app
}`, string(contentAfter))

}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"os/signal"
	"syscall"
)

func main() {
	var (
		flags   int = (os.O_WRONLY | os.O_CREATE)
		exitval int
		files   = []io.Writer{os.Stdout}
	)

	appendFlag := flag.Bool("a", false, "Append the output to the files rather than overwriting them.")
	interruptFlag := flag.Bool("i", false, "Ignore the SIGINT signal.")
	flag.Usage = usage

	flag.Parse()

	if *interruptFlag {
		signal.Ignore(syscall.SIGINT)
	}

	if *appendFlag {
		flags |= os.O_APPEND
	} else {
		flags |= os.O_TRUNC
	}

	for _, arg := range flag.Args() {
		if arg == "-" {
			continue
		}

		if f, err := os.OpenFile(arg, flags, os.ModePerm); err != nil {
			fmt.Fprintf(os.Stderr, "%s - %v", arg, err)
			exitval = 1
		} else {
			defer f.Close()
			files = append(files, f)
		}
	}

	if _, err := io.Copy(io.MultiWriter(files...), os.Stdin); err != nil {
		fmt.Fprintf(os.Stderr, "%v", err)
		exitval = 1
	}

	os.Exit(exitval)
}

func usage() {
	fmt.Fprintln(os.Stderr, "usage: tee [-ai] [file ...]")
	os.Exit(1)
}
<commit_msg>files convert to long var decleration<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"os/signal"
	"syscall"
)

func main() {
	var (
		flags   int = (os.O_WRONLY | os.O_CREATE)
		exitval int
		files   []io.Writer = []io.Writer{os.Stdout}
	)

	appendFlag := flag.Bool("a", false, "Append the output to the files rather than overwriting them.")
	interruptFlag := flag.Bool("i", false, "Ignore the SIGINT signal.")
	flag.Usage = usage

	flag.Parse()

	if *interruptFlag {
		signal.Ignore(syscall.SIGINT)
	}

	if *appendFlag {
		flags |= os.O_APPEND
	} else {
		flags |= os.O_TRUNC
	}

	for _, arg := range flag.Args() {
		if arg == "-" {
			continue
		}

		if f, err := os.OpenFile(arg, flags, os.ModePerm); err != nil {
			fmt.Fprintf(os.Stderr, "%s - %v", arg, err)
			exitval = 1
		} else {
			defer f.Close()
			files = append(files, f)
		}
	}

	if _, err := io.Copy(io.MultiWriter(files...), os.Stdin); err != nil {
		fmt.Fprintf(os.Stderr, "%v", err)
		exitval = 1
	}

	os.Exit(exitval)
}

func usage() {
	fmt.Fprintln(os.Stderr, "usage: tee [-ai] [file ...]")
	os.Exit(1)
}
<|endoftext|>


<commit_before>// The MIT License (MIT)

// Copyright (c) 2016 Maciej Borzecki

// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
package models

import (
	"github.com/jinzhu/gorm"
)

// Registered players
type Player struct {
	gorm.Model

	// user defined name, ex. joe
	Name string
}
<commit_msg>models/player: field for storing password hash<commit_after>// The MIT License (MIT)

// Copyright (c) 2016 Maciej Borzecki

// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
package models

import (
	"github.com/jinzhu/gorm"
)

// Registered players
type Player struct {
	gorm.Model

	// user defined name, ex. joe
	Name string

	// password hash
	PasswordHash string
}
<|endoftext|>


<commit_before>package model

import (
	gonet "code.google.com/p/go.net/html"
	"github.com/slyrz/newscat/html"
)

// A ClusterMap groups Clusters by HTML nodes.
type ClusterMap map[*gonet.Node]*Cluster

// A Cluster stores a group of html.Chunks and their scores.
type Cluster struct {
	Chunks  []*html.Chunk
	Scores  []float32
	Weights []float32
	// Unexported fields.
	average float32 // weighted average score
	changed bool    // denotes struct changes after average calculation
}

// NewCluster creates and initalizes a new Cluster.
func NewCluster() *Cluster {
	result := new(Cluster)
	result.Chunks = make([]*html.Chunk, 0)
	result.Scores = make([]float32, 0)
	result.Weights = make([]float32, 0)
	return result
}

// Add adds the html.Chunk chunk to the Cluster. The variadic float32 parameter
// args must either be (score,) or (score, weight).
func (cl *Cluster) Add(chunk *html.Chunk, args ...float32) {
	var score float32 = 0.0
	var weight float32 = 0.0
	switch len(args) {
	case 2:
		score, weight = args[0], args[1]
	case 1:
		score, weight = args[0], 1.0
	default:
		panic("call Add(chunk, score) or Add(chunk, score, weight)")
	}
	cl.Chunks = append(cl.Chunks, chunk)
	cl.Scores = append(cl.Scores, score)
	cl.Weights = append(cl.Weights, weight)
	cl.changed = true
}

// Score calculates the weighted average of all chunk scores in Cluster.
func (cl *Cluster) Score() float32 {
	if cl.changed {
		var s float32 = 0.0
		var w float32 = 0.0
		for i := range cl.Chunks {
			s += cl.Weights[i] * cl.Scores[i]
			w += cl.Weights[i]
		}
		cl.average = s / w
		cl.changed = true
	}
	return cl.average
}

// NewClusterMap creates and initalizes a new ClusterMap.
func NewClusterMap() ClusterMap {
	return make(ClusterMap)
}

// Add adds the html.Chunk chunk to the Cluster indexed by key.
func (cl ClusterMap) Add(key *gonet.Node, chunk *html.Chunk, args ...float32) {
	cluster, ok := cl[key]
	if !ok {
		cluster = NewCluster()
		cl[base] = cluster
	}
	cluster.Add(chunk, args...)
}
<commit_msg>wrong variable name<commit_after>package model

import (
	gonet "code.google.com/p/go.net/html"
	"github.com/slyrz/newscat/html"
)

// A ClusterMap groups Clusters by HTML nodes.
type ClusterMap map[*gonet.Node]*Cluster

// A Cluster stores a group of html.Chunks and their scores.
type Cluster struct {
	Chunks  []*html.Chunk
	Scores  []float32
	Weights []float32
	// Unexported fields.
	average float32 // weighted average score
	changed bool    // denotes struct changes after average calculation
}

// NewCluster creates and initalizes a new Cluster.
func NewCluster() *Cluster {
	result := new(Cluster)
	result.Chunks = make([]*html.Chunk, 0)
	result.Scores = make([]float32, 0)
	result.Weights = make([]float32, 0)
	return result
}

// Add adds the html.Chunk chunk to the Cluster. The variadic float32 parameter
// args must either be (score,) or (score, weight).
func (cl *Cluster) Add(chunk *html.Chunk, args ...float32) {
	var score float32 = 0.0
	var weight float32 = 0.0
	switch len(args) {
	case 2:
		score, weight = args[0], args[1]
	case 1:
		score, weight = args[0], 1.0
	default:
		panic("call Add(chunk, score) or Add(chunk, score, weight)")
	}
	cl.Chunks = append(cl.Chunks, chunk)
	cl.Scores = append(cl.Scores, score)
	cl.Weights = append(cl.Weights, weight)
	cl.changed = true
}

// Score calculates the weighted average of all chunk scores in Cluster.
func (cl *Cluster) Score() float32 {
	if cl.changed {
		var s float32 = 0.0
		var w float32 = 0.0
		for i := range cl.Chunks {
			s += cl.Weights[i] * cl.Scores[i]
			w += cl.Weights[i]
		}
		cl.average = s / w
		cl.changed = true
	}
	return cl.average
}

// NewClusterMap creates and initalizes a new ClusterMap.
func NewClusterMap() ClusterMap {
	return make(ClusterMap)
}

// Add adds the html.Chunk chunk to the Cluster indexed by key.
func (cl ClusterMap) Add(key *gonet.Node, chunk *html.Chunk, args ...float32) {
	cluster, ok := cl[key]
	if !ok {
		cluster = NewCluster()
		cl[key] = cluster
	}
	cluster.Add(chunk, args...)
}
<|endoftext|>


<commit_before>package models

import (
	"regexp"
	"strings"

	c "github.com/kotakanbe/goval-dictionary/config"
	"github.com/ymomoi/goval-parser/oval"
)

// ConvertUbuntuToModel Convert OVAL to models
func ConvertUbuntuToModel(root *oval.Root) (defs []Definition) {
	for _, d := range root.Definitions.Definitions {

		cveID := ""
		rs := []Reference{}
		for _, r := range d.References {
			rs = append(rs, Reference{
				Source: r.Source,
				RefID:  r.RefID,
				RefURL: r.RefURL,
			})
			if r.Source == "CVE" {
				cveID = r.RefID
			}
		}

		for _, r := range d.Advisory.Refs {
			rs = append(rs, Reference{
				Source: "Ref",
				RefURL: r.URL,
			})
		}

		for _, r := range d.Advisory.Bugs {
			rs = append(rs, Reference{
				Source: "Bug",
				RefURL: r.URL,
			})
		}

		def := Definition{
			DefinitionID: d.ID,
			Title:        d.Title,
			Description:  d.Description,
			Advisory: Advisory{
				Severity: d.Advisory.Severity,
			},
			Debian:        Debian{CveID: cveID},
			AffectedPacks: collectUbuntuPacks(d.Criteria),
			References:    rs,
		}

		if c.Conf.NoDetails {
			def.Title = ""
			def.Description = ""
			def.Advisory = Advisory{}
			def.References = []Reference{}
		}

		defs = append(defs, def)
	}
	return
}

func collectUbuntuPacks(cri oval.Criteria) []Package {
	return walkUbuntu(cri, []Package{})
}

func walkUbuntu(cri oval.Criteria, acc []Package) []Package {
	for _, c := range cri.Criterions {
		if c.Negate {
			continue
		}

		if pkg, ok := parseNotFixedYet(c.Comment); ok {
			acc = append(acc, *pkg)
		}
		if pkg, ok := parseNotDecided(c.Comment); ok {
			acc = append(acc, *pkg)
		}
		if pkg, ok := parseFixed(c.Comment); ok {
			acc = append(acc, *pkg)
		}

		// nop for now
		// <criterion test_ref="oval:com.ubuntu.xenial:tst:10" comment="The vulnerability of the 'brotli' package in xenial is not known (status: 'needs-triage'). It is pending evaluation." />
		// <criterion test_ref="oval:com.ubuntu.bionic:tst:201211480000000" comment="apache2: while related to the CVE in some way, a decision has been made to ignore this issue (note: 'code-not-compiled')." />

	}

	if len(cri.Criterias) == 0 {
		return acc
	}
	for _, c := range cri.Criterias {
		acc = walkUbuntu(c, acc)
	}
	return acc
}

var reNotFixed = regexp.MustCompile(`^(.+) package in .+ affected and needs fixing.$`)

func parseNotFixedYet(comment string) (*Package, bool) {
	// Ubuntu 14
	// The 'php-openid' package in trusty is affected and needs fixing.

	// Ubuntu 16, 18
	// xine-console package in bionic is affected and needs fixing. />
	res := reNotFixed.FindStringSubmatch(comment)
	if len(res) == 2 {
		return &Package{
			Name:        trimPkgName(res[1]),
			NotFixedYet: true,
		}, true
	}
	return nil, false
}

var reNotDecided = regexp.MustCompile(`^(.+) package in .+ is affected, but a decision has been made to defer addressing it .+$`)

func parseNotDecided(comment string) (*Package, bool) {
	// Ubuntu 14
	// The 'ruby1.9.1' package in trusty is affected, but a decision has been made to defer addressing it (note: '2019-04-10').

	// Ubuntu 16, 18
	// libxerces-c-samples package in bionic is affected, but a decision has been made to defer addressing it (note: '2019-01-01').
	res := reNotDecided.FindStringSubmatch(comment)
	if len(res) == 2 {
		return &Package{
			Name:        trimPkgName(res[1]),
			NotFixedYet: true,
		}, true
	}
	return nil, false
}

var reFixed = regexp.MustCompile(`^(.+) package in .+ has been fixed \(note: '([^\s]+).*'\).$`)

func parseFixed(comment string) (*Package, bool) {
	// Ubuntu 14
	// The 'poppler' package in trusty was vulnerable but has been fixed (note: '0.10.5-1ubuntu2').

	// Ubuntu 16, 18
	// iproute2 package in bionic, is related to the CVE in some way and has been fixed (note: '3.12.0-2').
	res := reFixed.FindStringSubmatch(comment)
	if len(res) == 3 {
		return &Package{
			Name:    trimPkgName(res[1]),
			Version: res[2],
		}, true
	}
	return nil, false
}

func trimPkgName(name string) string {
	name = strings.TrimPrefix(name, "The '")
	return strings.TrimSuffix(name, "'")
}
<commit_msg>Do not remove CVE in references of Ubuntu (#72)<commit_after>package models

import (
	"regexp"
	"strings"

	c "github.com/kotakanbe/goval-dictionary/config"
	"github.com/ymomoi/goval-parser/oval"
)

// ConvertUbuntuToModel Convert OVAL to models
func ConvertUbuntuToModel(root *oval.Root) (defs []Definition) {
	for _, d := range root.Definitions.Definitions {

		cveID := ""
		rs := []Reference{}
		for _, r := range d.References {
			rs = append(rs, Reference{
				Source: r.Source,
				RefID:  r.RefID,
				RefURL: r.RefURL,
			})
			if r.Source == "CVE" {
				cveID = r.RefID
			}
		}

		for _, r := range d.Advisory.Refs {
			rs = append(rs, Reference{
				Source: "Ref",
				RefURL: r.URL,
			})
		}

		for _, r := range d.Advisory.Bugs {
			rs = append(rs, Reference{
				Source: "Bug",
				RefURL: r.URL,
			})
		}

		def := Definition{
			DefinitionID: d.ID,
			Title:        d.Title,
			Description:  d.Description,
			Advisory: Advisory{
				Severity: d.Advisory.Severity,
			},
			Debian:        Debian{CveID: cveID},
			AffectedPacks: collectUbuntuPacks(d.Criteria),
			References:    rs,
		}

		if c.Conf.NoDetails {
			def.Title = ""
			def.Description = ""
			def.Advisory = Advisory{}

			var references []Reference
			for _, ref := range def.References {
				if ref.Source != "CVE" {
					continue
				}
				references = append(references, Reference{
					Source: ref.Source,
					RefID:  ref.RefID,
				})
			}
			def.References = references
		}

		defs = append(defs, def)
	}
	return
}

func collectUbuntuPacks(cri oval.Criteria) []Package {
	return walkUbuntu(cri, []Package{})
}

func walkUbuntu(cri oval.Criteria, acc []Package) []Package {
	for _, c := range cri.Criterions {
		if c.Negate {
			continue
		}

		if pkg, ok := parseNotFixedYet(c.Comment); ok {
			acc = append(acc, *pkg)
		}
		if pkg, ok := parseNotDecided(c.Comment); ok {
			acc = append(acc, *pkg)
		}
		if pkg, ok := parseFixed(c.Comment); ok {
			acc = append(acc, *pkg)
		}

		// nop for now
		// <criterion test_ref="oval:com.ubuntu.xenial:tst:10" comment="The vulnerability of the 'brotli' package in xenial is not known (status: 'needs-triage'). It is pending evaluation." />
		// <criterion test_ref="oval:com.ubuntu.bionic:tst:201211480000000" comment="apache2: while related to the CVE in some way, a decision has been made to ignore this issue (note: 'code-not-compiled')." />

	}

	if len(cri.Criterias) == 0 {
		return acc
	}
	for _, c := range cri.Criterias {
		acc = walkUbuntu(c, acc)
	}
	return acc
}

var reNotFixed = regexp.MustCompile(`^(.+) package in .+ affected and needs fixing.$`)

func parseNotFixedYet(comment string) (*Package, bool) {
	// Ubuntu 14
	// The 'php-openid' package in trusty is affected and needs fixing.

	// Ubuntu 16, 18
	// xine-console package in bionic is affected and needs fixing. />
	res := reNotFixed.FindStringSubmatch(comment)
	if len(res) == 2 {
		return &Package{
			Name:        trimPkgName(res[1]),
			NotFixedYet: true,
		}, true
	}
	return nil, false
}

var reNotDecided = regexp.MustCompile(`^(.+) package in .+ is affected, but a decision has been made to defer addressing it .+$`)

func parseNotDecided(comment string) (*Package, bool) {
	// Ubuntu 14
	// The 'ruby1.9.1' package in trusty is affected, but a decision has been made to defer addressing it (note: '2019-04-10').

	// Ubuntu 16, 18
	// libxerces-c-samples package in bionic is affected, but a decision has been made to defer addressing it (note: '2019-01-01').
	res := reNotDecided.FindStringSubmatch(comment)
	if len(res) == 2 {
		return &Package{
			Name:        trimPkgName(res[1]),
			NotFixedYet: true,
		}, true
	}
	return nil, false
}

var reFixed = regexp.MustCompile(`^(.+) package in .+ has been fixed \(note: '([^\s]+).*'\).$`)

func parseFixed(comment string) (*Package, bool) {
	// Ubuntu 14
	// The 'poppler' package in trusty was vulnerable but has been fixed (note: '0.10.5-1ubuntu2').

	// Ubuntu 16, 18
	// iproute2 package in bionic, is related to the CVE in some way and has been fixed (note: '3.12.0-2').
	res := reFixed.FindStringSubmatch(comment)
	if len(res) == 3 {
		return &Package{
			Name:    trimPkgName(res[1]),
			Version: res[2],
		}, true
	}
	return nil, false
}

func trimPkgName(name string) string {
	name = strings.TrimPrefix(name, "The '")
	return strings.TrimSuffix(name, "'")
}
<|endoftext|>


<commit_before>package spider

import (
	"math"
	"sync"
	"time"

	"github.com/henrylee2cn/pholcus/app/downloader/request"
	"github.com/henrylee2cn/pholcus/app/scheduler"
	"github.com/henrylee2cn/pholcus/common/util"
	"github.com/henrylee2cn/pholcus/logs"
	"github.com/henrylee2cn/pholcus/runtime/status"
)

const (
	KEYIN       = util.USE_KEYIN // 若使用Spider.Keyin，则须在规则中设置初始值为USE_KEYIN
	LIMIT       = math.MaxInt64  // 如希望在规则中自定义控制Limit，则Limit初始值必须为LIMIT
	ACTIVE_STOP = "——主动终止Spider——"
)

type (
	// 蜘蛛规则
	Spider struct {
		// 以下字段由用户定义
		Name            string                                                     // 用户界面显示的名称（应保证唯一性）
		Description     string                                                     // 用户界面显示的描述
		Pausetime       int64                                                      // 随机暂停区间(50%~200%)，若规则中直接定义，则不被界面传参覆盖
		Limit           int64                                                      // 默认限制请求数，0为不限；若规则中定义为LIMIT，则采用规则的自定义限制方案
		Keyin           string                                                     // 自定义输入的配置信息，使用前须在规则中设置初始值为KEYIN
		EnableCookie    bool                                                       // 所有请求是否使用cookie记录
		NotDefaultField bool                                                       // 是否禁止输出结果中的默认字段 Url/ParentUrl/DownloadTime
		Namespace       func(self *Spider) string                                  // 命名空间，用于输出文件、路径的命名
		SubNamespace    func(self *Spider, dataCell map[string]interface{}) string // 次级命名，用于输出文件、路径的命名，可依赖具体数据内容
		RuleTree        *RuleTree                                                  // 定义具体的采集规则树

		// 以下字段系统自动赋值
		id        int               // 自动分配的SpiderQueue中的索引
		subName   string            // 由Keyin转换为的二级标识名
		reqMatrix *scheduler.Matrix // 请求矩阵
		timer     *Timer            // 定时器
		status    int               // 执行状态
		lock      sync.RWMutex
		once      sync.Once
	}
	//采集规则树
	RuleTree struct {
		Root  func(*Context)   // 执行入口（树根）
		Trunk map[string]*Rule // 执行采集过程（树干）
	}
	// 采集规则单元
	Rule struct {
		ItemFields []string                                           // 输出结果的字段名列表
		ParseFunc  func(*Context)                                     // 内容解析函数
		AidFunc    func(*Context, map[string]interface{}) interface{} // 通用辅助函数
	}
)

// 添加自身到蜘蛛菜单
func (self Spider) Register() *Spider {
	self.status = status.STOPPED
	return Species.Add(&self)
}

// 指定规则的获取结果的字段名列表
func (self *Spider) GetItemFields(rule *Rule) []string {
	return rule.ItemFields
}

// 返回结果字段名的值
// 不存在时返回空字符串
func (self *Spider) GetItemField(rule *Rule, index int) (field string) {
	if index > len(rule.ItemFields)-1 || index < 0 {
		return ""
	}
	return rule.ItemFields[index]
}

// 返回结果字段名的其索引
// 不存在时索引为-1
func (self *Spider) GetItemFieldIndex(rule *Rule, field string) (index int) {
	for idx, v := range rule.ItemFields {
		if v == field {
			return idx
		}
	}
	return -1
}

// 为指定Rule动态追加结果字段名，并返回索引位置
// 已存在时返回原来索引位置
func (self *Spider) UpsertItemField(rule *Rule, field string) (index int) {
	for i, v := range rule.ItemFields {
		if v == field {
			return i
		}
	}
	rule.ItemFields = append(rule.ItemFields, field)
	return len(rule.ItemFields) - 1
}

// 获取蜘蛛名称
func (self *Spider) GetName() string {
	return self.Name
}

// 获取蜘蛛二级标识名
func (self *Spider) GetSubName() string {
	self.once.Do(func() {
		self.subName = self.GetKeyin()
		if len([]rune(self.subName)) > 6 {
			self.subName = util.MakeHash(self.subName)
		}
	})
	return self.subName
}

// 安全返回指定规则
func (self *Spider) GetRule(ruleName string) (*Rule, bool) {
	rule, found := self.RuleTree.Trunk[ruleName]
	return rule, found
}

// 返回指定规则
func (self *Spider) MustGetRule(ruleName string) *Rule {
	return self.RuleTree.Trunk[ruleName]
}

// 返回规则树
func (self *Spider) GetRules() map[string]*Rule {
	return self.RuleTree.Trunk
}

// 获取蜘蛛描述
func (self *Spider) GetDescription() string {
	return self.Description
}

// 获取蜘蛛ID
func (self *Spider) GetId() int {
	return self.id
}

// 设置蜘蛛ID
func (self *Spider) SetId(id int) {
	self.id = id
}

// 获取自定义配置信息
func (self *Spider) GetKeyin() string {
	return self.Keyin
}

// 设置自定义配置信息
func (self *Spider) SetKeyin(keyword string) {
	self.Keyin = keyword
}

// 获取采集上限
// <0 表示采用限制请求数的方案
// >0 表示采用规则中的自定义限制方案
func (self *Spider) GetLimit() int64 {
	return self.Limit
}

// 设置采集上限
// <0 表示采用限制请求数的方案
// >0 表示采用规则中的自定义限制方案
func (self *Spider) SetLimit(max int64) {
	self.Limit = max
}

// 控制所有请求是否使用cookie
func (self *Spider) GetEnableCookie() bool {
	return self.EnableCookie
}

// 自定义暂停时间 pause[0]~(pause[0]+pause[1])，优先级高于外部传参
// 当且仅当runtime[0]为true时可覆盖现有值
func (self *Spider) SetPausetime(pause int64, runtime ...bool) {
	if self.Pausetime == 0 || len(runtime) > 0 && runtime[0] {
		self.Pausetime = pause
	}
}

// 设置定时器
// @id为定时器唯一标识
// @bell==nil时为倒计时器，此时@tol为睡眠时长
// @bell!=nil时为闹铃，此时@tol用于指定醒来时刻（从now起遇到的第tol个bell）
func (self *Spider) SetTimer(id string, tol time.Duration, bell *Bell) bool {
	if self.timer == nil {
		self.timer = newTimer()
	}
	return self.timer.set(id, tol, bell)
}

// 启动定时器，并返回定时器是否可以继续使用
func (self *Spider) RunTimer(id string) bool {
	if self.timer == nil {
		return false
	}
	return self.timer.sleep(id)
}

// 返回一个自身复制品
func (self *Spider) Copy() *Spider {
	ghost := &Spider{}
	ghost.Name = self.Name
	ghost.subName = self.subName

	ghost.RuleTree = &RuleTree{
		Root:  self.RuleTree.Root,
		Trunk: make(map[string]*Rule, len(self.RuleTree.Trunk)),
	}
	for k, v := range self.RuleTree.Trunk {
		ghost.RuleTree.Trunk[k] = new(Rule)

		ghost.RuleTree.Trunk[k].ItemFields = make([]string, len(v.ItemFields))
		copy(ghost.RuleTree.Trunk[k].ItemFields, v.ItemFields)

		ghost.RuleTree.Trunk[k].ParseFunc = v.ParseFunc
		ghost.RuleTree.Trunk[k].AidFunc = v.AidFunc
	}

	ghost.Description = self.Description
	ghost.Pausetime = self.Pausetime
	ghost.EnableCookie = self.EnableCookie
	ghost.Limit = self.Limit
	ghost.Keyin = self.Keyin

	ghost.NotDefaultField = self.NotDefaultField
	ghost.Namespace = self.Namespace
	ghost.SubNamespace = self.SubNamespace

	ghost.timer = self.timer
	ghost.status = self.status

	return ghost
}

func (self *Spider) ReqmatrixInit() *Spider {
	if self.Limit < 0 {
		self.reqMatrix = scheduler.AddMatrix(self.GetName(), self.GetSubName(), self.Limit)
		self.SetLimit(0)
	} else {
		self.reqMatrix = scheduler.AddMatrix(self.GetName(), self.GetSubName(), math.MinInt64)
	}
	return self
}

// 返回是否作为新的失败请求被添加至队列尾部
func (self *Spider) DoHistory(req *request.Request, ok bool) bool {
	return self.reqMatrix.DoHistory(req, ok)
}

func (self *Spider) RequestPush(req *request.Request) {
	self.reqMatrix.Push(req)
}

func (self *Spider) RequestPull() *request.Request {
	return self.reqMatrix.Pull()
}

func (self *Spider) RequestUse() {
	self.reqMatrix.Use()
}

func (self *Spider) RequestFree() {
	self.reqMatrix.Free()
}

func (self *Spider) RequestLen() int {
	return self.reqMatrix.Len()
}

func (self *Spider) TryFlushSuccess() {
	self.reqMatrix.TryFlushSuccess()
}

func (self *Spider) TryFlushFailure() {
	self.reqMatrix.TryFlushFailure()
}

// 开始执行蜘蛛
func (self *Spider) Start() {
	defer func() {
		if p := recover(); p != nil {
			logs.Log.Error(" *     Panic  [root]: %v\n", p)
		}
		self.lock.Lock()
		self.status = status.RUN
		self.lock.Unlock()
	}()
	self.RuleTree.Root(GetContext(self, nil))
}

// 主动崩溃爬虫运行协程
func (self *Spider) Stop() {
	self.lock.Lock()
	self.status = status.STOP
	// 取消所有定时器
	if self.timer != nil {
		self.timer.drop()
		self.timer = nil
	}
	self.lock.Unlock()
}

func (self *Spider) CanStop() bool {
	return self.status != status.STOPPED && self.reqMatrix.CanStop()
}

// 若已主动终止任务，则崩溃爬虫协程
func (self *Spider) tryPanic() {
	self.lock.RLock()
	if self.status == status.STOP {
		self.lock.RUnlock()
		panic(ACTIVE_STOP)
	}
	self.lock.RUnlock()
}

// 退出任务前收尾工作
func (self *Spider) Defer() {
	// 取消所有定时器
	if self.timer != nil {
		self.timer.drop()
		self.timer = nil
	}
	// 等待处理中的请求完成
	self.reqMatrix.Wait()
	// 更新失败记录
	self.reqMatrix.TryFlushFailure()
}

// 是否输出默认添加的字段 Url/ParentUrl/DownloadTime
func (self *Spider) OutDefaultField() bool {
	return !self.NotDefaultField
}
<commit_msg>Optimal Comments<commit_after>package spider

import (
	"math"
	"sync"
	"time"

	"github.com/henrylee2cn/pholcus/app/downloader/request"
	"github.com/henrylee2cn/pholcus/app/scheduler"
	"github.com/henrylee2cn/pholcus/common/util"
	"github.com/henrylee2cn/pholcus/logs"
	"github.com/henrylee2cn/pholcus/runtime/status"
)

const (
	KEYIN       = util.USE_KEYIN // 若使用Spider.Keyin，则须在规则中设置初始值为USE_KEYIN
	LIMIT       = math.MaxInt64  // 如希望在规则中自定义控制Limit，则Limit初始值必须为LIMIT
	ACTIVE_STOP = "——主动终止Spider——"
)

type (
	// 蜘蛛规则
	Spider struct {
		// 以下字段由用户定义
		Name            string                                                     // 用户界面显示的名称（应保证唯一性）
		Description     string                                                     // 用户界面显示的描述
		Pausetime       int64                                                      // 随机暂停区间(50%~200%)，若规则中直接定义，则不被界面传参覆盖
		Limit           int64                                                      // 默认限制请求数，0为不限；若规则中定义为LIMIT，则采用规则的自定义限制方案
		Keyin           string                                                     // 自定义输入的配置信息，使用前须在规则中设置初始值为KEYIN
		EnableCookie    bool                                                       // 所有请求是否使用cookie记录
		NotDefaultField bool                                                       // 是否禁止输出结果中的默认字段 Url/ParentUrl/DownloadTime
		Namespace       func(self *Spider) string                                  // 命名空间，用于输出文件、路径的命名
		SubNamespace    func(self *Spider, dataCell map[string]interface{}) string // 次级命名，用于输出文件、路径的命名，可依赖具体数据内容
		RuleTree        *RuleTree                                                  // 定义具体的采集规则树

		// 以下字段系统自动赋值
		id        int               // 自动分配的SpiderQueue中的索引
		subName   string            // 由Keyin转换为的二级标识名
		reqMatrix *scheduler.Matrix // 请求矩阵
		timer     *Timer            // 定时器
		status    int               // 执行状态
		lock      sync.RWMutex
		once      sync.Once
	}
	//采集规则树
	RuleTree struct {
		Root  func(*Context)   // 根节点(执行入口)
		Trunk map[string]*Rule // 节点散列表(执行采集过程)
	}
	// 采集规则节点
	Rule struct {
		ItemFields []string                                           // 结果字段列表(选填，写上可保证字段顺序)
		ParseFunc  func(*Context)                                     // 内容解析函数
		AidFunc    func(*Context, map[string]interface{}) interface{} // 通用辅助函数
	}
)

// 添加自身到蜘蛛菜单
func (self Spider) Register() *Spider {
	self.status = status.STOPPED
	return Species.Add(&self)
}

// 指定规则的获取结果的字段名列表
func (self *Spider) GetItemFields(rule *Rule) []string {
	return rule.ItemFields
}

// 返回结果字段名的值
// 不存在时返回空字符串
func (self *Spider) GetItemField(rule *Rule, index int) (field string) {
	if index > len(rule.ItemFields)-1 || index < 0 {
		return ""
	}
	return rule.ItemFields[index]
}

// 返回结果字段名的其索引
// 不存在时索引为-1
func (self *Spider) GetItemFieldIndex(rule *Rule, field string) (index int) {
	for idx, v := range rule.ItemFields {
		if v == field {
			return idx
		}
	}
	return -1
}

// 为指定Rule动态追加结果字段名，并返回索引位置
// 已存在时返回原来索引位置
func (self *Spider) UpsertItemField(rule *Rule, field string) (index int) {
	for i, v := range rule.ItemFields {
		if v == field {
			return i
		}
	}
	rule.ItemFields = append(rule.ItemFields, field)
	return len(rule.ItemFields) - 1
}

// 获取蜘蛛名称
func (self *Spider) GetName() string {
	return self.Name
}

// 获取蜘蛛二级标识名
func (self *Spider) GetSubName() string {
	self.once.Do(func() {
		self.subName = self.GetKeyin()
		if len([]rune(self.subName)) > 6 {
			self.subName = util.MakeHash(self.subName)
		}
	})
	return self.subName
}

// 安全返回指定规则
func (self *Spider) GetRule(ruleName string) (*Rule, bool) {
	rule, found := self.RuleTree.Trunk[ruleName]
	return rule, found
}

// 返回指定规则
func (self *Spider) MustGetRule(ruleName string) *Rule {
	return self.RuleTree.Trunk[ruleName]
}

// 返回规则树
func (self *Spider) GetRules() map[string]*Rule {
	return self.RuleTree.Trunk
}

// 获取蜘蛛描述
func (self *Spider) GetDescription() string {
	return self.Description
}

// 获取蜘蛛ID
func (self *Spider) GetId() int {
	return self.id
}

// 设置蜘蛛ID
func (self *Spider) SetId(id int) {
	self.id = id
}

// 获取自定义配置信息
func (self *Spider) GetKeyin() string {
	return self.Keyin
}

// 设置自定义配置信息
func (self *Spider) SetKeyin(keyword string) {
	self.Keyin = keyword
}

// 获取采集上限
// <0 表示采用限制请求数的方案
// >0 表示采用规则中的自定义限制方案
func (self *Spider) GetLimit() int64 {
	return self.Limit
}

// 设置采集上限
// <0 表示采用限制请求数的方案
// >0 表示采用规则中的自定义限制方案
func (self *Spider) SetLimit(max int64) {
	self.Limit = max
}

// 控制所有请求是否使用cookie
func (self *Spider) GetEnableCookie() bool {
	return self.EnableCookie
}

// 自定义暂停时间 pause[0]~(pause[0]+pause[1])，优先级高于外部传参
// 当且仅当runtime[0]为true时可覆盖现有值
func (self *Spider) SetPausetime(pause int64, runtime ...bool) {
	if self.Pausetime == 0 || len(runtime) > 0 && runtime[0] {
		self.Pausetime = pause
	}
}

// 设置定时器
// @id为定时器唯一标识
// @bell==nil时为倒计时器，此时@tol为睡眠时长
// @bell!=nil时为闹铃，此时@tol用于指定醒来时刻（从now起遇到的第tol个bell）
func (self *Spider) SetTimer(id string, tol time.Duration, bell *Bell) bool {
	if self.timer == nil {
		self.timer = newTimer()
	}
	return self.timer.set(id, tol, bell)
}

// 启动定时器，并返回定时器是否可以继续使用
func (self *Spider) RunTimer(id string) bool {
	if self.timer == nil {
		return false
	}
	return self.timer.sleep(id)
}

// 返回一个自身复制品
func (self *Spider) Copy() *Spider {
	ghost := &Spider{}
	ghost.Name = self.Name
	ghost.subName = self.subName

	ghost.RuleTree = &RuleTree{
		Root:  self.RuleTree.Root,
		Trunk: make(map[string]*Rule, len(self.RuleTree.Trunk)),
	}
	for k, v := range self.RuleTree.Trunk {
		ghost.RuleTree.Trunk[k] = new(Rule)

		ghost.RuleTree.Trunk[k].ItemFields = make([]string, len(v.ItemFields))
		copy(ghost.RuleTree.Trunk[k].ItemFields, v.ItemFields)

		ghost.RuleTree.Trunk[k].ParseFunc = v.ParseFunc
		ghost.RuleTree.Trunk[k].AidFunc = v.AidFunc
	}

	ghost.Description = self.Description
	ghost.Pausetime = self.Pausetime
	ghost.EnableCookie = self.EnableCookie
	ghost.Limit = self.Limit
	ghost.Keyin = self.Keyin

	ghost.NotDefaultField = self.NotDefaultField
	ghost.Namespace = self.Namespace
	ghost.SubNamespace = self.SubNamespace

	ghost.timer = self.timer
	ghost.status = self.status

	return ghost
}

func (self *Spider) ReqmatrixInit() *Spider {
	if self.Limit < 0 {
		self.reqMatrix = scheduler.AddMatrix(self.GetName(), self.GetSubName(), self.Limit)
		self.SetLimit(0)
	} else {
		self.reqMatrix = scheduler.AddMatrix(self.GetName(), self.GetSubName(), math.MinInt64)
	}
	return self
}

// 返回是否作为新的失败请求被添加至队列尾部
func (self *Spider) DoHistory(req *request.Request, ok bool) bool {
	return self.reqMatrix.DoHistory(req, ok)
}

func (self *Spider) RequestPush(req *request.Request) {
	self.reqMatrix.Push(req)
}

func (self *Spider) RequestPull() *request.Request {
	return self.reqMatrix.Pull()
}

func (self *Spider) RequestUse() {
	self.reqMatrix.Use()
}

func (self *Spider) RequestFree() {
	self.reqMatrix.Free()
}

func (self *Spider) RequestLen() int {
	return self.reqMatrix.Len()
}

func (self *Spider) TryFlushSuccess() {
	self.reqMatrix.TryFlushSuccess()
}

func (self *Spider) TryFlushFailure() {
	self.reqMatrix.TryFlushFailure()
}

// 开始执行蜘蛛
func (self *Spider) Start() {
	defer func() {
		if p := recover(); p != nil {
			logs.Log.Error(" *     Panic  [root]: %v\n", p)
		}
		self.lock.Lock()
		self.status = status.RUN
		self.lock.Unlock()
	}()
	self.RuleTree.Root(GetContext(self, nil))
}

// 主动崩溃爬虫运行协程
func (self *Spider) Stop() {
	self.lock.Lock()
	self.status = status.STOP
	// 取消所有定时器
	if self.timer != nil {
		self.timer.drop()
		self.timer = nil
	}
	self.lock.Unlock()
}

func (self *Spider) CanStop() bool {
	return self.status != status.STOPPED && self.reqMatrix.CanStop()
}

// 若已主动终止任务，则崩溃爬虫协程
func (self *Spider) tryPanic() {
	self.lock.RLock()
	if self.status == status.STOP {
		self.lock.RUnlock()
		panic(ACTIVE_STOP)
	}
	self.lock.RUnlock()
}

// 退出任务前收尾工作
func (self *Spider) Defer() {
	// 取消所有定时器
	if self.timer != nil {
		self.timer.drop()
		self.timer = nil
	}
	// 等待处理中的请求完成
	self.reqMatrix.Wait()
	// 更新失败记录
	self.reqMatrix.TryFlushFailure()
}

// 是否输出默认添加的字段 Url/ParentUrl/DownloadTime
func (self *Spider) OutDefaultField() bool {
	return !self.NotDefaultField
}
<|endoftext|>