<commit_before>package gosecco

import (
	"fmt"
	"runtime"
	"syscall"

	"github.com/twtiger/gosecco/data"
	"github.com/twtiger/gosecco/native"

	"golang.org/x/sys/unix"
)

// CheckSupport checks for the required seccomp support in the kernel.
func CheckSupport() error {
	if err := native.CheckGetSeccomp(); err != nil {
		return fmt.Errorf("seccomp not available: %v", err)
	}
	if err := native.CheckSetSeccompModeFilter(); err != syscall.EFAULT {
		return fmt.Errorf("seccomp filter not available: %v", err)
	}
	if err := native.CheckSetSeccompModeFilterWithSeccomp(); err != syscall.EFAULT {
		return fmt.Errorf("seccomp syscall not available: %v", err)
	}
	if err := native.CheckSetSeccompModeTsync(); err != syscall.EFAULT {
		return fmt.Errorf("seccomp tsync not available: %v", err)
	}
	return nil
}

// SeccompSettings contains the extra settings necessary to tweak the
// behavior of the compilation process
type SeccompSettings struct {
	extraDefinitions      []string
	defaultPositiveAction string
	defaultNegativeAction string
}

// Prepare will take the given path and settings, parse and compile the given
// data, combined with the settings - and returns the bytecode
func Prepare(path string, s SeccompSettings) ([]unix.SockFilter, error) {
	// TODO: test when compiler is ready:
	// - test that parser errors come through
	// - test that unification works and that errors come through
	// - test that default pos and neg actions come through
	// - test that the type checker errors come through
	// - test that the simplifier is invoked and simplifies stuff
	// - test that simplifier errors come through
	// - test that the compiler works and returns the expected results
	// - test that compiler errors come through
	return nil, nil
}

// Compile provides the compatibility interface for gosecco - it has the same signature as
// Compile from the go-seccomp package and should provide the same behavior.
// However, the modern interface is through the Prepare function
func Compile(path string, enforce bool) ([]unix.SockFilter, error) {
	// TODO: test once compiler is done, light testing needed, since main testing
	// will be of the Prepare method
	return nil, nil
}

// CompileBlacklist provides the compatibility interface for gosecco, for blacklist mode
// It has the same signature as CompileBlacklist from Subgraphs go-seccomp and should provide the same behavior.
// However, the modern interface is through the Prepare function
func CompileBlacklist(path string, enforce bool) ([]unix.SockFilter, error) {
	// TODO: test once compiler is done, light testing needed, since main testing
	// will be of the Prepare method
	return nil, nil
}

// Load makes the seccomp system call to install the bpf filter for
// all threads (with tsync). Most users of this library should use
// Install instead of Load, since Install ensures that prctl(set_no_new_privs, 1)
// has been called
func Load(bpf []unix.SockFilter) error {
	if size, limit := len(bpf), 0xffff; size > limit {
		return fmt.Errorf("filter program too big: %d bpf instructions (limit = %d)", size, limit)
	}

	prog := &data.SockFprog{
		Filter: &bpf[0],
		Len:    uint16(len(bpf)),
	}

	return native.InstallSeccomp(prog)
}

// Install will install the given policy filters into the kernel
func Install(bpf []unix.SockFilter) error {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()
	if err := native.NoNewPrivs(); err != nil {
		return err
	}
	return Load(bpf)
}

// InstallBlacklist makes the necessary system calls to install the Seccomp-BPF
// filter for the current process (all threads). Install can be called
// multiple times to install additional filters.
func InstallBlacklist(bpf []unix.SockFilter) error {
	return Install(bpf)
}
<commit_msg>Adds compile blacklist and enforce flag for compile<commit_after>package gosecco

import (
	"fmt"
	"runtime"
	"syscall"

	"github.com/twtiger/gosecco/data"
	"github.com/twtiger/gosecco/native"

	"golang.org/x/sys/unix"
)

// CheckSupport checks for the required seccomp support in the kernel.
func CheckSupport() error {
	if err := native.CheckGetSeccomp(); err != nil {
		return fmt.Errorf("seccomp not available: %v", err)
	}
	if err := native.CheckSetSeccompModeFilter(); err != syscall.EFAULT {
		return fmt.Errorf("seccomp filter not available: %v", err)
	}
	if err := native.CheckSetSeccompModeFilterWithSeccomp(); err != syscall.EFAULT {
		return fmt.Errorf("seccomp syscall not available: %v", err)
	}
	if err := native.CheckSetSeccompModeTsync(); err != syscall.EFAULT {
		return fmt.Errorf("seccomp tsync not available: %v", err)
	}
	return nil
}

// SeccompSettings contains the extra settings necessary to tweak the
// behavior of the compilation process
type SeccompSettings struct {
	extraDefinitions      []string
	defaultPositiveAction string
	defaultNegativeAction string
}

// Prepare will take the given path and settings, parse and compile the given
// data, combined with the settings - and returns the bytecode
func Prepare(path string, s SeccompSettings) ([]unix.SockFilter, error) {
	// TODO: test when compiler is ready:
	// - test that parser errors come through
	// - test that unification works and that errors come through
	// - test that default pos and neg actions come through
	// - test that the type checker errors come through
	// - test that the simplifier is invoked and simplifies stuff
	// - test that simplifier errors come through
	// - test that the compiler works and returns the expected results
	// - test that compiler errors come through
	return nil, nil
}

// Compile provides the compatibility interface for gosecco - it has the same signature as
// Compile from the go-seccomp package and should provide the same behavior.
// However, the modern interface is through the Prepare function
func Compile(path string, enforce bool) ([]unix.SockFilter, error) {
	// TODO: test when compiler is done

	settings := SeccompSettings{}
	settings.defaultPositiveAction = "allow"
	if enforce {
		settings.defaultNegativeAction = "kill"
	} else {
		settings.defaultNegativeAction = "trace"
	}

	return Prepare(path, settings)
}

// CompileBlacklist provides the compatibility interface for gosecco, for blacklist mode
// It has the same signature as CompileBlacklist from Subgraphs go-seccomp and should provide the same behavior.
// However, the modern interface is through the Prepare function
func CompileBlacklist(path string, enforce bool) ([]unix.SockFilter, error) {
	// TODO: test when compiler is done

	settings := SeccompSettings{}
	settings.defaultNegativeAction = "allow"
	if enforce {
		settings.defaultPositiveAction = "kill"
	} else {
		settings.defaultPositiveAction = "trace"
	}

	return Prepare(path, settings)
}

// Load makes the seccomp system call to install the bpf filter for
// all threads (with tsync). Most users of this library should use
// Install instead of Load, since Install ensures that prctl(set_no_new_privs, 1)
// has been called
func Load(bpf []unix.SockFilter) error {
	if size, limit := len(bpf), 0xffff; size > limit {
		return fmt.Errorf("filter program too big: %d bpf instructions (limit = %d)", size, limit)
	}

	prog := &data.SockFprog{
		Filter: &bpf[0],
		Len:    uint16(len(bpf)),
	}

	return native.InstallSeccomp(prog)
}

// Install will install the given policy filters into the kernel
func Install(bpf []unix.SockFilter) error {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()
	if err := native.NoNewPrivs(); err != nil {
		return err
	}
	return Load(bpf)
}

// InstallBlacklist makes the necessary system calls to install the Seccomp-BPF
// filter for the current process (all threads). Install can be called
// multiple times to install additional filters.
func InstallBlacklist(bpf []unix.SockFilter) error {
	return Install(bpf)
}
<|endoftext|>


<commit_before>package lytics

import (
	"fmt"
	"net/url"
	"strconv"
	"strings"
	"time"
)

const (
	segmentEndpoint               = "segment/:id"
	segmentListEndpoint           = "segment"
	segmentSizeEndpoint           = "segment/:id/sizes"
	segmentSizesEndpoint          = "segment/sizes"       // ids
	segmentAttributionEndpoint    = "segment/attribution" // ids
	segmentScanEndpoint           = "segment/:id/scan"
	adHocsegmentScanEndpoint      = "segment/scan"
	segmentCollectionListEndpoint = "segmentcollection"
	segmentCollectionEndpoint     = "segmentcollection/:id"
	segmentCreateEndpoint         = segmentListEndpoint
	segmentValidateEndpoint       = "segment/validate"
)

type (
	// Segment is a logical expression to filter entity
	//  The normal concept is logical filter to find users, but the
	//  table also allows logical filter on content, or other entity types.
	Segment struct {
		Id            string    `json:"id"`
		AccountId     string    `json:"account_id"`
		Name          string    `json:"name"`
		IsPublic      bool      `json:"is_public"`
		SlugName      string    `json:"slug_name"`
		Description   string    `json:"description,omitempty"`
		SegKind       string    `json:"kind,omitempty"`
		Table         string    `json:"table,omitempty"`
		AuthorId      string    `json:"author_id"`
		Updated       time.Time `json:"updated"`
		Created       time.Time `json:"created"`
		Tags          []string  `json:"tags"`
		Category      string    `json:category,omitempty`
		Invalid       bool      `json:"invalid"`
		InvalidReason string    `json:"invalid_reason"`
		FilterQL      string    `json:"segment_ql,omitempty"`
		AST           *Expr     `json:"ast,omitempty"`
	}
	// SegmentSize request is just name, slug, id, size
	// - also filters out any non Kind="Segment" segments
	SegmentSize struct {
		Id       string  `json:"id"`
		Name     string  `json:"name"`
		SlugName string  `json:"slug_name"`
		Size     float64 `json:"size"`
	}
	// SegmentAttribution is segment size history
	SegmentAttribution struct {
		Id      string                      `json:"id"`
		Metrics []SegmentAttributionMetrics `json:"metrics"`
	}
	// Specific metric point for a segment at a given time
	SegmentAttributionMetrics struct {
		Value   int64   `json:"value"`
		Ts      string  `json:"ts"`
		Anomaly float64 `json:"anomaly"`
	}
	// SegmentCollection is a set of Segments logically grouped
	// and containing relations (ordering)
	SegmentCollection struct {
		AccountId     string            `json:"account_id"`
		Id            string            `json:"id"`
		Name          string            `json:"name"`
		Slug          string            `json:"slug_name"`
		Description   string            `json:"description,omitempty"`
		Table         string            `json:"table,omitempty"`
		AuthorId      string            `json:"author_id"`
		Updated       time.Time         `json:"updated"`
		Created       time.Time         `json:"created"`
		Internal      bool              `json:"internal"`
		Collection    []*SegColRelation `json:"collection""`
		ParentSegment string            `json:"parent_segment"`
	}
	// SegColRelation maps a segment relationship to a collection
	SegColRelation struct {
		Id    string `json:"id"`
		Order int    `json:"order"`
	}
	// SegmentScanner is a stateful, forward only pager to iterate through
	// entities in a Segment
	SegmentScanner struct {
		SegmentID string
		SegmentQl string
		next      string
		previous  string
		buffer    chan []Entity
		nextChan  chan Entity
		shutdown  chan bool
		Total     int
		Batches   []int
		err       error
	}

	// Expr is the AST structures of a SegmentQL statement
	Expr struct {
		// The token, and node expressions are non
		// nil if it is an expression
		Op   string  `json:"op,omitempty"`
		Args []*Expr `json:"args,omitempty"`

		// If op is 0, and args nil then exactly one of these should be set
		Identity string `json:"ident,omitempty"`
		Value    string `json:"val,omitempty"`
	}
)

func (s *SegmentScanner) Stop() {
	defer func() { recover() }()
	close(s.shutdown)
}

func (s *SegmentScanner) Err() error {
	return s.err
}

func (s *SegmentScanner) Next() Entity {
	select {
	case e, ok := <-s.nextChan:
		if !ok {
			return nil
		}
		return e
	case <-s.shutdown:
		return nil
	}
}

// Created is a helper method to convert the timestamp into human readable format for metrics
func (s *SegmentAttributionMetrics) Created() (time.Time, error) {
	return parseLyticsTime(s.Ts)
}

// PostSegment creates a Segment
// https://www.getlytics.com/developers/rest-api#segment
func (l *Client) PostSegment(segmentQL string) (Segment, error) {
	res := ApiResp{}
	data := Segment{}

	// make the request
	err := l.PostType("text/plain", "segment", nil, segmentQL, &res, &data)

	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegment returns the details for a single segment based on id
// https://www.getlytics.com/developers/rest-api#segment
func (l *Client) GetSegment(id string) (Segment, error) {
	res := ApiResp{}
	data := Segment{}

	// make the request
	err := l.Get(parseLyticsURL(segmentEndpoint, map[string]string{"id": id}), nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegments returns a list of all segments for an account
// https://www.getlytics.com/developers/rest-api#segment-list
func (l *Client) GetSegments(table string) ([]Segment, error) {
	res := ApiResp{}
	data := []Segment{}
	params := url.Values{}

	params.Add("table", table)

	// make the request
	err := l.Get(segmentListEndpoint, params, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentSize returns the segment size information for a single segment
// https://www.getlytics.com/developers/rest-api#segment-sizes
func (l *Client) GetSegmentSize(id string) (SegmentSize, error) {
	res := ApiResp{}
	data := SegmentSize{}

	// make the request
	err := l.Get(parseLyticsURL(segmentSizeEndpoint, map[string]string{"id": id}), nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentSizes returns the segment sizes for all segments on an account
// https://www.getlytics.com/developers/rest-api#segment-sizes
func (l *Client) GetSegmentSizes(segments []string) ([]SegmentSize, error) {
	params := url.Values{}
	res := ApiResp{}
	data := []SegmentSize{}

	// if we have specific segments to filter by add those to the params as comma separated string
	if len(segments) > 0 {
		params.Add("ids", strings.Join(segments, ","))
	}

	// make the request
	err := l.Get(segmentSizesEndpoint, params, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentAttribution returns the attribution (change over time) for segments
// method accepts a string slice of 1 or more segments to query.
// NOT CURRENTLY DOCUMENTED
func (l *Client) GetSegmentAttribution(segments []string) ([]SegmentAttribution, error) {
	params := url.Values{}

	res := ApiResp{}
	data := []SegmentAttribution{}

	// if the request is for a specific set of segments add that as comma separated param
	if len(segments) > 0 {
		params.Add("ids", strings.Join(segments, ","))
	}

	// make the request
	err := l.Get(parseLyticsURL(segmentAttributionEndpoint, nil), params, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentCollection returns a single collection of segments
// (a grouped/named lists of segments)
func (l *Client) GetSegmentCollection(id string) (SegmentCollection, error) {
	res := ApiResp{}
	data := SegmentCollection{}

	err := l.Get(parseLyticsURL(segmentCollectionEndpoint, map[string]string{"id": id}), nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentCollectionList returns a list of all segment
// collections for an account
func (l *Client) GetSegmentCollectionList() ([]SegmentCollection, error) {
	res := ApiResp{}
	data := []SegmentCollection{}

	err := l.Get(parseLyticsURL(segmentCollectionListEndpoint, nil), nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// Other Available Endpoints
// * DELETE  remove segment

// **************************** START OF SEGMENT SCAN METHODS ****************************

// GetSegmentEntities returns a single page of entities for the given segment
// also returns the next value if there are more than limit entities in the segment
// https://www.getlytics.com/developers/rest-api#segment-scan
func (l *Client) GetSegmentEntities(segment, next string, limit int) (interface{}, string, []Entity, error) {
	res := ApiResp{}
	data := []Entity{}
	params := url.Values{}

	params.Add("start", next)
	params.Add("limit", strconv.Itoa(limit))

	// make the request
	err := l.Get(parseLyticsURL(segmentScanEndpoint, map[string]string{"id": segment}), params, nil, &res, &data)
	if err != nil {
		return "", "", data, err
	}

	return res.Status, res.Next, data, nil
}

// GetAdHocSegmentEntities returns a single page of entities for the given Ad Hoc segment
// also returns the next value if there are more than limit entities in the segment
// https://www.getlytics.com/developers/rest-api#segment-scan
func (l *Client) GetAdHocSegmentEntities(ql, next string, limit int) (interface{}, string, []Entity, error) {

	res := ApiResp{}
	data := []Entity{}
	params := url.Values{}

	params.Add("start", next)
	params.Add("limit", strconv.Itoa(limit))

	err := l.Get(adHocsegmentScanEndpoint, params, ql, &res, &data)
	if err != nil {
		return "", "", data, err
	}

	return res.Status, res.Next, data, nil
}

func (s *SegmentScanner) run(c *Client) {
	var (
		entities []Entity
		fails    int
		maxTries int
		err      error
	)

	maxTries = 10

	go func() {
		defer func() { recover() }()
		// This drains the Buffer into output channel
		for {
			select {
			case <-s.shutdown:
				// we are shutdown
				return
			case el, ok := <-s.buffer:
				if !ok {
					// we have closed buffer, we are done
					close(s.shutdown)
					return
				}
				for _, e := range el {
					s.nextChan <- e
				}
			}
		}
	}()
	// make calls for next batch of segment entities until we run out of next pages
	for {

		select {
		case <-s.shutdown:
			// we are shutdown
			return
		default:
			// keep paging
		}
		switch {
		case s.SegmentQl != "":
			_, s.next, entities, err = c.GetAdHocSegmentEntities(s.SegmentQl, s.next, 100)
		case s.SegmentID != "":
			_, s.next, entities, err = c.GetSegmentEntities(s.SegmentID, s.next, 100)
		default:
			s.err = fmt.Errorf("Must have segment id or segmentql")
			return
		}

		if err != nil {
			fails++

			if fails > maxTries {
				s.err = err
				return
			}

			// if we fail and have not exceeded the limit, try again
			continue
		} else {
			maxTries = 10
		}

		// for logging add the batch details to the scanner
		s.Batches = append(s.Batches, len(entities))

		// for logging add the total entites returned to the scanner
		s.Total = s.Total + len(entities)

		// if buffer is full we will block here
		// thus preving getting too far ahead of consumption
		s.buffer <- entities

		// if there are no more pages we will have a blank next, just break and return
		if s.next == "" {
			close(s.buffer)
			break
		}
	}
}

// PageSegment Pages by either SegmentId or QL
func (l *Client) PageSegment(qlOrId string) *SegmentScanner {
	return l.pageSegment(qlOrId)
}

// PageSegmentId pages through each user in segment
func (l *Client) PageSegmentId(segmentid string) *SegmentScanner {
	return l.pageSegment(segmentid)
}
func (l *Client) pageSegment(qlOrId string) *SegmentScanner {

	segmentId, ql := "", ""

	if strings.Contains(qlOrId, " ") {
		// segmentql
		ql = qlOrId
	} else {
		segmentId = qlOrId
	}

	scanner := &SegmentScanner{
		buffer:    make(chan []Entity, 1),
		nextChan:  make(chan Entity, 1),
		shutdown:  make(chan bool),
		SegmentID: segmentId,
		SegmentQl: ql,
	}

	// fire up the go routine for paging entities
	go scanner.run(l)
	return scanner
}

// PageAdHocSegment sets the ad-hoc segment ql on the master scanner and initiates
// the main go routine for paging
func (l *Client) PageAdHocSegment(ql string) *SegmentScanner {
	return l.pageSegment(ql)
}

// CreateSegment creates a new segment from a Segment QL logic expression
// https://www.getlytics.com/developers/rest-api#segment
func (l *Client) CreateSegment(name, ql, slug string) (Segment, error) {
	res := ApiResp{}
	data := Segment{}

	payload := Segment{
		Name:     name,
		FilterQL: ql,
		SlugName: slug,
	}

	// make the request
	err := l.Post(segmentCreateEndpoint, nil, payload, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// ValidateSegment validates a single segment QL statement
// https://www.getlytics.com/developers/rest-api#segment-validate
func (l *Client) ValidateSegment(ql string) (bool, error) {
	res := ApiResp{}

	err := l.Post(segmentValidateEndpoint, nil, ql, &res, nil)
	if err != nil {
		return false, err
	}

	if res.Message == "success" {
		return true, nil
	}

	return false, nil
}
<commit_msg>Fix segment scan from dropping entities.<commit_after>package lytics

import (
	"fmt"
	"net/url"
	"strconv"
	"strings"
	"time"
)

const (
	segmentEndpoint               = "segment/:id"
	segmentListEndpoint           = "segment"
	segmentSizeEndpoint           = "segment/:id/sizes"
	segmentSizesEndpoint          = "segment/sizes"       // ids
	segmentAttributionEndpoint    = "segment/attribution" // ids
	segmentScanEndpoint           = "segment/:id/scan"
	adHocsegmentScanEndpoint      = "segment/scan"
	segmentCollectionListEndpoint = "segmentcollection"
	segmentCollectionEndpoint     = "segmentcollection/:id"
	segmentCreateEndpoint         = segmentListEndpoint
	segmentValidateEndpoint       = "segment/validate"
)

type (
	// Segment is a logical expression to filter entity
	//  The normal concept is logical filter to find users, but the
	//  table also allows logical filter on content, or other entity types.
	Segment struct {
		Id            string    `json:"id"`
		AccountId     string    `json:"account_id"`
		Name          string    `json:"name"`
		IsPublic      bool      `json:"is_public"`
		SlugName      string    `json:"slug_name"`
		Description   string    `json:"description,omitempty"`
		SegKind       string    `json:"kind,omitempty"`
		Table         string    `json:"table,omitempty"`
		AuthorId      string    `json:"author_id"`
		Updated       time.Time `json:"updated"`
		Created       time.Time `json:"created"`
		Tags          []string  `json:"tags"`
		Category      string    `json:category,omitempty`
		Invalid       bool      `json:"invalid"`
		InvalidReason string    `json:"invalid_reason"`
		FilterQL      string    `json:"segment_ql,omitempty"`
		AST           *Expr     `json:"ast,omitempty"`
	}
	// SegmentSize request is just name, slug, id, size
	// - also filters out any non Kind="Segment" segments
	SegmentSize struct {
		Id       string  `json:"id"`
		Name     string  `json:"name"`
		SlugName string  `json:"slug_name"`
		Size     float64 `json:"size"`
	}
	// SegmentAttribution is segment size history
	SegmentAttribution struct {
		Id      string                      `json:"id"`
		Metrics []SegmentAttributionMetrics `json:"metrics"`
	}
	// Specific metric point for a segment at a given time
	SegmentAttributionMetrics struct {
		Value   int64   `json:"value"`
		Ts      string  `json:"ts"`
		Anomaly float64 `json:"anomaly"`
	}
	// SegmentCollection is a set of Segments logically grouped
	// and containing relations (ordering)
	SegmentCollection struct {
		AccountId     string            `json:"account_id"`
		Id            string            `json:"id"`
		Name          string            `json:"name"`
		Slug          string            `json:"slug_name"`
		Description   string            `json:"description,omitempty"`
		Table         string            `json:"table,omitempty"`
		AuthorId      string            `json:"author_id"`
		Updated       time.Time         `json:"updated"`
		Created       time.Time         `json:"created"`
		Internal      bool              `json:"internal"`
		Collection    []*SegColRelation `json:"collection""`
		ParentSegment string            `json:"parent_segment"`
	}
	// SegColRelation maps a segment relationship to a collection
	SegColRelation struct {
		Id    string `json:"id"`
		Order int    `json:"order"`
	}
	// SegmentScanner is a stateful, forward only pager to iterate through
	// entities in a Segment
	SegmentScanner struct {
		SegmentID string
		SegmentQl string
		next      string
		previous  string
		buffer    chan []Entity
		nextChan  chan Entity
		shutdown  chan bool
		Total     int
		Batches   []int
		err       error
	}

	// Expr is the AST structures of a SegmentQL statement
	Expr struct {
		// The token, and node expressions are non
		// nil if it is an expression
		Op   string  `json:"op,omitempty"`
		Args []*Expr `json:"args,omitempty"`

		// If op is 0, and args nil then exactly one of these should be set
		Identity string `json:"ident,omitempty"`
		Value    string `json:"val,omitempty"`
	}
)

func (s *SegmentScanner) Stop() {
	defer func() { recover() }()
	close(s.shutdown)
}

func (s *SegmentScanner) Err() error {
	return s.err
}

func (s *SegmentScanner) Next() Entity {
	select {
	case e, ok := <-s.nextChan:
		if !ok {
			return nil
		}
		return e
	case <-s.shutdown:
		return nil
	}
}

// Created is a helper method to convert the timestamp into human readable format for metrics
func (s *SegmentAttributionMetrics) Created() (time.Time, error) {
	return parseLyticsTime(s.Ts)
}

// PostSegment creates a Segment
// https://www.getlytics.com/developers/rest-api#segment
func (l *Client) PostSegment(segmentQL string) (Segment, error) {
	res := ApiResp{}
	data := Segment{}

	// make the request
	err := l.PostType("text/plain", "segment", nil, segmentQL, &res, &data)

	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegment returns the details for a single segment based on id
// https://www.getlytics.com/developers/rest-api#segment
func (l *Client) GetSegment(id string) (Segment, error) {
	res := ApiResp{}
	data := Segment{}

	// make the request
	err := l.Get(parseLyticsURL(segmentEndpoint, map[string]string{"id": id}), nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegments returns a list of all segments for an account
// https://www.getlytics.com/developers/rest-api#segment-list
func (l *Client) GetSegments(table string) ([]Segment, error) {
	res := ApiResp{}
	data := []Segment{}
	params := url.Values{}

	params.Add("table", table)

	// make the request
	err := l.Get(segmentListEndpoint, params, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentSize returns the segment size information for a single segment
// https://www.getlytics.com/developers/rest-api#segment-sizes
func (l *Client) GetSegmentSize(id string) (SegmentSize, error) {
	res := ApiResp{}
	data := SegmentSize{}

	// make the request
	err := l.Get(parseLyticsURL(segmentSizeEndpoint, map[string]string{"id": id}), nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentSizes returns the segment sizes for all segments on an account
// https://www.getlytics.com/developers/rest-api#segment-sizes
func (l *Client) GetSegmentSizes(segments []string) ([]SegmentSize, error) {
	params := url.Values{}
	res := ApiResp{}
	data := []SegmentSize{}

	// if we have specific segments to filter by add those to the params as comma separated string
	if len(segments) > 0 {
		params.Add("ids", strings.Join(segments, ","))
	}

	// make the request
	err := l.Get(segmentSizesEndpoint, params, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentAttribution returns the attribution (change over time) for segments
// method accepts a string slice of 1 or more segments to query.
// NOT CURRENTLY DOCUMENTED
func (l *Client) GetSegmentAttribution(segments []string) ([]SegmentAttribution, error) {
	params := url.Values{}

	res := ApiResp{}
	data := []SegmentAttribution{}

	// if the request is for a specific set of segments add that as comma separated param
	if len(segments) > 0 {
		params.Add("ids", strings.Join(segments, ","))
	}

	// make the request
	err := l.Get(parseLyticsURL(segmentAttributionEndpoint, nil), params, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentCollection returns a single collection of segments
// (a grouped/named lists of segments)
func (l *Client) GetSegmentCollection(id string) (SegmentCollection, error) {
	res := ApiResp{}
	data := SegmentCollection{}

	err := l.Get(parseLyticsURL(segmentCollectionEndpoint, map[string]string{"id": id}), nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentCollectionList returns a list of all segment
// collections for an account
func (l *Client) GetSegmentCollectionList() ([]SegmentCollection, error) {
	res := ApiResp{}
	data := []SegmentCollection{}

	err := l.Get(parseLyticsURL(segmentCollectionListEndpoint, nil), nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// Other Available Endpoints
// * DELETE  remove segment

// **************************** START OF SEGMENT SCAN METHODS ****************************

// GetSegmentEntities returns a single page of entities for the given segment
// also returns the next value if there are more than limit entities in the segment
// https://www.getlytics.com/developers/rest-api#segment-scan
func (l *Client) GetSegmentEntities(segment, next string, limit int) (interface{}, string, []Entity, error) {
	res := ApiResp{}
	data := []Entity{}
	params := url.Values{}

	params.Add("start", next)
	params.Add("limit", strconv.Itoa(limit))

	// make the request
	err := l.Get(parseLyticsURL(segmentScanEndpoint, map[string]string{"id": segment}), params, nil, &res, &data)
	if err != nil {
		return "", "", data, err
	}

	return res.Status, res.Next, data, nil
}

// GetAdHocSegmentEntities returns a single page of entities for the given Ad Hoc segment
// also returns the next value if there are more than limit entities in the segment
// https://www.getlytics.com/developers/rest-api#segment-scan
func (l *Client) GetAdHocSegmentEntities(ql, next string, limit int) (interface{}, string, []Entity, error) {

	res := ApiResp{}
	data := []Entity{}
	params := url.Values{}

	params.Add("start", next)
	params.Add("limit", strconv.Itoa(limit))

	err := l.Get(adHocsegmentScanEndpoint, params, ql, &res, &data)
	if err != nil {
		return "", "", data, err
	}

	return res.Status, res.Next, data, nil
}

func (s *SegmentScanner) run(c *Client) {
	var (
		entities []Entity
		fails    int
		maxTries int
		err      error
	)

	maxTries = 10

	go func() {
		defer func() { recover() }()
		// This drains the Buffer into output channel
		for {
			select {
			case <-s.shutdown:
				// we are shutdown
				return
			case el, ok := <-s.buffer:
				if ok {
					for _, e := range el {
						s.nextChan <- e
					}
				} else {
					// buffer is closed, no more entities
					close(s.nextChan)
				}
			}
		}
	}()
	// make calls for next batch of segment entities until we run out of next pages
	for {

		select {
		case <-s.shutdown:
			// we are shutdown
			return
		default:
			// keep paging
		}
		switch {
		case s.SegmentQl != "":
			_, s.next, entities, err = c.GetAdHocSegmentEntities(s.SegmentQl, s.next, 100)
		case s.SegmentID != "":
			_, s.next, entities, err = c.GetSegmentEntities(s.SegmentID, s.next, 100)
		default:
			s.err = fmt.Errorf("Must have segment id or segmentql")
			return
		}

		if err != nil {
			fails++

			if fails > maxTries {
				s.err = err
				return
			}

			// if we fail and have not exceeded the limit, try again
			continue
		} else {
			maxTries = 10
		}

		// for logging add the batch details to the scanner
		s.Batches = append(s.Batches, len(entities))

		// for logging add the total entites returned to the scanner
		s.Total = s.Total + len(entities)

		// if buffer is full we will block here
		// thus preving getting too far ahead of consumption
		s.buffer <- entities

		// if there are no more pages we will have a blank next, just break and return
		if s.next == "" {
			close(s.buffer)
			break
		}
	}
}

// PageSegment Pages by either SegmentId or QL
func (l *Client) PageSegment(qlOrId string) *SegmentScanner {
	return l.pageSegment(qlOrId)
}

// PageSegmentId pages through each user in segment
func (l *Client) PageSegmentId(segmentid string) *SegmentScanner {
	return l.pageSegment(segmentid)
}
func (l *Client) pageSegment(qlOrId string) *SegmentScanner {

	segmentId, ql := "", ""

	if strings.Contains(qlOrId, " ") {
		// segmentql
		ql = qlOrId
	} else {
		segmentId = qlOrId
	}

	scanner := &SegmentScanner{
		buffer:    make(chan []Entity, 1),
		nextChan:  make(chan Entity, 1),
		shutdown:  make(chan bool),
		SegmentID: segmentId,
		SegmentQl: ql,
	}

	// fire up the go routine for paging entities
	go scanner.run(l)
	return scanner
}

// PageAdHocSegment sets the ad-hoc segment ql on the master scanner and initiates
// the main go routine for paging
func (l *Client) PageAdHocSegment(ql string) *SegmentScanner {
	return l.pageSegment(ql)
}

// CreateSegment creates a new segment from a Segment QL logic expression
// https://www.getlytics.com/developers/rest-api#segment
func (l *Client) CreateSegment(name, ql, slug string) (Segment, error) {
	res := ApiResp{}
	data := Segment{}

	payload := Segment{
		Name:     name,
		FilterQL: ql,
		SlugName: slug,
	}

	// make the request
	err := l.Post(segmentCreateEndpoint, nil, payload, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// ValidateSegment validates a single segment QL statement
// https://www.getlytics.com/developers/rest-api#segment-validate
func (l *Client) ValidateSegment(ql string) (bool, error) {
	res := ApiResp{}

	err := l.Post(segmentValidateEndpoint, nil, ql, &res, nil)
	if err != nil {
		return false, err
	}

	if res.Message == "success" {
		return true, nil
	}

	return false, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/tls"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"sync"

	"golang.org/x/net/html"
)

// Results -- struct returned by Crawl() to represent the entire crawl process
type Results struct {
	// Inherit the Resource struct
	Resource

	// Body represents a string implementation of the byte array returned by
	// http.Response
	Body string

	// Slice of Resource structs containing the needed resources for the given URL
	Resources []*Resource

	// ResourceTime shows how long it took to fetch all resources
	ResourceTime *TimerResult

	// TotalTime represents the time it took to crawl the site
	TotalTime *TimerResult
}

// Resource represents a single entity of many within a given crawl. These should
// only be of type css, js, jpg, png, etc (static resources).
type Resource struct {
	// connURL is the initial URL received by input
	connURL string

	// connIP is the initial IP address received by input
	connIP string

	// connHostname represents the original requested hostname for the resource
	connHostname string

	// URL represents the resulting static URL derived by the original result page
	URL string

	// Hostname represents the resulting hostname derived by the original returned
	// resource
	Hostname string

	// Remote represents if the resulting resource is remote to the original domain
	Remote bool

	// Error represents any errors that may have occurred when fetching the resource
	Error error

	// Code represents the numeric HTTP based status code
	Code int

	// Proto represents the end protocol used to fetch the page. For example, HTTP/2.0
	Proto string

	// Scheme represents the end scheme used to fetch the page. For example, https
	Scheme string

	// ContentLength represents the number of bytes in the body of the response
	ContentLength int64

	// TLS represents the SSL/TLS handshake/session if the resource was loaded over
	// SSL.
	TLS *tls.ConnectionState

	// Time represents the time it took to complete the request
	Time *TimerResult
}

var resourcePool sync.WaitGroup

// getAttr pulls a specific attribute from a token/element
func getAttr(attr string, attrs []html.Attribute) (val string) {
	for _, item := range attrs {
		if item.Key == attr {
			val = item.Val
			break
		}
	}

	return
}

// getSrc crawls the body of the Results page, yielding all img/script/link resources
// so they can later be fetched.
func getSrc(b io.ReadCloser, req *http.Request) (urls []string) {
	urls = []string{}

	z := html.NewTokenizer(b)

	for {
		// loop through all tokens in the html body response
		tt := z.Next()

		switch {
		case tt == html.ErrorToken:
			// this assumes that there are no further tokens -- end of document
			return
		case tt == html.StartTagToken || tt == html.SelfClosingTagToken:
			t := z.Token()

			var src string

			switch t.Data {
			case "link":
				src = getAttr("href", t.Attr)

				rel := getAttr("rel", t.Attr)

				if len(rel) > 0 && strings.ToLower(rel) != "stylesheet" && strings.ToLower(rel) != "shortcut icon" {
					continue
				}

			case "script":
				src = getAttr("src", t.Attr)

			case "img":
				src = getAttr("src", t.Attr)

			default:
				continue
			}

			if len(src) == 0 {
				continue
			}

			// this assumes that the resource is something along the lines of:
			//   http://something.com/ -- which we don't care about
			if len(src) == 0 || strings.HasSuffix(src, "/") {
				continue
			}

			// add trailing slash to the end of the path
			if len(req.URL.Path) == 0 {
				req.URL.Path = "/"
			}

			// site was developed using relative paths. E.g:
			//  - url: http://domain.com/sub/path and resource: ./something/main.js
			//    would equal http://domain.com/sub/path/something/main.js
			if strings.HasPrefix(src, "./") {
				src = req.URL.Scheme + "://" + req.URL.Host + req.URL.Path + strings.SplitN(src, "./", 2)[1]
			}

			// site is loading resources from a remote location that supports both
			// http and https. browsers should natively tack on the current sites
			// protocol to the url. E.g:
			//  - url: http://domain.com/ and resource: //other.com/some-resource.js
			//    generates: http://other.com/some-resource.js
			//  - url: https://domain.com/ and resource: //other.com/some-resource.js
			//    generates: https://other.com/some-resource.js
			if strings.HasPrefix(src, "//") {
				src = req.URL.Scheme + ":" + src
			}

			// non-host-absolute resource. E.g. resource is loaded based on the docroot
			// of the domain. E.g:
			//  - url: http://domain.com/ and resource: /some-resource.js
			//    generates: http://domain.com/some-resource.js
			//  - url: https://domain.com/sub/resource and resource: /some-resource.js
			//    generates: https://domain.com/some-resource.js
			if strings.HasPrefix(src, "/") {
				src = req.URL.Scheme + "://" + req.URL.Host + src
			}

			// ignore anything else that isn't http based. E.g. ftp://, and other svg-like
			// data urls, as we really can't fetch those.
			if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
				continue
			}

			urls = append(urls, src)
		}
	}
}

func connHostname(URL string) (host string, err error) {
	tmp, err := url.Parse(URL)

	if err != nil {
		return
	}

	host = tmp.Host
	return
}

// FetchResource fetches a singular resource from a page, returning a *Resource struct.
// As we don't care much about the body of the resource, that can safely be ignored. We
// must still close the body object, however.
func (rsrc *Resource) FetchResource() {
	var err error

	defer resourcePool.Done()

	// calculate the time it takes to fetch the request
	timer := NewTimer()
	resp, err := Get(rsrc.connURL, rsrc.connIP)
	rsrc.Time = timer.End()
	resp.Body.Close()

	if err != nil {
		rsrc.Error = err
		return
	}

	rsrc.connHostname, err = connHostname(rsrc.connURL)
	if err != nil {
		rsrc.Error = err
		return
	}

	rsrc.Hostname = resp.Request.Host
	rsrc.URL = resp.Request.URL.String()
	rsrc.Code = resp.StatusCode
	rsrc.Proto = resp.Proto
	rsrc.Scheme = resp.Request.URL.Scheme
	rsrc.ContentLength = resp.ContentLength
	rsrc.TLS = resp.TLS

	if rsrc.Hostname != rsrc.connHostname {
		rsrc.Remote = true
	}

	fmt.Printf("[%d] [%s] %s\n", rsrc.Code, rsrc.Proto, rsrc.URL)

	return
}

// Crawl manages the fetching of the main resource, as well as all child resources,
// providing a Results struct containing the entire crawl data needed
func Crawl(URL string, IP string) (res *Results) {
	res = &Results{}

	crawlTimer := NewTimer()
	reqTimer := NewTimer()

	// actually fetch the request
	resp, err := Get(URL, IP)

	res.Time = reqTimer.End()

	if err != nil {
		res.Error = err
		return
	}

	defer resp.Body.Close()

	res.connHostname, err = connHostname(URL)
	if err != nil {
		res.Error = err
		return
	}

	res.connURL = URL
	res.connIP = IP
	res.Hostname = resp.Request.Host
	res.URL = resp.Request.URL.String()
	res.Code = resp.StatusCode
	res.Proto = resp.Proto
	res.Scheme = resp.Request.URL.Scheme
	res.ContentLength = resp.ContentLength
	res.TLS = resp.TLS

	if res.Hostname != res.connHostname {
		res.Remote = true
	}

	buf, _ := ioutil.ReadAll(resp.Body)
	b := ioutil.NopCloser(bytes.NewReader(buf))
	defer b.Close()

	bbytes, err := ioutil.ReadAll(bytes.NewBuffer(buf))
	if len(bbytes) != 0 {
		res.Body = string(bbytes[:])
	}

	urls := getSrc(b, resp.Request)

	fmt.Printf("[%d] [%s] %s\n", res.Code, res.Proto, res.URL)

	resourceTime := NewTimer()

	for i := range urls {
		resourcePool.Add(1)

		rsrc := &Resource{connURL: urls[i], connIP: ""}
		res.Resources = append(res.Resources, rsrc)
		go res.Resources[i].FetchResource()
	}

	resourcePool.Wait()

	res.ResourceTime = resourceTime.End()
	res.TotalTime = crawlTimer.End()

	return
}
<commit_msg>take adv. of customer http resp. wrapper<commit_after>package main

import (
	"bytes"
	"crypto/tls"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"sync"

	"golang.org/x/net/html"
)

// Results -- struct returned by Crawl() to represent the entire crawl process
type Results struct {
	// Inherit the Resource struct
	Resource

	// Body represents a string implementation of the byte array returned by
	// http.Response
	Body string

	// Slice of Resource structs containing the needed resources for the given URL
	Resources []*Resource

	// ResourceTime shows how long it took to fetch all resources
	ResourceTime *TimerResult

	// TotalTime represents the time it took to crawl the site
	TotalTime *TimerResult
}

// Resource represents a single entity of many within a given crawl. These should
// only be of type css, js, jpg, png, etc (static resources).
type Resource struct {
	// connURL is the initial URL received by input
	connURL string

	// connIP is the initial IP address received by input
	connIP string

	// connHostname represents the original requested hostname for the resource
	connHostname string

	// URL represents the resulting static URL derived by the original result page
	URL string

	// Hostname represents the resulting hostname derived by the original returned
	// resource
	Hostname string

	// Remote represents if the resulting resource is remote to the original domain
	Remote bool

	// Error represents any errors that may have occurred when fetching the resource
	Error error

	// Code represents the numeric HTTP based status code
	Code int

	// Proto represents the end protocol used to fetch the page. For example, HTTP/2.0
	Proto string

	// Scheme represents the end scheme used to fetch the page. For example, https
	Scheme string

	// ContentLength represents the number of bytes in the body of the response
	ContentLength int64

	// TLS represents the SSL/TLS handshake/session if the resource was loaded over
	// SSL.
	TLS *tls.ConnectionState

	// Time represents the time it took to complete the request
	Time *TimerResult
}

var resourcePool sync.WaitGroup

// getAttr pulls a specific attribute from a token/element
func getAttr(attr string, attrs []html.Attribute) (val string) {
	for _, item := range attrs {
		if item.Key == attr {
			val = item.Val
			break
		}
	}

	return
}

// getSrc crawls the body of the Results page, yielding all img/script/link resources
// so they can later be fetched.
func getSrc(b io.ReadCloser, req *http.Request) (urls []string) {
	urls = []string{}

	z := html.NewTokenizer(b)

	for {
		// loop through all tokens in the html body response
		tt := z.Next()

		switch {
		case tt == html.ErrorToken:
			// this assumes that there are no further tokens -- end of document
			return
		case tt == html.StartTagToken || tt == html.SelfClosingTagToken:
			t := z.Token()

			var src string

			switch t.Data {
			case "link":
				src = getAttr("href", t.Attr)

				rel := getAttr("rel", t.Attr)

				if len(rel) > 0 && strings.ToLower(rel) != "stylesheet" && strings.ToLower(rel) != "shortcut icon" {
					continue
				}

			case "script":
				src = getAttr("src", t.Attr)

			case "img":
				src = getAttr("src", t.Attr)

			default:
				continue
			}

			if len(src) == 0 {
				continue
			}

			// this assumes that the resource is something along the lines of:
			//   http://something.com/ -- which we don't care about
			if len(src) == 0 || strings.HasSuffix(src, "/") {
				continue
			}

			// add trailing slash to the end of the path
			if len(req.URL.Path) == 0 {
				req.URL.Path = "/"
			}

			// site was developed using relative paths. E.g:
			//  - url: http://domain.com/sub/path and resource: ./something/main.js
			//    would equal http://domain.com/sub/path/something/main.js
			if strings.HasPrefix(src, "./") {
				src = req.URL.Scheme + "://" + req.URL.Host + req.URL.Path + strings.SplitN(src, "./", 2)[1]
			}

			// site is loading resources from a remote location that supports both
			// http and https. browsers should natively tack on the current sites
			// protocol to the url. E.g:
			//  - url: http://domain.com/ and resource: //other.com/some-resource.js
			//    generates: http://other.com/some-resource.js
			//  - url: https://domain.com/ and resource: //other.com/some-resource.js
			//    generates: https://other.com/some-resource.js
			if strings.HasPrefix(src, "//") {
				src = req.URL.Scheme + ":" + src
			}

			// non-host-absolute resource. E.g. resource is loaded based on the docroot
			// of the domain. E.g:
			//  - url: http://domain.com/ and resource: /some-resource.js
			//    generates: http://domain.com/some-resource.js
			//  - url: https://domain.com/sub/resource and resource: /some-resource.js
			//    generates: https://domain.com/some-resource.js
			if strings.HasPrefix(src, "/") {
				src = req.URL.Scheme + "://" + req.URL.Host + src
			}

			// ignore anything else that isn't http based. E.g. ftp://, and other svg-like
			// data urls, as we really can't fetch those.
			if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
				continue
			}

			urls = append(urls, src)
		}
	}
}

func connHostname(URL string) (host string, err error) {
	tmp, err := url.Parse(URL)

	if err != nil {
		return
	}

	host = tmp.Host
	return
}

// FetchResource fetches a singular resource from a page, returning a *Resource struct.
// As we don't care much about the body of the resource, that can safely be ignored. We
// must still close the body object, however.
func (rsrc *Resource) FetchResource() {
	var err error

	defer resourcePool.Done()

	// calculate the time it takes to fetch the request
	resp, err := Get(rsrc.connURL, rsrc.connIP)
	resp.Body.Close()

	if err != nil {
		rsrc.Error = err
		return
	}

	rsrc.connHostname, err = connHostname(rsrc.connURL)
	if err != nil {
		rsrc.Error = err
		return
	}

	rsrc.Hostname = resp.Request.Host
	rsrc.URL = resp.URL
	rsrc.Code = resp.StatusCode
	rsrc.Proto = resp.Proto
	rsrc.Scheme = resp.Request.URL.Scheme
	rsrc.ContentLength = resp.ContentLength
	rsrc.TLS = resp.TLS
	rsrc.Time = resp.Time

	if rsrc.Hostname != rsrc.connHostname {
		rsrc.Remote = true
	}

	fmt.Printf("[%d] [%s] %s\n", rsrc.Code, rsrc.Proto, rsrc.URL)

	return
}

// Crawl manages the fetching of the main resource, as well as all child resources,
// providing a Results struct containing the entire crawl data needed
func Crawl(URL string, IP string) (res *Results) {
	res = &Results{}

	crawlTimer := NewTimer()

	// actually fetch the request
	resp, err := Get(URL, IP)

	if err != nil {
		res.Error = err
		return
	}

	defer resp.Body.Close()

	res.connHostname, err = connHostname(URL)
	if err != nil {
		res.Error = err
		return
	}

	res.connURL = URL
	res.connIP = IP
	res.Hostname = resp.Request.Host
	res.URL = resp.URL
	res.Code = resp.StatusCode
	res.Proto = resp.Proto
	res.Scheme = resp.Request.URL.Scheme
	res.ContentLength = resp.ContentLength
	res.TLS = resp.TLS
	res.Time = resp.Time

	if res.Hostname != res.connHostname {
		res.Remote = true
	}

	buf, _ := ioutil.ReadAll(resp.Body)
	b := ioutil.NopCloser(bytes.NewReader(buf))
	defer b.Close()

	bbytes, err := ioutil.ReadAll(bytes.NewBuffer(buf))
	if len(bbytes) != 0 {
		res.Body = string(bbytes[:])
	}

	urls := getSrc(b, resp.Request)

	fmt.Printf("[%d] [%s] %s\n", res.Code, res.Proto, res.URL)

	resourceTime := NewTimer()

	for i := range urls {
		resourcePool.Add(1)

		rsrc := &Resource{connURL: urls[i], connIP: ""}
		res.Resources = append(res.Resources, rsrc)
		go res.Resources[i].FetchResource()
	}

	resourcePool.Wait()

	res.ResourceTime = resourceTime.End()
	res.TotalTime = crawlTimer.End()

	return
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path"
	"strings"

	"github.com/howeyc/gopass"
	"github.com/urfave/cli"
	"gopkg.in/yaml.v2"

	"code.tobolaski.com/brendan/ejrnl"
	"code.tobolaski.com/brendan/ejrnl/storage"
	"code.tobolaski.com/brendan/ejrnl/workflows"
)

var version = "0.0.1"

var tempFlag = cli.StringFlag{
	Name:  "temp-dir",
	Usage: "Specifies the temporary directory to write the temporary files to.",
	Value: os.TempDir(),
}

func main() {
	app := cli.NewApp()

	app.Name = "ejrnl"
	app.Usage = "An encrypted journal application"
	app.Version = version
	app.Authors = []cli.Author{cli.Author{
		Name:  "Brendan Tobolaski",
		Email: "brendan@tobolaski.com",
	}}

	var configPath string
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "config",
			Usage: "Specifies the config file to use",
			Value: "~/.config/ejrnl/ejrnl.yml",
		},
	}
	app.Before = func(c *cli.Context) error {
		user, err := user.Current()
		if err != nil {
			return err
		}
		configPath = strings.Replace(c.String("config"), "~", user.HomeDir, -1)
		return nil
	}
	app.Commands = []cli.Command{
		{
			Name:  "init",
			Usage: "Creates a new journal",
			Flags: []cli.Flag{
				cli.UintFlag{
					Name:  "pow",
					Usage: "Configures the workfactor for scrypt",
					Value: 19,
				},
				cli.StringFlag{
					Name:  "destination",
					Usage: "Configures where the journal is stored",
					Value: "~/journal",
				},
			},
			Action: func(c *cli.Context) error {
				if err := os.MkdirAll(path.Dir(configPath), 0750); err != nil {
					return err
				}
				if _, err := os.Stat(configPath); !os.IsNotExist(err) {
					return errors.New("Configuration directory already exists")
				}
				config := workflows.DefaultConfig()
				config.StorageDirectory = c.String("destination")
				config.Pow = c.Uint("pow")
				data, err := yaml.Marshal(config)
				if err != nil {
					return err
				}
				err = ioutil.WriteFile(configPath, data, 0600)
				if err != nil {
					return err
				}
				password, err := getPassword()
				if err != nil {
					return err
				}

				driver, err := storage.NewDriver(config, password)
				password = ""
				if _, ok := err.(*storage.NeedsInit); !ok {
					return err
				}
				return workflows.Init(driver)
			},
		},
		{
			Name:  "import",
			Usage: "Adds the specified file into the journal",
			Action: func(c *cli.Context) error {
				if len(c.Args()) != 1 {
					return errors.New("import requires 1 argument which is the file to import")
				}
				driver, err := standardLoad(configPath)
				if err != nil {
					return err
				}

				return workflows.Import(c.Args()[0], driver)
			},
		},
		{
			Name:  "print",
			Usage: "Prints out the most recent entries",
			Flags: []cli.Flag{
				cli.IntFlag{
					Name:  "count",
					Usage: "The number of entries to output. If it is 0 or less, all entries are output",
					Value: 0,
				},
			},
			Action: func(c *cli.Context) error {
				driver, err := standardLoad(configPath)
				if err != nil {
					return err
				}

				return workflows.Print(driver, c.Int("count"))
			},
		},
		{
			Name:  "list",
			Usage: "Lists the ids and dates of the most recent entries",
			Flags: []cli.Flag{
				cli.IntFlag{
					Name:  "count",
					Value: 0,
					Usage: "The number of results to return. If it is <= 0, it returns all of the entries",
				},
			},
			Action: func(c *cli.Context) error {
				driver, err := standardLoad(configPath)
				if err != nil {
					return err
				}

				return workflows.ListEntries(driver, c.Int("count"))
			},
		},
		{
			Name:  "new",
			Usage: "Creates a new entry",
			Flags: []cli.Flag{tempFlag},
			Action: func(c *cli.Context) error {
				driver, err := standardLoad(configPath)
				if err != nil {
					return err
				}
				return workflows.NewEntry(driver, c.String("temp-dir"))
			},
		},
		{
			Name:  "edit",
			Usage: "edits an existing journal entry. Takes an id as an argument.",
			Flags: []cli.Flag{tempFlag},
			Action: func(c *cli.Context) error {
				if len(c.Args()) != 1 {
					return errors.New("edit takes 1 argument which is an entry's id")
				}
				driver, err := standardLoad(configPath)
				if err != nil {
					return err
				}
				return workflows.EditEntry(driver, c.Args()[0], c.String("temp-dir"))
			},
		},
	}
	err := app.Run(os.Args)
	if err != nil {
		log.Printf("Failed to complete because %s", err)
		os.Exit(1)
	}
}

func getPassword() (string, error) {
	fmt.Printf("Password: ")
	raw, err := gopass.GetPasswd()
	return string(raw), err
}

func readConfig(path string) (ejrnl.Config, error) {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return ejrnl.Config{}, err
	}
	entry := &ejrnl.Config{}
	err = yaml.Unmarshal(data, entry)
	return *entry, err
}

func standardLoad(configPath string) (*storage.Driver, error) {
	config, err := readConfig(configPath)
	if err != nil {
		return &storage.Driver{}, err
	}
	password, err := getPassword()
	if err != nil {
		return &storage.Driver{}, err
	}
	return storage.NewDriver(config, password)
}
<commit_msg>Allow the prompt to vary<commit_after>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path"
	"strings"

	"github.com/howeyc/gopass"
	"github.com/urfave/cli"
	"gopkg.in/yaml.v2"

	"code.tobolaski.com/brendan/ejrnl"
	"code.tobolaski.com/brendan/ejrnl/storage"
	"code.tobolaski.com/brendan/ejrnl/workflows"
)

var version = "0.0.1"

var tempFlag = cli.StringFlag{
	Name:  "temp-dir",
	Usage: "Specifies the temporary directory to write the temporary files to.",
	Value: os.TempDir(),
}

func main() {
	app := cli.NewApp()

	app.Name = "ejrnl"
	app.Usage = "An encrypted journal application"
	app.Version = version
	app.Authors = []cli.Author{cli.Author{
		Name:  "Brendan Tobolaski",
		Email: "brendan@tobolaski.com",
	}}

	var configPath string
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "config",
			Usage: "Specifies the config file to use",
			Value: "~/.config/ejrnl/ejrnl.yml",
		},
	}
	app.Before = func(c *cli.Context) error {
		user, err := user.Current()
		if err != nil {
			return err
		}
		configPath = strings.Replace(c.String("config"), "~", user.HomeDir, -1)
		return nil
	}
	app.Commands = []cli.Command{
		{
			Name:  "init",
			Usage: "Creates a new journal",
			Flags: []cli.Flag{
				cli.UintFlag{
					Name:  "pow",
					Usage: "Configures the workfactor for scrypt",
					Value: 19,
				},
				cli.StringFlag{
					Name:  "destination",
					Usage: "Configures where the journal is stored",
					Value: "~/journal",
				},
			},
			Action: func(c *cli.Context) error {
				if err := os.MkdirAll(path.Dir(configPath), 0750); err != nil {
					return err
				}
				if _, err := os.Stat(configPath); !os.IsNotExist(err) {
					return errors.New("Configuration directory already exists")
				}
				config := workflows.DefaultConfig()
				config.StorageDirectory = c.String("destination")
				config.Pow = c.Uint("pow")
				data, err := yaml.Marshal(config)
				if err != nil {
					return err
				}
				err = ioutil.WriteFile(configPath, data, 0600)
				if err != nil {
					return err
				}
				password, err := getPassword("Password: ")
				if err != nil {
					return err
				}
				confirm, err := getPassword("Confirm:   ")
				if err != nil {
					return err
				}
				if password != confirm {
					return errors.New("Passwords didn't match")
				}
				confirm = ""

				driver, err := storage.NewDriver(config, password)
				password = ""
				if _, ok := err.(*storage.NeedsInit); !ok {
					return err
				}
				return workflows.Init(driver)
			},
		},
		{
			Name:  "import",
			Usage: "Adds the specified file into the journal",
			Action: func(c *cli.Context) error {
				if len(c.Args()) != 1 {
					return errors.New("import requires 1 argument which is the file to import")
				}
				driver, err := standardLoad(configPath)
				if err != nil {
					return err
				}

				return workflows.Import(c.Args()[0], driver)
			},
		},
		{
			Name:  "print",
			Usage: "Prints out the most recent entries",
			Flags: []cli.Flag{
				cli.IntFlag{
					Name:  "count",
					Usage: "The number of entries to output. If it is 0 or less, all entries are output",
					Value: 0,
				},
			},
			Action: func(c *cli.Context) error {
				driver, err := standardLoad(configPath)
				if err != nil {
					return err
				}

				return workflows.Print(driver, c.Int("count"))
			},
		},
		{
			Name:  "list",
			Usage: "Lists the ids and dates of the most recent entries",
			Flags: []cli.Flag{
				cli.IntFlag{
					Name:  "count",
					Value: 0,
					Usage: "The number of results to return. If it is <= 0, it returns all of the entries",
				},
			},
			Action: func(c *cli.Context) error {
				driver, err := standardLoad(configPath)
				if err != nil {
					return err
				}

				return workflows.ListEntries(driver, c.Int("count"))
			},
		},
		{
			Name:  "new",
			Usage: "Creates a new entry",
			Flags: []cli.Flag{tempFlag},
			Action: func(c *cli.Context) error {
				driver, err := standardLoad(configPath)
				if err != nil {
					return err
				}
				return workflows.NewEntry(driver, c.String("temp-dir"))
			},
		},
		{
			Name:  "edit",
			Usage: "edits an existing journal entry. Takes an id as an argument.",
			Flags: []cli.Flag{tempFlag},
			Action: func(c *cli.Context) error {
				if len(c.Args()) != 1 {
					return errors.New("edit takes 1 argument which is an entry's id")
				}
				driver, err := standardLoad(configPath)
				if err != nil {
					return err
				}
				return workflows.EditEntry(driver, c.Args()[0], c.String("temp-dir"))
			},
		},
	}
	err := app.Run(os.Args)
	if err != nil {
		log.Printf("Failed to complete because %s", err)
		os.Exit(1)
	}
}

func getPassword(prompt string) (string, error) {
	fmt.Printf(prompt)
	raw, err := gopass.GetPasswd()
	return string(raw), err
}

func readConfig(path string) (ejrnl.Config, error) {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return ejrnl.Config{}, err
	}
	entry := &ejrnl.Config{}
	err = yaml.Unmarshal(data, entry)
	return *entry, err
}

func standardLoad(configPath string) (*storage.Driver, error) {
	config, err := readConfig(configPath)
	if err != nil {
		return &storage.Driver{}, err
	}
	password, err := getPassword("Password: ")
	if err != nil {
		return &storage.Driver{}, err
	}
	return storage.NewDriver(config, password)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"github.com/yuin/gopher-lua"
	"github.com/yuin/gopher-lua/parse"
	"io"
	"os"
)

func main() {
	os.Exit(mainAux())
}

func mainAux() int {
	var opt_e, opt_l string
	var opt_i, opt_v, opt_dt, opt_dc bool
	var opt_m int
	flag.StringVar(&opt_e, "e", "", "")
	flag.StringVar(&opt_l, "l", "", "")
	flag.IntVar(&opt_m, "mx", 0, "")
	flag.BoolVar(&opt_i, "i", false, "")
	flag.BoolVar(&opt_v, "v", false, "")
	flag.BoolVar(&opt_dt, "dt", false, "")
	flag.BoolVar(&opt_dc, "dc", false, "")
	flag.Usage = func() {
		fmt.Println(`usage: glua.exe [options] [script [args]].
Available options are:
  -e stat  execute string 'stat'
  -l name  require library 'name'
  -mx MB   memory limit(default: unlimited)
  -dt      dump AST trees
  -dc      dump VM codes
  -i       enter interactive mode after executing 'script'
  -v       show version information
`)
	}
	flag.Parse()
	if len(opt_e) == 0 && !opt_i && !opt_v && flag.NArg() == 0 {
		opt_i = true
	}

	status := 0

	L := lua.NewState()
	defer L.Close()
	if opt_m > 0 {
		L.SetMx(opt_m)
	}

	if opt_v || opt_i {
		fmt.Println(lua.PackageCopyRight)
	}

	if len(opt_l) > 0 {
		if err := L.DoFile(opt_l); err != nil {
			fmt.Println(err.Error())
		}
	}

	if nargs := flag.NArg(); nargs > 0 {
		script := flag.Arg(0)
		argtb := L.NewTable()
		for i := 1; i < nargs; i++ {
			L.RawSet(argtb, lua.LNumber(i), lua.LString(flag.Arg(i)))
		}
		L.SetGlobal("arg", argtb)
		if opt_dt || opt_dc {
			file, err := os.Open(script)
			if err != nil {
				fmt.Println(err.Error())
				return 1
			}
			chunk, err2 := parse.Parse(file, script)
			if err2 != nil {
				fmt.Println(err2.Error())
				return 1
			}
			if opt_dt {
				fmt.Println(parse.Dump(chunk))
			}
			if opt_dc {
				proto, err3 := lua.Compile(chunk, script)
				if err3 != nil {
					fmt.Println(err3.Error())
					return 1
				}
				fmt.Println(proto.String())
			}
		}
		if err := L.DoFile(script); err != nil {
			fmt.Println(err.Error())
			status = 1
		}
	}

	if len(opt_e) > 0 {
		if err := L.DoString(opt_e); err != nil {
			fmt.Println(err.Error())
			status = 1
		}
	}

	if opt_i {
		reader := bufio.NewReader(os.Stdin)
		for {
			fmt.Print("> ")
			if buf, err := reader.ReadString('\n'); err == io.EOF {
				break
			} else if err != nil {
			} else {
				if err := L.DoString(buf); err != nil {
					fmt.Println(err.Error())
				}
			}
		}
	}
	return status
}
<commit_msg>Add a -p flag to generate cpu profiles<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"github.com/yuin/gopher-lua"
	"github.com/yuin/gopher-lua/parse"
	"io"
	"os"
	"runtime/pprof"
)

func main() {
	os.Exit(mainAux())
}

func mainAux() int {
	var opt_e, opt_l, opt_p string
	var opt_i, opt_v, opt_dt, opt_dc bool
	var opt_m int
	flag.StringVar(&opt_e, "e", "", "")
	flag.StringVar(&opt_l, "l", "", "")
	flag.StringVar(&opt_p, "p", "", "")
	flag.IntVar(&opt_m, "mx", 0, "")
	flag.BoolVar(&opt_i, "i", false, "")
	flag.BoolVar(&opt_v, "v", false, "")
	flag.BoolVar(&opt_dt, "dt", false, "")
	flag.BoolVar(&opt_dc, "dc", false, "")
	flag.Usage = func() {
		fmt.Println(`Usage: glua [options] [script [args]].
Available options are:
  -e stat  execute string 'stat'
  -l name  require library 'name'
  -mx MB   memory limit(default: unlimited)
  -dt      dump AST trees
  -dc      dump VM codes
  -i       enter interactive mode after executing 'script'
  -p file  write cpu profiles to the file
  -v       show version information
`)
	}
	flag.Parse()
	if len(opt_p) != 0 {
		f, err := os.Create(opt_p)
		if err != nil {
			fmt.Println(err.Error())
			os.Exit(1)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}
	if len(opt_e) == 0 && !opt_i && !opt_v && flag.NArg() == 0 {
		opt_i = true
	}

	status := 0

	L := lua.NewState()
	defer L.Close()
	if opt_m > 0 {
		L.SetMx(opt_m)
	}

	if opt_v || opt_i {
		fmt.Println(lua.PackageCopyRight)
	}

	if len(opt_l) > 0 {
		if err := L.DoFile(opt_l); err != nil {
			fmt.Println(err.Error())
		}
	}

	if nargs := flag.NArg(); nargs > 0 {
		script := flag.Arg(0)
		argtb := L.NewTable()
		for i := 1; i < nargs; i++ {
			L.RawSet(argtb, lua.LNumber(i), lua.LString(flag.Arg(i)))
		}
		L.SetGlobal("arg", argtb)
		if opt_dt || opt_dc {
			file, err := os.Open(script)
			if err != nil {
				fmt.Println(err.Error())
				return 1
			}
			chunk, err2 := parse.Parse(file, script)
			if err2 != nil {
				fmt.Println(err2.Error())
				return 1
			}
			if opt_dt {
				fmt.Println(parse.Dump(chunk))
			}
			if opt_dc {
				proto, err3 := lua.Compile(chunk, script)
				if err3 != nil {
					fmt.Println(err3.Error())
					return 1
				}
				fmt.Println(proto.String())
			}
		}
		if err := L.DoFile(script); err != nil {
			fmt.Println(err.Error())
			status = 1
		}
	}

	if len(opt_e) > 0 {
		if err := L.DoString(opt_e); err != nil {
			fmt.Println(err.Error())
			status = 1
		}
	}

	if opt_i {
		reader := bufio.NewReader(os.Stdin)
		for {
			fmt.Print("> ")
			if buf, err := reader.ReadString('\n'); err == io.EOF {
				break
			} else if err != nil {
			} else {
				if err := L.DoString(buf); err != nil {
					fmt.Println(err.Error())
				}
			}
		}
	}
	return status
}
<|endoftext|>


<commit_before>// Copyright (C) 2015 Nippon Telegraph and Telephone Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"io"
	"net"
	"strconv"

	api "github.com/osrg/gobgp/v3/api"
	"github.com/osrg/gobgp/v3/pkg/packet/bmp"
	"github.com/spf13/cobra"
)

func showStations() error {
	stream, err := client.ListBmp(ctx, &api.ListBmpRequest{})
	if err != nil {
		fmt.Println(err)
		return err
	}
	stations := make([]*api.ListBmpResponse_BmpStation, 0)
	for {
		rsp, err := stream.Recv()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}
		stations = append(stations, rsp.Station)
	}
	format := "%-23s %-6s %-10s\n"
	for _, r := range stations {
		s := "Down"
		uptime := "Never"
		if r.State.Uptime.AsTime().Unix() != 0 {
			uptime = fmt.Sprint(formatTimedelta(r.State.Uptime.AsTime()))
			if r.State.Uptime.AsTime().After(r.State.Downtime.AsTime()) {
				s = "Up"
			} else {
				s = "Down"
			}
		}
		fmt.Printf(format, net.JoinHostPort(r.Conf.Address, fmt.Sprintf("%d", r.Conf.Port)), s, uptime)
	}

	return nil
}

func modBmpServer(cmdType string, args []string) error {
	if len(args) < 1 {
		return fmt.Errorf("usage: gobgp bmp %s <addr>[:<port>] [{pre|post|both|local-rib|all}]", cmdType)
	}

	var address string
	port := uint32(bmp.BMP_DEFAULT_PORT)
	if host, p, err := net.SplitHostPort(args[0]); err != nil {
		ip := net.ParseIP(args[0])
		if ip == nil {
			return nil
		}
		address = args[0]
	} else {
		address = host
		// Note: BmpServerConfig.Port is uint32 type, but the TCP/UDP port is
		// 16-bit length.
		pn, _ := strconv.ParseUint(p, 10, 16)
		port = uint32(pn)
	}

	var err error
	switch cmdType {
	case cmdAdd:
		statisticsTimeout := 0
		if bmpOpts.StatisticsTimeout >= 0 && bmpOpts.StatisticsTimeout <= 65535 {
			statisticsTimeout = bmpOpts.StatisticsTimeout
		} else {
			return fmt.Errorf("invalid statistics-timeout value. it must be in the range 0-65535. default value is 0 and means disabled")
		}

		policyType := api.AddBmpRequest_PRE
		if len(args) > 1 {
			switch args[1] {
			case "pre":
				policyType = api.AddBmpRequest_PRE
			case "post":
				policyType = api.AddBmpRequest_POST
			case "both":
				policyType = api.AddBmpRequest_BOTH
			case "local-rib":
				policyType = api.AddBmpRequest_LOCAL
			case "all":
				policyType = api.AddBmpRequest_ALL
			default:
				return fmt.Errorf("invalid bmp policy type. valid type is {pre|post|both|local-rib|all}")
			}
		}
		_, err = client.AddBmp(ctx, &api.AddBmpRequest{
			Address:           address,
			Port:              port,
			Policy:            policyType,
			StatisticsTimeout: int32(statisticsTimeout),
		})
	case cmdDel:
		_, err = client.DeleteBmp(ctx, &api.DeleteBmpRequest{
			Address: address,
			Port:    port,
		})
	}
	return err
}

func newBmpCmd() *cobra.Command {
	bmpCmd := &cobra.Command{
		Use: cmdBMP,
		Run: func(cmd *cobra.Command, args []string) {
			showStations()
		},
	}

	for _, w := range []string{cmdAdd, cmdDel} {
		subcmd := &cobra.Command{
			Use: w,
			Run: func(cmd *cobra.Command, args []string) {
				err := modBmpServer(cmd.Use, args)
				if err != nil {
					exitWithError(err)
				}
			},
		}
		if w == cmdAdd {
			subcmd.PersistentFlags().IntVarP(&bmpOpts.StatisticsTimeout, "statistics-timeout", "s", 0, "Timeout of statistics report")
		}
		bmpCmd.AddCommand(subcmd)
	}

	return bmpCmd
}
<commit_msg>make output of gobgp mrt pretty<commit_after>// Copyright (C) 2015 Nippon Telegraph and Telephone Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"io"
	"net"
	"strconv"

	api "github.com/osrg/gobgp/v3/api"
	"github.com/osrg/gobgp/v3/pkg/packet/bmp"
	"github.com/spf13/cobra"
)

func showStations() error {
	stream, err := client.ListBmp(ctx, &api.ListBmpRequest{})
	if err != nil {
		fmt.Println(err)
		return err
	}
	stations := make([]*api.ListBmpResponse_BmpStation, 0)
	for {
		rsp, err := stream.Recv()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}
		stations = append(stations, rsp.Station)
	}
	format := "%-23s %-6s %-10s\n"
	fmt.Printf(format, "Session", "State", "Uptime")
	for _, r := range stations {
		s := "Down"
		uptime := "Never"
		if r.State.Uptime.AsTime().Unix() != 0 {
			uptime = fmt.Sprint(formatTimedelta(r.State.Uptime.AsTime()))
			if r.State.Uptime.AsTime().After(r.State.Downtime.AsTime()) {
				s = "Up"
			} else {
				s = "Down"
			}
		}
		fmt.Printf(format, net.JoinHostPort(r.Conf.Address, fmt.Sprintf("%d", r.Conf.Port)), s, uptime)
	}

	return nil
}

func modBmpServer(cmdType string, args []string) error {
	if len(args) < 1 {
		return fmt.Errorf("usage: gobgp bmp %s <addr>[:<port>] [{pre|post|both|local-rib|all}]", cmdType)
	}

	var address string
	port := uint32(bmp.BMP_DEFAULT_PORT)
	if host, p, err := net.SplitHostPort(args[0]); err != nil {
		ip := net.ParseIP(args[0])
		if ip == nil {
			return nil
		}
		address = args[0]
	} else {
		address = host
		// Note: BmpServerConfig.Port is uint32 type, but the TCP/UDP port is
		// 16-bit length.
		pn, _ := strconv.ParseUint(p, 10, 16)
		port = uint32(pn)
	}

	var err error
	switch cmdType {
	case cmdAdd:
		statisticsTimeout := 0
		if bmpOpts.StatisticsTimeout >= 0 && bmpOpts.StatisticsTimeout <= 65535 {
			statisticsTimeout = bmpOpts.StatisticsTimeout
		} else {
			return fmt.Errorf("invalid statistics-timeout value. it must be in the range 0-65535. default value is 0 and means disabled")
		}

		policyType := api.AddBmpRequest_PRE
		if len(args) > 1 {
			switch args[1] {
			case "pre":
				policyType = api.AddBmpRequest_PRE
			case "post":
				policyType = api.AddBmpRequest_POST
			case "both":
				policyType = api.AddBmpRequest_BOTH
			case "local-rib":
				policyType = api.AddBmpRequest_LOCAL
			case "all":
				policyType = api.AddBmpRequest_ALL
			default:
				return fmt.Errorf("invalid bmp policy type. valid type is {pre|post|both|local-rib|all}")
			}
		}
		_, err = client.AddBmp(ctx, &api.AddBmpRequest{
			Address:           address,
			Port:              port,
			Policy:            policyType,
			StatisticsTimeout: int32(statisticsTimeout),
		})
	case cmdDel:
		_, err = client.DeleteBmp(ctx, &api.DeleteBmpRequest{
			Address: address,
			Port:    port,
		})
	}
	return err
}

func newBmpCmd() *cobra.Command {
	bmpCmd := &cobra.Command{
		Use: cmdBMP,
		Run: func(cmd *cobra.Command, args []string) {
			showStations()
		},
	}

	for _, w := range []string{cmdAdd, cmdDel} {
		subcmd := &cobra.Command{
			Use: w,
			Run: func(cmd *cobra.Command, args []string) {
				err := modBmpServer(cmd.Use, args)
				if err != nil {
					exitWithError(err)
				}
			},
		}
		if w == cmdAdd {
			subcmd.PersistentFlags().IntVarP(&bmpOpts.StatisticsTimeout, "statistics-timeout", "s", 0, "Timeout of statistics report")
		}
		bmpCmd.AddCommand(subcmd)
	}

	return bmpCmd
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"os"

	"github.com/codegangsta/cli"
	"github.com/kubernetes/helm/pkg/client"
	"github.com/kubernetes/helm/pkg/format"
	"github.com/kubernetes/helm/pkg/version"
)

const desc = `Helm: the package and deployment manager for Kubernetes

   Helm is a tool for packaging, deploying, and managing Kubernetes
   applications. It has a client component (this tool) and several in-cluster
   components.

   Before you can use Helm to manage applications, you must install the
   in-cluster components into the target Kubernetes cluster:

      $ helm server install

   Once the in-cluster portion is running, you can use 'helm deploy' to
   deploy a new application:

      $ helm deploy CHARTNAME

   For more information on Helm commands, you can use the following tools:

      $ helm help          # top-level help
      $ helm CMD --help    # help for a particular command or set of commands
`

var commands []cli.Command

func init() {
	addCommands(cmds()...)
}

// debug indicates whether the process is in debug mode.
//
// This is set at app start-up time, based on the presence of the --debug
// flag.
var debug bool

func main() {
	app := cli.NewApp()
	app.Name = "helm"
	app.Version = version.Version
	app.Usage = desc
	app.Commands = commands

	// TODO: make better
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "host,u",
			Usage:  "The URL of the DM server",
			EnvVar: "HELM_HOST",
			Value:  "https://localhost:8000/",
		},
		cli.StringFlag{
			Name:   "kubectl",
			Usage:  "The path to the kubectl binary",
			EnvVar: "KUBECTL",
		},
		cli.IntFlag{
			Name:  "timeout",
			Usage: "Time in seconds to wait for response",
			Value: 10,
		},
		cli.BoolFlag{
			Name:  "debug",
			Usage: "Enable verbose debugging output",
		},
	}
	app.Before = func(c *cli.Context) error {
		debug = c.GlobalBool("debug")
		return nil
	}
	app.Run(os.Args)
}

func cmds() []cli.Command {
	return []cli.Command{
		{
			Name: "search",
		},
	}
}

func addCommands(cmds ...cli.Command) {
	commands = append(commands, cmds...)
}

func run(c *cli.Context, f func(c *cli.Context) error) {
	if err := f(c); err != nil {
		format.Err(err)
		os.Exit(1)
	}
}

// NewClient creates a new client instance preconfigured for CLI usage.
func NewClient(c *cli.Context) *client.Client {
	host := c.GlobalString("host")
	timeout := c.GlobalInt("timeout")
	return client.NewClient(host).SetDebug(debug).SetTimeout(timeout)
}
<commit_msg>fix(cli): remove 'helm search' for MVP<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"os"

	"github.com/codegangsta/cli"
	"github.com/kubernetes/helm/pkg/client"
	"github.com/kubernetes/helm/pkg/format"
	"github.com/kubernetes/helm/pkg/version"
)

const desc = `Helm: the package and deployment manager for Kubernetes

   Helm is a tool for packaging, deploying, and managing Kubernetes
   applications. It has a client component (this tool) and several in-cluster
   components.

   Before you can use Helm to manage applications, you must install the
   in-cluster components into the target Kubernetes cluster:

      $ helm server install

   Once the in-cluster portion is running, you can use 'helm deploy' to
   deploy a new application:

      $ helm deploy CHARTNAME

   For more information on Helm commands, you can use the following tools:

      $ helm help          # top-level help
      $ helm CMD --help    # help for a particular command or set of commands
`

var commands []cli.Command

func init() {
	addCommands(cmds()...)
}

// debug indicates whether the process is in debug mode.
//
// This is set at app start-up time, based on the presence of the --debug
// flag.
var debug bool

func main() {
	app := cli.NewApp()
	app.Name = "helm"
	app.Version = version.Version
	app.Usage = desc
	app.Commands = commands

	// TODO: make better
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "host,u",
			Usage:  "The URL of the DM server",
			EnvVar: "HELM_HOST",
			Value:  "https://localhost:8000/",
		},
		cli.StringFlag{
			Name:   "kubectl",
			Usage:  "The path to the kubectl binary",
			EnvVar: "KUBECTL",
		},
		cli.IntFlag{
			Name:  "timeout",
			Usage: "Time in seconds to wait for response",
			Value: 10,
		},
		cli.BoolFlag{
			Name:  "debug",
			Usage: "Enable verbose debugging output",
		},
	}
	app.Before = func(c *cli.Context) error {
		debug = c.GlobalBool("debug")
		return nil
	}
	app.Run(os.Args)
}

func cmds() []cli.Command {
	return []cli.Command{}
}

func addCommands(cmds ...cli.Command) {
	commands = append(commands, cmds...)
}

func run(c *cli.Context, f func(c *cli.Context) error) {
	if err := f(c); err != nil {
		format.Err(err)
		os.Exit(1)
	}
}

// NewClient creates a new client instance preconfigured for CLI usage.
func NewClient(c *cli.Context) *client.Client {
	host := c.GlobalString("host")
	timeout := c.GlobalInt("timeout")
	return client.NewClient(host).SetDebug(debug).SetTimeout(timeout)
}
<|endoftext|>


<commit_before>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"io"

	"github.com/spf13/cobra"

	"helm.sh/helm/v3/cmd/helm/require"
	experimental "helm.sh/helm/v3/internal/experimental/action"
	"helm.sh/helm/v3/pkg/action"
)

const pushDesc = `
Upload a package to a registry.

If the --with-prov flag is specified, the chart MUST have an associated
provenance file, which will also be uploaded.
`

func newPushCmd(cfg *action.Configuration, out io.Writer) *cobra.Command {
	client := experimental.NewPushWithOpts(experimental.WithPushConfig(cfg))

	cmd := &cobra.Command{
		Use:               "push [chart] [remote]",
		Short:             "push a chart to remote",
		Long:              pushDesc,
		Hidden:            !FeatureGateOCI.IsEnabled(),
		PersistentPreRunE: checkOCIFeatureGate(),
		Args:              require.MinimumNArgs(2),
		RunE: func(cmd *cobra.Command, args []string) error {
			chartRef := args[0]
			remote := args[1]
			client.Settings = settings
			output, err := client.Run(chartRef, remote)
			if err != nil {
				return err
			}
			fmt.Fprint(out, output)
			return nil
		},
	}

	return cmd
}
<commit_msg>modify helm push help text<commit_after>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"io"

	"github.com/spf13/cobra"

	"helm.sh/helm/v3/cmd/helm/require"
	experimental "helm.sh/helm/v3/internal/experimental/action"
	"helm.sh/helm/v3/pkg/action"
)

const pushDesc = `
Upload a chart to a registry.

If the chart has an associated provenance file,
it will also be uploaded.
`

func newPushCmd(cfg *action.Configuration, out io.Writer) *cobra.Command {
	client := experimental.NewPushWithOpts(experimental.WithPushConfig(cfg))

	cmd := &cobra.Command{
		Use:               "push [chart] [remote]",
		Short:             "push a chart to remote",
		Long:              pushDesc,
		Hidden:            !FeatureGateOCI.IsEnabled(),
		PersistentPreRunE: checkOCIFeatureGate(),
		Args:              require.MinimumNArgs(2),
		RunE: func(cmd *cobra.Command, args []string) error {
			chartRef := args[0]
			remote := args[1]
			client.Settings = settings
			output, err := client.Run(chartRef, remote)
			if err != nil {
				return err
			}
			fmt.Fprint(out, output)
			return nil
		},
	}

	return cmd
}
<|endoftext|>


<commit_before>// +build linux

package init

import (
	"fmt"

	"github.com/rancher/os/pkg/init/selinux"

	"github.com/rancher/os/pkg/init/docker"
	"github.com/rancher/os/pkg/init/one"

	"github.com/rancher/os/config"
	"github.com/rancher/os/pkg/dfs"
	"github.com/rancher/os/pkg/init/b2d"
	"github.com/rancher/os/pkg/init/cloudinit"
	"github.com/rancher/os/pkg/init/configfiles"
	"github.com/rancher/os/pkg/init/debug"
	"github.com/rancher/os/pkg/init/env"
	"github.com/rancher/os/pkg/init/fsmount"
	"github.com/rancher/os/pkg/init/hypervisor"
	"github.com/rancher/os/pkg/init/modules"
	"github.com/rancher/os/pkg/init/prepare"
	"github.com/rancher/os/pkg/init/recovery"
	"github.com/rancher/os/pkg/init/sharedroot"
	"github.com/rancher/os/pkg/init/switchroot"
	"github.com/rancher/os/pkg/log"
	"github.com/rancher/os/pkg/sysinit"
)

func MainInit() {
	log.InitLogger()
	// TODO: this breaks and does nothing if the cfg is invalid (or is it due to threading?)
	defer func() {
		if r := recover(); r != nil {
			fmt.Printf("Starting Recovery console: %v\n", r)
			recovery.Recovery(nil)
		}
	}()

	if err := RunInit(); err != nil {
		log.Fatal(err)
	}
}

func RunInit() error {
	initFuncs := config.CfgFuncs{
		{"set env", env.Init},
		{"preparefs", prepare.FS},
		{"save init cmdline", prepare.SaveCmdline},
		{"mount OEM", fsmount.MountOem},
		{"debug save cfg", debug.PrintAndLoadConfig},
		{"load modules", modules.LoadModules},
		{"recovery console", recovery.LoadRecoveryConsole},
		{"cloud-init", cloudinit.CloudInit},
		{"b2d env", b2d.B2D},
		{"mount STATE and bootstrap", fsmount.MountStateAndBootstrap},
		{"read cfg and log files", configfiles.ReadConfigFiles},
		{"switchroot", switchroot.SwitchRoot},
		{"mount OEM2", fsmount.MountOem},
		{"mount BOOT", fsmount.MountBoot},
		{"write cfg and log files", configfiles.WriteConfigFiles},
		{"hypervisor Env", hypervisor.Env},
		{"b2d Env", b2d.Env},
		{"hypervisor tools", hypervisor.Tools},
		{"preparefs2", prepare.FS},
		{"load modules2", modules.LoadModules},
		{"set proxy env", env.Proxy},
		{"init SELinux", selinux.Initialize},
		{"setupSharedRoot", sharedroot.Setup},
		{"sysinit", sysinit.RunSysInit},
	}

	cfg, err := config.ChainCfgFuncs(nil, initFuncs)
	if err != nil {
		recovery.Recovery(err)
	}

	launchConfig, args := docker.GetLaunchConfig(cfg, &cfg.Rancher.SystemDocker)
	launchConfig.Fork = !cfg.Rancher.SystemDocker.Exec
	//launchConfig.NoLog = true

	log.Info("Launching System Docker")
	_, err = dfs.LaunchDocker(launchConfig, config.SystemDockerBin, args...)
	if err != nil {
		log.Errorf("Error Launching System Docker: %s", err)
		recovery.Recovery(err)
		return err
	}
	// Code never gets here - rancher.system_docker.exec=true

	return one.PidOne()
}
<commit_msg>Revert commit 9bcab2f663929ef5dda6055c86aaca187d4c0b55<commit_after>// +build linux

package init

import (
	"fmt"

	"github.com/rancher/os/pkg/init/selinux"

	"github.com/rancher/os/pkg/init/docker"
	"github.com/rancher/os/pkg/init/one"

	"github.com/rancher/os/config"
	"github.com/rancher/os/pkg/dfs"
	"github.com/rancher/os/pkg/init/b2d"
	"github.com/rancher/os/pkg/init/cloudinit"
	"github.com/rancher/os/pkg/init/configfiles"
	"github.com/rancher/os/pkg/init/debug"
	"github.com/rancher/os/pkg/init/env"
	"github.com/rancher/os/pkg/init/fsmount"
	"github.com/rancher/os/pkg/init/hypervisor"
	"github.com/rancher/os/pkg/init/modules"
	"github.com/rancher/os/pkg/init/prepare"
	"github.com/rancher/os/pkg/init/recovery"
	"github.com/rancher/os/pkg/init/sharedroot"
	"github.com/rancher/os/pkg/init/switchroot"
	"github.com/rancher/os/pkg/log"
	"github.com/rancher/os/pkg/sysinit"
)

func MainInit() {
	log.InitLogger()
	// TODO: this breaks and does nothing if the cfg is invalid (or is it due to threading?)
	defer func() {
		if r := recover(); r != nil {
			fmt.Printf("Starting Recovery console: %v\n", r)
			recovery.Recovery(nil)
		}
	}()

	if err := RunInit(); err != nil {
		log.Fatal(err)
	}
}

func RunInit() error {
	initFuncs := config.CfgFuncs{
		{"set env", env.Init},
		{"preparefs", prepare.FS},
		{"save init cmdline", prepare.SaveCmdline},
		{"mount OEM", fsmount.MountOem},
		{"debug save cfg", debug.PrintAndLoadConfig},
		{"load modules", modules.LoadModules},
		{"recovery console", recovery.LoadRecoveryConsole},
		{"b2d env", b2d.B2D},
		{"mount STATE and bootstrap", fsmount.MountStateAndBootstrap},
		{"cloud-init", cloudinit.CloudInit},
		{"read cfg and log files", configfiles.ReadConfigFiles},
		{"switchroot", switchroot.SwitchRoot},
		{"mount OEM2", fsmount.MountOem},
		{"mount BOOT", fsmount.MountBoot},
		{"write cfg and log files", configfiles.WriteConfigFiles},
		{"hypervisor Env", hypervisor.Env},
		{"b2d Env", b2d.Env},
		{"hypervisor tools", hypervisor.Tools},
		{"preparefs2", prepare.FS},
		{"load modules2", modules.LoadModules},
		{"set proxy env", env.Proxy},
		{"init SELinux", selinux.Initialize},
		{"setupSharedRoot", sharedroot.Setup},
		{"sysinit", sysinit.RunSysInit},
	}

	cfg, err := config.ChainCfgFuncs(nil, initFuncs)
	if err != nil {
		recovery.Recovery(err)
	}

	launchConfig, args := docker.GetLaunchConfig(cfg, &cfg.Rancher.SystemDocker)
	launchConfig.Fork = !cfg.Rancher.SystemDocker.Exec
	//launchConfig.NoLog = true

	log.Info("Launching System Docker")
	_, err = dfs.LaunchDocker(launchConfig, config.SystemDockerBin, args...)
	if err != nil {
		log.Errorf("Error Launching System Docker: %s", err)
		recovery.Recovery(err)
		return err
	}
	// Code never gets here - rancher.system_docker.exec=true

	return one.PidOne()
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"

	"github.com/prydonius/karn"
	"github.com/urfave/cli"
)

func main() {
	log.SetFlags(0)
	app := cli.NewApp()
	app.Name = "karn"
	app.Usage = "manage multiple Git identities"
	app.Author = "Adnan Abdulhussein"
	app.Email = "adnan@prydoni.us"
	app.Version = "0.0.4"
	app.Commands = commands()
	app.Run(os.Args)
}

func commands() []cli.Command {
	return []cli.Command{
		{
			Name:  "update",
			Usage: "Update the current repository with a karn configured identity",
			Action: func(ctx *cli.Context) {
				karn.Update()
			},
		},
		{
			Name:  "init",
			Usage: "Initialise karn for use in a bash compatible shell",
			Action: func(ctx *cli.Context) {
				karn.Init()
			},
		},
		{
			Name:  "install",
			Usage: "Install karn for shell, and a sample configuration",
			Action: func(ctx *cli.Context) {
				karn.Install()
			},
		},
	}
}
<commit_msg>Bump karn to 0.0.5<commit_after>package main

import (
	"log"
	"os"

	"github.com/prydonius/karn"
	"github.com/urfave/cli"
)

func main() {
	log.SetFlags(0)
	app := cli.NewApp()
	app.Name = "karn"
	app.Usage = "manage multiple Git identities"
	app.Author = "Adnan Abdulhussein"
	app.Email = "adnan@prydoni.us"
	app.Version = "0.0.5"
	app.Commands = commands()
	app.Run(os.Args)
}

func commands() []cli.Command {
	return []cli.Command{
		{
			Name:  "update",
			Usage: "Update the current repository with a karn configured identity",
			Action: func(ctx *cli.Context) {
				karn.Update()
			},
		},
		{
			Name:  "init",
			Usage: "Initialise karn for use in a bash compatible shell",
			Action: func(ctx *cli.Context) {
				karn.Init()
			},
		},
		{
			Name:  "install",
			Usage: "Install karn for shell, and a sample configuration",
			Action: func(ctx *cli.Context) {
				karn.Install()
			},
		},
	}
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"

	"github.com/codegangsta/cli"
	"github.com/prydonius/karn"
)

func main() {
	log.SetFlags(0)
	app := cli.NewApp()
	app.Name = "karn"
	app.Usage = "manage multiple Git identities"
	app.Author = "Adnan Abdulhussein"
	app.Email = "adnan@prydoni.us"
	app.Version = "0.0.2"
	app.Commands = commands()
	app.Run(os.Args)
}

func commands() []cli.Command {
	return []cli.Command{
		{
			Name:  "update",
			Usage: "Update the current repository with a karn configured identity",
			Action: func(ctx *cli.Context) {
				karn.Update()
			},
		},
		{
			Name:  "init",
			Usage: "Initialise karn for use in a bash compatible shell",
			Action: func(ctx *cli.Context) {
				karn.Init()
			},
		},
		{
			Name:  "install",
			Usage: "Install karn for shell, and a sample configuration",
			Action: func(ctx *cli.Context) {
				karn.Install()
			},
		},
	}
}
<commit_msg>bump karn version to 0.0.3<commit_after>package main

import (
	"log"
	"os"

	"github.com/codegangsta/cli"
	"github.com/prydonius/karn"
)

func main() {
	log.SetFlags(0)
	app := cli.NewApp()
	app.Name = "karn"
	app.Usage = "manage multiple Git identities"
	app.Author = "Adnan Abdulhussein"
	app.Email = "adnan@prydoni.us"
	app.Version = "0.0.3"
	app.Commands = commands()
	app.Run(os.Args)
}

func commands() []cli.Command {
	return []cli.Command{
		{
			Name:  "update",
			Usage: "Update the current repository with a karn configured identity",
			Action: func(ctx *cli.Context) {
				karn.Update()
			},
		},
		{
			Name:  "init",
			Usage: "Initialise karn for use in a bash compatible shell",
			Action: func(ctx *cli.Context) {
				karn.Init()
			},
		},
		{
			Name:  "install",
			Usage: "Install karn for shell, and a sample configuration",
			Action: func(ctx *cli.Context) {
				karn.Install()
			},
		},
	}
}
<|endoftext|>