<commit_before>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package service

import (
	"fmt"
	"net/http"
	"regexp"
	"strings"

	"github.com/juju/cmd"
	"github.com/juju/errors"
	"github.com/juju/romulus/api/budget"
	"gopkg.in/macaroon-bakery.v1/httpbakery"
	"launchpad.net/gnuflag"

	"github.com/juju/juju/api"
	"github.com/juju/juju/api/charms"
	"github.com/juju/juju/apiserver/params"
)

var budgetWithLimitRe = regexp.MustCompile(`^[a-zA-Z0-9\-]+:[1-9][0-9]*$`)

// AllocateBudget implements the DeployStep interface.
type AllocateBudget struct {
	AllocationSpec string
	APIClient      apiClient
	allocated      bool
}

// SetFlags is part of the DeployStep interface.
func (a *AllocateBudget) SetFlags(f *gnuflag.FlagSet) {
	f.StringVar(&a.AllocationSpec, "budget", "", "budget and allocation limit")
}

// RunPre is part of the DeployStep interface.
func (a *AllocateBudget) RunPre(state api.Connection, client *http.Client, ctx *cmd.Context, deployInfo DeploymentInfo) error {
	if deployInfo.CharmURL.Schema == "local" {
		return nil
	}
	charmsClient := charms.NewClient(state)
	metered, err := charmsClient.IsMetered(deployInfo.CharmURL.String())
	if params.IsCodeNotImplemented(err) {
		// The state server is too old to support metering.  Warn
		// the user, but don't return an error.
		logger.Tracef("current state server version does not support charm metering")
		return nil
	} else if err != nil {
		return errors.Annotate(err, "could not determine charm type")
	}
	if !metered {
		return nil
	}

	allocBudget, allocLimit, err := parseBudgetWithLimit(a.AllocationSpec)
	if err != nil {
		return errors.Trace(err)
	}
	a.APIClient, err = getApiClient(client)
	if err != nil {
		return errors.Annotate(err, "could not create API client")
	}
	resp, err := a.APIClient.CreateAllocation(allocBudget, allocLimit, deployInfo.ModelUUID, []string{deployInfo.ServiceName})
	if err != nil {
		return errors.Annotate(err, "could not create budget allocation")
	}
	a.allocated = true
	fmt.Fprintf(ctx.Stdout, "%s\n", resp)
	return nil
}

func (a *AllocateBudget) RunPost(_ api.Connection, client *http.Client, ctx *cmd.Context, deployInfo DeploymentInfo, prevErr error) error {
	if prevErr == nil || !a.allocated {
		return nil
	}
	var err error
	if a.APIClient == nil {
		a.APIClient, err = getApiClient(client)
		if err != nil {
			return errors.Trace(err)
		}
	}
	resp, err := a.APIClient.DeleteAllocation(deployInfo.ModelUUID, deployInfo.ServiceName)
	if err != nil {
		return errors.Annotate(err, "failed to remove allocation")
	}
	fmt.Fprintf(ctx.Stdout, "%s\n", resp)
	return nil
}

func parseBudgetWithLimit(bl string) (string, string, error) {
	if !budgetWithLimitRe.MatchString(bl) {
		return "", "", errors.New("invalid budget specification, expecting <budget>:<limit>")
	}
	parts := strings.Split(bl, ":")
	return parts[0], parts[1], nil
}

var getApiClient = getApiClientImpl

func getApiClientImpl(client *http.Client) (apiClient, error) {
	bakeryClient := &httpbakery.Client{Client: client, VisitWebPage: httpbakery.OpenWebBrowser}
	c := budget.NewClient(bakeryClient)
	return c, nil
}

type apiClient interface {
	CreateAllocation(string, string, string, []string) (string, error)
	DeleteAllocation(string, string) (string, error)
}
<commit_msg>Default flag value.<commit_after>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package service

import (
	"fmt"
	"net/http"
	"regexp"
	"strings"

	"github.com/juju/cmd"
	"github.com/juju/errors"
	"github.com/juju/romulus/api/budget"
	"gopkg.in/macaroon-bakery.v1/httpbakery"
	"launchpad.net/gnuflag"

	"github.com/juju/juju/api"
	"github.com/juju/juju/api/charms"
	"github.com/juju/juju/apiserver/params"
)

var budgetWithLimitRe = regexp.MustCompile(`^[a-zA-Z0-9\-]+:[0-9]+$`)

// AllocateBudget implements the DeployStep interface.
type AllocateBudget struct {
	AllocationSpec string
	APIClient      apiClient
	allocated      bool
}

// SetFlags is part of the DeployStep interface.
func (a *AllocateBudget) SetFlags(f *gnuflag.FlagSet) {
	f.StringVar(&a.AllocationSpec, "budget", "personal:0", "budget and allocation limit")
}

// RunPre is part of the DeployStep interface.
func (a *AllocateBudget) RunPre(state api.Connection, client *http.Client, ctx *cmd.Context, deployInfo DeploymentInfo) error {
	if deployInfo.CharmURL.Schema == "local" {
		return nil
	}
	charmsClient := charms.NewClient(state)
	metered, err := charmsClient.IsMetered(deployInfo.CharmURL.String())
	if params.IsCodeNotImplemented(err) {
		// The state server is too old to support metering.  Warn
		// the user, but don't return an error.
		logger.Tracef("current state server version does not support charm metering")
		return nil
	} else if err != nil {
		return errors.Annotate(err, "could not determine charm type")
	}
	if !metered {
		return nil
	}

	allocBudget, allocLimit, err := parseBudgetWithLimit(a.AllocationSpec)
	if err != nil {
		return errors.Trace(err)
	}
	a.APIClient, err = getApiClient(client)
	if err != nil {
		return errors.Annotate(err, "could not create API client")
	}
	resp, err := a.APIClient.CreateAllocation(allocBudget, allocLimit, deployInfo.ModelUUID, []string{deployInfo.ServiceName})
	if err != nil {
		return errors.Annotate(err, "could not create budget allocation")
	}
	a.allocated = true
	fmt.Fprintf(ctx.Stdout, "%s\n", resp)
	return nil
}

func (a *AllocateBudget) RunPost(_ api.Connection, client *http.Client, ctx *cmd.Context, deployInfo DeploymentInfo, prevErr error) error {
	if prevErr == nil || !a.allocated {
		return nil
	}
	var err error
	if a.APIClient == nil {
		a.APIClient, err = getApiClient(client)
		if err != nil {
			return errors.Trace(err)
		}
	}
	resp, err := a.APIClient.DeleteAllocation(deployInfo.ModelUUID, deployInfo.ServiceName)
	if err != nil {
		return errors.Annotate(err, "failed to remove allocation")
	}
	fmt.Fprintf(ctx.Stdout, "%s\n", resp)
	return nil
}

func parseBudgetWithLimit(bl string) (string, string, error) {
	if !budgetWithLimitRe.MatchString(bl) {
		return "", "", errors.New("invalid budget specification, expecting <budget>:<limit>")
	}
	parts := strings.Split(bl, ":")
	return parts[0], parts[1], nil
}

var getApiClient = getApiClientImpl

func getApiClientImpl(client *http.Client) (apiClient, error) {
	bakeryClient := &httpbakery.Client{Client: client, VisitWebPage: httpbakery.OpenWebBrowser}
	c := budget.NewClient(bakeryClient)
	return c, nil
}

type apiClient interface {
	CreateAllocation(string, string, string, []string) (string, error)
	DeleteAllocation(string, string) (string, error)
}
<|endoftext|>


<commit_before>package disk

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"sync"
	"time"

	"github.com/funkygao/golib/sync2"
	log "github.com/funkygao/log4go"
)

// queue is a bounded, disk-backed, append-only type that combines queue and
// log semantics.
// key/value byte slices can be appended and read back in order through
// cursor.
//
// Internally, the queue writes key/value byte slices to multiple segment files so
// that disk space can be reclaimed. When a segment file is larger than
// the max segment size, a new file is created.   Segments are removed
// after cursor has advanced past the last entry.  The first
// segment is the head, and the last segment is the tail.  Reads are from
// the head segment and writes tail segment.
//
// queues can have a max size configured such that when the size of all
// segments on disk exceeds the size, write will fail.
//
// ┌─────┐
// │head │
// ├─────┘
// │
// ▼
// ┌─────────────────┐ ┌─────────────────┐┌─────────────────┐
// │segment 1 - 10MB │ │segment 2 - 10MB ││segment 3 - 10MB │
// └─────────────────┘ └─────────────────┘└─────────────────┘
//                          ▲                               ▲
//                          │                               │
//                          │                               │
//                       ┌───────┐                     ┌─────┐
//                       │cursor │                     │tail │
//                       └───────┘                     └─────┘
type queue struct {
	mu sync.RWMutex
	wg sync.WaitGroup

	dir          string // Directory to create segments
	clusterTopic clusterTopic

	// The maximum size in bytes of a segment file before a new one should be created
	maxSegmentSize int64

	// The maximum size allowed in bytes of all segments before writes will return an error
	// -1 means unlimited
	maxSize int64

	purgeInterval time.Duration
	maxAge        time.Duration

	cursor     *cursor
	head, tail *segment
	segments   segments

	quit          chan struct{}
	emptyInflight sync2.AtomicInt32
}

// newQueue create a queue that will store segments in dir and that will
// consume more than maxSize on disk.
func newQueue(ct clusterTopic, dir string, maxSize int64, purgeInterval, maxAge time.Duration) *queue {
	q := &queue{
		clusterTopic:   ct,
		dir:            dir,
		quit:           make(chan struct{}),
		maxSegmentSize: defaultSegmentSize,
		maxSize:        maxSize,
		purgeInterval:  purgeInterval,
		maxAge:         maxAge,
		segments:       segments{},
	}
	q.cursor = newCursor(q)
	return q
}

// Open opens the queue for reading and writing
func (q *queue) Open() error {
	q.mu.Lock()
	defer q.mu.Unlock()

	if err := mkdirIfNotExist(q.dir); err != nil {
		return err
	}

	segments, err := q.loadSegments()
	if err != nil {
		return err
	}
	q.segments = segments

	if len(q.segments) == 0 {
		// create the 1st segment
		if _, err = q.addSegment(); err != nil {
			return err
		}
	}

	q.head = q.segments[0]
	q.tail = q.segments[len(q.segments)-1]

	// cursor open must be placed below queue open
	if err = q.cursor.open(); err != nil {
		return err
	}

	return nil
}

func (q *queue) Start() {
	q.wg.Add(1)
	go q.housekeeping()

	q.wg.Add(1)
	go q.pump()
}

// Close stops the queue for reading and writing
func (q *queue) Close() error {
	close(q.quit)

	q.mu.Lock()
	defer q.mu.Unlock()

	for _, s := range q.segments {
		if err := s.Close(); err != nil {
			return err
		}
	}

	q.head = nil
	q.tail = nil
	q.segments = nil

	q.wg.Wait()
	if err := q.cursor.dump(); err != nil {
		return err
	}
	q.cursor = nil
	return nil
}

// Remove removes all underlying file-based resources for the queue.
// It is an error to call this on an open queue.
func (q *queue) Remove() error {
	q.mu.Lock()
	defer q.mu.Unlock()

	if q.head != nil || q.tail != nil || q.segments != nil {
		return ErrQueueOpen
	}

	return os.RemoveAll(q.dir)
}

// Purge garbage collects the segments that are behind cursor.
func (q *queue) Purge() error {
	log.Debug("queue[%s] purge...", q.ident())

	q.mu.Lock()
	defer q.mu.Unlock()

	if len(q.segments) <= 1 {
		// head, curror, tail are in the same segment
		return nil
	}

	for {
		if q.cursor.pos.SegmentID > q.head.id &&
			q.head.LastModified().Add(q.maxAge).Unix() < time.Now().Unix() {
			q.trimHead()
		} else {
			return nil
		}

	}
}

// LastModified returns the last time the queue was modified.
func (q *queue) LastModified() time.Time {
	q.mu.RLock()
	defer q.mu.RUnlock()

	return q.tail.LastModified()
}

// Append appends a block to the end of the queue
func (q *queue) Append(b *block) error {
	q.mu.Lock()
	defer q.mu.Unlock()

	if q.tail == nil {
		return ErrQueueNotOpen
	}

	if q.maxSize > 0 && q.diskUsage()+b.size() > q.maxSize {
		return ErrQueueFull
	}

	// Append the entry to the tail, if the segment is full,
	// try to create new segment and retry the append
	if err := q.tail.Append(b); err == ErrSegmentFull {
		segment, err := q.addSegment()
		if err != nil {
			return err
		}

		q.tail = segment
		return q.tail.Append(b)
	} else if err != nil {
		return err
	}
	return nil
}

func (q *queue) Rollback(b *block) (err error) {
	c := q.cursor
	if err = c.advanceOffset(-b.size()); err != nil {
		return
	}

	// rollback needn't consider cross segment case
	return c.seg.Seek(c.pos.Offset)
}

func (q *queue) Next(b *block) (err error) {
	q.mu.RLock()
	defer q.mu.RUnlock()

	c := q.cursor
	if c == nil {
		return ErrQueueNotOpen
	}
	err = c.seg.ReadOne(b)
	switch err {
	case nil:
		c.advanceOffset(b.size())
		return

	case io.EOF:
		// cursor might have:
		// 1. reached end of the current segment: will advance to next segment
		// 2. reached end of tail
		if ok := c.advanceSegment(); !ok {
			return ErrEOQ
		}

		// advanced to next segment, read one block
		err = c.seg.ReadOne(b)
		switch err {
		case nil:
			// bingo!
			c.advanceOffset(b.size())
			return

		case io.EOF:
			// tail is empty
			return ErrEOQ

		default:
			return
		}

	default:
		return
	}
}

func (q *queue) EmptyInflight() bool {
	return q.emptyInflight.Get() == 1
}

// diskUsage returns the total size on disk used by the queue
func (q *queue) diskUsage() int64 {
	var size int64
	for _, s := range q.segments {
		size += s.DiskUsage()
	}
	return size
}

// loadSegments loads all segments on disk
func (q *queue) loadSegments() (segments, error) {
	segments := []*segment{}

	files, err := ioutil.ReadDir(q.dir)
	if err != nil {
		return segments, err
	}

	for _, segment := range files {
		if segment.IsDir() || segment.Name() == cursorFile {
			continue
		}

		// Segments file names are all numeric
		id, err := strconv.ParseUint(segment.Name(), 10, 64)
		if err != nil {
			continue
		}

		segment, err := newSegment(id, filepath.Join(q.dir, segment.Name()), q.maxSegmentSize)
		if err != nil {
			return segments, err
		}

		segments = append(segments, segment)
	}
	return segments, nil
}

// addSegment creates a new empty segment file
// caller is responsible for the lock
func (q *queue) addSegment() (*segment, error) {
	nextID, err := q.nextSegmentID()
	if err != nil {
		return nil, err
	}

	path := filepath.Join(q.dir, fmt.Sprintf("%020d", nextID))
	segment, err := newSegment(nextID, path, q.maxSegmentSize)
	if err != nil {
		return nil, err
	}

	q.segments = append(q.segments, segment)
	return segment, nil
}

// nextSegmentID returns the next segment ID that is free
func (q *queue) nextSegmentID() (uint64, error) {
	segments, err := ioutil.ReadDir(q.dir)
	if err != nil {
		return 0, err
	}

	var maxID uint64
	for _, segment := range segments {
		if segment.IsDir() || segment.Name() == cursorFile {
			continue
		}

		// Segments file names are all numeric
		segmentID, err := strconv.ParseUint(segment.Name(), 10, 64)
		if err != nil {
			log.Warn("unexpected segment file: %s", filepath.Join(q.dir, segment.Name()))
			continue
		}

		if segmentID > maxID {
			maxID = segmentID
		}
	}

	return maxID + 1, nil
}

func (q *queue) ident() string {
	return q.dir
}

func (q *queue) trimHead() (err error) {
	q.segments = q.segments[1:]

	if err = q.head.Remove(); err != nil {
		return
	}

	q.head = q.segments[0]
	return
}

func (q *queue) nextDir() string {
	// find least loaded dir
	return ""
}

// skipCursorSegment skip the current corrupted cursor segment and
// advance to next segment.
// if tail corrupts, add new segment.
func (q *queue) skipCursorSegment() {

}
<commit_msg>discard the verbose useless debug log<commit_after>package disk

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"sync"
	"time"

	"github.com/funkygao/golib/sync2"
	log "github.com/funkygao/log4go"
)

// queue is a bounded, disk-backed, append-only type that combines queue and
// log semantics.
// key/value byte slices can be appended and read back in order through
// cursor.
//
// Internally, the queue writes key/value byte slices to multiple segment files so
// that disk space can be reclaimed. When a segment file is larger than
// the max segment size, a new file is created.   Segments are removed
// after cursor has advanced past the last entry.  The first
// segment is the head, and the last segment is the tail.  Reads are from
// the head segment and writes tail segment.
//
// queues can have a max size configured such that when the size of all
// segments on disk exceeds the size, write will fail.
//
// ┌─────┐
// │head │
// ├─────┘
// │
// ▼
// ┌─────────────────┐ ┌─────────────────┐┌─────────────────┐
// │segment 1 - 10MB │ │segment 2 - 10MB ││segment 3 - 10MB │
// └─────────────────┘ └─────────────────┘└─────────────────┘
//                          ▲                               ▲
//                          │                               │
//                          │                               │
//                       ┌───────┐                     ┌─────┐
//                       │cursor │                     │tail │
//                       └───────┘                     └─────┘
type queue struct {
	mu sync.RWMutex
	wg sync.WaitGroup

	dir          string // Directory to create segments
	clusterTopic clusterTopic

	// The maximum size in bytes of a segment file before a new one should be created
	maxSegmentSize int64

	// The maximum size allowed in bytes of all segments before writes will return an error
	// -1 means unlimited
	maxSize int64

	purgeInterval time.Duration
	maxAge        time.Duration

	cursor     *cursor
	head, tail *segment
	segments   segments

	quit          chan struct{}
	emptyInflight sync2.AtomicInt32
}

// newQueue create a queue that will store segments in dir and that will
// consume more than maxSize on disk.
func newQueue(ct clusterTopic, dir string, maxSize int64, purgeInterval, maxAge time.Duration) *queue {
	q := &queue{
		clusterTopic:   ct,
		dir:            dir,
		quit:           make(chan struct{}),
		maxSegmentSize: defaultSegmentSize,
		maxSize:        maxSize,
		purgeInterval:  purgeInterval,
		maxAge:         maxAge,
		segments:       segments{},
	}
	q.cursor = newCursor(q)
	return q
}

// Open opens the queue for reading and writing
func (q *queue) Open() error {
	q.mu.Lock()
	defer q.mu.Unlock()

	if err := mkdirIfNotExist(q.dir); err != nil {
		return err
	}

	segments, err := q.loadSegments()
	if err != nil {
		return err
	}
	q.segments = segments

	if len(q.segments) == 0 {
		// create the 1st segment
		if _, err = q.addSegment(); err != nil {
			return err
		}
	}

	q.head = q.segments[0]
	q.tail = q.segments[len(q.segments)-1]

	// cursor open must be placed below queue open
	if err = q.cursor.open(); err != nil {
		return err
	}

	return nil
}

func (q *queue) Start() {
	q.wg.Add(1)
	go q.housekeeping()

	q.wg.Add(1)
	go q.pump()
}

// Close stops the queue for reading and writing
func (q *queue) Close() error {
	close(q.quit)

	q.mu.Lock()
	defer q.mu.Unlock()

	for _, s := range q.segments {
		if err := s.Close(); err != nil {
			return err
		}
	}

	q.head = nil
	q.tail = nil
	q.segments = nil

	q.wg.Wait()
	if err := q.cursor.dump(); err != nil {
		return err
	}
	q.cursor = nil
	return nil
}

// Remove removes all underlying file-based resources for the queue.
// It is an error to call this on an open queue.
func (q *queue) Remove() error {
	q.mu.Lock()
	defer q.mu.Unlock()

	if q.head != nil || q.tail != nil || q.segments != nil {
		return ErrQueueOpen
	}

	return os.RemoveAll(q.dir)
}

// Purge garbage collects the segments that are behind cursor.
func (q *queue) Purge() error {
	q.mu.Lock()
	defer q.mu.Unlock()

	if len(q.segments) <= 1 {
		// head, curror, tail are in the same segment
		return nil
	}

	for {
		if q.cursor.pos.SegmentID > q.head.id &&
			q.head.LastModified().Add(q.maxAge).Unix() < time.Now().Unix() {
			q.trimHead()
		} else {
			return nil
		}

	}
}

// LastModified returns the last time the queue was modified.
func (q *queue) LastModified() time.Time {
	q.mu.RLock()
	defer q.mu.RUnlock()

	return q.tail.LastModified()
}

// Append appends a block to the end of the queue
func (q *queue) Append(b *block) error {
	q.mu.Lock()
	defer q.mu.Unlock()

	if q.tail == nil {
		return ErrQueueNotOpen
	}

	if q.maxSize > 0 && q.diskUsage()+b.size() > q.maxSize {
		return ErrQueueFull
	}

	// Append the entry to the tail, if the segment is full,
	// try to create new segment and retry the append
	if err := q.tail.Append(b); err == ErrSegmentFull {
		segment, err := q.addSegment()
		if err != nil {
			return err
		}

		q.tail = segment
		return q.tail.Append(b)
	} else if err != nil {
		return err
	}
	return nil
}

func (q *queue) Rollback(b *block) (err error) {
	c := q.cursor
	if err = c.advanceOffset(-b.size()); err != nil {
		return
	}

	// rollback needn't consider cross segment case
	return c.seg.Seek(c.pos.Offset)
}

func (q *queue) Next(b *block) (err error) {
	q.mu.RLock()
	defer q.mu.RUnlock()

	c := q.cursor
	if c == nil {
		return ErrQueueNotOpen
	}
	err = c.seg.ReadOne(b)
	switch err {
	case nil:
		c.advanceOffset(b.size())
		return

	case io.EOF:
		// cursor might have:
		// 1. reached end of the current segment: will advance to next segment
		// 2. reached end of tail
		if ok := c.advanceSegment(); !ok {
			return ErrEOQ
		}

		// advanced to next segment, read one block
		err = c.seg.ReadOne(b)
		switch err {
		case nil:
			// bingo!
			c.advanceOffset(b.size())
			return

		case io.EOF:
			// tail is empty
			return ErrEOQ

		default:
			return
		}

	default:
		return
	}
}

func (q *queue) EmptyInflight() bool {
	return q.emptyInflight.Get() == 1
}

// diskUsage returns the total size on disk used by the queue
func (q *queue) diskUsage() int64 {
	var size int64
	for _, s := range q.segments {
		size += s.DiskUsage()
	}
	return size
}

// loadSegments loads all segments on disk
func (q *queue) loadSegments() (segments, error) {
	segments := []*segment{}

	files, err := ioutil.ReadDir(q.dir)
	if err != nil {
		return segments, err
	}

	for _, segment := range files {
		if segment.IsDir() || segment.Name() == cursorFile {
			continue
		}

		// Segments file names are all numeric
		id, err := strconv.ParseUint(segment.Name(), 10, 64)
		if err != nil {
			continue
		}

		segment, err := newSegment(id, filepath.Join(q.dir, segment.Name()), q.maxSegmentSize)
		if err != nil {
			return segments, err
		}

		segments = append(segments, segment)
	}
	return segments, nil
}

// addSegment creates a new empty segment file
// caller is responsible for the lock
func (q *queue) addSegment() (*segment, error) {
	nextID, err := q.nextSegmentID()
	if err != nil {
		return nil, err
	}

	path := filepath.Join(q.dir, fmt.Sprintf("%020d", nextID))
	segment, err := newSegment(nextID, path, q.maxSegmentSize)
	if err != nil {
		return nil, err
	}

	q.segments = append(q.segments, segment)
	return segment, nil
}

// nextSegmentID returns the next segment ID that is free
func (q *queue) nextSegmentID() (uint64, error) {
	segments, err := ioutil.ReadDir(q.dir)
	if err != nil {
		return 0, err
	}

	var maxID uint64
	for _, segment := range segments {
		if segment.IsDir() || segment.Name() == cursorFile {
			continue
		}

		// Segments file names are all numeric
		segmentID, err := strconv.ParseUint(segment.Name(), 10, 64)
		if err != nil {
			log.Warn("unexpected segment file: %s", filepath.Join(q.dir, segment.Name()))
			continue
		}

		if segmentID > maxID {
			maxID = segmentID
		}
	}

	return maxID + 1, nil
}

func (q *queue) ident() string {
	return q.dir
}

func (q *queue) trimHead() (err error) {
	q.segments = q.segments[1:]

	if err = q.head.Remove(); err != nil {
		return
	}

	q.head = q.segments[0]
	return
}

func (q *queue) nextDir() string {
	// find least loaded dir
	return ""
}

// skipCursorSegment skip the current corrupted cursor segment and
// advance to next segment.
// if tail corrupts, add new segment.
func (q *queue) skipCursorSegment() {

}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"fmt"
	"io"
	"path"

	"github.com/lithammer/dedent"
	"github.com/spf13/cobra"
	flag "github.com/spf13/pflag"

	"k8s.io/apimachinery/pkg/util/sets"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/klog/v2"

	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
	kubeadmapiv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
	phases "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/reset"
	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
	utilruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime"
)

var (
	iptablesCleanupInstructions = dedent.Dedent(`
		The reset process does not reset or clean up iptables rules or IPVS tables.
		If you wish to reset iptables, you must do so manually by using the "iptables" command.

		If your cluster was setup to utilize IPVS, run ipvsadm --clear (or similar)
		to reset your system's IPVS tables.

		The reset process does not clean your kubeconfig files and you must remove them manually.
		Please, check the contents of the $HOME/.kube/config file.
	`)

	cniCleanupInstructions = dedent.Dedent(`
		The reset process does not clean CNI configuration. To do so, you must remove /etc/cni/net.d
	`)
)

// resetOptions defines all the options exposed via flags by kubeadm reset.
type resetOptions struct {
	certificatesDir       string
	criSocketPath         string
	forceReset            bool
	ignorePreflightErrors []string
	kubeconfigPath        string
	dryRun                bool
	cleanupTmpDir         bool
}

// resetData defines all the runtime information used when running the kubeadm reset workflow;
// this data is shared across all the phases that are included in the workflow.
type resetData struct {
	certificatesDir       string
	client                clientset.Interface
	criSocketPath         string
	forceReset            bool
	ignorePreflightErrors sets.String
	inputReader           io.Reader
	outputWriter          io.Writer
	cfg                   *kubeadmapi.InitConfiguration
	dryRun                bool
	cleanupTmpDir         bool
}

// newResetOptions returns a struct ready for being used for creating cmd join flags.
func newResetOptions() *resetOptions {
	return &resetOptions{
		certificatesDir: kubeadmapiv1.DefaultCertificatesDir,
		forceReset:      false,
		kubeconfigPath:  kubeadmconstants.GetAdminKubeConfigPath(),
		cleanupTmpDir:   false,
	}
}

// newResetData returns a new resetData struct to be used for the execution of the kubeadm reset workflow.
func newResetData(cmd *cobra.Command, options *resetOptions, in io.Reader, out io.Writer) (*resetData, error) {
	var cfg *kubeadmapi.InitConfiguration

	client, err := cmdutil.GetClientSet(options.kubeconfigPath, false)
	if err == nil {
		klog.V(1).Infof("[reset] Loaded client set from kubeconfig file: %s", options.kubeconfigPath)
		cfg, err = configutil.FetchInitConfigurationFromCluster(client, nil, "reset", false, false)
		if err != nil {
			klog.Warningf("[reset] Unable to fetch the kubeadm-config ConfigMap from cluster: %v", err)
		}
	} else {
		klog.V(1).Infof("[reset] Could not obtain a client set from the kubeconfig file: %s", options.kubeconfigPath)
	}

	ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(options.ignorePreflightErrors, ignorePreflightErrors(cfg))
	if err != nil {
		return nil, err
	}
	if cfg != nil {
		// Also set the union of pre-flight errors to InitConfiguration, to provide a consistent view of the runtime configuration:
		cfg.NodeRegistration.IgnorePreflightErrors = ignorePreflightErrorsSet.List()
	}

	var criSocketPath string
	if options.criSocketPath == "" {
		criSocketPath, err = resetDetectCRISocket(cfg)
		if err != nil {
			return nil, err
		}
		klog.V(1).Infof("[reset] Detected and using CRI socket: %s", criSocketPath)
	} else {
		criSocketPath = options.criSocketPath
		klog.V(1).Infof("[reset] Using specified CRI socket: %s", criSocketPath)
	}

	return &resetData{
		certificatesDir:       options.certificatesDir,
		client:                client,
		criSocketPath:         criSocketPath,
		forceReset:            options.forceReset,
		ignorePreflightErrors: ignorePreflightErrorsSet,
		inputReader:           in,
		outputWriter:          out,
		cfg:                   cfg,
		dryRun:                options.dryRun,
		cleanupTmpDir:         options.cleanupTmpDir,
	}, nil
}

func ignorePreflightErrors(cfg *kubeadmapi.InitConfiguration) []string {
	if cfg == nil {
		return []string{}
	}
	return cfg.NodeRegistration.IgnorePreflightErrors
}

// AddResetFlags adds reset flags
func AddResetFlags(flagSet *flag.FlagSet, resetOptions *resetOptions) {
	flagSet.StringVar(
		&resetOptions.certificatesDir, options.CertificatesDir, resetOptions.certificatesDir,
		`The path to the directory where the certificates are stored. If specified, clean this directory.`,
	)
	flagSet.BoolVarP(
		&resetOptions.forceReset, options.ForceReset, "f", false,
		"Reset the node without prompting for confirmation.",
	)
	flagSet.BoolVar(
		&resetOptions.dryRun, options.DryRun, resetOptions.dryRun,
		"Don't apply any changes; just output what would be done.",
	)
	flagSet.BoolVar(
		&resetOptions.cleanupTmpDir, options.CleanupTmpDir, resetOptions.cleanupTmpDir,
		fmt.Sprintf("Cleanup the %q directory", path.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.TempDirForKubeadm)),
	)

	options.AddKubeConfigFlag(flagSet, &resetOptions.kubeconfigPath)
	options.AddIgnorePreflightErrorsFlag(flagSet, &resetOptions.ignorePreflightErrors)
	cmdutil.AddCRISocketFlag(flagSet, &resetOptions.criSocketPath)
}

// newCmdReset returns the "kubeadm reset" command
func newCmdReset(in io.Reader, out io.Writer, resetOptions *resetOptions) *cobra.Command {
	if resetOptions == nil {
		resetOptions = newResetOptions()
	}
	resetRunner := workflow.NewRunner()

	cmd := &cobra.Command{
		Use:   "reset",
		Short: "Performs a best effort revert of changes made to this host by 'kubeadm init' or 'kubeadm join'",
		RunE: func(cmd *cobra.Command, args []string) error {
			err := resetRunner.Run(args)
			if err != nil {
				return err
			}

			// output help text instructing user how to remove cni folders
			fmt.Print(cniCleanupInstructions)
			// Output help text instructing user how to remove iptables rules
			fmt.Print(iptablesCleanupInstructions)
			return nil
		},
	}

	AddResetFlags(cmd.Flags(), resetOptions)

	// initialize the workflow runner with the list of phases
	resetRunner.AppendPhase(phases.NewPreflightPhase())
	resetRunner.AppendPhase(phases.NewRemoveETCDMemberPhase())
	resetRunner.AppendPhase(phases.NewCleanupNodePhase())

	// sets the data builder function, that will be used by the runner
	// both when running the entire workflow or single phases
	resetRunner.SetDataInitializer(func(cmd *cobra.Command, args []string) (workflow.RunData, error) {
		return newResetData(cmd, resetOptions, in, out)
	})

	// binds the Runner to kubeadm reset command by altering
	// command help, adding --skip-phases flag and by adding phases subcommands
	resetRunner.BindToCommand(cmd)

	return cmd
}

// Cfg returns the InitConfiguration.
func (r *resetData) Cfg() *kubeadmapi.InitConfiguration {
	return r.cfg
}

// DryRun returns the dryRun flag.
func (r *resetData) DryRun() bool {
	return r.dryRun
}

// CleanupTmpDir returns the cleanupTmpDir flag.
func (r *resetData) CleanupTmpDir() bool {
	return r.cleanupTmpDir
}

// CertificatesDir returns the CertificatesDir.
func (r *resetData) CertificatesDir() string {
	return r.certificatesDir
}

// Client returns the Client for accessing the cluster.
func (r *resetData) Client() clientset.Interface {
	return r.client
}

// ForceReset returns the forceReset flag.
func (r *resetData) ForceReset() bool {
	return r.forceReset
}

// InputReader returns the io.reader used to read messages.
func (r *resetData) InputReader() io.Reader {
	return r.inputReader
}

// IgnorePreflightErrors returns the list of preflight errors to ignore.
func (r *resetData) IgnorePreflightErrors() sets.String {
	return r.ignorePreflightErrors
}

// CRISocketPath returns the criSocketPath.
func (r *resetData) CRISocketPath() string {
	return r.criSocketPath
}

func resetDetectCRISocket(cfg *kubeadmapi.InitConfiguration) (string, error) {
	if cfg != nil {
		// first try to get the CRI socket from the cluster configuration
		return cfg.NodeRegistration.CRISocket, nil
	}

	// if this fails, try to detect it
	return utilruntime.DetectCRISocket()
}
<commit_msg>kubeadm: Don't reuse the `ignorePreflightErrors` from initCfg for `reset`<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"fmt"
	"io"
	"path"

	"github.com/lithammer/dedent"
	"github.com/spf13/cobra"
	flag "github.com/spf13/pflag"

	"k8s.io/apimachinery/pkg/util/sets"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/klog/v2"

	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
	kubeadmapiv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
	phases "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/reset"
	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
	utilruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime"
)

var (
	iptablesCleanupInstructions = dedent.Dedent(`
		The reset process does not reset or clean up iptables rules or IPVS tables.
		If you wish to reset iptables, you must do so manually by using the "iptables" command.

		If your cluster was setup to utilize IPVS, run ipvsadm --clear (or similar)
		to reset your system's IPVS tables.

		The reset process does not clean your kubeconfig files and you must remove them manually.
		Please, check the contents of the $HOME/.kube/config file.
	`)

	cniCleanupInstructions = dedent.Dedent(`
		The reset process does not clean CNI configuration. To do so, you must remove /etc/cni/net.d
	`)
)

// resetOptions defines all the options exposed via flags by kubeadm reset.
type resetOptions struct {
	certificatesDir       string
	criSocketPath         string
	forceReset            bool
	ignorePreflightErrors []string
	kubeconfigPath        string
	dryRun                bool
	cleanupTmpDir         bool
}

// resetData defines all the runtime information used when running the kubeadm reset workflow;
// this data is shared across all the phases that are included in the workflow.
type resetData struct {
	certificatesDir       string
	client                clientset.Interface
	criSocketPath         string
	forceReset            bool
	ignorePreflightErrors sets.String
	inputReader           io.Reader
	outputWriter          io.Writer
	cfg                   *kubeadmapi.InitConfiguration
	dryRun                bool
	cleanupTmpDir         bool
}

// newResetOptions returns a struct ready for being used for creating cmd join flags.
func newResetOptions() *resetOptions {
	return &resetOptions{
		certificatesDir: kubeadmapiv1.DefaultCertificatesDir,
		forceReset:      false,
		kubeconfigPath:  kubeadmconstants.GetAdminKubeConfigPath(),
		cleanupTmpDir:   false,
	}
}

// newResetData returns a new resetData struct to be used for the execution of the kubeadm reset workflow.
func newResetData(cmd *cobra.Command, options *resetOptions, in io.Reader, out io.Writer) (*resetData, error) {
	var cfg *kubeadmapi.InitConfiguration

	client, err := cmdutil.GetClientSet(options.kubeconfigPath, false)
	if err == nil {
		klog.V(1).Infof("[reset] Loaded client set from kubeconfig file: %s", options.kubeconfigPath)
		cfg, err = configutil.FetchInitConfigurationFromCluster(client, nil, "reset", false, false)
		if err != nil {
			klog.Warningf("[reset] Unable to fetch the kubeadm-config ConfigMap from cluster: %v", err)
		}
	} else {
		klog.V(1).Infof("[reset] Could not obtain a client set from the kubeconfig file: %s", options.kubeconfigPath)
	}

	ignorePreflightErrorsFromCfg := []string{}
	ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(options.ignorePreflightErrors, ignorePreflightErrorsFromCfg)
	if err != nil {
		return nil, err
	}
	if cfg != nil {
		// Also set the union of pre-flight errors to InitConfiguration, to provide a consistent view of the runtime configuration:
		cfg.NodeRegistration.IgnorePreflightErrors = ignorePreflightErrorsSet.List()
	}

	var criSocketPath string
	if options.criSocketPath == "" {
		criSocketPath, err = resetDetectCRISocket(cfg)
		if err != nil {
			return nil, err
		}
		klog.V(1).Infof("[reset] Detected and using CRI socket: %s", criSocketPath)
	} else {
		criSocketPath = options.criSocketPath
		klog.V(1).Infof("[reset] Using specified CRI socket: %s", criSocketPath)
	}

	return &resetData{
		certificatesDir:       options.certificatesDir,
		client:                client,
		criSocketPath:         criSocketPath,
		forceReset:            options.forceReset,
		ignorePreflightErrors: ignorePreflightErrorsSet,
		inputReader:           in,
		outputWriter:          out,
		cfg:                   cfg,
		dryRun:                options.dryRun,
		cleanupTmpDir:         options.cleanupTmpDir,
	}, nil
}

// AddResetFlags adds reset flags
func AddResetFlags(flagSet *flag.FlagSet, resetOptions *resetOptions) {
	flagSet.StringVar(
		&resetOptions.certificatesDir, options.CertificatesDir, resetOptions.certificatesDir,
		`The path to the directory where the certificates are stored. If specified, clean this directory.`,
	)
	flagSet.BoolVarP(
		&resetOptions.forceReset, options.ForceReset, "f", false,
		"Reset the node without prompting for confirmation.",
	)
	flagSet.BoolVar(
		&resetOptions.dryRun, options.DryRun, resetOptions.dryRun,
		"Don't apply any changes; just output what would be done.",
	)
	flagSet.BoolVar(
		&resetOptions.cleanupTmpDir, options.CleanupTmpDir, resetOptions.cleanupTmpDir,
		fmt.Sprintf("Cleanup the %q directory", path.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.TempDirForKubeadm)),
	)

	options.AddKubeConfigFlag(flagSet, &resetOptions.kubeconfigPath)
	options.AddIgnorePreflightErrorsFlag(flagSet, &resetOptions.ignorePreflightErrors)
	cmdutil.AddCRISocketFlag(flagSet, &resetOptions.criSocketPath)
}

// newCmdReset returns the "kubeadm reset" command
func newCmdReset(in io.Reader, out io.Writer, resetOptions *resetOptions) *cobra.Command {
	if resetOptions == nil {
		resetOptions = newResetOptions()
	}
	resetRunner := workflow.NewRunner()

	cmd := &cobra.Command{
		Use:   "reset",
		Short: "Performs a best effort revert of changes made to this host by 'kubeadm init' or 'kubeadm join'",
		RunE: func(cmd *cobra.Command, args []string) error {
			err := resetRunner.Run(args)
			if err != nil {
				return err
			}

			// output help text instructing user how to remove cni folders
			fmt.Print(cniCleanupInstructions)
			// Output help text instructing user how to remove iptables rules
			fmt.Print(iptablesCleanupInstructions)
			return nil
		},
	}

	AddResetFlags(cmd.Flags(), resetOptions)

	// initialize the workflow runner with the list of phases
	resetRunner.AppendPhase(phases.NewPreflightPhase())
	resetRunner.AppendPhase(phases.NewRemoveETCDMemberPhase())
	resetRunner.AppendPhase(phases.NewCleanupNodePhase())

	// sets the data builder function, that will be used by the runner
	// both when running the entire workflow or single phases
	resetRunner.SetDataInitializer(func(cmd *cobra.Command, args []string) (workflow.RunData, error) {
		return newResetData(cmd, resetOptions, in, out)
	})

	// binds the Runner to kubeadm reset command by altering
	// command help, adding --skip-phases flag and by adding phases subcommands
	resetRunner.BindToCommand(cmd)

	return cmd
}

// Cfg returns the InitConfiguration.
func (r *resetData) Cfg() *kubeadmapi.InitConfiguration {
	return r.cfg
}

// DryRun returns the dryRun flag.
func (r *resetData) DryRun() bool {
	return r.dryRun
}

// CleanupTmpDir returns the cleanupTmpDir flag.
func (r *resetData) CleanupTmpDir() bool {
	return r.cleanupTmpDir
}

// CertificatesDir returns the CertificatesDir.
func (r *resetData) CertificatesDir() string {
	return r.certificatesDir
}

// Client returns the Client for accessing the cluster.
func (r *resetData) Client() clientset.Interface {
	return r.client
}

// ForceReset returns the forceReset flag.
func (r *resetData) ForceReset() bool {
	return r.forceReset
}

// InputReader returns the io.reader used to read messages.
func (r *resetData) InputReader() io.Reader {
	return r.inputReader
}

// IgnorePreflightErrors returns the list of preflight errors to ignore.
func (r *resetData) IgnorePreflightErrors() sets.String {
	return r.ignorePreflightErrors
}

// CRISocketPath returns the criSocketPath.
func (r *resetData) CRISocketPath() string {
	return r.criSocketPath
}

func resetDetectCRISocket(cfg *kubeadmapi.InitConfiguration) (string, error) {
	if cfg != nil {
		// first try to get the CRI socket from the cluster configuration
		return cfg.NodeRegistration.CRISocket, nil
	}

	// if this fails, try to detect it
	return utilruntime.DetectCRISocket()
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"io/ioutil"
	"testing"

	"github.com/GoogleContainerTools/skaffold/testutil"
)

func TestPrintMan(t *testing.T) {
	var stdout bytes.Buffer
	var stderr bytes.Buffer

	err := printMan(&stdout, &stderr)
	output := stdout.String()

	// Sanity checks
	testutil.CheckError(t, false, err)
	testutil.CheckDeepEqual(t, "", stderr.String())
	testutil.CheckContains(t, "skaffold build", output)
	testutil.CheckContains(t, "skaffold run", output)
	testutil.CheckContains(t, "skaffold dev", output)
	testutil.CheckContains(t, "Env vars", output)

	// Compare to current man page
	header, err := ioutil.ReadFile("../../../docs/content/en/docs/references/cli/index_header")
	testutil.CheckError(t, false, err)

	expected, err := ioutil.ReadFile("../../../docs/content/en/docs/references/cli/_index.md")
	testutil.CheckError(t, false, err)
	if string(expected) != string(header)+output {
		t.Error("You have skaffold command changes but haven't generated the CLI reference docs. Please run ./hack/generate-man.sh and commit the results!")
	}
}
<commit_msg>Fix master branch<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"io/ioutil"
	"path/filepath"
	"testing"

	"github.com/GoogleContainerTools/skaffold/testutil"
)

func TestPrintMan(t *testing.T) {
	var stdout bytes.Buffer
	var stderr bytes.Buffer

	err := printMan(&stdout, &stderr)
	output := stdout.String()

	// Sanity checks
	testutil.CheckError(t, false, err)
	testutil.CheckDeepEqual(t, "", stderr.String())
	testutil.CheckContains(t, "skaffold build", output)
	testutil.CheckContains(t, "skaffold run", output)
	testutil.CheckContains(t, "skaffold dev", output)
	testutil.CheckContains(t, "Env vars", output)

	// Compare to current man page
	header, err := ioutil.ReadFile(filepath.Join("..", "..", "..", "docs", "content", "en", "docs", "references", "cli", "index_header"))
	testutil.CheckError(t, false, err)
	header = bytes.Replace(header, []byte("\r\n"), []byte("\n"), -1)

	expected, err := ioutil.ReadFile(filepath.Join("..", "..", "..", "docs", "content", "en", "docs", "references", "cli", "_index.md"))
	testutil.CheckError(t, false, err)
	expected = bytes.Replace(expected, []byte("\r\n"), []byte("\n"), -1)

	if string(expected) != string(header)+output {
		t.Error("You have skaffold command changes but haven't generated the CLI reference docs. Please run ./hack/generate-man.sh and commit the results!")
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

// Copied from github.com/broady/cdbuild.
// TODO(adg): clean this up.

package main

import (
	"archive/tar"
	"compress/gzip"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"github.com/satori/go.uuid"

	cstorage "cloud.google.com/go/storage"
	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/cloudbuild/v1"
	"google.golang.org/api/googleapi"
	"google.golang.org/api/storage/v1"
)

func cdbuild(dir, projectID, name string) error {
	stagingBucket := projectID + "-cdbuild"
	buildObject := fmt.Sprintf("build/%s-%s.tar.gz", name, uuid.NewV4())

	ctx := context.Background()
	hc, err := google.DefaultClient(ctx, storage.CloudPlatformScope)
	if err != nil {
		return fmt.Errorf("Could not get authenticated HTTP client: %v", err)
	}

	log.Printf("Pushing code to gs://%s/%s", stagingBucket, buildObject)

	if err := uploadTar(ctx, dir, hc, stagingBucket, buildObject); err != nil {
		return fmt.Errorf("Could not upload source: %v", err)
	}

	api, err := cloudbuild.New(hc)
	if err != nil {
		return fmt.Errorf("Could not get cloudbuild client: %v", err)
	}
	call := api.Projects.Builds.Create(projectID, &cloudbuild.Build{
		LogsBucket: stagingBucket,
		Source: &cloudbuild.Source{
			StorageSource: &cloudbuild.StorageSource{
				Bucket: stagingBucket,
				Object: buildObject,
			},
		},
		Steps: []*cloudbuild.BuildStep{
			{
				Name: "gcr.io/cloud-builders/dockerizer",
				Args: []string{"gcr.io/" + projectID + "/" + name},
			},
		},
		Images: []string{"gcr.io/" + projectID + "/" + name},
	})
	op, err := call.Context(ctx).Do()
	if err != nil {
		if gerr, ok := err.(*googleapi.Error); ok {
			if gerr.Code == 404 {
				// HACK(cbro): the API does not return a good error if the API is not enabled.
				fmt.Fprintln(os.Stderr, "Could not create build. It's likely the Cloud Container Builder API is not enabled.")
				fmt.Fprintf(os.Stderr, "Go here to enable it: https://console.cloud.google.com/apis/api/cloudbuild.googleapis.com/overview?project=%s\n", projectID)
				os.Exit(1)
			}
		}
		return fmt.Errorf("Could not create build: %#v", err)
	}
	remoteID, err := getBuildID(op)
	if err != nil {
		return fmt.Errorf("Could not get build ID from op: %v", err)
	}

	log.Printf("Logs at https://console.cloud.google.com/m/cloudstorage/b/%s/o/log-%s.txt", stagingBucket, remoteID)

	fail := false
	for {
		b, err := api.Projects.Builds.Get(projectID, remoteID).Do()
		if err != nil {
			return fmt.Errorf("Could not get build status: %v", err)
		}

		if s := b.Status; s != "WORKING" && s != "QUEUED" {
			if b.Status == "FAILURE" {
				fail = true
			}
			log.Printf("Build status: %v", s)
			break
		}

		time.Sleep(time.Second)
	}

	c, err := cstorage.NewClient(ctx)
	if err != nil {
		return fmt.Errorf("Could not make Cloud storage client: %v", err)
	}
	defer c.Close()
	if err := c.Bucket(stagingBucket).Object(buildObject).Delete(ctx); err != nil {
		return fmt.Errorf("Could not delete source tar.gz: %v", err)
	}
	log.Print("Cleaned up.")
	if fail {
		return fmt.Errorf("cdbuild failed")
	}

	return nil
}

// HACK: workaround for lack of type for "Metadata" field.
func getBuildID(op *cloudbuild.Operation) (string, error) {
	if op.Metadata == nil {
		return "", errors.New("missing Metadata in operation")
	}
	if m, ok := op.Metadata.(map[string]interface{}); ok {
		b, err := json.Marshal(m["build"])
		if err != nil {
			return "", err
		}
		build := &cloudbuild.Build{}
		if err := json.Unmarshal(b, &build); err != nil {
			return "", err
		}
		return build.Id, nil
	}
	return "", errors.New("unknown type for op")
}

func uploadTar(ctx context.Context, root string, hc *http.Client, bucket string, objectName string) error {
	c, err := cstorage.NewClient(ctx)
	if err != nil {
		return err
	}
	defer c.Close()

	w := c.Bucket(bucket).Object(objectName).NewWriter(ctx)
	gzw := gzip.NewWriter(w)
	tw := tar.NewWriter(gzw)

	if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		if path == root {
			return nil
		}
		relpath, err := filepath.Rel(root, path)
		if err != nil {
			return err
		}
		info = renamingFileInfo{info, relpath}

		hdr, err := tar.FileInfoHeader(info, "")
		if err != nil {
			return err
		}
		if err := tw.WriteHeader(hdr); err != nil {
			return err
		}
		f, err := os.Open(path)
		if err != nil {
			return err
		}
		defer f.Close()
		_, err = io.Copy(tw, f)
		return err
	}); err != nil {
		w.CloseWithError(err)
		return err
	}
	if err := tw.Close(); err != nil {
		w.CloseWithError(err)
		return err
	}
	if err := gzw.Close(); err != nil {
		w.CloseWithError(err)
		return err
	}
	return w.Close()
}

type renamingFileInfo struct {
	os.FileInfo
	name string
}

func (fi renamingFileInfo) Name() string {
	return fi.name
}
<commit_msg>cmd/upspin-deploy: drop dependency on uuid package<commit_after>// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

// Copied from github.com/broady/cdbuild.
// TODO(adg): clean this up.

package main

import (
	"archive/tar"
	"compress/gzip"
	"crypto/rand"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"time"

	cstorage "cloud.google.com/go/storage"
	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/cloudbuild/v1"
	"google.golang.org/api/googleapi"
	"google.golang.org/api/storage/v1"
)

func cdbuild(dir, projectID, name string) error {
	stagingBucket := projectID + "-cdbuild"
	buildObject := fmt.Sprintf("build/%s-%s.tar.gz", name, randomID())

	ctx := context.Background()
	hc, err := google.DefaultClient(ctx, storage.CloudPlatformScope)
	if err != nil {
		return fmt.Errorf("Could not get authenticated HTTP client: %v", err)
	}

	log.Printf("Pushing code to gs://%s/%s", stagingBucket, buildObject)

	if err := uploadTar(ctx, dir, hc, stagingBucket, buildObject); err != nil {
		return fmt.Errorf("Could not upload source: %v", err)
	}

	api, err := cloudbuild.New(hc)
	if err != nil {
		return fmt.Errorf("Could not get cloudbuild client: %v", err)
	}
	call := api.Projects.Builds.Create(projectID, &cloudbuild.Build{
		LogsBucket: stagingBucket,
		Source: &cloudbuild.Source{
			StorageSource: &cloudbuild.StorageSource{
				Bucket: stagingBucket,
				Object: buildObject,
			},
		},
		Steps: []*cloudbuild.BuildStep{
			{
				Name: "gcr.io/cloud-builders/dockerizer",
				Args: []string{"gcr.io/" + projectID + "/" + name},
			},
		},
		Images: []string{"gcr.io/" + projectID + "/" + name},
	})
	op, err := call.Context(ctx).Do()
	if err != nil {
		if gerr, ok := err.(*googleapi.Error); ok {
			if gerr.Code == 404 {
				// HACK(cbro): the API does not return a good error if the API is not enabled.
				fmt.Fprintln(os.Stderr, "Could not create build. It's likely the Cloud Container Builder API is not enabled.")
				fmt.Fprintf(os.Stderr, "Go here to enable it: https://console.cloud.google.com/apis/api/cloudbuild.googleapis.com/overview?project=%s\n", projectID)
				os.Exit(1)
			}
		}
		return fmt.Errorf("Could not create build: %#v", err)
	}
	remoteID, err := getBuildID(op)
	if err != nil {
		return fmt.Errorf("Could not get build ID from op: %v", err)
	}

	log.Printf("Logs at https://console.cloud.google.com/m/cloudstorage/b/%s/o/log-%s.txt", stagingBucket, remoteID)

	fail := false
	for {
		b, err := api.Projects.Builds.Get(projectID, remoteID).Do()
		if err != nil {
			return fmt.Errorf("Could not get build status: %v", err)
		}

		if s := b.Status; s != "WORKING" && s != "QUEUED" {
			if b.Status == "FAILURE" {
				fail = true
			}
			log.Printf("Build status: %v", s)
			break
		}

		time.Sleep(time.Second)
	}

	c, err := cstorage.NewClient(ctx)
	if err != nil {
		return fmt.Errorf("Could not make Cloud storage client: %v", err)
	}
	defer c.Close()
	if err := c.Bucket(stagingBucket).Object(buildObject).Delete(ctx); err != nil {
		return fmt.Errorf("Could not delete source tar.gz: %v", err)
	}
	log.Print("Cleaned up.")
	if fail {
		return fmt.Errorf("cdbuild failed")
	}

	return nil
}

// HACK: workaround for lack of type for "Metadata" field.
func getBuildID(op *cloudbuild.Operation) (string, error) {
	if op.Metadata == nil {
		return "", errors.New("missing Metadata in operation")
	}
	if m, ok := op.Metadata.(map[string]interface{}); ok {
		b, err := json.Marshal(m["build"])
		if err != nil {
			return "", err
		}
		build := &cloudbuild.Build{}
		if err := json.Unmarshal(b, &build); err != nil {
			return "", err
		}
		return build.Id, nil
	}
	return "", errors.New("unknown type for op")
}

func uploadTar(ctx context.Context, root string, hc *http.Client, bucket string, objectName string) error {
	c, err := cstorage.NewClient(ctx)
	if err != nil {
		return err
	}
	defer c.Close()

	w := c.Bucket(bucket).Object(objectName).NewWriter(ctx)
	gzw := gzip.NewWriter(w)
	tw := tar.NewWriter(gzw)

	if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		if path == root {
			return nil
		}
		relpath, err := filepath.Rel(root, path)
		if err != nil {
			return err
		}
		info = renamingFileInfo{info, relpath}

		hdr, err := tar.FileInfoHeader(info, "")
		if err != nil {
			return err
		}
		if err := tw.WriteHeader(hdr); err != nil {
			return err
		}
		f, err := os.Open(path)
		if err != nil {
			return err
		}
		defer f.Close()
		_, err = io.Copy(tw, f)
		return err
	}); err != nil {
		w.CloseWithError(err)
		return err
	}
	if err := tw.Close(); err != nil {
		w.CloseWithError(err)
		return err
	}
	if err := gzw.Close(); err != nil {
		w.CloseWithError(err)
		return err
	}
	return w.Close()
}

type renamingFileInfo struct {
	os.FileInfo
	name string
}

func (fi renamingFileInfo) Name() string {
	return fi.name
}

func randomID() string {
	b := make([]byte, 16)
	_, err := rand.Read(b)
	if err != nil {
		panic(err)
	}
	return fmt.Sprintf("%x", b)
}
<|endoftext|>


<commit_before>package sync

import (
	"autoscaler/db"
	"autoscaler/models"
	"os"
	"time"

	"code.cloudfoundry.org/lager"

	"github.com/tedsuo/ifrit"
)

type DatabaseLock struct {
	logger lager.Logger
}

func NewDatabaseLock(logger lager.Logger) *DatabaseLock {
	return &DatabaseLock{
		logger: logger,
	}
}

func (dblock *DatabaseLock) InitDBLockRunner(retryInterval time.Duration, ttl time.Duration, owner string, lockDB db.LockDB) ifrit.Runner {
	dbLockMaintainer := ifrit.RunFunc(func(signals <-chan os.Signal, ready chan<- struct{}) error {
		lockTicker := time.NewTicker(retryInterval)
		readyToAcquireLock := true
		if owner == "" {
			dblock.logger.Info("failed-to-get-owner-details")
			os.Exit(1)
		}
		lock := &models.Lock{Owner: owner, Ttl: ttl}
		isLockAcquired, lockErr := lockDB.Lock(lock)
		if lockErr != nil {
			dblock.logger.Error("failed-to-acquire-lock-in-first-attempt", lockErr)
		}
		if isLockAcquired {
			readyToAcquireLock = false
			dblock.logger.Info("lock-acquired-in-first-attempt", lager.Data{"owner": owner, "isLockAcquired": isLockAcquired})
			close(ready)
		}
		for {
			select {
			case <-signals:
				dblock.logger.Info("received-interrupt-signal", lager.Data{"owner": owner})
				lockTicker.Stop()
				releaseErr := lockDB.Release(owner)
				if releaseErr != nil {
					dblock.logger.Error("failed-to-release-lock ", releaseErr)
				} else {
					dblock.logger.Debug("successfully-released-lock", lager.Data{"owner": owner})
				}
				readyToAcquireLock = true
				return nil

			case <-lockTicker.C:
				dblock.logger.Debug("retry-acquiring-lock", lager.Data{"owner": owner})
				lock := &models.Lock{Owner: owner, Ttl: ttl}
				isLockAcquired, lockErr := lockDB.Lock(lock)
				if lockErr != nil {
					dblock.logger.Error("failed-to-acquire-lock", lockErr)
					releaseErr := lockDB.Release(owner)
					if releaseErr != nil {
						dblock.logger.Error("failed-to-release-lock ", releaseErr)
					} else {
						dblock.logger.Debug("successfully-released-lock", lager.Data{"owner": owner})
					}
					os.Exit(1)
				}
				if isLockAcquired && readyToAcquireLock {
					readyToAcquireLock = false
					dblock.logger.Debug("successfully-acquired-lock", lager.Data{"owner": owner})
					close(ready)
				}
			}
		}
	})
	return dbLockMaintainer
}
<commit_msg>update for comments<commit_after>package sync

import (
	"autoscaler/db"
	"autoscaler/models"
	"os"
	"time"

	"code.cloudfoundry.org/lager"

	"github.com/tedsuo/ifrit"
)

type DatabaseLock struct {
	logger lager.Logger
}

func NewDatabaseLock(logger lager.Logger) *DatabaseLock {
	return &DatabaseLock{
		logger: logger,
	}
}

func (dblock *DatabaseLock) InitDBLockRunner(retryInterval time.Duration, ttl time.Duration, owner string, lockDB db.LockDB) ifrit.Runner {
	dbLockMaintainer := ifrit.RunFunc(func(signals <-chan os.Signal, ready chan<- struct{}) error {
		lockTicker := time.NewTicker(retryInterval)
		readyToAcquireLock := true
		if owner == "" {
			dblock.logger.Info("failed-to-get-owner-details")
			os.Exit(1)
		}
		lock := &models.Lock{Owner: owner, Ttl: ttl}
		isLockAcquired, lockErr := lockDB.Lock(lock)
		if lockErr != nil {
			dblock.logger.Error("failed-to-acquire-lock-in-first-attempt", lockErr)
		}
		if isLockAcquired {
			readyToAcquireLock = false
			dblock.logger.Info("lock-acquired-in-first-attempt", lager.Data{"owner": owner, "isLockAcquired": isLockAcquired})
			close(ready)
		}
		for {
			select {
			case <-signals:
				dblock.logger.Info("received-interrupt-signal", lager.Data{"owner": owner})
				lockTicker.Stop()
				releaseErr := lockDB.Release(owner)
				if releaseErr != nil {
					dblock.logger.Error("failed-to-release-lock ", releaseErr)
				} else {
					dblock.logger.Debug("successfully-released-lock", lager.Data{"owner": owner})
				}
				readyToAcquireLock = true
				return nil

			case <-lockTicker.C:
				dblock.logger.Debug("retry-acquiring-lock", lager.Data{"owner": owner})
				lock := &models.Lock{Owner: owner, Ttl: ttl}
				isLockAcquired, lockErr := lockDB.Lock(lock)
				if lockErr != nil {
					dblock.logger.Error("failed-to-acquire-lock", lockErr)
					releaseErr := lockDB.Release(owner)
					if releaseErr != nil {
						dblock.logger.Error("failed-to-release-lock ", releaseErr)
					} else {
						dblock.logger.Info("successfully-released-lock", lager.Data{"owner": owner})
					}
					os.Exit(1)
				}
				if isLockAcquired && readyToAcquireLock {
					readyToAcquireLock = false
					dblock.logger.Info("successfully-acquired-lock", lager.Data{"owner": owner})
					close(ready)
				}
			}
		}
	})
	return dbLockMaintainer
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"github.com/DECK36/go-gelf/gelf"
	"io"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"time"
)

/*
	http://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html
	https://github.com/Graylog2/graylog2-docs/wiki/GELF
*/
type SystemdJournalEntry struct {
	Cursor                    string `json:"__CURSOR"`
	Realtime_timestamp        int64  `json:"__REALTIME_TIMESTAMP,string"`
	Monotonic_timestamp       string `json:"__MONOTONIC_TIMESTAMP"`
	Boot_id                   string `json:"_BOOT_ID"`
	Transport                 string `json:"_TRANSPORT"`
	Priority                  int32  `json:"PRIORITY,string"`
	Syslog_facility           string `json:"SYSLOG_FACILITY"`
	Syslog_identifier         string `json:"SYSLOG_IDENTIFIER"`
	Message                   string `json:"MESSAGE"`
	Pid                       string `json:"_PID"`
	Uid                       string `json:"_UID"`
	Gid                       string `json:"_GID"`
	Comm                      string `json:"_COMM"`
	Exe                       string `json:"_EXE"`
	Cmdline                   string `json:"_CMDLINE"`
	Systemd_cgroup            string `json:"_SYSTEMD_CGROUP"`
	Systemd_session           string `json:"_SYSTEMD_SESSION"`
	Systemd_owner_uid         string `json:"_SYSTEMD_OWNER_UID"`
	Systemd_unit              string `json:"_SYSTEMD_UNIT"`
	Source_realtime_timestamp string `json:"_SOURCE_REALTIME_TIMESTAMP"`
	Machine_id                string `json:"_MACHINE_ID"`
	Hostname                  string `json:"_HOSTNAME"`
	FullMessage               string
}

// Strip date from message-content. Use named subpatterns to override other fields
var messageReplace = map[string]*regexp.Regexp{
	"*":         regexp.MustCompile("^20[0-9][0-9][/\\-][01][0-9][/\\-][0123][0-9] [0-2]?[0-9]:[0-5][0-9]:[0-5][0-9][,0-9]{0-3} "),
	"nginx":     regexp.MustCompile("\\[(?P<Priority>[a-z]+)\\] "),
	"java":      regexp.MustCompile("(?P<Priority>[A-Z]+): "),
	"mysqld":    regexp.MustCompile("^[0-9]+ \\[(?P<Priority>[A-Z][a-z]+)\\] "),
	"searchd":   regexp.MustCompile("^\\[([A-Z][a-z]{2} ){2} [0-9]+ [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\\.[0-9]{3} 20[0-9][0-9]\\] \\[[ 0-9]+\\] "),
	"jenkins":   regexp.MustCompile("^[A-Z][a-z]{2} [01][0-9], 20[0-9][0-9] [0-2]?[0-9]:[0-5][0-9]:[0-5][0-9] [AP]M "),
	"php-fpm":   regexp.MustCompile("^pool [a-z_0-9\\[\\]]+: "),
	"syncthing": regexp.MustCompile("^\\[[0-9A-Z]{5}\\] [0-2][0-9]:[0-5][0-9]:[0-5][0-9] (?P<Priority>INFO): "),
}

var priorities = map[string]int32{
	"emergency": 0,
	"emerg":     0,
	"alert":     1,
	"critical":  2,
	"crit":      2,
	"error":     3,
	"err":       3,
	"warning":   4,
	"warn":      4,
	"notice":    5,
	"info":      6,
	"debug":     7,
}

func (this *SystemdJournalEntry) toGelf() *gelf.Message {
	var extra = map[string]interface{}{
		"Boot_id": this.Boot_id,
		"Pid":     this.Pid,
		"Uid":     this.Uid,
	}

	// php-fpm refuses to fill identifier
	facility := this.Syslog_identifier
	if "" == facility {
		facility = this.Comm
	}

	if this.isJsonMessage() {
		if err := json.Unmarshal([]byte(this.Message), &extra); err == nil {
			if m, ok := extra["Message"]; ok {
				this.Message = m.(string)
				delete(extra, "Message")
			}

			if f, ok := extra["FullMessage"]; ok {
				this.FullMessage = f.(string)
				delete(extra, "FullMessage")
			}
		}
	} else if -1 != strings.Index(this.Message, "\n") {
		this.FullMessage = this.Message
		this.Message = strings.Split(this.Message, "\n")[0]
	}

	return &gelf.Message{
		Version:  "1.1",
		Host:     this.Hostname,
		Short:    this.Message,
		Full:     this.FullMessage,
		TimeUnix: float64(this.Realtime_timestamp) / 1000 / 1000,
		Level:    this.Priority,
		Facility: facility,
		Extra:    extra,
	}
}

func (this *SystemdJournalEntry) process() {
	// Replace generic timestamp
	this.Message = messageReplace["*"].ReplaceAllString(this.Message, "")

	re := messageReplace[ this.Syslog_identifier ]
	if nil == re {
		re = messageReplace[ this.Comm ]
	}

	if nil == re {
		return
	}

	m := re.FindStringSubmatch(this.Message)
	if m == nil {
		return
	}

	// Store subpatterns in fields
	for idx, key := range re.SubexpNames() {
		if "Priority" == key {
			this.Priority = priorities[strings.ToLower(m[idx])]
		}
	}

	this.Message = re.ReplaceAllString(this.Message, "")
}

func (this *SystemdJournalEntry) sameSource(message *SystemdJournalEntry) bool {
	if this.Syslog_identifier != message.Syslog_identifier {
		return false
	}

	if this.Priority != message.Priority {
		return false
	}

	if this.Realtime_timestamp-message.Realtime_timestamp > SAMESOURCE_TIME_DIFFERENCE {
		return false
	}

	return true
}

func (this *SystemdJournalEntry) send() {
	message := this.toGelf()

	if err := writer.WriteMessage(message); err != nil {
		fmt.Fprintln(os.Stderr, err)
	}
}

func (this *SystemdJournalEntry) isJsonMessage() bool {
	return len(this.Message) > 64 && this.Message[0] == '{' && this.Message[1] == '"'
}

func (this *SystemdJournalEntry) extendWith(message *SystemdJournalEntry) {
	if this.FullMessage == "" {
		this.FullMessage = this.Message
	}

	this.FullMessage += "\n" + message.Message
}

var (
	pendingEntry *SystemdJournalEntry
	writer       *gelf.Writer
)

const (
	WRITE_INTERVAL             = 50 * time.Millisecond
	SAMESOURCE_TIME_DIFFERENCE = 100 * 1000
)

func main() {
	if len(os.Args) < 3 {
		fmt.Fprintln(os.Stderr, "Pass server:12201 as first argument and append journalctl parameters to use")
		os.Exit(1)
	}

	if w, err := gelf.NewWriter(os.Args[1]); err != nil {
		fmt.Fprintf(os.Stderr, "While connecting to Graylog server: %s\n", err)
		os.Exit(1)
	} else {
		writer = w
	}

	journalArgs := []string{"--all", "--output=json"}
	journalArgs = append(journalArgs, os.Args[2:]...)
	cmd := exec.Command("journalctl", journalArgs...)

	stderr, _ := cmd.StderrPipe()
	go io.Copy(os.Stderr, stderr)
	stdout, _ := cmd.StdoutPipe()
	s := bufio.NewScanner(stdout)

	go writePendingEntry()

	cmd.Start()

	for s.Scan() {
		line := s.Text()

		var entry = &SystemdJournalEntry{}
		if err := json.Unmarshal([]byte(line), &entry); err != nil {
			//fmt.Fprintf(os.Stderr, "Could not parse line, skipping: %s\n", line)
			continue
		}

		entry.process()

		if pendingEntry == nil {
			pendingEntry = entry
		} else if !pendingEntry.sameSource(entry) || pendingEntry.isJsonMessage() {
			pendingEntry.send()
			pendingEntry = entry
		} else {
			pendingEntry.extendWith(entry)

			// Keeps writePendingEntry waiting longer for us to append even more
			pendingEntry.Realtime_timestamp = entry.Realtime_timestamp
		}

		// Prevent saturation and throttling
		time.Sleep(1 * time.Millisecond)
	}

	if err := s.Err(); err != nil {
		fmt.Fprintf(os.Stderr, "Error from Scanner: %s\n", err)
		cmd.Process.Kill()
	}

	cmd.Wait()
	pendingEntry.send()
}

func writePendingEntry() {
	var entry *SystemdJournalEntry

	for {
		time.Sleep(WRITE_INTERVAL)

		if pendingEntry != nil && (time.Now().UnixNano()/1000-pendingEntry.Realtime_timestamp) > SAMESOURCE_TIME_DIFFERENCE {
			entry = pendingEntry
			pendingEntry = nil

			entry.send()
		}
	}
}
<commit_msg>When we exit due to a Scanner error, exit with non-zero exit-code so the service can restart us<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"github.com/DECK36/go-gelf/gelf"
	"io"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"time"
)

/*
	http://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html
	https://github.com/Graylog2/graylog2-docs/wiki/GELF
*/
type SystemdJournalEntry struct {
	Cursor                    string `json:"__CURSOR"`
	Realtime_timestamp        int64  `json:"__REALTIME_TIMESTAMP,string"`
	Monotonic_timestamp       string `json:"__MONOTONIC_TIMESTAMP"`
	Boot_id                   string `json:"_BOOT_ID"`
	Transport                 string `json:"_TRANSPORT"`
	Priority                  int32  `json:"PRIORITY,string"`
	Syslog_facility           string `json:"SYSLOG_FACILITY"`
	Syslog_identifier         string `json:"SYSLOG_IDENTIFIER"`
	Message                   string `json:"MESSAGE"`
	Pid                       string `json:"_PID"`
	Uid                       string `json:"_UID"`
	Gid                       string `json:"_GID"`
	Comm                      string `json:"_COMM"`
	Exe                       string `json:"_EXE"`
	Cmdline                   string `json:"_CMDLINE"`
	Systemd_cgroup            string `json:"_SYSTEMD_CGROUP"`
	Systemd_session           string `json:"_SYSTEMD_SESSION"`
	Systemd_owner_uid         string `json:"_SYSTEMD_OWNER_UID"`
	Systemd_unit              string `json:"_SYSTEMD_UNIT"`
	Source_realtime_timestamp string `json:"_SOURCE_REALTIME_TIMESTAMP"`
	Machine_id                string `json:"_MACHINE_ID"`
	Hostname                  string `json:"_HOSTNAME"`
	FullMessage               string
}

// Strip date from message-content. Use named subpatterns to override other fields
var messageReplace = map[string]*regexp.Regexp{
	"*":         regexp.MustCompile("^20[0-9][0-9][/\\-][01][0-9][/\\-][0123][0-9] [0-2]?[0-9]:[0-5][0-9]:[0-5][0-9][,0-9]{0-3} "),
	"nginx":     regexp.MustCompile("\\[(?P<Priority>[a-z]+)\\] "),
	"java":      regexp.MustCompile("(?P<Priority>[A-Z]+): "),
	"mysqld":    regexp.MustCompile("^[0-9]+ \\[(?P<Priority>[A-Z][a-z]+)\\] "),
	"searchd":   regexp.MustCompile("^\\[([A-Z][a-z]{2} ){2} [0-9]+ [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\\.[0-9]{3} 20[0-9][0-9]\\] \\[[ 0-9]+\\] "),
	"jenkins":   regexp.MustCompile("^[A-Z][a-z]{2} [01][0-9], 20[0-9][0-9] [0-2]?[0-9]:[0-5][0-9]:[0-5][0-9] [AP]M "),
	"php-fpm":   regexp.MustCompile("^pool [a-z_0-9\\[\\]]+: "),
	"syncthing": regexp.MustCompile("^\\[[0-9A-Z]{5}\\] [0-2][0-9]:[0-5][0-9]:[0-5][0-9] (?P<Priority>INFO): "),
}

var priorities = map[string]int32{
	"emergency": 0,
	"emerg":     0,
	"alert":     1,
	"critical":  2,
	"crit":      2,
	"error":     3,
	"err":       3,
	"warning":   4,
	"warn":      4,
	"notice":    5,
	"info":      6,
	"debug":     7,
}

func (this *SystemdJournalEntry) toGelf() *gelf.Message {
	var extra = map[string]interface{}{
		"Boot_id": this.Boot_id,
		"Pid":     this.Pid,
		"Uid":     this.Uid,
	}

	// php-fpm refuses to fill identifier
	facility := this.Syslog_identifier
	if "" == facility {
		facility = this.Comm
	}

	if this.isJsonMessage() {
		if err := json.Unmarshal([]byte(this.Message), &extra); err == nil {
			if m, ok := extra["Message"]; ok {
				this.Message = m.(string)
				delete(extra, "Message")
			}

			if f, ok := extra["FullMessage"]; ok {
				this.FullMessage = f.(string)
				delete(extra, "FullMessage")
			}
		}
	} else if -1 != strings.Index(this.Message, "\n") {
		this.FullMessage = this.Message
		this.Message = strings.Split(this.Message, "\n")[0]
	}

	return &gelf.Message{
		Version:  "1.1",
		Host:     this.Hostname,
		Short:    this.Message,
		Full:     this.FullMessage,
		TimeUnix: float64(this.Realtime_timestamp) / 1000 / 1000,
		Level:    this.Priority,
		Facility: facility,
		Extra:    extra,
	}
}

func (this *SystemdJournalEntry) process() {
	// Replace generic timestamp
	this.Message = messageReplace["*"].ReplaceAllString(this.Message, "")

	re := messageReplace[ this.Syslog_identifier ]
	if nil == re {
		re = messageReplace[ this.Comm ]
	}

	if nil == re {
		return
	}

	m := re.FindStringSubmatch(this.Message)
	if m == nil {
		return
	}

	// Store subpatterns in fields
	for idx, key := range re.SubexpNames() {
		if "Priority" == key {
			this.Priority = priorities[strings.ToLower(m[idx])]
		}
	}

	this.Message = re.ReplaceAllString(this.Message, "")
}

func (this *SystemdJournalEntry) sameSource(message *SystemdJournalEntry) bool {
	if this.Syslog_identifier != message.Syslog_identifier {
		return false
	}

	if this.Priority != message.Priority {
		return false
	}

	if this.Realtime_timestamp-message.Realtime_timestamp > SAMESOURCE_TIME_DIFFERENCE {
		return false
	}

	return true
}

func (this *SystemdJournalEntry) send() {
	message := this.toGelf()

	if err := writer.WriteMessage(message); err != nil {
		fmt.Fprintln(os.Stderr, err)
	}
}

func (this *SystemdJournalEntry) isJsonMessage() bool {
	return len(this.Message) > 64 && this.Message[0] == '{' && this.Message[1] == '"'
}

func (this *SystemdJournalEntry) extendWith(message *SystemdJournalEntry) {
	if this.FullMessage == "" {
		this.FullMessage = this.Message
	}

	this.FullMessage += "\n" + message.Message
}

var (
	pendingEntry *SystemdJournalEntry
	writer       *gelf.Writer
)

const (
	WRITE_INTERVAL             = 50 * time.Millisecond
	SAMESOURCE_TIME_DIFFERENCE = 100 * 1000
)

func main() {
	if len(os.Args) < 3 {
		fmt.Fprintln(os.Stderr, "Pass server:12201 as first argument and append journalctl parameters to use")
		os.Exit(1)
	}

	if w, err := gelf.NewWriter(os.Args[1]); err != nil {
		fmt.Fprintf(os.Stderr, "While connecting to Graylog server: %s\n", err)
		os.Exit(1)
	} else {
		writer = w
	}

	journalArgs := []string{"--all", "--output=json"}
	journalArgs = append(journalArgs, os.Args[2:]...)
	cmd := exec.Command("journalctl", journalArgs...)

	stderr, _ := cmd.StderrPipe()
	go io.Copy(os.Stderr, stderr)
	stdout, _ := cmd.StdoutPipe()
	s := bufio.NewScanner(stdout)

	go writePendingEntry()

	cmd.Start()

	for s.Scan() {
		line := s.Text()

		var entry = &SystemdJournalEntry{}
		if err := json.Unmarshal([]byte(line), &entry); err != nil {
			//fmt.Fprintf(os.Stderr, "Could not parse line, skipping: %s\n", line)
			continue
		}

		entry.process()

		if pendingEntry == nil {
			pendingEntry = entry
		} else if !pendingEntry.sameSource(entry) || pendingEntry.isJsonMessage() {
			pendingEntry.send()
			pendingEntry = entry
		} else {
			pendingEntry.extendWith(entry)

			// Keeps writePendingEntry waiting longer for us to append even more
			pendingEntry.Realtime_timestamp = entry.Realtime_timestamp
		}

		// Prevent saturation and throttling
		time.Sleep(1 * time.Millisecond)
	}

	if err := s.Err(); err != nil {
		fmt.Fprintf(os.Stderr, "Error from Scanner: %s\n", err)
		cmd.Process.Kill()
		os.Exit(1)
	}

	cmd.Wait()
	pendingEntry.send()
}

func writePendingEntry() {
	var entry *SystemdJournalEntry

	for {
		time.Sleep(WRITE_INTERVAL)

		if pendingEntry != nil && (time.Now().UnixNano()/1000-pendingEntry.Realtime_timestamp) > SAMESOURCE_TIME_DIFFERENCE {
			entry = pendingEntry
			pendingEntry = nil

			entry.send()
		}
	}
}
<|endoftext|>


<commit_before>package io

import (
	. "jvmgo/any"
	"jvmgo/jvm/rtda"
	rtc "jvmgo/jvm/rtda/class"
	"jvmgo/util"
	"os"
)

func init() {
	_fis(fis_initIDs, "initIDs", "()V")
	_fis(close0, "close0", "()V")
	_fis(readBytes, "readBytes", "([BII)I")
	_fis(open, "open", "(Ljava/lang/String;)V")
}

func _fis(method Any, name, desc string) {
	rtc.RegisterNativeMethod("java/io/FileInputStream", name, desc, method)
}

// private static native void initIDs();
// ()V
func fis_initIDs(frame *rtda.Frame) {
	// todo
}

// private native void close0() throws IOException;
// ()V
func close0(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()

	goFile := this.Extra().(*os.File)
	err := goFile.Close()
	if err != nil {
		// todo
		panic("IOException")
	}
}

// private native void open(String name) throws FileNotFoundException;
// (Ljava/lang/String;)V
func open(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()
	name := vars.GetRef(1)

	goName := rtda.GoString(name)
	goFile, err := os.Open(goName)
	if err != nil {
		frame.Thread().ThrowFileNotFoundException(goName)
		return
	}

	this.SetExtra(goFile)
}

// private native int readBytes(byte b[], int off, int len) throws IOException;
// ([BII)I
func readBytes(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()
	buf := vars.GetRef(1)
	off := vars.GetInt(2)
	_len := vars.GetInt(3)

	goFile := this.Extra().(*os.File)
	goBuf := util.CastInt8sToUint8s(buf.Fields().([]int8))
	goBuf = goBuf[:_len] // limit the maximum number of bytes read

	// func (f *File) ReadAt(b []byte, off int64) (n int, err error)
	n, err := goFile.ReadAt(goBuf, int64(off))
	if err == nil || n > 0 {
		frame.OperandStack().PushInt(int32(n))
	} else {
		// todo
		panic("IOException!" + err.Error())
	}
}
<commit_msg>implement native method: FileInputStream.available()<commit_after>package io

import (
	. "jvmgo/any"
	"jvmgo/jvm/rtda"
	rtc "jvmgo/jvm/rtda/class"
	"jvmgo/util"
	"os"
)

func init() {
	_fis(fis_initIDs, "initIDs", "()V")
	_fis(available, "available", "()I")
	_fis(close0, "close0", "()V")
	_fis(readBytes, "readBytes", "([BII)I")
	_fis(open, "open", "(Ljava/lang/String;)V")
}

func _fis(method Any, name, desc string) {
	rtc.RegisterNativeMethod("java/io/FileInputStream", name, desc, method)
}

// private static native void initIDs();
// ()V
func fis_initIDs(frame *rtda.Frame) {
	// todo
}

// public native int available() throws IOException;
// ()I
func available(frame *rtda.Frame) {
	// todo
	stack := frame.OperandStack()
	stack.PushInt(1)
}

// private native void close0() throws IOException;
// ()V
func close0(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()

	goFile := this.Extra().(*os.File)
	err := goFile.Close()
	if err != nil {
		// todo
		panic("IOException")
	}
}

// private native void open(String name) throws FileNotFoundException;
// (Ljava/lang/String;)V
func open(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()
	name := vars.GetRef(1)

	goName := rtda.GoString(name)
	goFile, err := os.Open(goName)
	if err != nil {
		frame.Thread().ThrowFileNotFoundException(goName)
		return
	}

	this.SetExtra(goFile)
}

// private native int readBytes(byte b[], int off, int len) throws IOException;
// ([BII)I
func readBytes(frame *rtda.Frame) {
	vars := frame.LocalVars()
	this := vars.GetThis()
	buf := vars.GetRef(1)
	off := vars.GetInt(2)
	_len := vars.GetInt(3)

	goFile := this.Extra().(*os.File)
	goBuf := util.CastInt8sToUint8s(buf.Fields().([]int8))
	goBuf = goBuf[off : off+_len]

	// func (f *File) Read(b []byte) (n int, err error)
	n, err := goFile.Read(goBuf)
	if err == nil || n > 0 {
		frame.OperandStack().PushInt(int32(n))
	} else {
		// todo
		panic("IOException!" + err.Error())
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/Netflix-Skunkworks/go-jira/jira/cli"
	"github.com/docopt/docopt-go"
	"github.com/op/go-logging"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"os"
	"strings"
)

var log = logging.MustGetLogger("jira")
var format = "%{color}%{time:2006-01-02T15:04:05.000Z07:00} %{level:-5s} [%{shortfile}]%{color:reset} %{message}"

func main() {
	user := os.Getenv("USER")
	home := os.Getenv("HOME")
	usage := fmt.Sprintf(`
Usage:
  jira [-v ...] [-u USER] [-e URI] [-t FILE] fields
  jira [-v ...] [-u USER] [-e URI] [-t FILE] login
  jira [-v ...] [-u USER] [-e URI] [-t FILE] (ls|list) ( [-q JQL] | [-p PROJECT] [-c COMPONENT] [-a ASSIGNEE] [-i ISSUETYPE])
  jira [-v ...] [-u USER] [-e URI] [-t FILE] view ISSUE
  jira [-v ...] [-u USER] [-e URI] [-t FILE] issuelinktypes
  jira [-v ...] [-u USER] [-e URI] [-t FILE] transmeta ISSUE
  jira [-v ...] [-u USER] [-e URI] [-t FILE] editmeta ISSUE
  jira [-v ...] export-templates [-d DIR]
  jira [-v ...] [-u USER] [-e URI] [-t FILE] ISSUE
  jira [-v ...] [-u USER] [-e URI] [-t FILE] edit ISSUE [-m COMMENT] [-o KEY=VAL]...
  jira [-v ...] [-u USER] [-e URI] [-t FILE] issuetypes [-p PROJECT]
  jira [-v ...] [-u USER] [-e URI] [-t FILE] createmeta [-p PROJECT] [-i ISSUETYPE]
  jira [-v ...] [-u USER] [-e URI] [-t FILE] transitions ISSUE
  jira [-v ...] [-u USER] [-e URI] [-t FILE] create [-p PROJECT] [-i ISSUETYPE] [-o KEY=VAL]...
  jira [-v ...] [-u USER] [-e URI] DUPLICATE dups ISSUE
  jira [-v ...] [-u USER] [-e URI] BLOCKER blocks ISSUE
  jira [-v ...] [-u USER] [-e URI] watch ISSUE [-w WATCHER]
  jira [-v ...] [-u USER] [-e URI] (trans|transition) TRANSITION ISSUE [-m COMMENT]
  jira [-v ...] [-u USER] [-e URI] ack ISSUE [-m COMMENT]
  jira [-v ...] [-u USER] [-e URI] close ISSUE [-m COMMENT]
  jira [-v ...] [-u USER] [-e URI] resolve ISSUE [-m COMMENT]
  jira [-v ...] [-u USER] [-e URI] reopen ISSUE [-m COMMENT]
  jira [-v ...] [-u USER] [-e URI] start ISSUE [-m COMMENT]
  jira [-v ...] [-u USER] [-e URI] stop ISSUE [-m COMMENT]
  jira [-v ...] [-u USER] [-e URI] [-t FILE] comment ISSUE [-m COMMENT]
  jira [-v ...] [-u USER] [-e URI] take ISSUE
  jira [-v ...] [-u USER] [-e URI] (assign|give) ISSUE ASSIGNEE

General Options:
  -e --endpoint=URI   URI to use for jira (default: https://jira)
  -h --help           Show this usage
  -t --template=FILE  Template file to use for output/editing
  -u --user=USER      Username to use for authenticaion (default: %s)
  -v --verbose        Increase output logging
  --version           Show this version

Command Options:
  -a --assignee=USER        Username assigned the issue
  -c --component=COMPONENT  Component to Search for
  -d --directory=DIR        Directory to export templates to (default: %s)
  -i --issuetype=ISSUETYPE  Jira Issue Type (default: Bug)
  -m --comment=COMMENT      Comment message for transition
  -o --override=KEY:VAL     Set custom key/value pairs
  -p --project=PROJECT      Project to Search for
  -q --query=JQL            Jira Query Language expression for the search
  -w --watcher=USER         Watcher to add to issue (default: %s)
`, user, fmt.Sprintf("%s/.jira.d/templates", home), user)

	args, err := docopt.Parse(usage, nil, true, "0.0.1", false, false)
	if err != nil {
		log.Error("Failed to parse options: %s", err)
		os.Exit(1)
	}
	logBackend := logging.NewLogBackend(os.Stderr, "", 0)
	logging.SetBackend(
		logging.NewBackendFormatter(
			logBackend,
			logging.MustStringFormatter(format),
		),
	)
	logging.SetLevel(logging.NOTICE, "")
	if verbose, ok := args["--verbose"]; ok {
		if verbose.(int) > 1 {
			logging.SetLevel(logging.DEBUG, "")
		} else if verbose.(int) > 0 {
			logging.SetLevel(logging.INFO, "")
		}
	}

	log.Info("Args: %v", args)

	opts := make(map[string]string)
	loadConfigs(opts)

	// strip the "--" off the command line options
	// and populate the opts that we pass to the cli ctor
	for key, val := range args {
		if val != nil && strings.HasPrefix(key, "--") {
			opt := key[2:]
			if opt == "override" {
				for _, v := range val.([]string) {
					if strings.Contains(v, "=") {
						kv := strings.SplitN(v, "=", 2)
						opts[kv[0]] = kv[1]
					} else {
						log.Error("Malformed override, expected KEY=VALUE, got %s", v)
						os.Exit(1)
					}
				}
			} else {
				switch v := val.(type) {
				case string:
					opts[opt] = v
				case int:
					opts[opt] = fmt.Sprintf("%d", v)
				}
			}
		}
	}

	// cant use proper [default:x] syntax in docopt
	// because only want to default if the option is not
	// already specified in some .jira.d/config.yml file
	if _, ok := opts["endpoint"]; !ok {
		opts["endpoint"] = "https://jira"
	}
	if _, ok := opts["user"]; !ok {
		opts["user"] = user
	}
	if _, ok := opts["issuetype"]; !ok {
		opts["issuetype"] = "Bug"
	}
	if _, ok := opts["directory"]; !ok {
		opts["directory"] = fmt.Sprintf("%s/.jira.d/templates", home)
	}

	c := cli.New(opts)

	log.Debug("opts: %s", opts)

	validCommand := func(cmd string) bool {
		if val, ok := args[cmd]; ok && val.(bool) {
			return true
		}
		return false
	}

	validOpt := func(opt string, dflt interface{}) interface{} {
		if val, ok := opts[opt]; ok {
			return val
		}
		if dflt == nil {
			log.Error("Missing required option --%s or \"%s\" property override in the config file", opt, opt)
			os.Exit(1)
		}
		return dflt
	}

	if validCommand("login") {
		err = c.CmdLogin()
	} else if validCommand("fields") {
		err = c.CmdFields()
	} else if validCommand("ls") || validCommand("list") {
		err = c.CmdList()
	} else if validCommand("edit") {
		err = c.CmdEdit(args["ISSUE"].(string))
	} else if validCommand("editmeta") {
		err = c.CmdEditMeta(args["ISSUE"].(string))
	} else if validCommand("transmeta") {
		err = c.CmdTransitionMeta(args["ISSUE"].(string))
	} else if validCommand("issuelinktypes") {
		err = c.CmdIssueLinkTypes()
	} else if validCommand("issuetypes") {
		err = c.CmdIssueTypes(validOpt("project", nil).(string))
	} else if validCommand("createmeta") {
		err = c.CmdCreateMeta(
			validOpt("project", nil).(string),
			validOpt("issuetype", "Bug").(string),
		)
	} else if validCommand("create") {
		err = c.CmdCreate(
			validOpt("project", nil).(string),
			validOpt("issuetype", "Bug").(string),
		)
	} else if validCommand("transitions") {
		err = c.CmdTransitions(args["ISSUE"].(string))
	} else if validCommand("blocks") {
		err = c.CmdBlocks(
			args["BLOCKER"].(string),
			args["ISSUE"].(string),
		)
	} else if validCommand("dups") {
		err = c.CmdDups(
			args["DUPLICATE"].(string),
			args["ISSUE"].(string),
		)
	} else if validCommand("watch") {
		err = c.CmdWatch(
			args["ISSUE"].(string),
			validOpt("watcher", user).(string),
		)
	} else if validCommand("trans") || validCommand("transition") {
		err = c.CmdTransition(
			args["ISSUE"].(string),
			args["TRANSITION"].(string),
		)
	} else if validCommand("close") {
		err = c.CmdTransition(args["ISSUE"].(string), "close")
	} else if validCommand("ack") {
		err = c.CmdTransition(args["ISSUE"].(string), "acknowledge")
	} else if validCommand("reopen") {
		err = c.CmdTransition(args["ISSUE"].(string), "reopen")
	} else if validCommand("resolve") {
		err = c.CmdTransition(args["ISSUE"].(string), "resolve")
	} else if validCommand("start") {
		err = c.CmdTransition(args["ISSUE"].(string), "start")
	} else if validCommand("stop") {
		err = c.CmdTransition(args["ISSUE"].(string), "stop")
	} else if validCommand("comment") {
		err = c.CmdComment(args["ISSUE"].(string))
	} else if validCommand("take") {
		err = c.CmdAssign(args["ISSUE"].(string), user)
	} else if validCommand("export-templates") {
		err = c.CmdExportTemplates()
	} else if validCommand("assign") || validCommand("give") {
		err = c.CmdAssign(
			args["ISSUE"].(string),
			args["ASSIGNEE"].(string),
		)
	} else if val, ok := args["ISSUE"]; ok {
		err = c.CmdView(val.(string))
	}

	if err != nil {
		os.Exit(1)
	}
	os.Exit(0)
}

func parseYaml(file string, opts map[string]string) {
	if fh, err := ioutil.ReadFile(file); err == nil {
		log.Debug("Found Config file: %s", file)
		yaml.Unmarshal(fh, &opts)
	}
}

func loadConfigs(opts map[string]string) {
	paths := cli.FindParentPaths(".jira.d/config.yml")
	// prepend
	paths = append([]string{"/etc/jira-cli.yml"}, paths...)

	for _, file := range paths {
		parseYaml(file, opts)
	}
}
<commit_msg>dont default endpoint make users specify it on command line or in config<commit_after>package main

import (
	"fmt"
	"github.com/Netflix-Skunkworks/go-jira/jira/cli"
	"github.com/docopt/docopt-go"
	"github.com/op/go-logging"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"os"
	"strings"
)

var log = logging.MustGetLogger("jira")
var format = "%{color}%{time:2006-01-02T15:04:05.000Z07:00} %{level:-5s} [%{shortfile}]%{color:reset} %{message}"

func main() {
	user := os.Getenv("USER")
	home := os.Getenv("HOME")
	usage := fmt.Sprintf(`
Usage:
  jira [-v ...] [-u USER] [-e URI] [-t FILE] fields
  jira [-v ...] [-u USER] [-e URI] [-t FILE] login
  jira [-v ...] [-u USER] [-e URI] [-t FILE] (ls|list) ( [-q JQL] | [-p PROJECT] [-c COMPONENT] [-a ASSIGNEE] [-i ISSUETYPE])
  jira [-v ...] [-u USER] [-e URI] [-t FILE] view ISSUE
  jira [-v ...] [-u USER] [-e URI] [-t FILE] issuelinktypes
  jira [-v ...] [-u USER] [-e URI] [-t FILE] transmeta ISSUE
  jira [-v ...] [-u USER] [-e URI] [-t FILE] editmeta ISSUE
  jira [-v ...] export-templates [-d DIR]
  jira [-v ...] [-u USER] [-e URI] [-t FILE] ISSUE
  jira [-v ...] [-u USER] [-e URI] [-t FILE] edit ISSUE [-m COMMENT] [-o KEY=VAL]...
  jira [-v ...] [-u USER] [-e URI] [-t FILE] issuetypes [-p PROJECT]
  jira [-v ...] [-u USER] [-e URI] [-t FILE] createmeta [-p PROJECT] [-i ISSUETYPE]
  jira [-v ...] [-u USER] [-e URI] [-t FILE] transitions ISSUE
  jira [-v ...] [-u USER] [-e URI] [-t FILE] create [-p PROJECT] [-i ISSUETYPE] [-o KEY=VAL]...
  jira [-v ...] [-u USER] [-e URI] DUPLICATE dups ISSUE
  jira [-v ...] [-u USER] [-e URI] BLOCKER blocks ISSUE
  jira [-v ...] [-u USER] [-e URI] watch ISSUE [-w WATCHER]
  jira [-v ...] [-u USER] [-e URI] (trans|transition) TRANSITION ISSUE [-m COMMENT]
  jira [-v ...] [-u USER] [-e URI] ack ISSUE [-m COMMENT]
  jira [-v ...] [-u USER] [-e URI] close ISSUE [-m COMMENT]
  jira [-v ...] [-u USER] [-e URI] resolve ISSUE [-m COMMENT]
  jira [-v ...] [-u USER] [-e URI] reopen ISSUE [-m COMMENT]
  jira [-v ...] [-u USER] [-e URI] start ISSUE [-m COMMENT]
  jira [-v ...] [-u USER] [-e URI] stop ISSUE [-m COMMENT]
  jira [-v ...] [-u USER] [-e URI] [-t FILE] comment ISSUE [-m COMMENT]
  jira [-v ...] [-u USER] [-e URI] take ISSUE
  jira [-v ...] [-u USER] [-e URI] (assign|give) ISSUE ASSIGNEE

General Options:
  -e --endpoint=URI   URI to use for jira
  -h --help           Show this usage
  -t --template=FILE  Template file to use for output/editing
  -u --user=USER      Username to use for authenticaion (default: %s)
  -v --verbose        Increase output logging
  --version           Show this version

Command Options:
  -a --assignee=USER        Username assigned the issue
  -c --component=COMPONENT  Component to Search for
  -d --directory=DIR        Directory to export templates to (default: %s)
  -i --issuetype=ISSUETYPE  Jira Issue Type (default: Bug)
  -m --comment=COMMENT      Comment message for transition
  -o --override=KEY:VAL     Set custom key/value pairs
  -p --project=PROJECT      Project to Search for
  -q --query=JQL            Jira Query Language expression for the search
  -w --watcher=USER         Watcher to add to issue (default: %s)
`, user, fmt.Sprintf("%s/.jira.d/templates", home), user)

	args, err := docopt.Parse(usage, nil, true, "0.0.1", false, false)
	if err != nil {
		log.Error("Failed to parse options: %s", err)
		os.Exit(1)
	}
	logBackend := logging.NewLogBackend(os.Stderr, "", 0)
	logging.SetBackend(
		logging.NewBackendFormatter(
			logBackend,
			logging.MustStringFormatter(format),
		),
	)
	logging.SetLevel(logging.NOTICE, "")
	if verbose, ok := args["--verbose"]; ok {
		if verbose.(int) > 1 {
			logging.SetLevel(logging.DEBUG, "")
		} else if verbose.(int) > 0 {
			logging.SetLevel(logging.INFO, "")
		}
	}

	log.Info("Args: %v", args)

	opts := make(map[string]string)
	loadConfigs(opts)

	// strip the "--" off the command line options
	// and populate the opts that we pass to the cli ctor
	for key, val := range args {
		if val != nil && strings.HasPrefix(key, "--") {
			opt := key[2:]
			if opt == "override" {
				for _, v := range val.([]string) {
					if strings.Contains(v, "=") {
						kv := strings.SplitN(v, "=", 2)
						opts[kv[0]] = kv[1]
					} else {
						log.Error("Malformed override, expected KEY=VALUE, got %s", v)
						os.Exit(1)
					}
				}
			} else {
				switch v := val.(type) {
				case string:
					opts[opt] = v
				case int:
					opts[opt] = fmt.Sprintf("%d", v)
				}
			}
		}
	}

	// cant use proper [default:x] syntax in docopt
	// because only want to default if the option is not
	// already specified in some .jira.d/config.yml file
	if _, ok := opts["user"]; !ok {
		opts["user"] = user
	}
	if _, ok := opts["issuetype"]; !ok {
		opts["issuetype"] = "Bug"
	}
	if _, ok := opts["directory"]; !ok {
		opts["directory"] = fmt.Sprintf("%s/.jira.d/templates", home)
	}

	if _, ok := opts["endpoint"]; !ok {
		log.Error("endpoint option required.  Either use --endpoint or set a enpoint option in your ~/.jira.d/config.yml file")
		os.Exit(1)
	}

	c := cli.New(opts)

	log.Debug("opts: %s", opts)

	validCommand := func(cmd string) bool {
		if val, ok := args[cmd]; ok && val.(bool) {
			return true
		}
		return false
	}

	validOpt := func(opt string, dflt interface{}) interface{} {
		if val, ok := opts[opt]; ok {
			return val
		}
		if dflt == nil {
			log.Error("Missing required option --%s or \"%s\" property override in the config file", opt, opt)
			os.Exit(1)
		}
		return dflt
	}

	if validCommand("login") {
		err = c.CmdLogin()
	} else if validCommand("fields") {
		err = c.CmdFields()
	} else if validCommand("ls") || validCommand("list") {
		err = c.CmdList()
	} else if validCommand("edit") {
		err = c.CmdEdit(args["ISSUE"].(string))
	} else if validCommand("editmeta") {
		err = c.CmdEditMeta(args["ISSUE"].(string))
	} else if validCommand("transmeta") {
		err = c.CmdTransitionMeta(args["ISSUE"].(string))
	} else if validCommand("issuelinktypes") {
		err = c.CmdIssueLinkTypes()
	} else if validCommand("issuetypes") {
		err = c.CmdIssueTypes(validOpt("project", nil).(string))
	} else if validCommand("createmeta") {
		err = c.CmdCreateMeta(
			validOpt("project", nil).(string),
			validOpt("issuetype", "Bug").(string),
		)
	} else if validCommand("create") {
		err = c.CmdCreate(
			validOpt("project", nil).(string),
			validOpt("issuetype", "Bug").(string),
		)
	} else if validCommand("transitions") {
		err = c.CmdTransitions(args["ISSUE"].(string))
	} else if validCommand("blocks") {
		err = c.CmdBlocks(
			args["BLOCKER"].(string),
			args["ISSUE"].(string),
		)
	} else if validCommand("dups") {
		err = c.CmdDups(
			args["DUPLICATE"].(string),
			args["ISSUE"].(string),
		)
	} else if validCommand("watch") {
		err = c.CmdWatch(
			args["ISSUE"].(string),
			validOpt("watcher", user).(string),
		)
	} else if validCommand("trans") || validCommand("transition") {
		err = c.CmdTransition(
			args["ISSUE"].(string),
			args["TRANSITION"].(string),
		)
	} else if validCommand("close") {
		err = c.CmdTransition(args["ISSUE"].(string), "close")
	} else if validCommand("ack") {
		err = c.CmdTransition(args["ISSUE"].(string), "acknowledge")
	} else if validCommand("reopen") {
		err = c.CmdTransition(args["ISSUE"].(string), "reopen")
	} else if validCommand("resolve") {
		err = c.CmdTransition(args["ISSUE"].(string), "resolve")
	} else if validCommand("start") {
		err = c.CmdTransition(args["ISSUE"].(string), "start")
	} else if validCommand("stop") {
		err = c.CmdTransition(args["ISSUE"].(string), "stop")
	} else if validCommand("comment") {
		err = c.CmdComment(args["ISSUE"].(string))
	} else if validCommand("take") {
		err = c.CmdAssign(args["ISSUE"].(string), user)
	} else if validCommand("export-templates") {
		err = c.CmdExportTemplates()
	} else if validCommand("assign") || validCommand("give") {
		err = c.CmdAssign(
			args["ISSUE"].(string),
			args["ASSIGNEE"].(string),
		)
	} else if val, ok := args["ISSUE"]; ok {
		err = c.CmdView(val.(string))
	}

	if err != nil {
		os.Exit(1)
	}
	os.Exit(0)
}

func parseYaml(file string, opts map[string]string) {
	if fh, err := ioutil.ReadFile(file); err == nil {
		log.Debug("Found Config file: %s", file)
		yaml.Unmarshal(fh, &opts)
	}
}

func loadConfigs(opts map[string]string) {
	paths := cli.FindParentPaths(".jira.d/config.yml")
	// prepend
	paths = append([]string{"/etc/jira-cli.yml"}, paths...)

	for _, file := range paths {
		parseYaml(file, opts)
	}
}
<|endoftext|>