<commit_before>package newt

/*
#cgo CFLAGS: -I/opt/local/include
#cgo LDFLAGS: -L/opt/local/lib -lnewt

#include <stdio.h>
#include <stdlib.h>
#include <newt.h>
*/
import "C"

import (
    "unsafe"
)

func WinMessage(title, buttonText, text string) {
    t1 := C.CString(title)
    b1 := C.CString(buttonText)
    t2 := C.CString(text)
    defer func() {
        C.free(unsafe.Pointer(t1))
        C.free(unsafe.Pointer(b1))
        C.free(unsafe.Pointer(t2))
    }()
    C.newtWinMessage(t1, b1, t2)
}

func WinMessageV(title, buttonText, text string) {
    panic("not implemented")
}

func WinChoice(title, button1, button2, text string) int {
    t1 := C.CString(title)
    b1 := C.CString(button1)
    b2 := C.CString(button2)
    t2 := C.CString(text)
    defer func() {
        C.free(unsafe.Pointer(t1))
        C.free(unsafe.Pointer(b1))
        C.free(unsafe.Pointer(b2))
        C.free(unsafe.Pointer(t2))
    }()
    return int(C.newtWinChoice(t1, b1, b2, t2))
}

func WinTernary(title, button1, button2, button3, message string) int {
    t1 := C.CString(title)
    b1 := C.CString(button1)
    b2 := C.CString(button2)
    b3 := C.CString(button3)
    m1 := C.CString(message)
    defer func() {
        C.free(unsafe.Pointer(t1))
        C.free(unsafe.Pointer(b1))
        C.free(unsafe.Pointer(b2))
        C.free(unsafe.Pointer(b3))
        C.free(unsafe.Pointer(m1))
    }()
    return int(C.newtWinTernary(t1, b1, b2, b3, m1))
}

func WinMenu(title, text string, suggestedWidth, flexDown, flexUp, maxListHeight int, items []string, button1 string) (int, int) {
    t1 := C.CString(title)
    t2 := C.CString(text)
    b1 := C.CString(button1)
    defer func() {
        C.free(unsafe.Pointer(t1))
        C.free(unsafe.Pointer(t2))
        C.free(unsafe.Pointer(b1))
    }()
    Citems := make([]*C.char, len(items))
    for i, v := range(items) {
        Citems[i] = C.CString(v)
    }
    defer func() {
        for i := range(Citems) {
            C.free(unsafe.Pointer(Citems[i]))
        }
    }()

    panic("not implemented")
    //var listItem C.int
    // r := C.newtWinMenu(t1, t2, C.int(suggestedWidth), C.int(flexDown), C.int(flexUp), C.int(maxListHeight), (**C.char)(unsafe.Pointer(&Citems[0])), &listItem, b1, nil)
    // return int(r), int(listItem)
}
<commit_msg>WinMenu() rewritten<commit_after>package newt

/*
#cgo CFLAGS: -I/opt/local/include
#cgo LDFLAGS: -L/opt/local/lib -lnewt

#include <stdio.h>
#include <stdlib.h>
#include <newt.h>
*/
import "C"

import (
    "unsafe"
)

func WinMessage(title, buttonText, text string) {
    t1 := C.CString(title)
    b1 := C.CString(buttonText)
    t2 := C.CString(text)
    defer func() {
        C.free(unsafe.Pointer(t1))
        C.free(unsafe.Pointer(b1))
        C.free(unsafe.Pointer(t2))
    }()
    C.newtWinMessage(t1, b1, t2)
}

func WinMessageV(title, buttonText, text string) {
    panic("not implemented")
}

func WinChoice(title, button1, button2, text string) int {
    t1 := C.CString(title)
    b1 := C.CString(button1)
    b2 := C.CString(button2)
    t2 := C.CString(text)
    defer func() {
        C.free(unsafe.Pointer(t1))
        C.free(unsafe.Pointer(b1))
        C.free(unsafe.Pointer(b2))
        C.free(unsafe.Pointer(t2))
    }()
    return int(C.newtWinChoice(t1, b1, b2, t2))
}

func WinTernary(title, button1, button2, button3, message string) int {
    t1 := C.CString(title)
    b1 := C.CString(button1)
    b2 := C.CString(button2)
    b3 := C.CString(button3)
    m1 := C.CString(message)
    defer func() {
        C.free(unsafe.Pointer(t1))
        C.free(unsafe.Pointer(b1))
        C.free(unsafe.Pointer(b2))
        C.free(unsafe.Pointer(b3))
        C.free(unsafe.Pointer(m1))
    }()
    return int(C.newtWinTernary(t1, b1, b2, b3, m1))
}

func WinMenu(title, text string, suggestedWidth, flexDown, flexUp, maxListHeight int, items []string, button1 ...string) (int, int) {
    textbox := TextboxReflowed(-1, -1, text, suggestedWidth, flexDown, flexUp, 0)
    if len(items) < maxListHeight {
        maxListHeight = len(items)
    }
    needScroll := 0
    if len(items) > maxListHeight {
        needScroll = FLAG_SCROLL
    }

    listbox := Listbox(-1, -1, maxListHeight, needScroll | FLAG_RETURNEXIT);
    for i, v := range items {
        u := uint(i)
        ListboxAddEntry(listbox, v, uintptr(unsafe.Pointer(&u)))
    }

    var listItem int
    ListboxSetCurrent(listbox, listItem)

    buttons := make([]Component, len(button1))
    buttonBar := CreateGrid(len(button1), 1)
    for i, v := range button1 {
        j := 1
        if i == 0 { j = 0 }
        buttons[i] = Button(-1, -1, v)
        GridSetField(buttonBar, i, 0, GRID_COMPONENT, buttons[i], j, 0, 0, 0, 0, 0)
    }

    grid := GridSimpleWindow(textbox, listbox, buttonBar)
    GridWrappedWindow(grid, title)

    form := Form(nil, "", 0)
    GridAddComponentsToForm(grid, form, 1)
    GridFree(grid, 1)

    result := RunForm(form)
    listItemPtr := ListboxGetCurrent(listbox)
    listItemC := *(*C.int)(unsafe.Pointer(listItemPtr))
    listItem = int(listItemC)

    rc := 0
    for _, v := range(buttons) {
        if result == v {
            break
        }
        rc++
    }
    if rc == len(buttons) { rc = 0 } else { rc++ }

    FormDestroy(form)
    PopWindow()

    return rc, int(listItem)

    //t1 := C.CString(title)
    //t2 := C.CString(text)
    //b1 := C.CString(button1)
    //defer func() {
    //    C.free(unsafe.Pointer(t1))
    //    C.free(unsafe.Pointer(t2))
    //    C.free(unsafe.Pointer(b1))
    //}()
    //Citems := make([]*C.char, len(items))
    //for i, v := range(items) {
    //    Citems[i] = C.CString(v)
   // }
    //defer func() {
    //    for i := range(Citems) {
    //        C.free(unsafe.Pointer(Citems[i]))
    //    }
    //}()

    //panic("not implemented")
    //var listItem C.int
    // r := C.newtWinMenu(t1, t2, C.int(suggestedWidth), C.int(flexDown), C.int(flexUp), C.int(maxListHeight), (**C.char)(unsafe.Pointer(&Citems[0])), &listItem, b1, nil)
    // return int(r), int(listItem)
}
<|endoftext|>


<commit_before>// 11 february 2014

package ui

import (
	"fmt"
)

// Window represents an on-screen window.
type Window struct {
	created    bool
	sysData    *sysData
	initTitle  string
	initWidth  int
	initHeight int
	shownOnce  bool
	spaced	bool
	handler	WindowHandler
}

// WindowHandler represents an event handler for a Window and all its child Controls.
//
// When an event on a Window or one of its child Controls comes in, the respect Window's handler's Event() method is called. The method call occurs on the main thread, and thus any call to any package ui method can be performed.
//
// Each Event() call takes two parameters: the event ID and a data argument. For most events, the data argument is a pointer to the Control that triggered the event.
//
// For Closing, the data argument is [TODO].
//
// For any event >= CustomEvent, the data argument is the argument passed to the Window's SendEvent() method.
type WindowHandler interface {
	Event(e Event, data interface{})
}

// Event represents an event; see WindowHandler for details.
// All event values >= CustomEvent are available for program use.
type Event int
const (
	Closing Event = iota			// Window close
	Clicked					// Button click
	CustomEvent = 5000		// very high number; higher than the package would ever need, anyway
)

// NewWindow allocates a new Window with the given title and size. The window is not created until a call to Create() or Open().
func NewWindow(title string, width int, height int, handler WindowHandler) *Window {
	return &Window{
		sysData:    mksysdata(c_window),
		initTitle:  title,
		initWidth:  width,
		initHeight: height,
		handler:		handler,
	}
}

// SetTitle sets the window's title.
func (w *Window) SetTitle(title string) {
	if w.created {
		w.sysData.setText(title)
		return
	}
	w.initTitle = title
}

// SetSize sets the window's size.
func (w *Window) SetSize(width int, height int) (err error) {
	if w.created {
		err := w.sysData.setWindowSize(width, height)
		if err != nil {
			return fmt.Errorf("error setting window size: %v", err)
		}
		return nil
	}
	w.initWidth = width
	w.initHeight = height
	return nil
}

// SetSpaced sets whether the Window's child control takes padding and spacing into account.
// That is, with w.SetSpaced(true), w's child will have a margin around the window frame and will have sub-controls separated by an implementation-defined amount.
// Currently, only Stack and Grid explicitly understand this property.
// This property is visible recursively throughout the widget hierarchy of the Window.
// This property cannot be set after the Window has been created.
func (w *Window) SetSpaced(spaced bool) {
	if w.created {
		panic(fmt.Errorf("Window.SetSpaced() called after window created"))
	}
	w.spaced = spaced
}

// Open creates the Window with Create and then shows the Window with Show. As with Create, you cannot call Open more than once per window.
func (w *Window) Open(control Control) {
	w.Create(control)
	w.Show()
}

// Create creates the Window, setting its control to the given control. It does not show the window. This can only be called once per window, and finalizes all initialization of the control.
func (w *Window) Create(control Control) {
	if w.created {
		panic("window already open")
	}
	w.sysData.spaced = w.spaced
	w.sysData.winhandler = w.handler
	w.sysData.close = func(b *bool) {
		w.sysData.winhandler.Event(Closing, b)
	}
	err := w.sysData.make(nil)
	if err != nil {
		panic(fmt.Errorf("error opening window: %v", err))
	}
	if control != nil {
		w.sysData.allocate = control.allocate
		err = control.make(w.sysData)
		if err != nil {
			panic(fmt.Errorf("error adding window's control: %v", err))
		}
	}
	err = w.sysData.setWindowSize(w.initWidth, w.initHeight)
	if err != nil {
		panic(fmt.Errorf("error setting window size (in Window.Open()): %v", err))
	}
	w.sysData.setText(w.initTitle)
	w.created = true
}

// Show shows the window.
func (w *Window) Show() {
	if !w.shownOnce {
		w.shownOnce = true
		err := w.sysData.firstShow()
		if err != nil {
			panic(fmt.Errorf("error showing window for the first time: %v", err))
		}
		return
	}
	w.sysData.show()
}

// Hide hides the window.
func (w *Window) Hide() {
	w.sysData.hide()
}

// Center centers the Window on-screen.
// The concept of "screen" in the case of a multi-monitor setup is implementation-defined.
// It presently panics if the Window has not been created.
func (w *Window) Center() {
	if !w.created {
		panic("attempt to center Window before it has been created")
	}
	w.sysData.center()
}
<commit_msg>Documented the new Window.Closing behavior.<commit_after>// 11 february 2014

package ui

import (
	"fmt"
)

// Window represents an on-screen window.
type Window struct {
	created    bool
	sysData    *sysData
	initTitle  string
	initWidth  int
	initHeight int
	shownOnce  bool
	spaced	bool
	handler	WindowHandler
}

// WindowHandler represents an event handler for a Window and all its child Controls.
//
// When an event on a Window or one of its child Controls comes in, the respect Window's handler's Event() method is called. The method call occurs on the main thread, and thus any call to any package ui method can be performed.
//
// Each Event() call takes two parameters: the event ID and a data argument. For most events, the data argument is a pointer to the Control that triggered the event.
//
// For Closing, the data argument is a pointer to a bool variable. If, after returning from Event, the value of this variable is true, the Window is closed; if false, the Window is not closed. The default value on entry to the function is [TODO].
//
// For any event >= CustomEvent, the data argument is the argument passed to the Window's SendEvent() method.
type WindowHandler interface {
	Event(e Event, data interface{})
}

// Event represents an event; see WindowHandler for details.
// All event values >= CustomEvent are available for program use.
type Event int
const (
	Closing Event = iota			// Window close
	Clicked					// Button click
	CustomEvent = 5000		// very high number; higher than the package would ever need, anyway
)

// NewWindow allocates a new Window with the given title and size. The window is not created until a call to Create() or Open().
func NewWindow(title string, width int, height int, handler WindowHandler) *Window {
	return &Window{
		sysData:    mksysdata(c_window),
		initTitle:  title,
		initWidth:  width,
		initHeight: height,
		handler:		handler,
	}
}

// SetTitle sets the window's title.
func (w *Window) SetTitle(title string) {
	if w.created {
		w.sysData.setText(title)
		return
	}
	w.initTitle = title
}

// SetSize sets the window's size.
func (w *Window) SetSize(width int, height int) (err error) {
	if w.created {
		err := w.sysData.setWindowSize(width, height)
		if err != nil {
			return fmt.Errorf("error setting window size: %v", err)
		}
		return nil
	}
	w.initWidth = width
	w.initHeight = height
	return nil
}

// SetSpaced sets whether the Window's child control takes padding and spacing into account.
// That is, with w.SetSpaced(true), w's child will have a margin around the window frame and will have sub-controls separated by an implementation-defined amount.
// Currently, only Stack and Grid explicitly understand this property.
// This property is visible recursively throughout the widget hierarchy of the Window.
// This property cannot be set after the Window has been created.
func (w *Window) SetSpaced(spaced bool) {
	if w.created {
		panic(fmt.Errorf("Window.SetSpaced() called after window created"))
	}
	w.spaced = spaced
}

// Open creates the Window with Create and then shows the Window with Show. As with Create, you cannot call Open more than once per window.
func (w *Window) Open(control Control) {
	w.Create(control)
	w.Show()
}

// Create creates the Window, setting its control to the given control. It does not show the window. This can only be called once per window, and finalizes all initialization of the control.
func (w *Window) Create(control Control) {
	if w.created {
		panic("window already open")
	}
	w.sysData.spaced = w.spaced
	w.sysData.winhandler = w.handler
	w.sysData.close = func(b *bool) {
		w.sysData.winhandler.Event(Closing, b)
	}
	err := w.sysData.make(nil)
	if err != nil {
		panic(fmt.Errorf("error opening window: %v", err))
	}
	if control != nil {
		w.sysData.allocate = control.allocate
		err = control.make(w.sysData)
		if err != nil {
			panic(fmt.Errorf("error adding window's control: %v", err))
		}
	}
	err = w.sysData.setWindowSize(w.initWidth, w.initHeight)
	if err != nil {
		panic(fmt.Errorf("error setting window size (in Window.Open()): %v", err))
	}
	w.sysData.setText(w.initTitle)
	w.created = true
}

// Show shows the window.
func (w *Window) Show() {
	if !w.shownOnce {
		w.shownOnce = true
		err := w.sysData.firstShow()
		if err != nil {
			panic(fmt.Errorf("error showing window for the first time: %v", err))
		}
		return
	}
	w.sysData.show()
}

// Hide hides the window.
func (w *Window) Hide() {
	w.sysData.hide()
}

// Center centers the Window on-screen.
// The concept of "screen" in the case of a multi-monitor setup is implementation-defined.
// It presently panics if the Window has not been created.
func (w *Window) Center() {
	if !w.created {
		panic("attempt to center Window before it has been created")
	}
	w.sysData.center()
}
<|endoftext|>


<commit_before>package task

import (
	"encoding/json"
	"fmt"
	"log"
	"os"

	"github.com/jinzhu/gorm"
	"github.com/velocity-ci/velocity/backend/velocity"
)

type gormTask struct {
	ID       string `gorm:"primary_key"`
	CommitID string
	Name     string
	VTask    []byte // JSON of task for storage
}

func gormTaskFromTask(t Task) gormTask {
	jsonTask, err := json.Marshal(t.Task)
	if err != nil {
		log.Println("Could not marshal task")
		log.Fatal(err)
	}

	return gormTask{
		ID:       t.ID,
		CommitID: t.CommitID,
		Name:     t.Name,
		VTask:    jsonTask,
	}
}

func taskFromGormTask(g gormTask) Task {
	var vTask velocity.Task
	err := json.Unmarshal(g.VTask, &vTask)
	if err != nil {
		log.Println("Could not unmarshal task from %v", g.VTask)
		log.Fatal(err)
	}

	return Task{
		ID:       g.ID,
		CommitID: g.CommitID,
		Task:     vTask,
	}
}

// Expose CRUD operations (implement interface?) Implement repository funcs, as they will be used when we have caching.
type gormRepository struct {
	logger *log.Logger
	gorm   *gorm.DB
}

func newGORMRepository(db *gorm.DB) *gormRepository {
	db.AutoMigrate(gormTask{})
	return &gormRepository{
		logger: log.New(os.Stdout, "[gorm:task]", log.Lshortfile),
		gorm:   db,
	}
}

func (r *gormRepository) Save(t Task) Task {
	tx := r.gorm.Begin()

	gT := gormTaskFromTask(t)

	err := tx.Where(&gormTask{
		ID: t.ID,
	}).First(&gormTask{}).Error
	if err != nil {
		err = tx.Create(&gT).Error
	} else {
		tx.Save(&gT)
	}

	tx.Commit()
	r.logger.Printf("saved task %s", t.ID)
	return taskFromGormTask(gT)
}

func (r *gormRepository) Delete(t Task) {
	tx := r.gorm.Begin()

	gT := gormTaskFromTask(t)

	if err := tx.Delete(gT).Error; err != nil {
		tx.Rollback()
		log.Fatal(err)
	}

	tx.Commit()
}

func (r *gormRepository) GetByTaskID(taskID string) (Task, error) {
	gT := gormTask{}

	if r.gorm.
		Where(&gormTask{
			ID: taskID,
		}).
		First(&gT).RecordNotFound() {
		log.Printf("could not find task %s", taskID)
		return Task{}, fmt.Errorf("could not find task %s", taskID)
	}

	return taskFromGormTask(gT), nil
}

func (r *gormRepository) GetByCommitIDAndTaskName(commitID string, name string) (Task, error) {
	gT := gormTask{}

	if r.gorm.
		Where(&gormTask{
			CommitID: commitID,
			Name:     name,
		}).
		First(&gT).RecordNotFound() {
		log.Printf("could not find commit:task %s:%s", commitID, name)
		return Task{}, fmt.Errorf("could not find commit:task %s:%s", commitID, name)
	}

	return taskFromGormTask(gT), nil
}

func (r *gormRepository) GetAllByCommitID(commitID string, q Query) ([]Task, uint64) {
	gTs := []gormTask{}
	var count uint64

	r.gorm.
		Where(&gormTask{
			CommitID: commitID,
		}).
		Find(&gTs).
		Count(&count)

	tasks := []Task{}
	for _, gT := range gTs {
		tasks = append(tasks, taskFromGormTask(gT))
	}

	return tasks, count
}
<commit_msg>[backend] Fixed table name for tasks<commit_after>package task

import (
	"encoding/json"
	"fmt"
	"log"
	"os"

	"github.com/jinzhu/gorm"
	"github.com/velocity-ci/velocity/backend/velocity"
)

type gormTask struct {
	ID       string `gorm:"primary_key"`
	CommitID string
	Name     string
	VTask    []byte // JSON of task for storage
}

func (gormTask) TableName() string {
	return "tasks"
}

func gormTaskFromTask(t Task) gormTask {
	jsonTask, err := json.Marshal(t.Task)
	if err != nil {
		log.Println("Could not marshal task")
		log.Fatal(err)
	}

	return gormTask{
		ID:       t.ID,
		CommitID: t.CommitID,
		Name:     t.Name,
		VTask:    jsonTask,
	}
}

func taskFromGormTask(g gormTask) Task {
	var vTask velocity.Task
	err := json.Unmarshal(g.VTask, &vTask)
	if err != nil {
		log.Println("Could not unmarshal task from %v", g.VTask)
		log.Fatal(err)
	}

	return Task{
		ID:       g.ID,
		CommitID: g.CommitID,
		Task:     vTask,
	}
}

// Expose CRUD operations (implement interface?) Implement repository funcs, as they will be used when we have caching.
type gormRepository struct {
	logger *log.Logger
	gorm   *gorm.DB
}

func newGORMRepository(db *gorm.DB) *gormRepository {
	db.AutoMigrate(gormTask{})
	return &gormRepository{
		logger: log.New(os.Stdout, "[gorm:task]", log.Lshortfile),
		gorm:   db,
	}
}

func (r *gormRepository) Save(t Task) Task {
	tx := r.gorm.Begin()

	gT := gormTaskFromTask(t)

	err := tx.Where(&gormTask{
		ID: t.ID,
	}).First(&gormTask{}).Error
	if err != nil {
		err = tx.Create(&gT).Error
	} else {
		tx.Save(&gT)
	}

	tx.Commit()
	r.logger.Printf("saved task %s", t.ID)
	return taskFromGormTask(gT)
}

func (r *gormRepository) Delete(t Task) {
	tx := r.gorm.Begin()

	gT := gormTaskFromTask(t)

	if err := tx.Delete(gT).Error; err != nil {
		tx.Rollback()
		log.Fatal(err)
	}

	tx.Commit()
}

func (r *gormRepository) GetByTaskID(taskID string) (Task, error) {
	gT := gormTask{}

	if r.gorm.
		Where(&gormTask{
			ID: taskID,
		}).
		First(&gT).RecordNotFound() {
		log.Printf("could not find task %s", taskID)
		return Task{}, fmt.Errorf("could not find task %s", taskID)
	}

	return taskFromGormTask(gT), nil
}

func (r *gormRepository) GetByCommitIDAndTaskName(commitID string, name string) (Task, error) {
	gT := gormTask{}

	if r.gorm.
		Where(&gormTask{
			CommitID: commitID,
			Name:     name,
		}).
		First(&gT).RecordNotFound() {
		log.Printf("could not find commit:task %s:%s", commitID, name)
		return Task{}, fmt.Errorf("could not find commit:task %s:%s", commitID, name)
	}

	return taskFromGormTask(gT), nil
}

func (r *gormRepository) GetAllByCommitID(commitID string, q Query) ([]Task, uint64) {
	gTs := []gormTask{}
	var count uint64

	r.gorm.
		Where(&gormTask{
			CommitID: commitID,
		}).
		Find(&gTs).
		Count(&count)

	tasks := []Task{}
	for _, gT := range gTs {
		tasks = append(tasks, taskFromGormTask(gT))
	}

	return tasks, count
}
<|endoftext|>


<commit_before>package xing

import (
	"fmt"
	"sync"
	"time"

	"github.com/rs/zerolog/log"
	"github.com/streadway/amqp"
	wk "github.com/wzhliang/xing/pkg/worker"
	"github.com/wzhliang/xing/utils"
)

// Worker ...
type Worker struct {
	sync.Mutex
	id         int
	ch         *amqp.Channel    // incoming channel
	sch        *amqp.Channel    // outgoing channel
	queue      amqp.Queue       // queue for consumption, for RPC client, it holds results
	conn       *amqp.Connection // own copy for detecting reconnection
	d          wk.Dispatcher
	c          *Client // the AMQP client
	rpcCounter uint
}

// SetDispatcher ...
func (w *Worker) SetDispatcher(dis wk.Dispatcher) {
	w.d = dis
}

// Close ... should be called after stopped
func (w *Worker) Close() {
	// at this time, the channel is already closed
	var err error
	if w.c.conn == nil {
		log.Error().Msg("no connection to AMQP server. can't do much")
		return
	}
	w.ch, err = w.conn.Channel()
	if err != nil {
		log.Error().Err(err).Msg("failed to create channel")
		return
	}

	n, err := w.ch.QueueDelete(w.queue.Name, false, false, false)
	if err != nil {
		log.Error().Int("n", n).Err(err).Msg("failed to delete queue")
	}
}

func (w *Worker) corrid() string {
	return fmt.Sprintf("rpc-%s-%02d-%d", w.c.name, w.id, w.rpcCounter)
}

// Loop ...
func (w *Worker) Loop() {
	for {
		req := w.d.Dispatch()
		if req.In == nil {
			break
		}
		var results chan interface{}
		var errs chan error
		ret := func(v interface{}, err error) {
			results <- v
			errs <- err
		}

		results = req.Result

		errs = req.Err
		if w.c.conn == nil {
			log.Info().Msg("connection not available, sleeping a while...")
			time.Sleep(time.Duration(utils.Random(2000, 3000)) * time.Millisecond)
			// prevent mutlple works from trying to reconnect at the same time
			_, err := w.c.ensureConnection()
			if err == nil {
				w.onConnect(w.c.conn)
			} else {
				ret(nil, fmt.Errorf("connection lost"))
				continue
			}
		} else if w.c.conn != w.conn {
			log.Info().Msg("new conection")
			w.onConnect(w.c.conn)
		}
		ret(w.processJob(req))
	}
	log.Info().Int("id", w.id).Msg("I am done")
	w.c.wg.Done() // FIXME: should not have access to wg
}

func newWorker(conn *amqp.Connection, c *Client, id int) *Worker {
	w := Worker{
		id:         id,
		c:          c,
		conn:       nil,
		rpcCounter: uint(utils.Random(1000, 9999)),
	}
	// onConnect() will be called on demand due to the fact that our own conn is
	// different from Client's conn
	// this makes sure that worker always has a consumer to its queue so that
	// the queue will not be TTLed
	return &w
}

func (w *Worker) onConnect(conn *amqp.Connection) error {
	log.Info().Msg("connected")
	var err error
	w.Lock()
	w.conn = conn // refreshing own copy
	w.Unlock()
	w.ch, err = conn.Channel()
	if err != nil {
		return nil
	}
	w.sch, err = conn.Channel()
	if err != nil {
		return nil
	}
	err = w.ch.ExchangeDeclare(
		RPCExchange, // name
		"topic",     // kind
		true,        // durable
		false,       // autodelete
		false,       // internal
		false,       // noWait
		nil,
	)
	if err != nil {
		return err
	}
	// I have to be a client
	w.queue, err = w.ch.QueueDeclare(
		// for client queue, we don't care it's name
		// also, changing it will cause crash
		fmt.Sprintf("%s-%02d", w.c.name, w.id), // name
		true,  // durable
		false, // autoDelete
		true,  // exclusive
		false, // noWait
		amqp.Table{
			"x-expires":     ResultQueueTTL,
			"x-message-ttl": RPCTTL,
		}, // args
	)
	if err != nil {
		log.Error().Err(err).Msg("setup failed")
		return nil
	}
	key := resultTopicName(w.queue.Name) // for client, queuename is the client's name
	log.Info().Str("queue", w.queue.Name).Str("exchange", RPCExchange).Str("key", key).
		Msg("Subscribing")
	err = w.ch.QueueBind(w.queue.Name, key, RPCExchange, false, nil)
	if err != nil {
		log.Error().Err(err).Msg("setup failed")
		return err
	}

	return nil
}

func (w *Worker) sendEx(target string, _type string, event string, replyTo string, payload interface{}) error {
	var cor string

	if _type == Command { // FIXME: actuall worker should only handle Command
		w.Lock()
		w.rpcCounter++
		w.Unlock()
		cor = w.corrid()
	} else {
		cor = "N/A"
	}

	ex := w.c.exchange(_type)
	var key string
	if topicLength(target) == 2 { // load balanced rpc
		key = fmt.Sprintf("%s.*.%s.%s", target, _type, event)
	} else if topicLength(target) == 3 {
		key = fmt.Sprintf("%s.%s.%s", target, _type, event)
	} else {
		return fmt.Errorf("Invalid target: %s", target)
	}
	return w._sendEx(ex, key, cor, event, replyTo, payload)
}

func (w *Worker) _sendEx(ex string, key string, corrid string, typ string, replyTo string, payload interface{}) error {
	pl, err := w.c.serializer.Marshal(payload)
	if err != nil {
		return err
	}
	msg := amqp.Publishing{
		DeliveryMode:  amqp.Persistent,
		Timestamp:     time.Now(),
		ContentType:   w.c.serializer.ContentType(),
		ReplyTo:       replyTo,
		CorrelationId: corrid,
		Body:          []byte(pl),
	}

	log.Info().Int("id", w.id).Str("to", ex).Str("key", key).Str("corid", corrid).Str("type", typ).Str("replyTo", replyTo).
		Msg("Sending")
	return w.sch.Publish(ex, key, false, false, msg)
}

func (w *Worker) newChannel() (*amqp.Channel, error) {
	_, err := w.c.ensureConnection()
	if err != nil {
		return nil, err
	}

	return w.c.conn.Channel()
}

// this deals with a job blockingly
func (w *Worker) processJob(req wk.Request) (interface{}, error) {
	in := req.In.(Request)
	{
		var err error
		var msgs <-chan amqp.Delivery
		// receive
		// this has to be done before sending, otherwise it's too late
		w.ch.Cancel(w.queue.Name, false)
		msgs, err = w.ch.Consume(
			w.queue.Name, // queue
			w.queue.Name, // consumer
			true,         // autoAck
			true,         // exclusive
			false,        // noLocal
			false,        // noWait
			nil,
		)
		if err != nil {
			log.Error().Err(err).Msg("failed to consume")
			return nil, err
		}
		// send
		err = w.sendEx(in.target, Command, in.method, w.queue.Name, in.payload)
		if err != nil {
			return nil, err
		}
		select {
		case msg := <-msgs:
			if w.corrid() != msg.CorrelationId {
				log.Warn().Str("corrid", msg.CorrelationId).Msg("It looks like you're using multiple thread to access a single channel.")
				return nil, fmt.Errorf("unexpected message received")
			}
			res, err := w.c.toResult(&msg)
			log.Info().Int("id", w.id).Str("corrid", w.corrid()).Msg("Response")
			return res, err
		case <-in.ctx.Done():
			log.Error().Int("id", w.id).Str("corrid", w.corrid()).Str("method", in.method).Msg("RPC timeout")
			err := fmt.Errorf("Your RPC has timedout, commander: %s", in.method)
			return nil, err
		}
	}
}
<commit_msg>removing unecessary lock<commit_after>package xing

import (
	"fmt"
	"time"

	"github.com/rs/zerolog/log"
	"github.com/streadway/amqp"
	wk "github.com/wzhliang/xing/pkg/worker"
	"github.com/wzhliang/xing/utils"
)

// Worker ...
type Worker struct {
	id         int
	ch         *amqp.Channel    // incoming channel
	sch        *amqp.Channel    // outgoing channel
	queue      amqp.Queue       // queue for consumption, for RPC client, it holds results
	conn       *amqp.Connection // own copy for detecting reconnection
	d          wk.Dispatcher
	c          *Client // the AMQP client
	rpcCounter uint
}

// SetDispatcher ...
func (w *Worker) SetDispatcher(dis wk.Dispatcher) {
	w.d = dis
}

// Close ... should be called after stopped
func (w *Worker) Close() {
	// at this time, the channel is already closed
	var err error
	if w.c.conn == nil {
		log.Error().Msg("no connection to AMQP server. can't do much")
		return
	}
	w.ch, err = w.conn.Channel()
	if err != nil {
		log.Error().Err(err).Msg("failed to create channel")
		return
	}

	n, err := w.ch.QueueDelete(w.queue.Name, false, false, false)
	if err != nil {
		log.Error().Int("n", n).Err(err).Msg("failed to delete queue")
	}
}

func (w *Worker) corrid() string {
	return fmt.Sprintf("rpc-%s-%02d-%d", w.c.name, w.id, w.rpcCounter)
}

// Loop ...
func (w *Worker) Loop() {
	for {
		req := w.d.Dispatch()
		if req.In == nil {
			break
		}
		var results chan interface{}
		var errs chan error
		ret := func(v interface{}, err error) {
			results <- v
			errs <- err
		}

		results = req.Result

		errs = req.Err
		if w.c.conn == nil {
			log.Info().Msg("connection not available, sleeping a while...")
			time.Sleep(time.Duration(utils.Random(2000, 3000)) * time.Millisecond)
			// prevent mutlple works from trying to reconnect at the same time
			_, err := w.c.ensureConnection()
			if err == nil {
				w.onConnect(w.c.conn)
			} else {
				ret(nil, fmt.Errorf("connection lost"))
				continue
			}
		} else if w.c.conn != w.conn {
			log.Info().Msg("new conection")
			w.onConnect(w.c.conn)
		}
		ret(w.processJob(req))
	}
	log.Info().Int("id", w.id).Msg("I am done")
	w.c.wg.Done() // FIXME: should not have access to wg
}

func newWorker(conn *amqp.Connection, c *Client, id int) *Worker {
	w := Worker{
		id:         id,
		c:          c,
		conn:       nil,
		rpcCounter: uint(utils.Random(1000, 9999)),
	}
	// onConnect() will be called on demand due to the fact that our own conn is
	// different from Client's conn
	// this makes sure that worker always has a consumer to its queue so that
	// the queue will not be TTLed
	return &w
}

func (w *Worker) onConnect(conn *amqp.Connection) error {
	log.Info().Msg("connected")
	var err error
	w.conn = conn // refreshing own copy
	w.ch, err = conn.Channel()
	if err != nil {
		return nil
	}
	w.sch, err = conn.Channel()
	if err != nil {
		return nil
	}
	err = w.ch.ExchangeDeclare(
		RPCExchange, // name
		"topic",     // kind
		true,        // durable
		false,       // autodelete
		false,       // internal
		false,       // noWait
		nil,
	)
	if err != nil {
		return err
	}
	// I have to be a client
	w.queue, err = w.ch.QueueDeclare(
		// for client queue, we don't care it's name
		// also, changing it will cause crash
		fmt.Sprintf("%s-%02d", w.c.name, w.id), // name
		true,  // durable
		false, // autoDelete
		true,  // exclusive
		false, // noWait
		amqp.Table{
			"x-expires":     ResultQueueTTL,
			"x-message-ttl": RPCTTL,
		}, // args
	)
	if err != nil {
		log.Error().Err(err).Msg("setup failed")
		return nil
	}
	key := resultTopicName(w.queue.Name) // for client, queuename is the client's name
	log.Info().Str("queue", w.queue.Name).Str("exchange", RPCExchange).Str("key", key).
		Msg("Subscribing")
	err = w.ch.QueueBind(w.queue.Name, key, RPCExchange, false, nil)
	if err != nil {
		log.Error().Err(err).Msg("setup failed")
		return err
	}

	return nil
}

func (w *Worker) sendEx(target string, _type string, event string, replyTo string, payload interface{}) error {
	var cor string

	if _type == Command { // FIXME: actuall worker should only handle Command
		w.rpcCounter++
		cor = w.corrid()
	} else {
		cor = "N/A"
	}

	ex := w.c.exchange(_type)
	var key string
	if topicLength(target) == 2 { // load balanced rpc
		key = fmt.Sprintf("%s.*.%s.%s", target, _type, event)
	} else if topicLength(target) == 3 {
		key = fmt.Sprintf("%s.%s.%s", target, _type, event)
	} else {
		return fmt.Errorf("Invalid target: %s", target)
	}
	return w._sendEx(ex, key, cor, event, replyTo, payload)
}

func (w *Worker) _sendEx(ex string, key string, corrid string, typ string, replyTo string, payload interface{}) error {
	pl, err := w.c.serializer.Marshal(payload)
	if err != nil {
		return err
	}
	msg := amqp.Publishing{
		DeliveryMode:  amqp.Persistent,
		Timestamp:     time.Now(),
		ContentType:   w.c.serializer.ContentType(),
		ReplyTo:       replyTo,
		CorrelationId: corrid,
		Body:          []byte(pl),
	}

	log.Info().Int("id", w.id).Str("to", ex).Str("key", key).Str("corid", corrid).Str("type", typ).Str("replyTo", replyTo).
		Msg("Sending")
	return w.sch.Publish(ex, key, false, false, msg)
}

func (w *Worker) newChannel() (*amqp.Channel, error) {
	_, err := w.c.ensureConnection()
	if err != nil {
		return nil, err
	}

	return w.c.conn.Channel()
}

// this deals with a job blockingly
func (w *Worker) processJob(req wk.Request) (interface{}, error) {
	in := req.In.(Request)
	{
		var err error
		var msgs <-chan amqp.Delivery
		// receive
		// this has to be done before sending, otherwise it's too late
		w.ch.Cancel(w.queue.Name, false)
		msgs, err = w.ch.Consume(
			w.queue.Name, // queue
			w.queue.Name, // consumer
			true,         // autoAck
			true,         // exclusive
			false,        // noLocal
			false,        // noWait
			nil,
		)
		if err != nil {
			log.Error().Err(err).Msg("failed to consume")
			return nil, err
		}
		// send
		err = w.sendEx(in.target, Command, in.method, w.queue.Name, in.payload)
		if err != nil {
			return nil, err
		}
		select {
		case msg := <-msgs:
			if w.corrid() != msg.CorrelationId {
				log.Warn().Str("corrid", msg.CorrelationId).Msg("It looks like you're using multiple thread to access a single channel.")
				return nil, fmt.Errorf("unexpected message received")
			}
			res, err := w.c.toResult(&msg)
			log.Info().Int("id", w.id).Str("corrid", w.corrid()).Msg("Response")
			return res, err
		case <-in.ctx.Done():
			log.Error().Int("id", w.id).Str("corrid", w.corrid()).Str("method", in.method).Msg("RPC timeout")
			err := fmt.Errorf("Your RPC has timedout, commander: %s", in.method)
			return nil, err
		}
	}
}
<|endoftext|>


<commit_before>package backend

import (
	"bytes"
	"errors"
	"fmt"
	"log"
	"sort"
	"strings"
	"time"

	"github.com/cblomart/vsphere-graphite/utils"
	"github.com/valyala/fasthttp"
)

// ThinPrometheusClient tries to export metrics to prometheus as simply as possible
type ThinPrometheusClient struct {
	Hostname string
	Port     int
	address  string
}

const defaultPort = 9155

// NewThinPrometheusClient creates a new thin prometheus
func NewThinPrometheusClient(server string, port int) (ThinPrometheusClient, error) {
	//create the port
	if port == 0 {
		port = defaultPort
	} else if port < 1000 || port > 65535 {
		return ThinPrometheusClient{}, errors.New("port is not in a user range")
	}
	address := ""
	if len(server) > 0 {
		address = server
	}
	address = fmt.Sprintf("%s:%d", address, port)
	return ThinPrometheusClient{Hostname: server, Port: port, address: address}, nil
}

// ListenAndServe will start the listen thead for metric requests
func (client *ThinPrometheusClient) ListenAndServe() error {
	log.Printf("Start listening for metric request at %s\n", client.address)
	return fasthttp.ListenAndServe(client.address, fasthttp.CompressHandlerLevel(requestHandler, 9))
}

func requestHandler(ctx *fasthttp.RequestCtx) {
	if string(ctx.Path()) != "/metrics" {
		ctx.Error("Unsupported path", fasthttp.StatusNotFound)
		return
	}
	// prepare the channels for the request
	request := make(chan Point, 100)
	done := make(chan bool)
	channels := Channels{Request: &request, Done: &done}
	// create a buffer to organise metrics per type
	buffer := map[string][]string{}
	log.Println("Thin Prometheus Sending Query Request")
	// start the queriess
	select {
	case *queries <- channels:
		log.Println("Thin Prometheus Sent Query Request")
	default:
		ctx.Error("Query buffer full", fasthttp.StatusConflict)
		return
	}
	// start a timeout
	timeout := time.NewTimer(300 * time.Millisecond)
	// recieve done
	recdone := false
	log.Println("Thin Prometheus is waiting for query results")
	for {
		select {
		case point := <-*channels.Request:
			// reset timer
			if !timeout.Stop() {
				<-timeout.C
			}
			timeout.Reset(300 * time.Millisecond)
			// add point to the buffer
			addToThinPrometheusBuffer(buffer, &point)
		case <-*channels.Done:
			// finish consuming metrics and break loop
			log.Println("Thin Prometheus was signaled the end of the collection")
			recdone = true
		case <-timeout.C:
			// stop timer
			if recdone {
				timeout.Stop()
				log.Println("Thin Prometheus was signaled a timeout")
				break
			}
		}
		if recdone {
			break
		}
	}
	ctx.SetContentType("text/plain; charset=utf8")
	var outbuff bytes.Buffer
	for key, vals := range buffer {
		utils.MustWriteString(&outbuff, "#HELP ")
		utils.MustWriteString(&outbuff, key)
		utils.MustWriteString(&outbuff, " ")
		utils.MustWriteString(&outbuff, strings.Replace(key, "_", " ", -1))
		utils.MustWriteString(&outbuff, "\n")
		utils.MustWriteString(&outbuff, "#TYPE ")
		utils.MustWriteString(&outbuff, key)
		utils.MustWriteString(&outbuff, " gauge\n")
		for _, val := range vals {
			utils.MustWriteString(&outbuff, key)
			utils.MustWriteString(&outbuff, val)
			utils.MustWriteString(&outbuff, "\n")
		}
		_, err := ctx.Write(outbuff.Bytes())
		if err != nil {
			log.Printf("Error writing to buffer %s\n", err)
		}
		outbuff.Reset()
	}
	log.Println("Thin Prometheus Sended Response to request")
}

func addToThinPrometheusBuffer(metrics map[string][]string, point *Point) {
	var buffer bytes.Buffer
	utils.MustWriteString(&buffer, prefix)
	utils.MustWriteString(&buffer, "_")
	utils.MustWriteString(&buffer, point.Group)
	utils.MustWriteString(&buffer, "_")
	utils.MustWriteString(&buffer, point.Counter)
	utils.MustWriteString(&buffer, "_")
	utils.MustWriteString(&buffer, point.Rollup)
	metric := buffer.String()
	buffer.Reset()
	tags := point.GetTags(false, ",")
	var keys = make([]string, len(tags))
	for key := range tags {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	var tmp []string
	for _, key := range keys {
		if len(tags[key]) == 0 {
			continue
		}
		utils.MustWriteString(&buffer, key)
		utils.MustWriteString(&buffer, "=\"")
		utils.MustWriteString(&buffer, tags[key])
		utils.MustWriteString(&buffer, "\"")
		tmp = append(tmp, buffer.String())
		buffer.Reset()
	}
	strtags := strings.Join(tmp, ",")
	utils.MustWriteString(&buffer, "{")
	utils.MustWriteString(&buffer, strtags)
	utils.MustWriteString(&buffer, "} ")
	utils.MustWriteString(&buffer, utils.ValToString(point.Value, ",", false))

	if metrics[metric] == nil {
		metrics[metric] = []string{buffer.String()}
	} else {
		metrics[metric] = append(metrics[metric], buffer.String())
	}
	buffer.Reset()
}
<commit_msg>remove unnecessary break<commit_after>package backend

import (
	"bytes"
	"errors"
	"fmt"
	"log"
	"sort"
	"strings"
	"time"

	"github.com/cblomart/vsphere-graphite/utils"
	"github.com/valyala/fasthttp"
)

// ThinPrometheusClient tries to export metrics to prometheus as simply as possible
type ThinPrometheusClient struct {
	Hostname string
	Port     int
	address  string
}

const defaultPort = 9155

// NewThinPrometheusClient creates a new thin prometheus
func NewThinPrometheusClient(server string, port int) (ThinPrometheusClient, error) {
	//create the port
	if port == 0 {
		port = defaultPort
	} else if port < 1000 || port > 65535 {
		return ThinPrometheusClient{}, errors.New("port is not in a user range")
	}
	address := ""
	if len(server) > 0 {
		address = server
	}
	address = fmt.Sprintf("%s:%d", address, port)
	return ThinPrometheusClient{Hostname: server, Port: port, address: address}, nil
}

// ListenAndServe will start the listen thead for metric requests
func (client *ThinPrometheusClient) ListenAndServe() error {
	log.Printf("Start listening for metric request at %s\n", client.address)
	return fasthttp.ListenAndServe(client.address, fasthttp.CompressHandlerLevel(requestHandler, 9))
}

func requestHandler(ctx *fasthttp.RequestCtx) {
	if string(ctx.Path()) != "/metrics" {
		ctx.Error("Unsupported path", fasthttp.StatusNotFound)
		return
	}
	// prepare the channels for the request
	request := make(chan Point, 100)
	done := make(chan bool)
	channels := Channels{Request: &request, Done: &done}
	// create a buffer to organise metrics per type
	buffer := map[string][]string{}
	log.Println("Thin Prometheus Sending Query Request")
	// start the queriess
	select {
	case *queries <- channels:
		log.Println("Thin Prometheus Sent Query Request")
	default:
		ctx.Error("Query buffer full", fasthttp.StatusConflict)
		return
	}
	// start a timeout
	timeout := time.NewTimer(300 * time.Millisecond)
	// recieve done
	recdone := false
	log.Println("Thin Prometheus is waiting for query results")
	for {
		select {
		case point := <-*channels.Request:
			// reset timer
			if !timeout.Stop() {
				<-timeout.C
			}
			timeout.Reset(300 * time.Millisecond)
			// add point to the buffer
			addToThinPrometheusBuffer(buffer, &point)
		case <-*channels.Done:
			// finish consuming metrics and break loop
			log.Println("Thin Prometheus was signaled the end of the collection")
			recdone = true
		case <-timeout.C:
			// stop timer
			if recdone {
				timeout.Stop()
				log.Println("Thin Prometheus was signaled a timeout")
			}
		}
		if recdone {
			break
		}
	}
	ctx.SetContentType("text/plain; charset=utf8")
	var outbuff bytes.Buffer
	for key, vals := range buffer {
		utils.MustWriteString(&outbuff, "#HELP ")
		utils.MustWriteString(&outbuff, key)
		utils.MustWriteString(&outbuff, " ")
		utils.MustWriteString(&outbuff, strings.Replace(key, "_", " ", -1))
		utils.MustWriteString(&outbuff, "\n")
		utils.MustWriteString(&outbuff, "#TYPE ")
		utils.MustWriteString(&outbuff, key)
		utils.MustWriteString(&outbuff, " gauge\n")
		for _, val := range vals {
			utils.MustWriteString(&outbuff, key)
			utils.MustWriteString(&outbuff, val)
			utils.MustWriteString(&outbuff, "\n")
		}
		_, err := ctx.Write(outbuff.Bytes())
		if err != nil {
			log.Printf("Error writing to buffer %s\n", err)
		}
		outbuff.Reset()
	}
	log.Println("Thin Prometheus Sended Response to request")
}

func addToThinPrometheusBuffer(metrics map[string][]string, point *Point) {
	var buffer bytes.Buffer
	utils.MustWriteString(&buffer, prefix)
	utils.MustWriteString(&buffer, "_")
	utils.MustWriteString(&buffer, point.Group)
	utils.MustWriteString(&buffer, "_")
	utils.MustWriteString(&buffer, point.Counter)
	utils.MustWriteString(&buffer, "_")
	utils.MustWriteString(&buffer, point.Rollup)
	metric := buffer.String()
	buffer.Reset()
	tags := point.GetTags(false, ",")
	var keys = make([]string, len(tags))
	for key := range tags {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	var tmp []string
	for _, key := range keys {
		if len(tags[key]) == 0 {
			continue
		}
		utils.MustWriteString(&buffer, key)
		utils.MustWriteString(&buffer, "=\"")
		utils.MustWriteString(&buffer, tags[key])
		utils.MustWriteString(&buffer, "\"")
		tmp = append(tmp, buffer.String())
		buffer.Reset()
	}
	strtags := strings.Join(tmp, ",")
	utils.MustWriteString(&buffer, "{")
	utils.MustWriteString(&buffer, strtags)
	utils.MustWriteString(&buffer, "} ")
	utils.MustWriteString(&buffer, utils.ValToString(point.Value, ",", false))

	if metrics[metric] == nil {
		metrics[metric] = []string{buffer.String()}
	} else {
		metrics[metric] = append(metrics[metric], buffer.String())
	}
	buffer.Reset()
}
<|endoftext|>


<commit_before>package backend

import (
	"bytes"
	"errors"
	"fmt"
	"log"
	"sort"
	"strings"
	"time"

	"github.com/cblomart/vsphere-graphite/utils"
	"github.com/valyala/fasthttp"
)

// ThinPrometheusClient tries to export metrics to prometheus as simply as possible
type ThinPrometheusClient struct {
	Hostname string
	Port     int
	address  string
}

const defaultPort = 9155

// NewThinPrometheusClient creates a new thin prometheus
func NewThinPrometheusClient(server string, port int) (ThinPrometheusClient, error) {
	//create the port
	if port == 0 {
		port = defaultPort
	} else if port < 1000 || port > 65535 {
		return ThinPrometheusClient{}, errors.New("Port is not in a user range")
	}
	address := ""
	if len(server) > 0 {
		address = server
	}
	address = fmt.Sprintf("%s:%d", address, port)
	return ThinPrometheusClient{Hostname: server, Port: port, address: address}, nil
}

// ListenAndServe will start the listen thead for metric requests
func (client *ThinPrometheusClient) ListenAndServe() error {
	log.Printf("Start listening for metric request at %s\n", client.address)
	return fasthttp.ListenAndServe(client.address, fasthttp.CompressHandlerLevel(requestHandler, 9))
}

func requestHandler(ctx *fasthttp.RequestCtx) {
	if string(ctx.Path()) != "/metrics" {
		ctx.Error("Unsupported path", fasthttp.StatusNotFound)
		return
	}
	// prepare the channels for the request
	request := make(chan Point, 100)
	done := make(chan bool)
	channels := Channels{Request: &request, Done: &done}
	// create a buffer to organise metrics per type
	buffer := map[string][]string{}
	log.Println("Thin Prometheus Sending Query Request")
	// start the queriess
	select {
	case *queries <- channels:
	default:
		ctx.Error("Query buffer full", fasthttp.StatusConflict)
		return
	}
	// start a timeout
	timeout := time.NewTimer(10 * time.Second)
	// wait for the results
	wait := true
	log.Println("Thin Prometheus is waiting for query results")
	for wait {
		select {
		case point := <-*channels.Request:
			// reset timer
			if !timeout.Stop() {
				<-timeout.C
			}
			timeout.Reset(10 * time.Second)
			// add point to the buffer
			addToThinPrometheusBuffer(buffer, &point)
		case <-*channels.Done:
			// finish consuming metrics and break loop
			log.Println("Thin Prometheus was signaled the end of the collection")
			wait = false
		case <-timeout.C:
			// stop timer
			timeout.Stop()
			log.Println("Thin Prometheus was signaled a timeout")
			wait = false
			ctx.Error("Timeout while gathering data", fasthttp.StatusRequestTimeout)
			return
		}
	}
	ctx.SetContentType("text/plain; charset=utf8")
	var outbuff bytes.Buffer
	for key, vals := range buffer {
		utils.MustWriteString(&outbuff, "#HELP ")
		utils.MustWriteString(&outbuff, key)
		utils.MustWriteString(&outbuff, " ")
		utils.MustWriteString(&outbuff, strings.Replace(key, "_", " ", -1))
		utils.MustWriteString(&outbuff, "\n")
		utils.MustWriteString(&outbuff, "#TYPE ")
		utils.MustWriteString(&outbuff, key)
		utils.MustWriteString(&outbuff, " gauge\n")
		for _, val := range vals {
			utils.MustWriteString(&outbuff, key)
			utils.MustWriteString(&outbuff, val)
			utils.MustWriteString(&outbuff, "\n")
		}
		_, err := ctx.Write(outbuff.Bytes())
		if err != nil {
			log.Printf("Error writing to buffer %s\n", err)
		}
		outbuff.Reset()
	}
	log.Println("Thin Prometheus Sended Response to request")
}

func addToThinPrometheusBuffer(metrics map[string][]string, point *Point) {
	var buffer bytes.Buffer
	utils.MustWriteString(&buffer, prefix)
	utils.MustWriteString(&buffer, "_")
	utils.MustWriteString(&buffer, point.Group)
	utils.MustWriteString(&buffer, "_")
	utils.MustWriteString(&buffer, point.Counter)
	utils.MustWriteString(&buffer, "_")
	utils.MustWriteString(&buffer, point.Rollup)
	metric := buffer.String()
	buffer.Reset()
	tags := point.GetTags(false, ",")
	var keys = make([]string, len(tags))
	for key := range tags {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	var tmp []string
	for _, key := range keys {
		if len(tags[key]) == 0 {
			continue
		}
		utils.MustWriteString(&buffer, key)
		utils.MustWriteString(&buffer, "=\"")
		utils.MustWriteString(&buffer, tags[key])
		utils.MustWriteString(&buffer, "\"")
		tmp = append(tmp, buffer.String())
		buffer.Reset()
	}
	strtags := strings.Join(tmp, ",")
	utils.MustWriteString(&buffer, "{")
	utils.MustWriteString(&buffer, strtags)
	utils.MustWriteString(&buffer, "} ")
	utils.MustWriteString(&buffer, utils.ValToString(point.Value, ",", false))

	if metrics[metric] == nil {
		metrics[metric] = []string{buffer.String()}
	} else {
		metrics[metric] = append(metrics[metric], buffer.String())
	}
	buffer.Reset()
}
<commit_msg>log query sent<commit_after>package backend

import (
	"bytes"
	"errors"
	"fmt"
	"log"
	"sort"
	"strings"
	"time"

	"github.com/cblomart/vsphere-graphite/utils"
	"github.com/valyala/fasthttp"
)

// ThinPrometheusClient tries to export metrics to prometheus as simply as possible
type ThinPrometheusClient struct {
	Hostname string
	Port     int
	address  string
}

const defaultPort = 9155

// NewThinPrometheusClient creates a new thin prometheus
func NewThinPrometheusClient(server string, port int) (ThinPrometheusClient, error) {
	//create the port
	if port == 0 {
		port = defaultPort
	} else if port < 1000 || port > 65535 {
		return ThinPrometheusClient{}, errors.New("Port is not in a user range")
	}
	address := ""
	if len(server) > 0 {
		address = server
	}
	address = fmt.Sprintf("%s:%d", address, port)
	return ThinPrometheusClient{Hostname: server, Port: port, address: address}, nil
}

// ListenAndServe will start the listen thead for metric requests
func (client *ThinPrometheusClient) ListenAndServe() error {
	log.Printf("Start listening for metric request at %s\n", client.address)
	return fasthttp.ListenAndServe(client.address, fasthttp.CompressHandlerLevel(requestHandler, 9))
}

func requestHandler(ctx *fasthttp.RequestCtx) {
	if string(ctx.Path()) != "/metrics" {
		ctx.Error("Unsupported path", fasthttp.StatusNotFound)
		return
	}
	// prepare the channels for the request
	request := make(chan Point, 100)
	done := make(chan bool)
	channels := Channels{Request: &request, Done: &done}
	// create a buffer to organise metrics per type
	buffer := map[string][]string{}
	log.Println("Thin Prometheus Sending Query Request")
	// start the queriess
	select {
	case *queries <- channels:
		log.Println("Thin Prometheus Sent Query Request")
	default:
		ctx.Error("Query buffer full", fasthttp.StatusConflict)
		return
	}
	// start a timeout
	timeout := time.NewTimer(10 * time.Second)
	// wait for the results
	wait := true
	log.Println("Thin Prometheus is waiting for query results")
	for wait {
		select {
		case point := <-*channels.Request:
			// reset timer
			if !timeout.Stop() {
				<-timeout.C
			}
			timeout.Reset(10 * time.Second)
			// add point to the buffer
			addToThinPrometheusBuffer(buffer, &point)
		case <-*channels.Done:
			// finish consuming metrics and break loop
			log.Println("Thin Prometheus was signaled the end of the collection")
			wait = false
		case <-timeout.C:
			// stop timer
			timeout.Stop()
			log.Println("Thin Prometheus was signaled a timeout")
			wait = false
			ctx.Error("Timeout while gathering data", fasthttp.StatusRequestTimeout)
			return
		}
	}
	ctx.SetContentType("text/plain; charset=utf8")
	var outbuff bytes.Buffer
	for key, vals := range buffer {
		utils.MustWriteString(&outbuff, "#HELP ")
		utils.MustWriteString(&outbuff, key)
		utils.MustWriteString(&outbuff, " ")
		utils.MustWriteString(&outbuff, strings.Replace(key, "_", " ", -1))
		utils.MustWriteString(&outbuff, "\n")
		utils.MustWriteString(&outbuff, "#TYPE ")
		utils.MustWriteString(&outbuff, key)
		utils.MustWriteString(&outbuff, " gauge\n")
		for _, val := range vals {
			utils.MustWriteString(&outbuff, key)
			utils.MustWriteString(&outbuff, val)
			utils.MustWriteString(&outbuff, "\n")
		}
		_, err := ctx.Write(outbuff.Bytes())
		if err != nil {
			log.Printf("Error writing to buffer %s\n", err)
		}
		outbuff.Reset()
	}
	log.Println("Thin Prometheus Sended Response to request")
}

func addToThinPrometheusBuffer(metrics map[string][]string, point *Point) {
	var buffer bytes.Buffer
	utils.MustWriteString(&buffer, prefix)
	utils.MustWriteString(&buffer, "_")
	utils.MustWriteString(&buffer, point.Group)
	utils.MustWriteString(&buffer, "_")
	utils.MustWriteString(&buffer, point.Counter)
	utils.MustWriteString(&buffer, "_")
	utils.MustWriteString(&buffer, point.Rollup)
	metric := buffer.String()
	buffer.Reset()
	tags := point.GetTags(false, ",")
	var keys = make([]string, len(tags))
	for key := range tags {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	var tmp []string
	for _, key := range keys {
		if len(tags[key]) == 0 {
			continue
		}
		utils.MustWriteString(&buffer, key)
		utils.MustWriteString(&buffer, "=\"")
		utils.MustWriteString(&buffer, tags[key])
		utils.MustWriteString(&buffer, "\"")
		tmp = append(tmp, buffer.String())
		buffer.Reset()
	}
	strtags := strings.Join(tmp, ",")
	utils.MustWriteString(&buffer, "{")
	utils.MustWriteString(&buffer, strtags)
	utils.MustWriteString(&buffer, "} ")
	utils.MustWriteString(&buffer, utils.ValToString(point.Value, ",", false))

	if metrics[metric] == nil {
		metrics[metric] = []string{buffer.String()}
	} else {
		metrics[metric] = append(metrics[metric], buffer.String())
	}
	buffer.Reset()
}
<|endoftext|>


<commit_before>package buildbox

import (
  "fmt"
  "log"
  "time"
  "path"
)

// The Job struct uses strings for StartedAt and FinishedAt because
// if they were actual date objects, then when this struct is
// initialized they would have a default value of: 00:00:00.000000000.
// This causes problems for the Buildbox Agent API because it looks for
// the presence of values in these properties to determine if the build
// has finished.
type Job struct {
  ID string
  State string
  Env map[string]string
  Output string `json:"output,omitempty"`
  ExitStatus string `json:"exit_status,omitempty"`
  StartedAt string `json:"started_at,omitempty"`
  FinishedAt string `json:"finished_at,omitempty"`
}

func (b Job) String() string {
  return fmt.Sprintf("Job{ID: %s, State: %s}", b.ID, b.State)
}

func (c *Client) JobNext() (*Job, error) {
  // Create a new instance of a job that will be populated
  // by the client.
  var job Job

  // Return the job.
  return &job, c.Get(&job, "jobs/next")
}

func (c *Client) JobFindAndAssign(id string) (*Job, error) {
  // Create a new instance of a job that will be populated
  // by the client.
  var job Job

  // Find the job
  return &job, c.Get(&job, "jobs/" + id + "/assign")
}

func (c *Client) JobUpdate(job *Job) (*Job, error) {
  // Create a new instance of a job that will be populated
  // with the updated data by the client
  var updatedJob Job

  // Return the job.
  return &updatedJob, c.Put(&updatedJob, "jobs/" + job.ID, job)
}

func (j *Job) Run(agent *Agent) error {
  log.Printf("Starting job %s", j.ID)

  // Create the environment that the script will use
  env := []string{}

  // Add the environment variables from the API to the process
  for key, value := range j.Env {
    env = append(env, fmt.Sprintf("%s=%s", key, value))
  }

  // Mark the build as started
  j.StartedAt = time.Now().Format(time.RFC3339)
  _, err := agent.Client.JobUpdate(j)
  if err != nil {
    // We don't care if the HTTP request failed here. We hope that it
    // starts working during the actual build.
  }

  // This callback is called every second the build is running. This lets
  // us do a lazy-person's method of streaming data to Buildbox.
  callback := func(process Process) {
    j.Output = process.Output

    // Post the update to the API
    updatedJob, err := agent.Client.JobUpdate(j)
    if err != nil {
      // We don't really care if the job couldn't update at this point.
      // This is just a partial update. We'll just let the job run
      // and hopefully the host will fix itself before we finish.
      log.Printf("Problem with updating job %s", j.ID, err)
    } else if updatedJob.State == "canceled" {
      log.Printf("Cancelling job %s", j.ID)
      process.Kill()
    }
  }

  // Run the bootstrap script
  scriptPath := path.Dir(agent.BootstrapScript)
  scriptName := path.Base(agent.BootstrapScript)
  process, err := RunScript(scriptPath, scriptName, env, callback)

  // Did the process succeed?
  if err == nil {
    // Store the final output
    j.Output = process.Output
  } else {
    j.Output = fmt.Sprintf("%s", err)
  }

  // Mark the build as finished
  j.FinishedAt = time.Now().Format(time.RFC3339)

  // Use the last processes exit status. ExitStatus is a string
  // on the Job struct because 0 is considerered an empty value
  // and won't be marshalled. We only want to send the exit status
  // when the build has finsihed.
  j.ExitStatus = fmt.Sprintf("%d", process.ExitStatus)

  // Keep trying this call until it works. This is the most important one.
  for {
    _, err = agent.Client.JobUpdate(j)
    if err != nil {
      log.Printf("Problem with updating final job information %s (%s)", j.ID, err)

      // How long should we wait until we try again?
      idleSeconds := 5

      // Sleep for a while
      sleepTime := time.Duration(idleSeconds * 1000) * time.Millisecond
      time.Sleep(sleepTime)
    } else {
      break
    }
  }

  log.Printf("Finished job %s", j.ID)

  return nil
}
<commit_msg>Added a mechanism to find a job.<commit_after>package buildbox

import (
  "fmt"
  "log"
  "time"
  "path"
)

// The Job struct uses strings for StartedAt and FinishedAt because
// if they were actual date objects, then when this struct is
// initialized they would have a default value of: 00:00:00.000000000.
// This causes problems for the Buildbox Agent API because it looks for
// the presence of values in these properties to determine if the build
// has finished.
type Job struct {
  ID string
  State string
  Env map[string]string
  Output string `json:"output,omitempty"`
  ExitStatus string `json:"exit_status,omitempty"`
  StartedAt string `json:"started_at,omitempty"`
  FinishedAt string `json:"finished_at,omitempty"`
}

func (b Job) String() string {
  return fmt.Sprintf("Job{ID: %s, State: %s}", b.ID, b.State)
}

func (c *Client) JobNext() (*Job, error) {
  // Create a new instance of a job that will be populated
  // by the client.
  var job Job

  // Return the job.
  return &job, c.Get(&job, "jobs/next")
}

func (c *Client) JobFind(id string) (*Job, error) {
  // Create a new instance of a job that will be populated
  // by the client.
  var job Job

  // Find the job
  return &job, c.Get(&job, "jobs/" + id)
}

func (c *Client) JobFindAndAssign(id string) (*Job, error) {
  // Create a new instance of a job that will be populated
  // by the client.
  var job Job

  // Find the job
  return &job, c.Get(&job, "jobs/" + id + "/assign")
}

func (c *Client) JobUpdate(job *Job) (*Job, error) {
  // Create a new instance of a job that will be populated
  // with the updated data by the client
  var updatedJob Job

  // Return the job.
  return &updatedJob, c.Put(&updatedJob, "jobs/" + job.ID, job)
}

func (j *Job) Run(agent *Agent) error {
  log.Printf("Starting job %s", j.ID)

  // Create the environment that the script will use
  env := []string{}

  // Add the environment variables from the API to the process
  for key, value := range j.Env {
    env = append(env, fmt.Sprintf("%s=%s", key, value))
  }

  // Mark the build as started
  j.StartedAt = time.Now().Format(time.RFC3339)
  _, err := agent.Client.JobUpdate(j)
  if err != nil {
    // We don't care if the HTTP request failed here. We hope that it
    // starts working during the actual build.
  }

  // This callback is called every second the build is running. This lets
  // us do a lazy-person's method of streaming data to Buildbox.
  callback := func(process Process) {
    j.Output = process.Output

    // Post the update to the API
    updatedJob, err := agent.Client.JobUpdate(j)
    if err != nil {
      // We don't really care if the job couldn't update at this point.
      // This is just a partial update. We'll just let the job run
      // and hopefully the host will fix itself before we finish.
      log.Printf("Problem with updating job %s", j.ID, err)
    } else if updatedJob.State == "canceled" {
      log.Printf("Cancelling job %s", j.ID)
      process.Kill()
    }
  }

  // Run the bootstrap script
  scriptPath := path.Dir(agent.BootstrapScript)
  scriptName := path.Base(agent.BootstrapScript)
  process, err := RunScript(scriptPath, scriptName, env, callback)

  // Did the process succeed?
  if err == nil {
    // Store the final output
    j.Output = process.Output
  } else {
    j.Output = fmt.Sprintf("%s", err)
  }

  // Mark the build as finished
  j.FinishedAt = time.Now().Format(time.RFC3339)

  // Use the last processes exit status. ExitStatus is a string
  // on the Job struct because 0 is considerered an empty value
  // and won't be marshalled. We only want to send the exit status
  // when the build has finsihed.
  j.ExitStatus = fmt.Sprintf("%d", process.ExitStatus)

  // Keep trying this call until it works. This is the most important one.
  for {
    _, err = agent.Client.JobUpdate(j)
    if err != nil {
      log.Printf("Problem with updating final job information %s (%s)", j.ID, err)

      // How long should we wait until we try again?
      idleSeconds := 5

      // Sleep for a while
      sleepTime := time.Duration(idleSeconds * 1000) * time.Millisecond
      time.Sleep(sleepTime)
    } else {
      break
    }
  }

  log.Printf("Finished job %s", j.ID)

  return nil
}
<|endoftext|>


<commit_before>package cli

import (
	"context"

	"github.com/cosmos/cosmos-sdk/client"
	"github.com/cosmos/cosmos-sdk/client/flags"
	sdk "github.com/cosmos/cosmos-sdk/types"
	"github.com/spf13/cobra"
	"github.com/zigbee-alliance/distributed-compliance-ledger/utils/cli"
	"github.com/zigbee-alliance/distributed-compliance-ledger/x/validator/types"
)

func CmdListDisabledValidator() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "all-disabled-validators",
		Short: "Query the list of all disabled validators",
		RunE: func(cmd *cobra.Command, args []string) error {
			clientCtx := client.GetClientContextFromCmd(cmd)

			pageReq, err := client.ReadPageRequest(cmd.Flags())
			if err != nil {
				return err
			}

			queryClient := types.NewQueryClient(clientCtx)

			params := &types.QueryAllDisabledValidatorRequest{
				Pagination: pageReq,
			}

			res, err := queryClient.DisabledValidatorAll(context.Background(), params)
			if cli.IsKeyNotFoundRPCError(err) {
				return clientCtx.PrintString(cli.LightClientProxyForListQueries)
			}
			if err != nil {
				return err
			}

			return clientCtx.PrintProto(res)
		},
	}

	flags.AddPaginationFlagsToCmd(cmd, cmd.Use)
	flags.AddQueryFlagsToCmd(cmd)

	return cmd
}

func CmdShowDisabledValidator() *cobra.Command {
	var address string

	cmd := &cobra.Command{
		Use:   "disabled-validator --address [address]",
		Short: "Query disabled validator by address",
		Args:  cobra.ExactArgs(0),
		RunE: func(cmd *cobra.Command, args []string) (err error) {
			clientCtx := client.GetClientContextFromCmd(cmd)

			var res types.DisabledValidator

			addr, err := sdk.ValAddressFromBech32(address)
			if err != nil {
				addr2, err2 := sdk.AccAddressFromBech32(address)
				if err2 != nil {
					return err2
				}
				addr = sdk.ValAddress(addr2)
			}

			return cli.QueryWithProof(
				clientCtx,
				types.StoreKey,
				types.DisabledValidatorKeyPrefix,
				types.DisabledValidatorKey(addr.String()),
				&res,
			)
		},
	}

	cmd.Flags().StringVar(&address, FlagAddress, "", "Validator address")

	_ = cmd.MarkFlagRequired(FlagAddress)

	flags.AddQueryFlagsToCmd(cmd)

	return cmd
}
<commit_msg>Rename variable name 'addr2' to 'owner'<commit_after>package cli

import (
	"context"

	"github.com/cosmos/cosmos-sdk/client"
	"github.com/cosmos/cosmos-sdk/client/flags"
	sdk "github.com/cosmos/cosmos-sdk/types"
	"github.com/spf13/cobra"
	"github.com/zigbee-alliance/distributed-compliance-ledger/utils/cli"
	"github.com/zigbee-alliance/distributed-compliance-ledger/x/validator/types"
)

func CmdListDisabledValidator() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "all-disabled-validators",
		Short: "Query the list of all disabled validators",
		RunE: func(cmd *cobra.Command, args []string) error {
			clientCtx := client.GetClientContextFromCmd(cmd)

			pageReq, err := client.ReadPageRequest(cmd.Flags())
			if err != nil {
				return err
			}

			queryClient := types.NewQueryClient(clientCtx)

			params := &types.QueryAllDisabledValidatorRequest{
				Pagination: pageReq,
			}

			res, err := queryClient.DisabledValidatorAll(context.Background(), params)
			if cli.IsKeyNotFoundRPCError(err) {
				return clientCtx.PrintString(cli.LightClientProxyForListQueries)
			}
			if err != nil {
				return err
			}

			return clientCtx.PrintProto(res)
		},
	}

	flags.AddPaginationFlagsToCmd(cmd, cmd.Use)
	flags.AddQueryFlagsToCmd(cmd)

	return cmd
}

func CmdShowDisabledValidator() *cobra.Command {
	var address string

	cmd := &cobra.Command{
		Use:   "disabled-validator --address [address]",
		Short: "Query disabled validator by address",
		Args:  cobra.ExactArgs(0),
		RunE: func(cmd *cobra.Command, args []string) (err error) {
			clientCtx := client.GetClientContextFromCmd(cmd)

			var res types.DisabledValidator

			addr, err := sdk.ValAddressFromBech32(address)
			if err != nil {
				owner, err2 := sdk.AccAddressFromBech32(address)
				if err2 != nil {
					return err2
				}
				addr = sdk.ValAddress(owner)
			}

			return cli.QueryWithProof(
				clientCtx,
				types.StoreKey,
				types.DisabledValidatorKeyPrefix,
				types.DisabledValidatorKey(addr.String()),
				&res,
			)
		},
	}

	cmd.Flags().StringVar(&address, FlagAddress, "", "Validator address")

	_ = cmd.MarkFlagRequired(FlagAddress)

	flags.AddQueryFlagsToCmd(cmd)

	return cmd
}
<|endoftext|>


<commit_before>// Copyright 2016 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package alpine implements a vulnerability Fetcher using the alpine-secdb
// git repository.
package alpine

import (
	"errors"
	"io"
	"io/ioutil"
	"os"
	"strings"

	"gopkg.in/yaml.v2"

	"github.com/coreos/pkg/capnslog"

	"github.com/coreos/clair/database"
	"github.com/coreos/clair/updater"
	"github.com/coreos/clair/utils"
	cerrors "github.com/coreos/clair/utils/errors"
	"github.com/coreos/clair/utils/types"
)

const (
	secdbGitURL  = "http://git.alpinelinux.org/cgit/alpine-secdb"
	updaterFlag  = "alpine-secdbUpdater"
	nvdURLPrefix = "https://cve.mitre.org/cgi-bin/cvename.cgi?name="
)

var (
	// ErrFilesystem is returned when a fetcher fails to interact with the local filesystem.
	ErrFilesystem = errors.New("updater/fetchers: something went wrong when interacting with the fs")

	// ErrGitFailure is returned when a fetcher fails to interact with git.
	ErrGitFailure = errors.New("updater/fetchers: something went wrong when interacting with git")

	log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/alpine")
)

func init() {
	updater.RegisterFetcher("alpine", &fetcher{})
}

type fetcher struct {
	repositoryLocalPath string
}

func (f *fetcher) FetchUpdate(db database.Datastore) (resp updater.FetcherResponse, err error) {
	log.Info("fetching Alpine vulnerabilities")

	// Pull the master branch.
	var commit string
	commit, err = f.pullRepository()
	if err != nil {
		return
	}

	// Ask the database for the latest commit we successfully applied.
	var dbCommit string
	dbCommit, err = db.GetKeyValue(updaterFlag)
	if err != nil {
		return
	}

	// Set the updaterFlag to equal the commit processed.
	resp.FlagName = updaterFlag
	resp.FlagValue = commit

	// Short-circuit if there have been no updates.
	if commit == dbCommit {
		log.Debug("no alpine update")
		return
	}

	// Append any changed vulnerabilities to the response.
	for _, namespace := range []string{"v3.3", "v3.4"} {
		var file io.ReadCloser
		file, err = os.Open(f.repositoryLocalPath + "/" + namespace + "/main.yaml")
		if err != nil {
			return
		}
		log.Debug(namespace)

		var vulns []database.Vulnerability
		switch namespace {
		case "v3.3":
			vulns, err = parse33YAML(file)
		case "v3.4":
			vulns, err = parse34YAML(file)
		}
		if err != nil {
			return
		}

		resp.Vulnerabilities = append(resp.Vulnerabilities, vulns...)
		file.Close()
	}

	return
}

func (f *fetcher) pullRepository() (commit string, err error) {
	if _, pathExists := os.Stat(f.repositoryLocalPath); f.repositoryLocalPath == "" || os.IsNotExist(pathExists) {
		if f.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "alpine-secdb"); err != nil {
			return "", ErrFilesystem
		}

		if out, err := utils.Exec(f.repositoryLocalPath, "git", "clone", secdbGitURL, "."); err != nil {
			f.Clean()
			log.Errorf("could not pull alpine-secdb repository: %s. output: %s", err, out)
			return "", cerrors.ErrCouldNotDownload
		}
	}

	out, err := utils.Exec(f.repositoryLocalPath, "git", "rev-parse", "HEAD")
	if err != nil {
		return "", ErrGitFailure
	}

	commit = strings.TrimSpace(string(out))
	return
}

func (f *fetcher) Clean() {
	if f.repositoryLocalPath != "" {
		os.RemoveAll(f.repositoryLocalPath)
	}
}

type secdb33File struct {
	Distro   string `yaml:"distroversion"`
	Packages []struct {
		Pkg struct {
			Name    string   `yaml:"name"`
			Version string   `yaml:"ver"`
			Fixes   []string `yaml:"fixes"`
		} `yaml:"pkg"`
	} `yaml:"packages"`
}

func parse33YAML(r io.Reader) (vulns []database.Vulnerability, err error) {
	var rBytes []byte
	rBytes, err = ioutil.ReadAll(r)
	if err != nil {
		return
	}

	var file secdb33File
	err = yaml.Unmarshal(rBytes, &file)
	if err != nil {
		return
	}
	for _, pack := range file.Packages {
		pkg := pack.Pkg
		for _, fix := range pkg.Fixes {
			version, err := types.NewVersion(pkg.Version)
			if err != nil {
				log.Warningf("could not parse package version '%s': %s. skipping", pkg.Version, err.Error())
				continue
			}

			var vuln database.Vulnerability
			vuln.Severity = types.Unknown
			vuln.Name = fix
			vuln.Link = nvdURLPrefix + fix
			vuln.FixedIn = []database.FeatureVersion{
				{
					Feature: database.Feature{
						Namespace: database.Namespace{Name: "alpine:" + file.Distro},
						Name:      pkg.Name,
					},
					Version: version,
				},
			}
			vulns = append(vulns, vuln)
		}
	}

	return
}

type secdb34File struct {
	Distro   string `yaml:"distroversion"`
	Packages []struct {
		Pkg struct {
			Name  string              `yaml:"name"`
			Fixes map[string][]string `yaml:"secfixes"`
		} `yaml:"pkg"`
	} `yaml:"packages"`
}

func parse34YAML(r io.Reader) (vulns []database.Vulnerability, err error) {
	var rBytes []byte
	rBytes, err = ioutil.ReadAll(r)
	if err != nil {
		return
	}

	var file secdb34File
	err = yaml.Unmarshal(rBytes, &file)
	if err != nil {
		return
	}

	for _, pack := range file.Packages {
		pkg := pack.Pkg
		for versionStr, vulnStrs := range pkg.Fixes {
			version, err := types.NewVersion(versionStr)
			if err != nil {
				log.Warningf("could not parse package version '%s': %s. skipping", versionStr, err.Error())
				continue
			}

			for _, vulnStr := range vulnStrs {
				var vuln database.Vulnerability
				vuln.Severity = types.Unknown
				vuln.Name = vulnStr
				vuln.Link = nvdURLPrefix + vulnStr
				vuln.FixedIn = []database.FeatureVersion{
					{
						Feature: database.Feature{
							Namespace: database.Namespace{Name: "alpine:" + file.Distro},
							Name:      pkg.Name,
						},
						Version: version,
					},
				}
				vulns = append(vulns, vuln)
			}
		}
	}

	return
}
<commit_msg>alpine: refactor fetcher & git pull on update<commit_after>// Copyright 2016 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package alpine implements a vulnerability Fetcher using the alpine-secdb
// git repository.
package alpine

import (
	"errors"
	"io"
	"io/ioutil"
	"os"
	"strings"

	"gopkg.in/yaml.v2"

	"github.com/coreos/pkg/capnslog"

	"github.com/coreos/clair/database"
	"github.com/coreos/clair/updater"
	"github.com/coreos/clair/utils"
	cerrors "github.com/coreos/clair/utils/errors"
	"github.com/coreos/clair/utils/types"
)

const (
	// When available, this should be updated to use HTTPS.
	secdbGitURL  = "http://git.alpinelinux.org/cgit/alpine-secdb"
	updaterFlag  = "alpine-secdbUpdater"
	nvdURLPrefix = "https://cve.mitre.org/cgi-bin/cvename.cgi?name="
)

var (
	// ErrFilesystem is returned when a fetcher fails to interact with the local filesystem.
	ErrFilesystem = errors.New("updater/fetchers: something went wrong when interacting with the fs")

	// ErrGitFailure is returned when a fetcher fails to interact with git.
	ErrGitFailure = errors.New("updater/fetchers: something went wrong when interacting with git")

	log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/alpine")
)

func init() {
	updater.RegisterFetcher("alpine", &fetcher{})
}

type fetcher struct {
	repositoryLocalPath string
}

func (f *fetcher) FetchUpdate(db database.Datastore) (resp updater.FetcherResponse, err error) {
	log.Info("fetching Alpine vulnerabilities")

	// Pull the master branch.
	var commit string
	commit, err = f.pullRepository()
	if err != nil {
		return
	}

	// Ask the database for the latest commit we successfully applied.
	var dbCommit string
	dbCommit, err = db.GetKeyValue(updaterFlag)
	if err != nil {
		return
	}

	// Set the updaterFlag to equal the commit processed.
	resp.FlagName = updaterFlag
	resp.FlagValue = commit

	// Short-circuit if there have been no updates.
	if commit == dbCommit {
		log.Debug("no alpine update")
		return
	}

	// Append any changed vulnerabilities to the response.
	for _, namespace := range []string{"v3.3", "v3.4"} {
		var vulns []database.Vulnerability
		vulns, err = parseVulnsFromNamespace(f.repositoryLocalPath, namespace)
		if err != nil {
			return
		}
		resp.Vulnerabilities = append(resp.Vulnerabilities, vulns...)
	}

	return
}

func parseVulnsFromNamespace(repositoryPath, namespace string) (vulns []database.Vulnerability, err error) {
	var file io.ReadCloser
	file, err = os.Open(repositoryPath + "/" + namespace + "/main.yaml")
	if err != nil {
		return
	}
	defer file.Close()

	switch namespace {
	case "v3.3":
		vulns, err = parse33YAML(file)
	case "v3.4":
		vulns, err = parse34YAML(file)
	}

	return
}

func (f *fetcher) pullRepository() (commit string, err error) {
	// If the repository doesn't exist, clone it.
	if _, pathExists := os.Stat(f.repositoryLocalPath); f.repositoryLocalPath == "" || os.IsNotExist(pathExists) {
		if f.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "alpine-secdb"); err != nil {
			return "", ErrFilesystem
		}

		if out, err := utils.Exec(f.repositoryLocalPath, "git", "clone", secdbGitURL, "."); err != nil {
			f.Clean()
			log.Errorf("could not pull alpine-secdb repository: %s. output: %s", err, out)
			return "", cerrors.ErrCouldNotDownload
		}
	} else {
		// The repository exists and it needs to be refreshed via a pull.
		_, err := utils.Exec(f.repositoryLocalPath, "git", "pull")
		if err != nil {
			return "", ErrGitFailure
		}
	}

	out, err := utils.Exec(f.repositoryLocalPath, "git", "rev-parse", "HEAD")
	if err != nil {
		return "", ErrGitFailure
	}

	commit = strings.TrimSpace(string(out))
	return
}

func (f *fetcher) Clean() {
	if f.repositoryLocalPath != "" {
		os.RemoveAll(f.repositoryLocalPath)
	}
}

type secdb33File struct {
	Distro   string `yaml:"distroversion"`
	Packages []struct {
		Pkg struct {
			Name    string   `yaml:"name"`
			Version string   `yaml:"ver"`
			Fixes   []string `yaml:"fixes"`
		} `yaml:"pkg"`
	} `yaml:"packages"`
}

func parse33YAML(r io.Reader) (vulns []database.Vulnerability, err error) {
	var rBytes []byte
	rBytes, err = ioutil.ReadAll(r)
	if err != nil {
		return
	}

	var file secdb33File
	err = yaml.Unmarshal(rBytes, &file)
	if err != nil {
		return
	}
	for _, pack := range file.Packages {
		pkg := pack.Pkg
		for _, fix := range pkg.Fixes {
			version, err := types.NewVersion(pkg.Version)
			if err != nil {
				log.Warningf("could not parse package version '%s': %s. skipping", pkg.Version, err.Error())
				continue
			}

			vulns = append(vulns, database.Vulnerability{
				Name:     fix,
				Severity: types.Unknown,
				Link:     nvdURLPrefix + fix,
				FixedIn: []database.FeatureVersion{
					{
						Feature: database.Feature{
							Namespace: database.Namespace{Name: "alpine:" + file.Distro},
							Name:      pkg.Name,
						},
						Version: version,
					},
				},
			})
		}
	}
	return
}

type secdb34File struct {
	Distro   string `yaml:"distroversion"`
	Packages []struct {
		Pkg struct {
			Name  string              `yaml:"name"`
			Fixes map[string][]string `yaml:"secfixes"`
		} `yaml:"pkg"`
	} `yaml:"packages"`
}

func parse34YAML(r io.Reader) (vulns []database.Vulnerability, err error) {
	var rBytes []byte
	rBytes, err = ioutil.ReadAll(r)
	if err != nil {
		return
	}

	var file secdb34File
	err = yaml.Unmarshal(rBytes, &file)
	if err != nil {
		return
	}

	for _, pack := range file.Packages {
		pkg := pack.Pkg
		for versionStr, vulnStrs := range pkg.Fixes {
			version, err := types.NewVersion(versionStr)
			if err != nil {
				log.Warningf("could not parse package version '%s': %s. skipping", versionStr, err.Error())
				continue
			}

			for _, vulnStr := range vulnStrs {
				var vuln database.Vulnerability
				vuln.Severity = types.Unknown
				vuln.Name = vulnStr
				vuln.Link = nvdURLPrefix + vulnStr
				vuln.FixedIn = []database.FeatureVersion{
					{
						Feature: database.Feature{
							Namespace: database.Namespace{Name: "alpine:" + file.Distro},
							Name:      pkg.Name,
						},
						Version: version,
					},
				}
				vulns = append(vulns, vuln)
			}
		}
	}

	return
}
<|endoftext|>