<commit_before>package main

import (
	"go/ast"
	"go/parser"
	"unicode"
	"unicode/utf8"
)

type cursor_context struct {
	decl    *decl
	partial string
}

type bytes_iterator struct {
	data   []byte
	cursor int
}

// return the character under the cursor
func (this *bytes_iterator) char() byte {
	return this.data[this.cursor]
}

// return the rune under the cursor
func (this *bytes_iterator) rune() rune {
	r, _ := utf8.DecodeRune(this.data[this.cursor:])
	return r
}

// move cursor backwards to the next valid utf8 rune start, or 0
func (this *bytes_iterator) move_backwards() {
	for this.cursor != 0 {
		this.cursor--
		if utf8.RuneStart(this.char()) {
			return
		}
	}
}

var g_unicode_ident_set = []*unicode.RangeTable{
	unicode.Letter,
	unicode.Digit,
	{R16: []unicode.Range16{{'_', '_', 1}}},
}

// move cursor backwards, stop at the first rune that is not from
// 'g_unicode_ident_set', or 0
func (this *bytes_iterator) skip_ident() {
	for this.cursor != 0 {
		r := this.rune()

		// stop if 'r' is not [a-zA-Z0-9_] (unicode correct though)
		if !unicode.IsOneOf(g_unicode_ident_set, r) {
			return
		}
		this.move_backwards()
	}
}

var g_bracket_pairs = map[byte]byte{
	')': '(',
	']': '[',
}

// when the cursor is at the ')' or ']', move the cursor to an opposite bracket
// pair, this functions takes inner bracker pairs into account
func (this *bytes_iterator) skip_to_bracket_pair() {
	right := this.char()
	left := g_bracket_pairs[right]
	balance := 1
	for balance != 0 {
		this.cursor--
		if this.cursor == 0 {
			return
		}
		switch this.char() {
		case right:
			balance++
		case left:
			balance--
		}
	}
}

// starting from the end of the 'file', move backwards and return a slice of a
// valid Go expression
func (this *bytes_iterator) extract_go_expr() []byte {
	const (
		last_none = iota
		last_dot
		last_paren
		last_ident
	)
	last := last_none
	orig := this.cursor
	this.move_backwards()
loop:
	for {
		if this.cursor == 0 {
			return this.data[:orig]
		}
		r := this.rune()
		switch r {
		case '.':
			this.move_backwards()
			last = last_dot
		case ')', ']':
			if last == last_ident {
				break loop
			}
			this.skip_to_bracket_pair()
			this.move_backwards()
			last = last_paren
		default:
			if unicode.IsOneOf(g_unicode_ident_set, r) {
				this.skip_ident()
				last = last_ident
			} else {
				break loop
			}
		}
	}
	return this.data[this.cursor+1 : orig]
}

// this function is called when the cursor is at the '.' and you need to get the
// declaration before that dot
func (c *auto_complete_context) deduce_cursor_decl(iter *bytes_iterator) *decl {
	e := string(iter.extract_go_expr())
	expr, err := parser.ParseExpr(e)
	if err != nil {
		return nil
	}
	return expr_to_decl(expr, c.current.scope)
}

// deduce cursor context, it includes the declaration under the cursor and partial identifier
// (usually a part of the name of the child declaration)
func (c *auto_complete_context) deduce_cursor_context(file []byte, cursor int) (cursor_context, bool) {
	if cursor <= 0 {
		return cursor_context{nil, ""}, true
	}

	orig := cursor
	iter := bytes_iterator{file, cursor}

	// figure out what is just before the cursor
	iter.move_backwards()
	if iter.char() == '.' {
		// we're '<whatever>.'
		// figure out decl, Parital is ""
		decl := c.deduce_cursor_decl(&iter)
		return cursor_context{decl, ""}, decl != nil
	}

	r := iter.rune()
	if unicode.IsOneOf(g_unicode_ident_set, r) {
		// we're '<whatever>.<ident>'
		// parse <ident> as Partial and figure out decl
		iter.skip_ident()
		partial := string(iter.data[iter.cursor+1 : orig])
		if iter.char() == '.' {
			decl := c.deduce_cursor_decl(&iter)
			return cursor_context{decl, partial}, decl != nil
		} else {
			return cursor_context{nil, partial}, true
		}
	}

	return cursor_context{nil, ""}, true
}

// deduce the type of the expression under the cursor, a bit of copy & paste from the method
// above, returns true if deduction was successful (even if the result of it is nil)
func (c *auto_complete_context) deduce_cursor_type_pkg(file []byte, cursor int) (ast.Expr, string, bool) {
	deduce := func(iter *bytes_iterator) (ast.Expr, string) {
		e := string(iter.extract_go_expr())
		expr, err := parser.ParseExpr(e)
		if err != nil {
			return nil, ""
		}
		t, scope, _ := infer_type(expr, c.current.scope, -1)
		return t, lookup_pkg(get_type_path(t), scope)
	}

	if cursor <= 0 {
		return nil, "", true
	}

	iter := bytes_iterator{file, cursor}

	// figure out what is just before the cursor
	iter.move_backwards()
	if iter.char() == '.' {
		// we're '<whatever>.'
		// figure out decl, Parital is ""
		decl, pkg := deduce(&iter)
		return decl, pkg, decl != nil
	}

	r := iter.rune()
	if unicode.IsOneOf(g_unicode_ident_set, r) {
		// we're '<whatever>.<ident>'
		// parse <ident> as Partial and figure out decl
		iter.skip_ident()
		if iter.char() == '.' {
			decl, pkg := deduce(&iter)
			return decl, pkg, decl != nil
		} else {
			return nil, "", true
		}
	}

	return nil, "", true
}
<commit_msg>Rework cursortype behaviour to better match its name<commit_after>package main

import (
	"go/ast"
	"go/parser"
	"unicode"
	"unicode/utf8"
)

type cursor_context struct {
	decl    *decl
	partial string
}

type bytes_iterator struct {
	data   []byte
	cursor int
}

// return the character under the cursor
func (this *bytes_iterator) char() byte {
	return this.data[this.cursor]
}

// return the rune under the cursor
func (this *bytes_iterator) rune() rune {
	r, _ := utf8.DecodeRune(this.data[this.cursor:])
	return r
}

// move cursor backwards to the next valid utf8 rune start, or 0
func (this *bytes_iterator) move_backwards() {
	for this.cursor != 0 {
		this.cursor--
		if utf8.RuneStart(this.char()) {
			return
		}
	}
}

// move cursor forwards to the next valid utf8 rune start
func (this *bytes_iterator) move_forwards() {
	for this.cursor < len(this.data) {
		this.cursor++
		if utf8.RuneStart(this.char()) {
			return
		}
	}
}

var g_unicode_ident_set = []*unicode.RangeTable{
	unicode.Letter,
	unicode.Digit,
	{R16: []unicode.Range16{{'_', '_', 1}}},
}

// move cursor backwards, stop at the first rune that is not from
// 'g_unicode_ident_set', or 0
func (this *bytes_iterator) skip_ident() {
	for this.cursor != 0 {
		r := this.rune()

		// stop if 'r' is not [a-zA-Z0-9_] (unicode correct though)
		if !unicode.IsOneOf(g_unicode_ident_set, r) {
			return
		}
		this.move_backwards()
	}
}

func (this *bytes_iterator) skip_forward_ident() {
	for this.cursor < len(this.data) {
		r := this.rune()

		// stop if 'r' is not [a-zA-Z0-9_] (unicode correct though)
		if !unicode.IsOneOf(g_unicode_ident_set, r) {
			return
		}
		this.move_forwards()
	}
}

var g_bracket_pairs = map[byte]byte{
	')': '(',
	']': '[',
}

// when the cursor is at the ')' or ']', move the cursor to an opposite bracket
// pair, this functions takes inner bracker pairs into account
func (this *bytes_iterator) skip_to_bracket_pair() {
	right := this.char()
	left := g_bracket_pairs[right]
	balance := 1
	for balance != 0 {
		this.cursor--
		if this.cursor == 0 {
			return
		}
		switch this.char() {
		case right:
			balance++
		case left:
			balance--
		}
	}
}

// starting from the end of the 'file', move backwards and return a slice of a
// valid Go expression
func (this *bytes_iterator) extract_go_expr() []byte {
	const (
		last_none = iota
		last_dot
		last_paren
		last_ident
	)
	last := last_none
	orig := this.cursor
	this.move_backwards()
loop:
	for {
		if this.cursor == 0 {
			return this.data[:orig]
		}
		r := this.rune()
		switch r {
		case '.':
			this.move_backwards()
			last = last_dot
		case ')', ']':
			if last == last_ident {
				break loop
			}
			this.skip_to_bracket_pair()
			this.move_backwards()
			last = last_paren
		default:
			if unicode.IsOneOf(g_unicode_ident_set, r) {
				this.skip_ident()
				last = last_ident
			} else {
				break loop
			}
		}
	}
	return this.data[this.cursor+1 : orig]
}

// this function is called when the cursor is at the '.' and you need to get the
// declaration before that dot
func (c *auto_complete_context) deduce_cursor_decl(iter *bytes_iterator) *decl {
	e := string(iter.extract_go_expr())
	expr, err := parser.ParseExpr(e)
	if err != nil {
		return nil
	}
	return expr_to_decl(expr, c.current.scope)
}

// deduce cursor context, it includes the declaration under the cursor and partial identifier
// (usually a part of the name of the child declaration)
func (c *auto_complete_context) deduce_cursor_context(file []byte, cursor int) (cursor_context, bool) {
	if cursor <= 0 {
		return cursor_context{nil, ""}, true
	}

	orig := cursor
	iter := bytes_iterator{file, cursor}

	// figure out what is just before the cursor
	iter.move_backwards()
	if iter.char() == '.' {
		// we're '<whatever>.'
		// figure out decl, Parital is ""
		decl := c.deduce_cursor_decl(&iter)
		return cursor_context{decl, ""}, decl != nil
	}

	r := iter.rune()
	if unicode.IsOneOf(g_unicode_ident_set, r) {
		// we're '<whatever>.<ident>'
		// parse <ident> as Partial and figure out decl
		iter.skip_ident()
		partial := string(iter.data[iter.cursor+1 : orig])
		if iter.char() == '.' {
			decl := c.deduce_cursor_decl(&iter)
			return cursor_context{decl, partial}, decl != nil
		} else {
			return cursor_context{nil, partial}, true
		}
	}

	return cursor_context{nil, ""}, true
}

// deduce the type of the expression under the cursor, a bit of copy & paste from the method
// above, returns true if deduction was successful (even if the result of it is nil)
func (c *auto_complete_context) deduce_cursor_type_pkg(file []byte, cursor int) (ast.Expr, string, bool) {
	if cursor <= 0 {
		return nil, "", true
	}

	iter := bytes_iterator{file, cursor}

	// move forward to the end of the current identifier
	// so that we can grab the whole identifier when
	// reading backwards
	iter.skip_forward_ident()

	// read backwards to extract expression
	e := string(iter.extract_go_expr())

	expr, err := parser.ParseExpr(e)
	if err != nil {
		return nil, "", false
	} else {
		t, scope, _ := infer_type(expr, c.current.scope, -1)
		return t, lookup_pkg(get_type_path(t), scope), t != nil
	}
}
<|endoftext|>


<commit_before>package hipchat

import (
	"fmt"
	"net/http"
	"net/url"
	"strconv"
)

// RoomService gives access to the room related methods of the API.
type RoomService struct {
	client *Client
}

// Rooms represents a HipChat room list.
type Rooms struct {
	Items      []Room    `json:"items"`
	StartIndex int       `json:"startIndex"`
	MaxResults int       `json:"maxResults"`
	Links      PageLinks `json:"links"`
}

// Room represents a HipChat room.
type Room struct {
	ID                int            `json:"id"`
	Links             RoomLinks      `json:"links"`
	Name              string         `json:"name"`
	XmppJid           string         `json:"xmpp_jid"`
	Statistics        RoomStatistics `json:"statistics"`
	Created           string         `json:"created"`
	IsArchived        bool           `json:"is_archived"`
	Privacy           string         `json:"privacy"`
	IsGuestAccessible bool           `json:"is_guess_accessible"`
	Topic             string         `json:"topic"`
	Participants      []User         `json:"participants"`
	Owner             User           `json:"owner"`
	GuestAccessURL    string         `json:"guest_access_url"`
}

// RoomStatistics represents the HipChat room statistics.
type RoomStatistics struct {
	Links Links `json:"links"`
}

// CreateRoomRequest represents a HipChat room creation request.
type CreateRoomRequest struct {
	Topic       string `json:"topic,omitempty"`
	GuestAccess bool   `json:"guest_access,omitempty"`
	Name        string `json:"name,omitempty"`
	OwnerUserID string `json:"owner_user_id,omitempty"`
	Privacy     string `json:"privacy,omitempty"`
}

// UpdateRoomRequest represents a HipChat room update request.
type UpdateRoomRequest struct {
	Name          string `json:"name"`
	Topic         string `json:"topic"`
	IsGuestAccess bool   `json:"is_guest_access"`
	IsArchived    bool   `json:"is_archived"`
	Privacy       string `json:"privacy"`
	Owner         ID     `json:"owner"`
}

// RoomLinks represents the HipChat room links.
type RoomLinks struct {
	Links
	Webhooks     string `json:"webhooks"`
	Members      string `json:"members"`
	Participants string `json:"participants"`
}

// NotificationRequest represents a HipChat room notification request.
type NotificationRequest struct {
	Color         string `json:"color,omitempty"`
	Message       string `json:"message,omitempty"`
	Notify        bool   `json:"notify,omitempty"`
	MessageFormat string `json:"message_format,omitempty"`
	Card          Card   `json:"card,omitempty"`
}

// Card represents a HipChat card
type Card struct {
	Style       string      `json:"style"`
	ID          string      `json:"id"`
	URL         string      `json:"url"`
	Title       string      `json:"title"`
	Description string      `json:"description"`
	Site        string      `json:"site,omitempty"`
	Icon        Icon        `json:"icon,omitempty"`
	Date        uint32      `json:"date,omitempty"`
	Attributes  []Attribute `json:"attributes,omitempty"`
}

// Icon represents an icon
type Icon struct {
	URL string `json:"url"`
}

// Attribute represents an attribute on a card
type Attribute struct {
	Label string         `json:"label,omitempty"`
	Value AttributeValue `json:"value"`
}

// AttributeValue represents the value of an attribute
type AttributeValue struct {
	Icon  string `json:"icon,omitempty"`
	Label string `json:"label,string"`
	URL   string `json:"url,omitempty"`
}

// ShareFileRequest represents a HipChat room file share request.
type ShareFileRequest struct {
	Path     string `json:"path"`
	Filename string `json:"filename,omitempty"`
	Message  string `json:"message,omitempty"`
}

// HistoryRequest represents a HipChat room chat history request.
type HistoryRequest struct {
	Date       string `json:"date"`
	Timezone   string `json:"timezone"`
	StartIndex int    `json:"start-index"`
	MaxResults int    `json:"max-results"`
	Reverse    bool   `json:"reverse"`
}

// History represents a HipChat room chat history.
type History struct {
	Items      []Message `json:"items"`
	StartIndex int       `json:"startIndex"`
	MaxResults int       `json:"maxResults"`
	Links      PageLinks `json:"links"`
}

// Message represents a HipChat message.
type Message struct {
	Date          string      `json:"date"`
	From          interface{} `json:"from"` // string | obj <- weak
	Id            string      `json:"id"`
	Mentions      []User      `json:"mentions"`
	Message       string      `json:"message"`
	MessageFormat string      `json:"message_format"`
	Type          string      `json:"type"`
}

// SetTopicRequest represents a hipchat update topic request
type SetTopicRequest struct {
	Topic string `json:"topic"`
}

// InviteRequest represents a hipchat invite to room request
type InviteRequest struct {
	Reason string `json:"reason"`
}

// AddAttribute adds an attribute to a Card
func (c *Card) AddAttribute(mainLabel, subLabel, url, iconURL string) {
	attr := Attribute{Label: mainLabel}
	attr.Value = AttributeValue{Label: subLabel, URL: url, Icon: iconURL}

	c.Attributes = append(c.Attributes, attr)
}

// List returns all the rooms authorized.
//
// HipChat API docs: https://www.hipchat.com/docs/apiv2/method/get_all_rooms
func (r *RoomService) List() (*Rooms, *http.Response, error) {
	req, err := r.client.NewRequest("GET", "room", nil)
	if err != nil {
		return nil, nil, err
	}

	rooms := new(Rooms)
	resp, err := r.client.Do(req, rooms)
	if err != nil {
		return nil, resp, err
	}
	return rooms, resp, nil
}

// Get returns the room specified by the id.
//
// HipChat API docs: https://www.hipchat.com/docs/apiv2/method/get_room
func (r *RoomService) Get(id string) (*Room, *http.Response, error) {
	req, err := r.client.NewRequest("GET", fmt.Sprintf("room/%s", id), nil)
	if err != nil {
		return nil, nil, err
	}

	room := new(Room)
	resp, err := r.client.Do(req, room)
	if err != nil {
		return nil, resp, err
	}
	return room, resp, nil
}

// Notification sends a notification to the room specified by the id.
//
// HipChat API docs: https://www.hipchat.com/docs/apiv2/method/send_room_notification
func (r *RoomService) Notification(id string, notifReq *NotificationRequest) (*http.Response, error) {
	req, err := r.client.NewRequest("POST", fmt.Sprintf("room/%s/notification", id), notifReq)
	if err != nil {
		return nil, err
	}

	return r.client.Do(req, nil)
}

// ShareFile sends a file to the room specified by the id.
//
// HipChat API docs: https://www.hipchat.com/docs/apiv2/method/share_file_with_room
func (r *RoomService) ShareFile(id string, shareFileReq *ShareFileRequest) (*http.Response, error) {
	req, err := r.client.NewFileUploadRequest("POST", fmt.Sprintf("room/%s/share/file", id), shareFileReq)
	if err != nil {
		return nil, err
	}

	return r.client.Do(req, nil)
}

// Create creates a new room.
//
// HipChat API docs: https://www.hipchat.com/docs/apiv2/method/create_room
func (r *RoomService) Create(roomReq *CreateRoomRequest) (*Room, *http.Response, error) {
	req, err := r.client.NewRequest("POST", "room", roomReq)
	if err != nil {
		return nil, nil, err
	}

	room := new(Room)
	resp, err := r.client.Do(req, room)
	if err != nil {
		return nil, resp, err
	}
	return room, resp, nil
}

// Delete deletes an existing room.
//
// HipChat API docs: https://www.hipchat.com/docs/apiv2/method/delete_room
func (r *RoomService) Delete(id string) (*http.Response, error) {
	req, err := r.client.NewRequest("DELETE", fmt.Sprintf("room/%s", id), nil)
	if err != nil {
		return nil, err
	}

	return r.client.Do(req, nil)
}

// Update updates an existing room.
//
// HipChat API docs: https://www.hipchat.com/docs/apiv2/method/update_room
func (r *RoomService) Update(id string, roomReq *UpdateRoomRequest) (*http.Response, error) {
	req, err := r.client.NewRequest("PUT", fmt.Sprintf("room/%s", id), roomReq)
	if err != nil {
		return nil, err
	}

	return r.client.Do(req, nil)
}

// History fetches a room's chat history.
//
// HipChat API docs: https://www.hipchat.com/docs/apiv2/method/view_room_history
func (r *RoomService) History(id string, roomReq *HistoryRequest) (*History, *http.Response, error) {
	u := fmt.Sprintf("room/%s/history", id)
	// Form query parameters
	if roomReq != nil {
		p := url.Values{}
		if roomReq.Date != "" {
			p.Add("date", roomReq.Date)
		}
		if roomReq.Timezone != "" {
			p.Add("timezone", roomReq.Timezone)
		}
		if roomReq.StartIndex != 0 {
			p.Add("start-index", strconv.FormatInt(int64(roomReq.StartIndex), 10))
		}
		if roomReq.MaxResults != 0 {
			p.Add("max-results", strconv.FormatInt(int64(roomReq.MaxResults), 10))
		}
		// There's no way to tell whether caller set a boolean or not. We have to always set
		// it.
		p.Add("reverse", strconv.FormatBool(roomReq.Reverse))
		u += "?" + p.Encode()
	}
	req, err := r.client.NewRequest("GET", u, nil)
	h := new(History)
	resp, err := r.client.Do(req, &h)
	if err != nil {
		return nil, resp, err
	}
	return h, resp, nil
}

// Set Room topic.
//
// HipChat API docs: https://www.hipchat.com/docs/apiv2/method/set_topic
func (r *RoomService) SetTopic(id string, topic string) (*http.Response, error) {
	topicReq := &SetTopicRequest{Topic: topic}

	req, err := r.client.NewRequest("PUT", fmt.Sprintf("room/%s/topic", id), topicReq)
	if err != nil {
		return nil, err
	}

	return r.client.Do(req, nil)
}

// Invite someone to the Room.
//
// HipChat API docs: https://www.hipchat.com/docs/apiv2/method/invite_user
func (r *RoomService) Invite(room string, user string, reason string) (*http.Response, error) {
	reasonReq := &InviteRequest{Reason: reason}

	req, err := r.client.NewRequest("POST", fmt.Sprintf("room/%s/invite/%s", room, user), reasonReq)
	if err != nil {
		return nil, err
	}

	return r.client.Do(req, nil)
}
<commit_msg>More stuff<commit_after>package hipchat

import (
	"fmt"
	"net/http"
	"net/url"
	"strconv"
)

// RoomService gives access to the room related methods of the API.
type RoomService struct {
	client *Client
}

// Rooms represents a HipChat room list.
type Rooms struct {
	Items      []Room    `json:"items"`
	StartIndex int       `json:"startIndex"`
	MaxResults int       `json:"maxResults"`
	Links      PageLinks `json:"links"`
}

// Room represents a HipChat room.
type Room struct {
	ID                int            `json:"id"`
	Links             RoomLinks      `json:"links"`
	Name              string         `json:"name"`
	XmppJid           string         `json:"xmpp_jid"`
	Statistics        RoomStatistics `json:"statistics"`
	Created           string         `json:"created"`
	IsArchived        bool           `json:"is_archived"`
	Privacy           string         `json:"privacy"`
	IsGuestAccessible bool           `json:"is_guess_accessible"`
	Topic             string         `json:"topic"`
	Participants      []User         `json:"participants"`
	Owner             User           `json:"owner"`
	GuestAccessURL    string         `json:"guest_access_url"`
}

// RoomStatistics represents the HipChat room statistics.
type RoomStatistics struct {
	Links Links `json:"links"`
}

// CreateRoomRequest represents a HipChat room creation request.
type CreateRoomRequest struct {
	Topic       string `json:"topic,omitempty"`
	GuestAccess bool   `json:"guest_access,omitempty"`
	Name        string `json:"name,omitempty"`
	OwnerUserID string `json:"owner_user_id,omitempty"`
	Privacy     string `json:"privacy,omitempty"`
}

// UpdateRoomRequest represents a HipChat room update request.
type UpdateRoomRequest struct {
	Name          string `json:"name"`
	Topic         string `json:"topic"`
	IsGuestAccess bool   `json:"is_guest_access"`
	IsArchived    bool   `json:"is_archived"`
	Privacy       string `json:"privacy"`
	Owner         ID     `json:"owner"`
}

// RoomLinks represents the HipChat room links.
type RoomLinks struct {
	Links
	Webhooks     string `json:"webhooks"`
	Members      string `json:"members"`
	Participants string `json:"participants"`
}

// NotificationRequest represents a HipChat room notification request.
type NotificationRequest struct {
	Color         string `json:"color,omitempty"`
	Message       string `json:"message,omitempty"`
	Notify        bool   `json:"notify,omitempty"`
	MessageFormat string `json:"message_format,omitempty"`
	Card          Card   `json:"card,omitempty"`
}

// Card represents a HipChat card
type Card struct {
	Style       string      `json:"style"`
	ID          string      `json:"id"`
	URL         string      `json:"url"`
	Title       string      `json:"title"`
	Description string      `json:"description"`
	Site        string      `json:"site,omitempty"`
	Icon        Icon        `json:"icon,omitempty"`
	Date        uint32      `json:"date,omitempty"`
	Attributes  []Attribute `json:"attributes,omitempty"`
}

// Icon represents an icon
type Icon struct {
	URL string `json:"url"`
}

// Attribute represents an attribute on a card
type Attribute struct {
	Label string         `json:"label,omitempty"`
	Value AttributeValue `json:"value"`
}

// AttributeValue represents the value of an attribute
type AttributeValue struct {
	Icon  string `json:"icon,omitempty"`
	Label string `json:"label,omitempty"`
	URL   string `json:"url,omitempty"`
}

// ShareFileRequest represents a HipChat room file share request.
type ShareFileRequest struct {
	Path     string `json:"path"`
	Filename string `json:"filename,omitempty"`
	Message  string `json:"message,omitempty"`
}

// HistoryRequest represents a HipChat room chat history request.
type HistoryRequest struct {
	Date       string `json:"date"`
	Timezone   string `json:"timezone"`
	StartIndex int    `json:"start-index"`
	MaxResults int    `json:"max-results"`
	Reverse    bool   `json:"reverse"`
}

// History represents a HipChat room chat history.
type History struct {
	Items      []Message `json:"items"`
	StartIndex int       `json:"startIndex"`
	MaxResults int       `json:"maxResults"`
	Links      PageLinks `json:"links"`
}

// Message represents a HipChat message.
type Message struct {
	Date          string      `json:"date"`
	From          interface{} `json:"from"` // string | obj <- weak
	Id            string      `json:"id"`
	Mentions      []User      `json:"mentions"`
	Message       string      `json:"message"`
	MessageFormat string      `json:"message_format"`
	Type          string      `json:"type"`
}

// SetTopicRequest represents a hipchat update topic request
type SetTopicRequest struct {
	Topic string `json:"topic"`
}

// InviteRequest represents a hipchat invite to room request
type InviteRequest struct {
	Reason string `json:"reason"`
}

// AddAttribute adds an attribute to a Card
func (c *Card) AddAttribute(mainLabel, subLabel, url, iconURL string) {
	attr := Attribute{Label: mainLabel}
	attr.Value = AttributeValue{Label: subLabel, URL: url, Icon: iconURL}

	c.Attributes = append(c.Attributes, attr)
}

// List returns all the rooms authorized.
//
// HipChat API docs: https://www.hipchat.com/docs/apiv2/method/get_all_rooms
func (r *RoomService) List() (*Rooms, *http.Response, error) {
	req, err := r.client.NewRequest("GET", "room", nil)
	if err != nil {
		return nil, nil, err
	}

	rooms := new(Rooms)
	resp, err := r.client.Do(req, rooms)
	if err != nil {
		return nil, resp, err
	}
	return rooms, resp, nil
}

// Get returns the room specified by the id.
//
// HipChat API docs: https://www.hipchat.com/docs/apiv2/method/get_room
func (r *RoomService) Get(id string) (*Room, *http.Response, error) {
	req, err := r.client.NewRequest("GET", fmt.Sprintf("room/%s", id), nil)
	if err != nil {
		return nil, nil, err
	}

	room := new(Room)
	resp, err := r.client.Do(req, room)
	if err != nil {
		return nil, resp, err
	}
	return room, resp, nil
}

// Notification sends a notification to the room specified by the id.
//
// HipChat API docs: https://www.hipchat.com/docs/apiv2/method/send_room_notification
func (r *RoomService) Notification(id string, notifReq *NotificationRequest) (*http.Response, error) {
	req, err := r.client.NewRequest("POST", fmt.Sprintf("room/%s/notification", id), notifReq)
	if err != nil {
		return nil, err
	}

	return r.client.Do(req, nil)
}

// ShareFile sends a file to the room specified by the id.
//
// HipChat API docs: https://www.hipchat.com/docs/apiv2/method/share_file_with_room
func (r *RoomService) ShareFile(id string, shareFileReq *ShareFileRequest) (*http.Response, error) {
	req, err := r.client.NewFileUploadRequest("POST", fmt.Sprintf("room/%s/share/file", id), shareFileReq)
	if err != nil {
		return nil, err
	}

	return r.client.Do(req, nil)
}

// Create creates a new room.
//
// HipChat API docs: https://www.hipchat.com/docs/apiv2/method/create_room
func (r *RoomService) Create(roomReq *CreateRoomRequest) (*Room, *http.Response, error) {
	req, err := r.client.NewRequest("POST", "room", roomReq)
	if err != nil {
		return nil, nil, err
	}

	room := new(Room)
	resp, err := r.client.Do(req, room)
	if err != nil {
		return nil, resp, err
	}
	return room, resp, nil
}

// Delete deletes an existing room.
//
// HipChat API docs: https://www.hipchat.com/docs/apiv2/method/delete_room
func (r *RoomService) Delete(id string) (*http.Response, error) {
	req, err := r.client.NewRequest("DELETE", fmt.Sprintf("room/%s", id), nil)
	if err != nil {
		return nil, err
	}

	return r.client.Do(req, nil)
}

// Update updates an existing room.
//
// HipChat API docs: https://www.hipchat.com/docs/apiv2/method/update_room
func (r *RoomService) Update(id string, roomReq *UpdateRoomRequest) (*http.Response, error) {
	req, err := r.client.NewRequest("PUT", fmt.Sprintf("room/%s", id), roomReq)
	if err != nil {
		return nil, err
	}

	return r.client.Do(req, nil)
}

// History fetches a room's chat history.
//
// HipChat API docs: https://www.hipchat.com/docs/apiv2/method/view_room_history
func (r *RoomService) History(id string, roomReq *HistoryRequest) (*History, *http.Response, error) {
	u := fmt.Sprintf("room/%s/history", id)
	// Form query parameters
	if roomReq != nil {
		p := url.Values{}
		if roomReq.Date != "" {
			p.Add("date", roomReq.Date)
		}
		if roomReq.Timezone != "" {
			p.Add("timezone", roomReq.Timezone)
		}
		if roomReq.StartIndex != 0 {
			p.Add("start-index", strconv.FormatInt(int64(roomReq.StartIndex), 10))
		}
		if roomReq.MaxResults != 0 {
			p.Add("max-results", strconv.FormatInt(int64(roomReq.MaxResults), 10))
		}
		// There's no way to tell whether caller set a boolean or not. We have to always set
		// it.
		p.Add("reverse", strconv.FormatBool(roomReq.Reverse))
		u += "?" + p.Encode()
	}
	req, err := r.client.NewRequest("GET", u, nil)
	h := new(History)
	resp, err := r.client.Do(req, &h)
	if err != nil {
		return nil, resp, err
	}
	return h, resp, nil
}

// Set Room topic.
//
// HipChat API docs: https://www.hipchat.com/docs/apiv2/method/set_topic
func (r *RoomService) SetTopic(id string, topic string) (*http.Response, error) {
	topicReq := &SetTopicRequest{Topic: topic}

	req, err := r.client.NewRequest("PUT", fmt.Sprintf("room/%s/topic", id), topicReq)
	if err != nil {
		return nil, err
	}

	return r.client.Do(req, nil)
}

// Invite someone to the Room.
//
// HipChat API docs: https://www.hipchat.com/docs/apiv2/method/invite_user
func (r *RoomService) Invite(room string, user string, reason string) (*http.Response, error) {
	reasonReq := &InviteRequest{Reason: reason}

	req, err := r.client.NewRequest("POST", fmt.Sprintf("room/%s/invite/%s", room, user), reasonReq)
	if err != nil {
		return nil, err
	}

	return r.client.Do(req, nil)
}
<|endoftext|>


<commit_before>package hlib

type Wscript_t struct {
	Package   Package_t
	Options   Options_t
	Configure Configure_t
	Build     Build_t
}

type Stmt interface {
	is_stmt()
}

type Package_t struct {
	Name     string
	Authors  []Author
	Managers []Manager
	Version  Version
	Deps     []Dep_t
}

type Author string
type Manager string
type Version string

type Dep_t struct {
	Name    string
	Version string
	Type    DepType
}

type DepType int

const (
	PublicDep DepType = iota << 1
	PrivateDep
	RuntimeDep
)

type Visibility int

const (
	Local    Visibility = 0
	Exported Visibility = 1
)

type Options_t struct {
	Tools    []string
	HwafCall []string
	Stmts    []Stmt
}

type Configure_t struct {
	Tools    []string
	HwafCall []string
	Env      Env_t
	//Tags  []Value
	Stmts []Stmt
}

type Env_t map[string]Value

type Build_t struct {
	Tools    []string
	HwafCall []string
	Targets  Targets_t
	Stmts    []Stmt
	Env      Env_t
}

type Targets_t []Target_t

type Target_t struct {
	Name           string
	Features       string
	Source         []Value
	Use            []Value
	Defines        []Value
	CFlags         []Value
	CxxFlags       []Value
	LinkFlags      []Value
	ShlibFlags     []Value
	StlibFlags     []Value
	RPath          []Value
	Includes       []Value
	ExportIncludes []Value
}

type KeyValue struct {
	Tag   string
	Value []string
}

type Value struct {
	Name string
	Set  []KeyValue // first item is the "default"
}

// EOF
<commit_msg>hlib: fix definition of hlib.DepType to get a correct mask<commit_after>package hlib

type Wscript_t struct {
	Package   Package_t
	Options   Options_t
	Configure Configure_t
	Build     Build_t
}

type Stmt interface {
	is_stmt()
}

type Package_t struct {
	Name     string
	Authors  []Author
	Managers []Manager
	Version  Version
	Deps     []Dep_t
}

type Author string
type Manager string
type Version string

type Dep_t struct {
	Name    string
	Version string
	Type    DepType
}

type DepType int

const (
	UnknownDep DepType = 1 << iota
	PublicDep
	PrivateDep
	RuntimeDep
)

func (d DepType) HasMask(mask DepType) bool {
	return bool((d & mask) != 0)
}

type Visibility int

const (
	Local    Visibility = 0
	Exported Visibility = 1
)

type Options_t struct {
	Tools    []string
	HwafCall []string
	Stmts    []Stmt
}

type Configure_t struct {
	Tools    []string
	HwafCall []string
	Env      Env_t
	//Tags  []Value
	Stmts []Stmt
}

type Env_t map[string]Value

type Build_t struct {
	Tools    []string
	HwafCall []string
	Targets  Targets_t
	Stmts    []Stmt
	Env      Env_t
}

type Targets_t []Target_t

type Target_t struct {
	Name           string
	Features       string
	Source         []Value
	Use            []Value
	Defines        []Value
	CFlags         []Value
	CxxFlags       []Value
	LinkFlags      []Value
	ShlibFlags     []Value
	StlibFlags     []Value
	RPath          []Value
	Includes       []Value
	ExportIncludes []Value
}

type KeyValue struct {
	Tag   string
	Value []string
}

type Value struct {
	Name string
	Set  []KeyValue // first item is the "default"
}

// EOF
<|endoftext|>


<commit_before>package home

import (
	"errors"
	"github.com/belogik/goes"
	_ "log"
	"net/http"
	"net/url"
	"strconv"
	"time"

	"github.com/firstrow/logvoyage/common"
	"github.com/firstrow/logvoyage/web/render"
	"github.com/firstrow/logvoyage/web/widgets"
)

type DateTimeRange struct {
	Start string
	Stop  string
}

func (this *DateTimeRange) IsValid() bool {
	return this.Start != "" && this.Stop != ""
}

// Represents search request to perform in ES
type SearchRequest struct {
	Text      string   // test to search
	Indexes   []string // ES indexeses to perform search
	Types     []string // search types
	Size      int      // home much objects ES must return
	From      int      // how much objects should ES skip from first
	TimeRange DateTimeRange
}

func buildSearchRequest(text string, indexes []string, size int, from int, datetime DateTimeRange) SearchRequest {
	req := SearchRequest{
		Text:      text,
		Indexes:   indexes,
		From:      from,
		Types:     []string{"logs"},
		Size:      10,
		TimeRange: datetime,
	}
	return req
}

// Detects time range from request and returns
// elastic compatible format string
func buildTimeRange(req *http.Request) DateTimeRange {
	layout := "2006/01/02 15:04"
	var timeRange DateTimeRange

	switch req.URL.Query().Get("time") {
	case "15m":
		timeRange.Start = "now-15m"
	case "30m":
		timeRange.Start = "now-30m"
	case "60m":
		timeRange.Start = "now-60m"
	case "12h":
		timeRange.Start = "now-12h"
	case "24h":
		timeRange.Start = "now-24h"
	case "week":
		timeRange.Start = "now-1d"
	case "custom":
		timeStart, err := time.Parse(layout, req.URL.Query().Get("time_start"))
		if err != nil {
			return timeRange
		}
		timeStop, err := time.Parse(layout, req.URL.Query().Get("time_stop"))
		if err != nil {
			return timeRange
		}
		timeRange.Start = timeStart.Format(time.RFC3339)
		timeRange.Stop = timeStop.Format(time.RFC3339)
	}

	return timeRange
}

// Search logs in elastic.
func search(searchRequest SearchRequest) (goes.Response, error) {
	conn := common.GetConnection()

	if len(searchRequest.Text) > 0 {
		strconv.Quote(searchRequest.Text)
	} else {
		searchRequest.Text = "*"
	}

	var query = map[string]interface{}{
		"query": map[string]interface{}{
			"query_string": map[string]string{
				"default_field": "message",
				"query":         searchRequest.Text,
			},
		},
		"from": searchRequest.From,
		"size": searchRequest.Size,
		"sort": map[string]string{
			"datetime": "desc",
		},
	}

	if searchRequest.TimeRange.IsValid() {
		query["filter"] = map[string]interface{}{
			"range": map[string]interface{}{
				"datetime": map[string]string{
					"gte": searchRequest.TimeRange.Start,
					"lte": searchRequest.TimeRange.Stop,
				},
			},
		}
	}

	extraArgs := make(url.Values, 1)
	searchResults, err := conn.Search(query, searchRequest.Indexes, searchRequest.Types, extraArgs)

	if err != nil {
		return goes.Response{}, errors.New("No records found.")
	} else {
		return searchResults, nil
	}
}

func Index(req *http.Request, r *render.Render) {
	query_text := req.URL.Query().Get("q")
	user := common.FindUserByEmail(r.Context["email"].(string))

	// Pagination
	pagination := widgets.NewPagination(req)
	pagination.SetPerPage(100)

	// Load records
	searchRequest := buildSearchRequest(
		query_text,
		[]string{user.GetIndexName()},
		pagination.GetPerPage(),
		pagination.DetectFrom(),
		buildTimeRange(req),
	)
	// Search data in elastic
	data, err := search(searchRequest)

	pagination.SetTotalRecords(data.Hits.Total)

	var viewName string
	if data.Hits.Total > 0 && err == nil {
		viewName = "home/index"
	} else {
		viewName = "home/no_records"
	}

	r.HTML(viewName, render.ViewData{
		"logs":       data.Hits.Hits,
		"total":      data.Hits.Total,
		"query_text": query_text,
		"pagination": pagination,
	})
}
<commit_msg>Fixed range support<commit_after>package home

import (
	"errors"
	"github.com/belogik/goes"
	_ "log"
	"net/http"
	"net/url"
	"strconv"
	"time"

	"github.com/firstrow/logvoyage/common"
	"github.com/firstrow/logvoyage/web/render"
	"github.com/firstrow/logvoyage/web/widgets"
)

type DateTimeRange struct {
	Start string
	Stop  string
}

func (this *DateTimeRange) IsValid() bool {
	return this.Start != "" || this.Stop != ""
}

// Represents search request to perform in ES
type SearchRequest struct {
	Text      string   // test to search
	Indexes   []string // ES indexeses to perform search
	Types     []string // search types
	Size      int      // home much objects ES must return
	From      int      // how much objects should ES skip from first
	TimeRange DateTimeRange
}

func buildSearchRequest(text string, indexes []string, size int, from int, datetime DateTimeRange) SearchRequest {
	req := SearchRequest{
		Text:      text,
		Indexes:   indexes,
		From:      from,
		Types:     []string{"logs"},
		Size:      10,
		TimeRange: datetime,
	}
	return req
}

// Detects time range from request and returns
// elastic compatible format string
func buildTimeRange(req *http.Request) DateTimeRange {
	layout := "2006/01/02 15:04"
	var timeRange DateTimeRange

	switch req.URL.Query().Get("time") {
	case "15m":
		timeRange.Start = "now-15m"
	case "30m":
		timeRange.Start = "now-30m"
	case "60m":
		timeRange.Start = "now-60m"
	case "12h":
		timeRange.Start = "now-12h"
	case "24h":
		timeRange.Start = "now-24h"
	case "week":
		timeRange.Start = "now-1d"
	case "custom":
		timeStart, err := time.Parse(layout, req.URL.Query().Get("time_start"))
		if err != nil {
			return timeRange
		}
		timeStop, err := time.Parse(layout, req.URL.Query().Get("time_stop"))
		if err != nil {
			return timeRange
		}
		timeRange.Start = timeStart.Format(time.RFC3339)
		timeRange.Stop = timeStop.Format(time.RFC3339)
	}

	return timeRange
}

// Search logs in elastic.
func search(searchRequest SearchRequest) (goes.Response, error) {
	conn := common.GetConnection()

	if len(searchRequest.Text) > 0 {
		strconv.Quote(searchRequest.Text)
	} else {
		searchRequest.Text = "*"
	}

	var query = map[string]interface{}{
		"query": map[string]interface{}{
			"query_string": map[string]string{
				"default_field": "message",
				"query":         searchRequest.Text,
			},
		},
		"from": searchRequest.From,
		"size": searchRequest.Size,
		"sort": map[string]string{
			"datetime": "desc",
		},
	}

	if searchRequest.TimeRange.IsValid() {
		datetime := make(map[string]string)
		if searchRequest.TimeRange.Start != "" {
			datetime["gte"] = searchRequest.TimeRange.Start
		}
		if searchRequest.TimeRange.Stop != "" {
			datetime["lte"] = searchRequest.TimeRange.Stop
		}
		query["filter"] = map[string]interface{}{
			"range": map[string]interface{}{
				"datetime": datetime,
			},
		}
	}

	extraArgs := make(url.Values, 1)
	searchResults, err := conn.Search(query, searchRequest.Indexes, searchRequest.Types, extraArgs)

	if err != nil {
		return goes.Response{}, errors.New("No records found.")
	} else {
		return searchResults, nil
	}
}

func Index(req *http.Request, r *render.Render) {
	query_text := req.URL.Query().Get("q")
	user := common.FindUserByEmail(r.Context["email"].(string))

	// Pagination
	pagination := widgets.NewPagination(req)
	pagination.SetPerPage(100)

	// Load records
	searchRequest := buildSearchRequest(
		query_text,
		[]string{user.GetIndexName()},
		pagination.GetPerPage(),
		pagination.DetectFrom(),
		buildTimeRange(req),
	)
	// Search data in elastic
	data, err := search(searchRequest)

	pagination.SetTotalRecords(data.Hits.Total)

	var viewName string
	if data.Hits.Total > 0 && err == nil {
		viewName = "home/index"
	} else {
		viewName = "home/no_records"
	}

	r.HTML(viewName, render.ViewData{
		"logs":       data.Hits.Hits,
		"total":      data.Hits.Total,
		"query_text": query_text,
		"pagination": pagination,
	})
}
<|endoftext|>


<commit_before>package sgload

import "sync"

type GateLoadRunner struct {
	LoadRunner
	WriteLoadRunner
	ReadLoadRunner
	GateLoadSpec GateLoadSpec
}

func NewGateLoadRunner(gls GateLoadSpec) *GateLoadRunner {

	gls.MustValidate()

	loadRunner := LoadRunner{
		LoadSpec: gls.LoadSpec,
	}
	loadRunner.CreateStatsdClient()

	writeLoadRunner := WriteLoadRunner{
		LoadRunner:    loadRunner,
		WriteLoadSpec: gls.WriteLoadSpec,
	}

	readLoadRunner := ReadLoadRunner{
		LoadRunner:   loadRunner,
		ReadLoadSpec: gls.ReadLoadSpec,
	}

	return &GateLoadRunner{
		LoadRunner:      loadRunner,
		WriteLoadRunner: writeLoadRunner,
		ReadLoadRunner:  readLoadRunner,
		GateLoadSpec:    gls,
	}

}

func (glr GateLoadRunner) Run() error {

	// TODO -------------------

	// TODO: 1) writers need to add timestamp in doc of when they wrote the doc
	// TODO: 2) readers need to calculate RT latency delta and push to statsd
	// TODO: 3) instead of finishing when writers finish, block until readers have read all docs written)

	// Start Writers
	writerWaitGroup, writers, err := glr.startWriters()
	if err != nil {
		return err
	}

	// Start Readers
	if err := glr.startReaders(); err != nil {
		return err
	}

	// Start Doc Feeder
	if err := glr.startDocFeeder(writers); err != nil {
		return err
	}

	// Wait until writers finish
	if err := glr.waitUntilWritersFinish(writerWaitGroup); err != nil {
		return err
	}

	return nil
}

func (glr GateLoadRunner) startWriters() (*sync.WaitGroup, []*Writer, error) {

	// Create a wait group to see when all the writer goroutines have finished
	wg := sync.WaitGroup{}

	// Create writer goroutines
	writers, err := glr.createWriters(&wg)
	if err != nil {
		return nil, nil, err
	}
	for _, writer := range writers {
		go writer.Run()
	}

	return &wg, writers, nil
}

func (glr GateLoadRunner) startReaders() error {
	return nil
}

func (glr GateLoadRunner) startDocFeeder(writers []*Writer) error {
	// Create doc feeder goroutine
	go glr.feedDocsToWriters(writers)
	return nil
}

func (glr GateLoadRunner) waitUntilWritersFinish(writerWaitGroup *sync.WaitGroup) error {
	writerWaitGroup.Wait()
	return nil
}
<commit_msg>Start readers<commit_after>package sgload

import (
	"fmt"
	"sync"
)

type GateLoadRunner struct {
	LoadRunner
	WriteLoadRunner
	ReadLoadRunner
	GateLoadSpec GateLoadSpec
}

func NewGateLoadRunner(gls GateLoadSpec) *GateLoadRunner {

	gls.MustValidate()

	loadRunner := LoadRunner{
		LoadSpec: gls.LoadSpec,
	}
	loadRunner.CreateStatsdClient()

	writeLoadRunner := WriteLoadRunner{
		LoadRunner:    loadRunner,
		WriteLoadSpec: gls.WriteLoadSpec,
	}

	readLoadRunner := ReadLoadRunner{
		LoadRunner:   loadRunner,
		ReadLoadSpec: gls.ReadLoadSpec,
	}

	return &GateLoadRunner{
		LoadRunner:      loadRunner,
		WriteLoadRunner: writeLoadRunner,
		ReadLoadRunner:  readLoadRunner,
		GateLoadSpec:    gls,
	}

}

func (glr GateLoadRunner) Run() error {

	// TODO -------------------

	// TODO: 1) writers need to add timestamp in doc of when they wrote the doc
	// TODO: 2) readers need to calculate RT latency delta and push to statsd
	// TODO: 3) instead of finishing when writers finish, block until readers have read all docs written)

	// Start Writers
	writerWaitGroup, writers, err := glr.startWriters()
	if err != nil {
		return err
	}

	// Start Readers
	if err := glr.startReaders(); err != nil {
		return err
	}

	// Start Doc Feeder
	if err := glr.startDocFeeder(writers); err != nil {
		return err
	}

	// Wait until writers finish
	if err := glr.waitUntilWritersFinish(writerWaitGroup); err != nil {
		return err
	}

	return nil
}

func (glr GateLoadRunner) startWriters() (*sync.WaitGroup, []*Writer, error) {

	// Create a wait group to see when all the writer goroutines have finished
	wg := sync.WaitGroup{}

	// Create writer goroutines
	writers, err := glr.createWriters(&wg)
	if err != nil {
		return nil, nil, err
	}
	for _, writer := range writers {
		go writer.Run()
	}

	return &wg, writers, nil
}

func (glr GateLoadRunner) startReaders() error {

	// Create a wait group that is currently ignored
	var wg sync.WaitGroup

	// Create reader goroutines
	readers, err := glr.createReaders(&wg)
	if err != nil {
		return fmt.Errorf("Error creating readers: %v", err)
	}
	for _, reader := range readers {
		go reader.Run()
	}

	return nil
}

func (glr GateLoadRunner) startDocFeeder(writers []*Writer) error {
	// Create doc feeder goroutine
	go glr.feedDocsToWriters(writers)
	return nil
}

func (glr GateLoadRunner) waitUntilWritersFinish(writerWaitGroup *sync.WaitGroup) error {
	writerWaitGroup.Wait()
	return nil
}
<|endoftext|>


<commit_before>package disque

import (
	"errors"
	"fmt"
	"github.com/garyburd/redigo/redis"
	"time"
)

type client struct {
	pools []*redis.Pool
}

type DisqueClientOptions struct {
	// Password for Disque
	Password string
	// Maximum number of idle connections in the pool.
	MaxIdle int

	// Maximum number of connections allocated by the pool at a given time.
	// When zero, there is no limit on the number of connections in the pool.
	MaxActive int

	// Close connections after remaining idle for this duration. If the value
	// is zero, then idle connections are not closed. Applications should set
	// the timeout to a value less than the server's timeout.
	IdleTimeout time.Duration

	// If Wait is true and the pool is at the MaxActive limit, then Get() waits
	// for a connection to be returned to the pool before returning.
	Wait bool
}
type job struct {
	QueueName string
	Id        string
	Body      []byte
}
type jobWithCounters struct {
	QueueName            string
	Id                   string
	Body                 []byte
	Nack                 int64
	AdditionalDeliveries int64
}

type AddJobOptions struct {
	Timeout   time.Duration
	Replicate int
	Delay     time.Duration
	Retry     time.Duration
	TTL       time.Duration
	MaxLen    int
	Async     bool
}

type GetJobOptions struct {
	NoHang  bool
	Timeout time.Duration
	Count   int
}

type QScanOptions struct {
	Count      int
	BusyLoop   bool
	MinLen     int
	MaxLen     int
	ImportRate int
}

func NewDisqueClient(options DisqueClientOptions, servers ...string) *client {
	c := &client{
		pools: make([]*redis.Pool, len(servers)),
	}

	for i, server := range servers {
		c.pools[i] = &redis.Pool{
			MaxIdle:     options.MaxIdle,
			IdleTimeout: options.IdleTimeout * time.Second,
			Dial: func() (redis.Conn, error) {
				c, err := redis.Dial("tcp", server)
				if err != nil {
					return nil, err
				}
				if options.Password != "" {
					if _, err := c.Do("AUTH", options.Password); err != nil {
						c.Close()
						return nil, err
					}
					return c, err

				}
				return c, err

			},
			TestOnBorrow: func(c redis.Conn, t time.Time) error {
				if time.Since(t) < time.Minute {
					return nil
				}
				_, err := c.Do("PING")
				return err
			},
			Wait:      options.Wait,
			MaxActive: options.MaxActive,
		}
	}
	return c
}
func (c *client) get() (redis.Conn, error) {
	var err error
	for _, pool := range c.pools {
		conn := pool.Get()
		if conn.Err() == nil {
			return conn, nil
		} else {
			err = conn.Err()

		}

	}
	return nil, err
}

func (c *client) Close() {

	for _, pool := range c.pools {
		pool.Close()
	}
}

func (c *client) AddJob(queueName string, job []byte, options AddJobOptions) (string, error) {
	conn, err := c.get()

	if err != nil {
		return "", err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	args := redis.Args{queueName, job, int(options.Timeout.Nanoseconds() / 1000000)}
	if options.Replicate > 0 {
		args = args.Add("REPLICATE", options.Replicate)
	}
	if options.Delay > 0 {
		args = args.Add("DELAY", options.Delay.Seconds())
	}
	if options.Retry >= 0 {
		args = args.Add("RETRY", options.Retry.Seconds())
	}
	if options.TTL > 0 {
		args = args.Add("TTL", options.TTL.Seconds())
	}
	if options.MaxLen > 0 {
		args = args.Add("MAXLEN", options.MaxLen)
	}
	if options.Async == true {
		args = args.Add("ASYNC")
	}
	return redis.String(conn.Do("ADDJOB", args...))
}

//GetJob returns a job
func (c *client) GetJob(options GetJobOptions, queueNames ...string) ([]job, error) {
	conn, err := c.get()
	if err != nil {
		return nil, err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	args := redis.Args{}

	if options.NoHang == true {
		args = args.Add("NOHANG")
	}
	if options.Timeout.Nanoseconds() > 0 {
		args = args.Add("TIMEOUT", int(options.Timeout.Nanoseconds()/1000000))
	}
	if options.Count > 0 {
		args = args.Add("COUNT", options.Count)
	}
	args = args.Add("FROM")
	for _, queueName := range queueNames {
		args = args.Add(queueName)
	}
	reply, err := redis.Values(conn.Do("GETJOB", args...))
	if err != nil {
		return nil, err
	}

	result := make([]job, 0, len(reply))
	for _, v := range reply {
		if value, err := redis.Values(v, nil); err != nil {
			return nil, err
		} else {
			queueName, err := redis.String(value[0], nil)
			id, err := redis.String(value[1], err)
			body, err := redis.Bytes(value[2], err)
			if err != nil {
				return nil, err
			}
			result = append(result, job{QueueName: queueName, Id: id, Body: body})
		}
	}
	return result, nil
}

//GetJobWithCounters returns a job details with counters details
func (c *client) GetJobWithCounters(options GetJobOptions, queueNames ...string) ([]jobWithCounters, error) {
	conn, err := c.get()
	if err != nil {
		return nil, err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	args := redis.Args{}

	if options.NoHang == true {
		args = args.Add("NOHANG")
	}
	if options.Timeout.Nanoseconds() > 0 {
		args = args.Add("TIMEOUT", int(options.Timeout.Nanoseconds()/1000000))
	}
	if options.Count > 0 {
		args = args.Add("COUNT", options.Count)
	}
	args = args.Add("WITHCOUNTERS")
	args = args.Add("FROM")
	for _, queueName := range queueNames {
		args = args.Add(queueName)
	}
	reply, err := redis.Values(conn.Do("GETJOB", args...))
	if err != nil {
		return nil, err
	}

	result := make([]jobWithCounters, 0, len(reply))
	for _, v := range reply {
		if value, err := redis.Values(v, nil); err != nil {
			return nil, err
		} else {
			queueName, err := redis.String(value[0], nil)
			id, err := redis.String(value[1], err)
			body, err := redis.Bytes(value[2], err)
			var nack, additionalDeliveries int64
			nack, err = redis.Int64(value[4], err)
			additionalDeliveries, err = redis.Int64(value[6], err)
			if err != nil {
				return nil, err
			}
			result = append(result, jobWithCounters{QueueName: queueName, Id: id, Body: body, Nack: nack, AdditionalDeliveries: additionalDeliveries})
		}
	}
	return result, nil
}

func (client *client) AckJob(jobIds ...string) (int, error) {
	conn, err := client.get()
	if err != nil {
		return 0, err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	args := redis.Args{}
	for _, jobId := range jobIds {
		args = args.Add(jobId)
	}
	return redis.Int(conn.Do("ACKJOB", args...))

}

func (client *client) FastAck(jobIds ...string) (int, error) {
	conn, err := client.get()
	if err != nil {
		return 0, err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	args := redis.Args{}
	for _, jobId := range jobIds {
		args = args.Add(jobId)
	}
	return redis.Int(conn.Do("FASTACK", args...))

}
func (client *client) Working(jobId string) (int, error) {
	conn, err := client.get()
	if err != nil {
		return 0, err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	args := redis.Args{}
	args = args.Add(jobId)

	if err != nil {
		return 0, err
	}
	return redis.Int(conn.Do("WORKING", args...))

}

func (client *client) Nack(jobIds ...string) (int, error) {
	conn, err := client.get()
	if err != nil {
		return 0, err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	args := redis.Args{}
	for _, jobId := range jobIds {
		args = args.Add(jobId)
	}

	return redis.Int(conn.Do("NACK", args...))

}

func (c *client) QLen(queueName string) (int, error) {
	conn, err := c.get()
	if err != nil {
		return 0, err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	return redis.Int(conn.Do("QLEN", queueName))
}
func (c *client) QPeek(queueName string, count int) ([]job, error) {
	conn, err := c.get()
	if err != nil {
		return nil, err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	reply, err := redis.Values(conn.Do("QPEEK", queueName, count))
	if err != nil {
		return nil, err
	}

	result := make([]job, 0, len(reply))
	for _, v := range reply {
		if value, err := redis.Values(v, nil); err != nil {
			return nil, err
		} else {
			queueName, err := redis.String(value[0], nil)
			id, err := redis.String(value[1], err)
			data, err := redis.Bytes(value[2], err)
			if err != nil {
				return nil, err
			}
			result = append(result, job{QueueName: queueName, Id: id, Body: data})
		}
	}
	return result, nil
}

func (c *client) Enqueue(jobIds ...string) (int, error) {
	conn, err := c.get()
	if err != nil {
		return 0, err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	args := redis.Args{}
	for _, jobId := range jobIds {
		args = args.Add(jobId)
	}
	return redis.Int(conn.Do("ENQUEUE", args...))
}

func (c *client) Dequeue(jobIds ...string) (int, error) {
	conn, err := c.get()
	if err != nil {
		return 0, err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	args := redis.Args{}
	for _, jobId := range jobIds {
		args = args.Add(jobId)
	}
	return redis.Int(conn.Do("DEQUEUE", args...))
}

func (c *client) DelJob(jobIds ...string) (int, error) {
	conn, err := c.get()
	if err != nil {
		return 0, err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	args := redis.Args{}
	for _, jobId := range jobIds {
		args = args.Add(jobId)
	}
	return redis.Int(conn.Do("DELJOB", args...))
}

func (c *client) Show(jobId string) (map[string]interface{}, error) {
	conn, err := c.get()
	if err != nil {
		return nil, err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	reply, err := redis.Values(conn.Do("SHOW", jobId))
	if err != nil {
		return nil, err
	}

	result := make(map[string]interface{})
	for i := 0; i < len(reply); i += 2 {
		if key, ok := reply[i].(string); ok {
			result[key] = reply[i+1]
		} else {
			return nil, errors.New(fmt.Sprintf("interface can not case string: %v", reply[i]))
		}
	}

	return result, nil
}

func (c *client) QScan(options QScanOptions) ([]interface{}, error) {
	conn, err := c.get()
	if err != nil {
		return nil, err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	args := redis.Args{}
	if options.Count > 0 {
		args = args.Add("COUNT", options.Count)
	}
	if options.BusyLoop == true {
		args = args.Add("BUSYLOOP")
	}
	if options.MinLen > 0 {
		args = args.Add("MINLEN", options.MinLen)
	}
	if options.MaxLen > 0 {
		args = args.Add("MAXLEN", options.MaxLen)
	}
	if options.ImportRate > 0 {
		args = args.Add("IMPORTRATE", options.ImportRate)
	}

	return redis.Values(conn.Do("QSCAN", args...))
}
<commit_msg>Added Comment<commit_after>package disque

import (
	"errors"
	"fmt"
	"github.com/garyburd/redigo/redis"
	"time"
)

type client struct {
	pools []*redis.Pool
}

//DisqueClientOptions has all the configuration for new connection
type DisqueClientOptions struct {
	// Password for Disque
	Password string
	// Maximum number of idle connections in the pool.
	MaxIdle int

	// Maximum number of connections allocated by the pool at a given time.
	// When zero, there is no limit on the number of connections in the pool.
	MaxActive int

	// Close connections after remaining idle for this duration. If the value
	// is zero, then idle connections are not closed. Applications should set
	// the timeout to a value less than the server's timeout.
	IdleTimeout time.Duration

	// If Wait is true and the pool is at the MaxActive limit, then Get() waits
	// for a connection to be returned to the pool before returning.
	Wait bool
}
type job struct {
	QueueName string
	Id        string
	Body      []byte
}
type jobWithCounters struct {
	QueueName            string
	Id                   string
	Body                 []byte
	Nack                 int64
	AdditionalDeliveries int64
}

type AddJobOptions struct {
	Timeout   time.Duration
	Replicate int
	Delay     time.Duration
	Retry     time.Duration
	TTL       time.Duration
	MaxLen    int
	Async     bool
}

type GetJobOptions struct {
	NoHang  bool
	Timeout time.Duration
	Count   int
}

type QScanOptions struct {
	Count      int
	BusyLoop   bool
	MinLen     int
	MaxLen     int
	ImportRate int
}

func NewDisqueClient(options DisqueClientOptions, servers ...string) *client {
	c := &client{
		pools: make([]*redis.Pool, len(servers)),
	}

	for i, server := range servers {
		c.pools[i] = &redis.Pool{
			MaxIdle:     options.MaxIdle,
			IdleTimeout: options.IdleTimeout * time.Second,
			Dial: func() (redis.Conn, error) {
				c, err := redis.Dial("tcp", server)
				if err != nil {
					return nil, err
				}
				if options.Password != "" {
					if _, err := c.Do("AUTH", options.Password); err != nil {
						c.Close()
						return nil, err
					}
					return c, err

				}
				return c, err

			},
			TestOnBorrow: func(c redis.Conn, t time.Time) error {
				if time.Since(t) < time.Minute {
					return nil
				}
				_, err := c.Do("PING")
				return err
			},
			Wait:      options.Wait,
			MaxActive: options.MaxActive,
		}
	}
	return c
}
func (c *client) get() (redis.Conn, error) {
	var err error
	for _, pool := range c.pools {
		conn := pool.Get()
		if conn.Err() == nil {
			return conn, nil
		} else {
			err = conn.Err()

		}

	}
	return nil, err
}

func (c *client) Close() {

	for _, pool := range c.pools {
		pool.Close()
	}
}

func (c *client) AddJob(queueName string, job []byte, options AddJobOptions) (string, error) {
	conn, err := c.get()

	if err != nil {
		return "", err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	args := redis.Args{queueName, job, int(options.Timeout.Nanoseconds() / 1000000)}
	if options.Replicate > 0 {
		args = args.Add("REPLICATE", options.Replicate)
	}
	if options.Delay > 0 {
		args = args.Add("DELAY", options.Delay.Seconds())
	}
	if options.Retry >= 0 {
		args = args.Add("RETRY", options.Retry.Seconds())
	}
	if options.TTL > 0 {
		args = args.Add("TTL", options.TTL.Seconds())
	}
	if options.MaxLen > 0 {
		args = args.Add("MAXLEN", options.MaxLen)
	}
	if options.Async == true {
		args = args.Add("ASYNC")
	}
	return redis.String(conn.Do("ADDJOB", args...))
}

//GetJob returns a job
func (c *client) GetJob(options GetJobOptions, queueNames ...string) ([]job, error) {
	conn, err := c.get()
	if err != nil {
		return nil, err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	args := redis.Args{}

	if options.NoHang == true {
		args = args.Add("NOHANG")
	}
	if options.Timeout.Nanoseconds() > 0 {
		args = args.Add("TIMEOUT", int(options.Timeout.Nanoseconds()/1000000))
	}
	if options.Count > 0 {
		args = args.Add("COUNT", options.Count)
	}
	args = args.Add("FROM")
	for _, queueName := range queueNames {
		args = args.Add(queueName)
	}
	reply, err := redis.Values(conn.Do("GETJOB", args...))
	if err != nil {
		return nil, err
	}

	result := make([]job, 0, len(reply))
	for _, v := range reply {
		if value, err := redis.Values(v, nil); err != nil {
			return nil, err
		} else {
			queueName, err := redis.String(value[0], nil)
			id, err := redis.String(value[1], err)
			body, err := redis.Bytes(value[2], err)
			if err != nil {
				return nil, err
			}
			result = append(result, job{QueueName: queueName, Id: id, Body: body})
		}
	}
	return result, nil
}

//GetJobWithCounters returns a job details with counters details
func (c *client) GetJobWithCounters(options GetJobOptions, queueNames ...string) ([]jobWithCounters, error) {
	conn, err := c.get()
	if err != nil {
		return nil, err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	args := redis.Args{}

	if options.NoHang == true {
		args = args.Add("NOHANG")
	}
	if options.Timeout.Nanoseconds() > 0 {
		args = args.Add("TIMEOUT", int(options.Timeout.Nanoseconds()/1000000))
	}
	if options.Count > 0 {
		args = args.Add("COUNT", options.Count)
	}
	args = args.Add("WITHCOUNTERS")
	args = args.Add("FROM")
	for _, queueName := range queueNames {
		args = args.Add(queueName)
	}
	reply, err := redis.Values(conn.Do("GETJOB", args...))
	if err != nil {
		return nil, err
	}

	result := make([]jobWithCounters, 0, len(reply))
	for _, v := range reply {
		if value, err := redis.Values(v, nil); err != nil {
			return nil, err
		} else {
			queueName, err := redis.String(value[0], nil)
			id, err := redis.String(value[1], err)
			body, err := redis.Bytes(value[2], err)
			var nack, additionalDeliveries int64
			nack, err = redis.Int64(value[4], err)
			additionalDeliveries, err = redis.Int64(value[6], err)
			if err != nil {
				return nil, err
			}
			result = append(result, jobWithCounters{QueueName: queueName, Id: id, Body: body, Nack: nack, AdditionalDeliveries: additionalDeliveries})
		}
	}
	return result, nil
}

func (client *client) AckJob(jobIds ...string) (int, error) {
	conn, err := client.get()
	if err != nil {
		return 0, err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	args := redis.Args{}
	for _, jobId := range jobIds {
		args = args.Add(jobId)
	}
	return redis.Int(conn.Do("ACKJOB", args...))

}

func (client *client) FastAck(jobIds ...string) (int, error) {
	conn, err := client.get()
	if err != nil {
		return 0, err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	args := redis.Args{}
	for _, jobId := range jobIds {
		args = args.Add(jobId)
	}
	return redis.Int(conn.Do("FASTACK", args...))

}
func (client *client) Working(jobId string) (int, error) {
	conn, err := client.get()
	if err != nil {
		return 0, err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	args := redis.Args{}
	args = args.Add(jobId)

	if err != nil {
		return 0, err
	}
	return redis.Int(conn.Do("WORKING", args...))

}

func (client *client) Nack(jobIds ...string) (int, error) {
	conn, err := client.get()
	if err != nil {
		return 0, err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	args := redis.Args{}
	for _, jobId := range jobIds {
		args = args.Add(jobId)
	}

	return redis.Int(conn.Do("NACK", args...))

}

func (c *client) QLen(queueName string) (int, error) {
	conn, err := c.get()
	if err != nil {
		return 0, err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	return redis.Int(conn.Do("QLEN", queueName))
}
func (c *client) QPeek(queueName string, count int) ([]job, error) {
	conn, err := c.get()
	if err != nil {
		return nil, err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	reply, err := redis.Values(conn.Do("QPEEK", queueName, count))
	if err != nil {
		return nil, err
	}

	result := make([]job, 0, len(reply))
	for _, v := range reply {
		if value, err := redis.Values(v, nil); err != nil {
			return nil, err
		} else {
			queueName, err := redis.String(value[0], nil)
			id, err := redis.String(value[1], err)
			data, err := redis.Bytes(value[2], err)
			if err != nil {
				return nil, err
			}
			result = append(result, job{QueueName: queueName, Id: id, Body: data})
		}
	}
	return result, nil
}

func (c *client) Enqueue(jobIds ...string) (int, error) {
	conn, err := c.get()
	if err != nil {
		return 0, err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	args := redis.Args{}
	for _, jobId := range jobIds {
		args = args.Add(jobId)
	}
	return redis.Int(conn.Do("ENQUEUE", args...))
}

func (c *client) Dequeue(jobIds ...string) (int, error) {
	conn, err := c.get()
	if err != nil {
		return 0, err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	args := redis.Args{}
	for _, jobId := range jobIds {
		args = args.Add(jobId)
	}
	return redis.Int(conn.Do("DEQUEUE", args...))
}

func (c *client) DelJob(jobIds ...string) (int, error) {
	conn, err := c.get()
	if err != nil {
		return 0, err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	args := redis.Args{}
	for _, jobId := range jobIds {
		args = args.Add(jobId)
	}
	return redis.Int(conn.Do("DELJOB", args...))
}

func (c *client) Show(jobId string) (map[string]interface{}, error) {
	conn, err := c.get()
	if err != nil {
		return nil, err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	reply, err := redis.Values(conn.Do("SHOW", jobId))
	if err != nil {
		return nil, err
	}

	result := make(map[string]interface{})
	for i := 0; i < len(reply); i += 2 {
		if key, ok := reply[i].(string); ok {
			result[key] = reply[i+1]
		} else {
			return nil, errors.New(fmt.Sprintf("interface can not case string: %v", reply[i]))
		}
	}

	return result, nil
}

func (c *client) QScan(options QScanOptions) ([]interface{}, error) {
	conn, err := c.get()
	if err != nil {
		return nil, err
	} else {
		if conn != nil {
			defer conn.Close()
		}

	}

	args := redis.Args{}
	if options.Count > 0 {
		args = args.Add("COUNT", options.Count)
	}
	if options.BusyLoop == true {
		args = args.Add("BUSYLOOP")
	}
	if options.MinLen > 0 {
		args = args.Add("MINLEN", options.MinLen)
	}
	if options.MaxLen > 0 {
		args = args.Add("MAXLEN", options.MaxLen)
	}
	if options.ImportRate > 0 {
		args = args.Add("IMPORTRATE", options.ImportRate)
	}

	return redis.Values(conn.Do("QSCAN", args...))
}
<|endoftext|>


<commit_before>package bolt

import (
	"fmt"
	"sort"
	"unsafe"
)

// freelist represents a list of all pages that are available for allocation.
// It also tracks pages that have been freed but are still in use by open transactions.
type freelist struct {
	ids     []pgid          // all free and available free page ids.
	pending map[txid][]pgid // mapping of soon-to-be free page ids by tx.
	cache   map[pgid]bool   // fast lookup of all free and pending page ids.
}

// newFreelist returns an empty, initialized freelist.
func newFreelist() *freelist {
	return &freelist{
		pending: make(map[txid][]pgid),
		cache:   make(map[pgid]bool),
	}
}

// size returns the size of the page after serialization.
func (f *freelist) size() int {
	return pageHeaderSize + (int(unsafe.Sizeof(pgid(0))) * f.count())
}

// count returns count of pages on the freelist
func (f *freelist) count() int {
	return f.free_count() + f.pending_count()
}

// free_count returns count of free pages
func (f *freelist) free_count() int {
	return len(f.ids)
}

// pending_count returns count of pending pages
func (f *freelist) pending_count() int {
	var count int
	for _, list := range f.pending {
		count += len(list)
	}
	return count
}

// all returns a list of all free ids and all pending ids in one sorted list.
func (f *freelist) all() []pgid {
	m := make(pgids, 0)

	for _, list := range f.pending {
		m = append(m, list...)
	}

	sort.Sort(m)
	return pgids(f.ids).merge(m)
}

// allocate returns the starting page id of a contiguous list of pages of a given size.
// If a contiguous block cannot be found then 0 is returned.
func (f *freelist) allocate(n int) pgid {
	if len(f.ids) == 0 {
		return 0
	}

	var initial, previd pgid
	for i, id := range f.ids {
		if id <= 1 {
			panic(fmt.Sprintf("invalid page allocation: %d", id))
		}

		// Reset initial page if this is not contiguous.
		if previd == 0 || id-previd != 1 {
			initial = id
		}

		// If we found a contiguous block then remove it and return it.
		if (id-initial)+1 == pgid(n) {
			// If we're allocating off the beginning then take the fast path
			// and just adjust the existing slice. This will use extra memory
			// temporarily but the append() in free() will realloc the slice
			// as is necessary.
			if (i + 1) == n {
				f.ids = f.ids[i+1:]
			} else {
				copy(f.ids[i-n+1:], f.ids[i+1:])
				f.ids = f.ids[:len(f.ids)-n]
			}

			// Remove from the free cache.
			for i := pgid(0); i < pgid(n); i++ {
				delete(f.cache, initial+i)
			}

			return initial
		}

		previd = id
	}
	return 0
}

// free releases a page and its overflow for a given transaction id.
// If the page is already free then a panic will occur.
func (f *freelist) free(txid txid, p *page) {
	if p.id <= 1 {
		panic(fmt.Sprintf("cannot free page 0 or 1: %d", p.id))
	}

	// Free page and all its overflow pages.
	var ids = f.pending[txid]
	for id := p.id; id <= p.id+pgid(p.overflow); id++ {
		// Verify that page is not already free.
		if f.cache[id] {
			panic(fmt.Sprintf("page %d already freed", id))
		}

		// Add to the freelist and cache.
		ids = append(ids, id)
		f.cache[id] = true
	}
	f.pending[txid] = ids
}

// release moves all page ids for a transaction id (or older) to the freelist.
func (f *freelist) release(txid txid) {
	m := make(pgids, 0)
	for tid, ids := range f.pending {
		if tid <= txid {
			// Move transaction's pending pages to the available freelist.
			// Don't remove from the cache since the page is still free.
			m = append(m, ids...)
			delete(f.pending, tid)
		}
	}
	sort.Sort(m)
	f.ids = pgids(f.ids).merge(m)
}

// rollback removes the pages from a given pending tx.
func (f *freelist) rollback(txid txid) {
	// Remove page ids from cache.
	for _, id := range f.pending[txid] {
		delete(f.cache, id)
	}

	// Remove pages from pending list.
	delete(f.pending, txid)
}

// freed returns whether a given page is in the free list.
func (f *freelist) freed(pgid pgid) bool {
	return f.cache[pgid]
}

// read initializes the freelist from a freelist page.
func (f *freelist) read(p *page) {
	// If the page.count is at the max uint16 value (64k) then it's considered
	// an overflow and the size of the freelist is stored as the first element.
	idx, count := 0, int(p.count)
	if count == 0xFFFF {
		idx = 1
		count = int(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[0])
	}

	// Copy the list of page ids from the freelist.
	if count == 0 {
		f.ids = nil
	} else {
		ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx:count]
		f.ids = make([]pgid, len(ids))
		copy(f.ids, ids)

		// Make sure they're sorted.
		sort.Sort(pgids(f.ids))
	}

	// Rebuild the page cache.
	f.reindex()
}

// write writes the page ids onto a freelist page. All free and pending ids are
// saved to disk since in the event of a program crash, all pending ids will
// become free.
func (f *freelist) write(p *page) error {
	// Combine the old free pgids and pgids waiting on an open transaction.
	ids := f.all()

	// Update the header flag.
	p.flags |= freelistPageFlag

	// The page.count can only hold up to 64k elements so if we overflow that
	// number then we handle it by putting the size in the first element.
	if len(ids) == 0 {
		p.count = uint16(len(ids))
	} else if len(ids) < 0xFFFF {
		p.count = uint16(len(ids))
		copy(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[:], ids)
	} else {
		p.count = 0xFFFF
		((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[0] = pgid(len(ids))
		copy(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[1:], ids)
	}

	return nil
}

// reload reads the freelist from a page and filters out pending items.
func (f *freelist) reload(p *page) {
	f.read(p)

	// Build a cache of only pending pages.
	pcache := make(map[pgid]bool)
	for _, pendingIDs := range f.pending {
		for _, pendingID := range pendingIDs {
			pcache[pendingID] = true
		}
	}

	// Check each page in the freelist and build a new available freelist
	// with any pages not in the pending lists.
	var a []pgid
	for _, id := range f.ids {
		if !pcache[id] {
			a = append(a, id)
		}
	}
	f.ids = a

	// Once the available list is rebuilt then rebuild the free cache so that
	// it includes the available and pending free pages.
	f.reindex()
}

// reindex rebuilds the free cache based on available and pending free lists.
func (f *freelist) reindex() {
	f.cache = make(map[pgid]bool)
	for _, id := range f.ids {
		f.cache[id] = true
	}
	for _, pendingIDs := range f.pending {
		for _, pendingID := range pendingIDs {
			f.cache[pendingID] = true
		}
	}
}
<commit_msg>Lower number of allocation in freelist.reindex()<commit_after>package bolt

import (
	"fmt"
	"sort"
	"unsafe"
)

// freelist represents a list of all pages that are available for allocation.
// It also tracks pages that have been freed but are still in use by open transactions.
type freelist struct {
	ids     []pgid          // all free and available free page ids.
	pending map[txid][]pgid // mapping of soon-to-be free page ids by tx.
	cache   map[pgid]bool   // fast lookup of all free and pending page ids.
}

// newFreelist returns an empty, initialized freelist.
func newFreelist() *freelist {
	return &freelist{
		pending: make(map[txid][]pgid),
		cache:   make(map[pgid]bool),
	}
}

// size returns the size of the page after serialization.
func (f *freelist) size() int {
	return pageHeaderSize + (int(unsafe.Sizeof(pgid(0))) * f.count())
}

// count returns count of pages on the freelist
func (f *freelist) count() int {
	return f.free_count() + f.pending_count()
}

// free_count returns count of free pages
func (f *freelist) free_count() int {
	return len(f.ids)
}

// pending_count returns count of pending pages
func (f *freelist) pending_count() int {
	var count int
	for _, list := range f.pending {
		count += len(list)
	}
	return count
}

// all returns a list of all free ids and all pending ids in one sorted list.
func (f *freelist) all() []pgid {
	m := make(pgids, 0)

	for _, list := range f.pending {
		m = append(m, list...)
	}

	sort.Sort(m)
	return pgids(f.ids).merge(m)
}

// allocate returns the starting page id of a contiguous list of pages of a given size.
// If a contiguous block cannot be found then 0 is returned.
func (f *freelist) allocate(n int) pgid {
	if len(f.ids) == 0 {
		return 0
	}

	var initial, previd pgid
	for i, id := range f.ids {
		if id <= 1 {
			panic(fmt.Sprintf("invalid page allocation: %d", id))
		}

		// Reset initial page if this is not contiguous.
		if previd == 0 || id-previd != 1 {
			initial = id
		}

		// If we found a contiguous block then remove it and return it.
		if (id-initial)+1 == pgid(n) {
			// If we're allocating off the beginning then take the fast path
			// and just adjust the existing slice. This will use extra memory
			// temporarily but the append() in free() will realloc the slice
			// as is necessary.
			if (i + 1) == n {
				f.ids = f.ids[i+1:]
			} else {
				copy(f.ids[i-n+1:], f.ids[i+1:])
				f.ids = f.ids[:len(f.ids)-n]
			}

			// Remove from the free cache.
			for i := pgid(0); i < pgid(n); i++ {
				delete(f.cache, initial+i)
			}

			return initial
		}

		previd = id
	}
	return 0
}

// free releases a page and its overflow for a given transaction id.
// If the page is already free then a panic will occur.
func (f *freelist) free(txid txid, p *page) {
	if p.id <= 1 {
		panic(fmt.Sprintf("cannot free page 0 or 1: %d", p.id))
	}

	// Free page and all its overflow pages.
	var ids = f.pending[txid]
	for id := p.id; id <= p.id+pgid(p.overflow); id++ {
		// Verify that page is not already free.
		if f.cache[id] {
			panic(fmt.Sprintf("page %d already freed", id))
		}

		// Add to the freelist and cache.
		ids = append(ids, id)
		f.cache[id] = true
	}
	f.pending[txid] = ids
}

// release moves all page ids for a transaction id (or older) to the freelist.
func (f *freelist) release(txid txid) {
	m := make(pgids, 0)
	for tid, ids := range f.pending {
		if tid <= txid {
			// Move transaction's pending pages to the available freelist.
			// Don't remove from the cache since the page is still free.
			m = append(m, ids...)
			delete(f.pending, tid)
		}
	}
	sort.Sort(m)
	f.ids = pgids(f.ids).merge(m)
}

// rollback removes the pages from a given pending tx.
func (f *freelist) rollback(txid txid) {
	// Remove page ids from cache.
	for _, id := range f.pending[txid] {
		delete(f.cache, id)
	}

	// Remove pages from pending list.
	delete(f.pending, txid)
}

// freed returns whether a given page is in the free list.
func (f *freelist) freed(pgid pgid) bool {
	return f.cache[pgid]
}

// read initializes the freelist from a freelist page.
func (f *freelist) read(p *page) {
	// If the page.count is at the max uint16 value (64k) then it's considered
	// an overflow and the size of the freelist is stored as the first element.
	idx, count := 0, int(p.count)
	if count == 0xFFFF {
		idx = 1
		count = int(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[0])
	}

	// Copy the list of page ids from the freelist.
	if count == 0 {
		f.ids = nil
	} else {
		ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx:count]
		f.ids = make([]pgid, len(ids))
		copy(f.ids, ids)

		// Make sure they're sorted.
		sort.Sort(pgids(f.ids))
	}

	// Rebuild the page cache.
	f.reindex()
}

// write writes the page ids onto a freelist page. All free and pending ids are
// saved to disk since in the event of a program crash, all pending ids will
// become free.
func (f *freelist) write(p *page) error {
	// Combine the old free pgids and pgids waiting on an open transaction.
	ids := f.all()

	// Update the header flag.
	p.flags |= freelistPageFlag

	// The page.count can only hold up to 64k elements so if we overflow that
	// number then we handle it by putting the size in the first element.
	if len(ids) == 0 {
		p.count = uint16(len(ids))
	} else if len(ids) < 0xFFFF {
		p.count = uint16(len(ids))
		copy(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[:], ids)
	} else {
		p.count = 0xFFFF
		((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[0] = pgid(len(ids))
		copy(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[1:], ids)
	}

	return nil
}

// reload reads the freelist from a page and filters out pending items.
func (f *freelist) reload(p *page) {
	f.read(p)

	// Build a cache of only pending pages.
	pcache := make(map[pgid]bool)
	for _, pendingIDs := range f.pending {
		for _, pendingID := range pendingIDs {
			pcache[pendingID] = true
		}
	}

	// Check each page in the freelist and build a new available freelist
	// with any pages not in the pending lists.
	var a []pgid
	for _, id := range f.ids {
		if !pcache[id] {
			a = append(a, id)
		}
	}
	f.ids = a

	// Once the available list is rebuilt then rebuild the free cache so that
	// it includes the available and pending free pages.
	f.reindex()
}

// reindex rebuilds the free cache based on available and pending free lists.
func (f *freelist) reindex() {
	f.cache = make(map[pgid]bool, len(f.ids))
	for _, id := range f.ids {
		f.cache[id] = true
	}
	for _, pendingIDs := range f.pending {
		for _, pendingID := range pendingIDs {
			f.cache[pendingID] = true
		}
	}
}
<|endoftext|>


<commit_before>package qbt

type FileInterface interface {
	Blocks() []BlockInterface
	Run() (err error)
}

type TestFile struct {
}

func (t *TestFile) Blocks() []BlockInterface {
	return t.structToInterface([]*TestBlock{&TestBlock{}})
}

func (t *TestFile) structToInterface(blockStruct []*TestBlock) []BlockInterface {
	blockInterface := make([]BlockInterface, len(blockStruct))
	for i := range blockStruct {
		blockInterface[i] = blockStruct[i]
	}
	return blockInterface
}
<commit_msg>comment added to structToInterface<commit_after>package qbt

type FileInterface interface {
	Blocks() []BlockInterface
	Run() (err error)
}

type TestFile struct {
}

func (t *TestFile) Blocks() []BlockInterface {
	return t.structToInterface([]*TestBlock{&TestBlock{}})
}

// Converts an array of TestBlock structs into one of BlockInterfaces
func (t *TestFile) structToInterface(blockStruct []*TestBlock) []BlockInterface {
	blockInterface := make([]BlockInterface, len(blockStruct))
	for i := range blockStruct {
		blockInterface[i] = blockStruct[i]
	}
	return blockInterface
}
<|endoftext|>