<commit_before>// Copyright 2014 The go-github AUTHORS. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package github

import (
	"context"
	"fmt"
	"time"
)

// IssueEvent represents an event that occurred around an Issue or Pull Request.
type IssueEvent struct {
	ID  *int    `json:"id,omitempty"`
	URL *string `json:"url,omitempty"`

	// The User that generated this event.
	Actor *User `json:"actor,omitempty"`

	// Event identifies the actual type of Event that occurred. Possible
	// values are:
	//
	//     closed
	//       The Actor closed the issue.
	//       If the issue was closed by commit message, CommitID holds the SHA1 hash of the commit.
	//
	//     merged
	//       The Actor merged into master a branch containing a commit mentioning the issue.
	//       CommitID holds the SHA1 of the merge commit.
	//
	//     referenced
	//       The Actor committed to master a commit mentioning the issue in its commit message.
	//       CommitID holds the SHA1 of the commit.
	//
	//     reopened, locked, unlocked
	//       The Actor did that to the issue.
	//
	//     renamed
	//       The Actor changed the issue title from Rename.From to Rename.To.
	//
	//     mentioned
	//       Someone unspecified @mentioned the Actor [sic] in an issue comment body.
	//
	//     assigned, unassigned
	//       The Actor assigned the issue to or removed the assignment from the Assignee.
	//
	//     labeled, unlabeled
	//       The Actor added or removed the Label from the issue.
	//
	//     milestoned, demilestoned
	//       The Actor added or removed the issue from the Milestone.
	//
	//     subscribed, unsubscribed
	//       The Actor subscribed to or unsubscribed from notifications for an issue.
	//
	//     head_ref_deleted, head_ref_restored
	//       The pull request’s branch was deleted or restored.
	//
	Event *string `json:"event,omitempty"`

	CreatedAt *time.Time `json:"created_at,omitempty"`
	Issue     *Issue     `json:"issue,omitempty"`

	// Only present on certain events; see above.
	Assignee  *User      `json:"assignee,omitempty"`
	Assigner  *User      `json:"assigner,omitempty"`
	CommitID  *string    `json:"commit_id,omitempty"`
	Milestone *Milestone `json:"milestone,omitempty"`
	Label     *Label     `json:"label,omitempty"`
	Rename    *Rename    `json:"rename,omitempty"`
}

// ListIssueEvents lists events for the specified issue.
//
// GitHub API docs: https://developer.github.com/v3/issues/events/#list-events-for-an-issue
func (s *IssuesService) ListIssueEvents(ctx context.Context, owner, repo string, number int, opt *ListOptions) ([]*IssueEvent, *Response, error) {
	u := fmt.Sprintf("repos/%v/%v/issues/%v/events", owner, repo, number)
	u, err := addOptions(u, opt)
	if err != nil {
		return nil, nil, err
	}

	req, err := s.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, nil, err
	}

	var events []*IssueEvent
	resp, err := s.client.Do(ctx, req, &events)
	if err != nil {
		return nil, resp, err
	}

	return events, resp, nil
}

// ListRepositoryEvents lists events for the specified repository.
//
// GitHub API docs: https://developer.github.com/v3/issues/events/#list-events-for-a-repository
func (s *IssuesService) ListRepositoryEvents(ctx context.Context, owner, repo string, opt *ListOptions) ([]*IssueEvent, *Response, error) {
	u := fmt.Sprintf("repos/%v/%v/issues/events", owner, repo)
	u, err := addOptions(u, opt)
	if err != nil {
		return nil, nil, err
	}

	req, err := s.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, nil, err
	}

	var events []*IssueEvent
	resp, err := s.client.Do(ctx, req, &events)
	if err != nil {
		return nil, resp, err
	}

	return events, resp, nil
}

// GetEvent returns the specified issue event.
//
// GitHub API docs: https://developer.github.com/v3/issues/events/#get-a-single-event
func (s *IssuesService) GetEvent(ctx context.Context, owner, repo string, id int) (*IssueEvent, *Response, error) {
	u := fmt.Sprintf("repos/%v/%v/issues/events/%v", owner, repo, id)

	req, err := s.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, nil, err
	}

	event := new(IssueEvent)
	resp, err := s.client.Do(ctx, req, event)
	if err != nil {
		return nil, resp, err
	}

	return event, resp, nil
}

// Rename contains details for 'renamed' events.
type Rename struct {
	From *string `json:"from,omitempty"`
	To   *string `json:"to,omitempty"`
}

func (r Rename) String() string {
	return Stringify(r)
}
<commit_msg>Update IssueEvent.Event assigned/unassigned doc. (#615)<commit_after>// Copyright 2014 The go-github AUTHORS. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package github

import (
	"context"
	"fmt"
	"time"
)

// IssueEvent represents an event that occurred around an Issue or Pull Request.
type IssueEvent struct {
	ID  *int    `json:"id,omitempty"`
	URL *string `json:"url,omitempty"`

	// The User that generated this event.
	Actor *User `json:"actor,omitempty"`

	// Event identifies the actual type of Event that occurred. Possible
	// values are:
	//
	//     closed
	//       The Actor closed the issue.
	//       If the issue was closed by commit message, CommitID holds the SHA1 hash of the commit.
	//
	//     merged
	//       The Actor merged into master a branch containing a commit mentioning the issue.
	//       CommitID holds the SHA1 of the merge commit.
	//
	//     referenced
	//       The Actor committed to master a commit mentioning the issue in its commit message.
	//       CommitID holds the SHA1 of the commit.
	//
	//     reopened, locked, unlocked
	//       The Actor did that to the issue.
	//
	//     renamed
	//       The Actor changed the issue title from Rename.From to Rename.To.
	//
	//     mentioned
	//       Someone unspecified @mentioned the Actor [sic] in an issue comment body.
	//
	//     assigned, unassigned
	//       The Assigner assigned the issue to or removed the assignment from the Assignee.
	//
	//     labeled, unlabeled
	//       The Actor added or removed the Label from the issue.
	//
	//     milestoned, demilestoned
	//       The Actor added or removed the issue from the Milestone.
	//
	//     subscribed, unsubscribed
	//       The Actor subscribed to or unsubscribed from notifications for an issue.
	//
	//     head_ref_deleted, head_ref_restored
	//       The pull request’s branch was deleted or restored.
	//
	Event *string `json:"event,omitempty"`

	CreatedAt *time.Time `json:"created_at,omitempty"`
	Issue     *Issue     `json:"issue,omitempty"`

	// Only present on certain events; see above.
	Assignee  *User      `json:"assignee,omitempty"`
	Assigner  *User      `json:"assigner,omitempty"`
	CommitID  *string    `json:"commit_id,omitempty"`
	Milestone *Milestone `json:"milestone,omitempty"`
	Label     *Label     `json:"label,omitempty"`
	Rename    *Rename    `json:"rename,omitempty"`
}

// ListIssueEvents lists events for the specified issue.
//
// GitHub API docs: https://developer.github.com/v3/issues/events/#list-events-for-an-issue
func (s *IssuesService) ListIssueEvents(ctx context.Context, owner, repo string, number int, opt *ListOptions) ([]*IssueEvent, *Response, error) {
	u := fmt.Sprintf("repos/%v/%v/issues/%v/events", owner, repo, number)
	u, err := addOptions(u, opt)
	if err != nil {
		return nil, nil, err
	}

	req, err := s.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, nil, err
	}

	var events []*IssueEvent
	resp, err := s.client.Do(ctx, req, &events)
	if err != nil {
		return nil, resp, err
	}

	return events, resp, nil
}

// ListRepositoryEvents lists events for the specified repository.
//
// GitHub API docs: https://developer.github.com/v3/issues/events/#list-events-for-a-repository
func (s *IssuesService) ListRepositoryEvents(ctx context.Context, owner, repo string, opt *ListOptions) ([]*IssueEvent, *Response, error) {
	u := fmt.Sprintf("repos/%v/%v/issues/events", owner, repo)
	u, err := addOptions(u, opt)
	if err != nil {
		return nil, nil, err
	}

	req, err := s.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, nil, err
	}

	var events []*IssueEvent
	resp, err := s.client.Do(ctx, req, &events)
	if err != nil {
		return nil, resp, err
	}

	return events, resp, nil
}

// GetEvent returns the specified issue event.
//
// GitHub API docs: https://developer.github.com/v3/issues/events/#get-a-single-event
func (s *IssuesService) GetEvent(ctx context.Context, owner, repo string, id int) (*IssueEvent, *Response, error) {
	u := fmt.Sprintf("repos/%v/%v/issues/events/%v", owner, repo, id)

	req, err := s.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, nil, err
	}

	event := new(IssueEvent)
	resp, err := s.client.Do(ctx, req, event)
	if err != nil {
		return nil, resp, err
	}

	return event, resp, nil
}

// Rename contains details for 'renamed' events.
type Rename struct {
	From *string `json:"from,omitempty"`
	To   *string `json:"to,omitempty"`
}

func (r Rename) String() string {
	return Stringify(r)
}
<|endoftext|>


<commit_before>// Code generated by protoc-gen-go.
// source: push_event.proto
// DO NOT EDIT!

package github

import proto "github.com/golang/protobuf/proto"
import fmt "fmt"
import math "math"

// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf

type PushCommit struct {
	Id        string      `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"`
	TreeId    string      `protobuf:"bytes,2,opt,name=tree_id,json=treeId" json:"tree_id,omitempty"`
	Distinct  bool        `protobuf:"varint,3,opt,name=distinct" json:"distinct,omitempty"`
	Message   string      `protobuf:"bytes,4,opt,name=message" json:"message,omitempty"`
	Timestamp string      `protobuf:"bytes,5,opt,name=timestamp" json:"timestamp,omitempty"`
	Url       string      `protobuf:"bytes,6,opt,name=url" json:"url,omitempty"`
	Author    *CommitUser `protobuf:"bytes,7,opt,name=author" json:"author,omitempty"`
	Committer *CommitUser `protobuf:"bytes,8,opt,name=committer" json:"committer,omitempty"`
	Added     []string    `protobuf:"bytes,9,rep,name=added" json:"added,omitempty"`
	Removed   []string    `protobuf:"bytes,10,rep,name=removed" json:"removed,omitempty"`
	Modified  []string    `protobuf:"bytes,11,rep,name=modified" json:"modified,omitempty"`
}

func (m *PushCommit) Reset()                    { *m = PushCommit{} }
func (m *PushCommit) String() string            { return proto.CompactTextString(m) }
func (*PushCommit) ProtoMessage()               {}
func (*PushCommit) Descriptor() ([]byte, []int) { return fileDescriptor27, []int{0} }

func (m *PushCommit) GetAuthor() *CommitUser {
	if m != nil {
		return m.Author
	}
	return nil
}

func (m *PushCommit) GetCommitter() *CommitUser {
	if m != nil {
		return m.Committer
	}
	return nil
}

type PushEvent struct {
	Ref        string        `protobuf:"bytes,1,opt,name=ref" json:"ref,omitempty"`
	Before     string        `protobuf:"bytes,2,opt,name=before" json:"before,omitempty"`
	After      string        `protobuf:"bytes,3,opt,name=after" json:"after,omitempty"`
	Created    bool          `protobuf:"varint,4,opt,name=created" json:"created,omitempty"`
	Deleted    bool          `protobuf:"varint,5,opt,name=deleted" json:"deleted,omitempty"`
	Forced     bool          `protobuf:"varint,6,opt,name=forced" json:"forced,omitempty"`
	BaseRef    string        `protobuf:"bytes,7,opt,name=base_ref,json=baseRef" json:"base_ref,omitempty"`
	Compare    string        `protobuf:"bytes,8,opt,name=compare" json:"compare,omitempty"`
	Commits    []*PushCommit `protobuf:"bytes,9,rep,name=commits" json:"commits,omitempty"`
	HeadCommit *PushCommit   `protobuf:"bytes,10,opt,name=head_commit,json=headCommit" json:"head_commit,omitempty"`
	Repository *Repository   `protobuf:"bytes,11,opt,name=repository" json:"repository,omitempty"`
	Pusher     *CommitUser   `protobuf:"bytes,12,opt,name=pusher" json:"pusher,omitempty"`
	Sender     *User         `protobuf:"bytes,13,opt,name=sender" json:"sender,omitempty"`
}

func (m *PushEvent) Reset()                    { *m = PushEvent{} }
func (m *PushEvent) String() string            { return proto.CompactTextString(m) }
func (*PushEvent) ProtoMessage()               {}
func (*PushEvent) Descriptor() ([]byte, []int) { return fileDescriptor27, []int{1} }

func (m *PushEvent) GetCommits() []*PushCommit {
	if m != nil {
		return m.Commits
	}
	return nil
}

func (m *PushEvent) GetHeadCommit() *PushCommit {
	if m != nil {
		return m.HeadCommit
	}
	return nil
}

func (m *PushEvent) GetRepository() *Repository {
	if m != nil {
		return m.Repository
	}
	return nil
}

func (m *PushEvent) GetPusher() *CommitUser {
	if m != nil {
		return m.Pusher
	}
	return nil
}

func (m *PushEvent) GetSender() *User {
	if m != nil {
		return m.Sender
	}
	return nil
}

func init() {
	proto.RegisterType((*PushCommit)(nil), "github.PushCommit")
	proto.RegisterType((*PushEvent)(nil), "github.PushEvent")
}

func init() { proto.RegisterFile("push_event.proto", fileDescriptor27) }

var fileDescriptor27 = []byte{
	// 439 bytes of a gzipped FileDescriptorProto
	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x74, 0x53, 0xcd, 0x8e, 0xd3, 0x30,
	0x10, 0x56, 0x9b, 0x6d, 0x9a, 0x4c, 0x17, 0x54, 0x46, 0x08, 0x4c, 0xc5, 0x61, 0xb5, 0xe2, 0x80,
	0x10, 0xaa, 0xd0, 0xee, 0x23, 0x20, 0x0e, 0xdc, 0x50, 0x24, 0xce, 0x51, 0x5a, 0x4f, 0xb7, 0x96,
	0x36, 0x75, 0x64, 0x3b, 0x95, 0x78, 0x30, 0x1e, 0x81, 0xf7, 0x62, 0xc6, 0x4e, 0xda, 0xee, 0xa1,
	0x37, 0x7f, 0x3f, 0xce, 0xcc, 0x7c, 0x9e, 0xc0, 0xb2, 0xeb, 0xfd, 0xbe, 0xa6, 0x23, 0x1d, 0xc2,
	0xba, 0x73, 0x36, 0x58, 0xcc, 0x9f, 0x4c, 0xd8, 0xf7, 0x9b, 0x15, 0xf4, 0x9e, 0x5c, 0xe2, 0x56,
	0x6f, 0xb6, 0xb6, 0x6d, 0x4d, 0xa8, 0x2f, 0xa8, 0xa5, 0xa3, 0xce, 0x7a, 0x13, 0xac, 0xfb, 0x93,
	0x98, 0xfb, 0x7f, 0x53, 0x80, 0x5f, 0xfc, 0xb5, 0xef, 0xd1, 0x8b, 0xaf, 0x61, 0x6a, 0xb4, 0x9a,
	0xdc, 0x4d, 0x3e, 0x97, 0x15, 0x9f, 0xf0, 0x3d, 0xcc, 0x83, 0x23, 0xaa, 0x99, 0x9c, 0x46, 0x32,
	0x17, 0xf8, 0x53, 0xe3, 0x0a, 0x0a, 0x6d, 0x7c, 0x30, 0x87, 0x6d, 0x50, 0x19, 0x2b, 0x45, 0x75,
	0xc2, 0xa8, 0x60, 0xde, 0x92, 0xf7, 0xcd, 0x13, 0xa9, 0x9b, 0x78, 0x69, 0x84, 0xf8, 0x11, 0xca,
	0x60, 0x18, 0x84, 0xa6, 0xed, 0xd4, 0x2c, 0x6a, 0x67, 0x02, 0x97, 0x90, 0xf5, 0xee, 0x59, 0xe5,
	0x91, 0x97, 0x23, 0x7e, 0x81, 0xbc, 0xe9, 0xc3, 0xde, 0x3a, 0x35, 0x67, 0x72, 0xf1, 0x80, 0xeb,
	0x34, 0xe7, 0x3a, 0xb5, 0xfb, 0x9b, 0x27, 0xab, 0x06, 0x07, 0x7e, 0x83, 0x32, 0x0d, 0x1c, 0xc8,
	0xa9, 0xe2, 0xaa, 0xfd, 0x6c, 0xc2, 0xb7, 0x30, 0x6b, 0xb4, 0x26, 0xad, 0xca, 0xbb, 0x8c, 0x2b,
	0x26, 0x20, 0xdd, 0x3b, 0x6a, 0xed, 0x91, 0x79, 0x88, 0xfc, 0x08, 0x65, 0xe6, 0xd6, 0x6a, 0xb3,
	0x33, 0x2c, 0x2d, 0xa2, 0x74, 0xc2, 0xf7, 0x7f, 0x33, 0x28, 0x25, 0xc7, 0x1f, 0xf2, 0x28, 0x32,
	0x89, 0xa3, 0xdd, 0x90, 0xa3, 0x1c, 0xf1, 0x1d, 0xe4, 0x1b, 0xda, 0x59, 0x47, 0x63, 0x8e, 0x09,
	0xc5, 0x1e, 0x76, 0xd2, 0x71, 0x16, 0xe9, 0x04, 0xa4, 0x87, 0xad, 0xa3, 0x26, 0x70, 0xa1, 0x9b,
	0x18, 0xee, 0x08, 0x45, 0xd1, 0xf4, 0x4c, 0xa2, 0xcc, 0x92, 0x32, 0x40, 0xa9, 0xc0, 0x5f, 0xdc,
	0xb2, 0x90, 0x47, 0x61, 0x40, 0xf8, 0x01, 0x8a, 0x4d, 0xe3, 0xa9, 0x96, 0x86, 0xe6, 0xe9, 0x39,
	0x04, 0x57, 0xdc, 0x94, 0x94, 0xb1, 0x6d, 0xd7, 0x70, 0x57, 0x45, 0x52, 0x06, 0x88, 0x5f, 0xa3,
	0xc2, 0x39, 0xf9, 0x18, 0xce, 0x45, 0x94, 0xe7, 0x65, 0xa9, 0x46, 0x0b, 0x3e, 0xc2, 0x62, 0x4f,
	0x8d, 0xae, 0x13, 0xe6, 0xd8, 0x26, 0x57, 0x6e, 0x80, 0xd8, 0x86, 0x55, 0x7b, 0x00, 0x38, 0x6f,
	0x23, 0xe7, 0xf9, 0xe2, 0x4e, 0x75, 0x52, 0xaa, 0x0b, 0x97, 0xec, 0x83, 0xac, 0x3e, 0xc7, 0x75,
	0x7b, 0x7d, 0x1f, 0x92, 0x03, 0x3f, 0x41, 0xee, 0xe9, 0xa0, 0xd9, 0xfb, 0x2a, 0x7a, 0x6f, 0x47,
	0x6f, 0x72, 0x25, 0x6d, 0x93, 0xc7, 0xdf, 0xe0, 0xf1, 0x7f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x64,
	0x7f, 0xc9, 0x6d, 0x53, 0x03, 0x00, 0x00,
}
<commit_msg>id should really be sha<commit_after>// Code generated by protoc-gen-go.
// source: push_event.proto
// DO NOT EDIT!

package github

import proto "github.com/golang/protobuf/proto"
import fmt "fmt"
import math "math"

// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf

type PushCommit struct {
	Sha       string      `protobuf:"bytes,1,opt,name=sha" json:"sha,omitempty"`
	TreeId    string      `protobuf:"bytes,2,opt,name=tree_id,json=treeId" json:"tree_id,omitempty"`
	Distinct  bool        `protobuf:"varint,3,opt,name=distinct" json:"distinct,omitempty"`
	Message   string      `protobuf:"bytes,4,opt,name=message" json:"message,omitempty"`
	Timestamp string      `protobuf:"bytes,5,opt,name=timestamp" json:"timestamp,omitempty"`
	Url       string      `protobuf:"bytes,6,opt,name=url" json:"url,omitempty"`
	Author    *CommitUser `protobuf:"bytes,7,opt,name=author" json:"author,omitempty"`
	Committer *CommitUser `protobuf:"bytes,8,opt,name=committer" json:"committer,omitempty"`
	Added     []string    `protobuf:"bytes,9,rep,name=added" json:"added,omitempty"`
	Removed   []string    `protobuf:"bytes,10,rep,name=removed" json:"removed,omitempty"`
	Modified  []string    `protobuf:"bytes,11,rep,name=modified" json:"modified,omitempty"`
}

func (m *PushCommit) Reset()                    { *m = PushCommit{} }
func (m *PushCommit) String() string            { return proto.CompactTextString(m) }
func (*PushCommit) ProtoMessage()               {}
func (*PushCommit) Descriptor() ([]byte, []int) { return fileDescriptor27, []int{0} }

func (m *PushCommit) GetAuthor() *CommitUser {
	if m != nil {
		return m.Author
	}
	return nil
}

func (m *PushCommit) GetCommitter() *CommitUser {
	if m != nil {
		return m.Committer
	}
	return nil
}

type PushEvent struct {
	Ref        string        `protobuf:"bytes,1,opt,name=ref" json:"ref,omitempty"`
	Before     string        `protobuf:"bytes,2,opt,name=before" json:"before,omitempty"`
	After      string        `protobuf:"bytes,3,opt,name=after" json:"after,omitempty"`
	Created    bool          `protobuf:"varint,4,opt,name=created" json:"created,omitempty"`
	Deleted    bool          `protobuf:"varint,5,opt,name=deleted" json:"deleted,omitempty"`
	Forced     bool          `protobuf:"varint,6,opt,name=forced" json:"forced,omitempty"`
	BaseRef    string        `protobuf:"bytes,7,opt,name=base_ref,json=baseRef" json:"base_ref,omitempty"`
	Compare    string        `protobuf:"bytes,8,opt,name=compare" json:"compare,omitempty"`
	Commits    []*PushCommit `protobuf:"bytes,9,rep,name=commits" json:"commits,omitempty"`
	HeadCommit *PushCommit   `protobuf:"bytes,10,opt,name=head_commit,json=headCommit" json:"head_commit,omitempty"`
	Repository *Repository   `protobuf:"bytes,11,opt,name=repository" json:"repository,omitempty"`
	Pusher     *CommitUser   `protobuf:"bytes,12,opt,name=pusher" json:"pusher,omitempty"`
	Sender     *User         `protobuf:"bytes,13,opt,name=sender" json:"sender,omitempty"`
}

func (m *PushEvent) Reset()                    { *m = PushEvent{} }
func (m *PushEvent) String() string            { return proto.CompactTextString(m) }
func (*PushEvent) ProtoMessage()               {}
func (*PushEvent) Descriptor() ([]byte, []int) { return fileDescriptor27, []int{1} }

func (m *PushEvent) GetCommits() []*PushCommit {
	if m != nil {
		return m.Commits
	}
	return nil
}

func (m *PushEvent) GetHeadCommit() *PushCommit {
	if m != nil {
		return m.HeadCommit
	}
	return nil
}

func (m *PushEvent) GetRepository() *Repository {
	if m != nil {
		return m.Repository
	}
	return nil
}

func (m *PushEvent) GetPusher() *CommitUser {
	if m != nil {
		return m.Pusher
	}
	return nil
}

func (m *PushEvent) GetSender() *User {
	if m != nil {
		return m.Sender
	}
	return nil
}

func init() {
	proto.RegisterType((*PushCommit)(nil), "github.PushCommit")
	proto.RegisterType((*PushEvent)(nil), "github.PushEvent")
}

func init() { proto.RegisterFile("push_event.proto", fileDescriptor27) }

var fileDescriptor27 = []byte{
	// 438 bytes of a gzipped FileDescriptorProto
	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x74, 0x53, 0xcd, 0x8a, 0xdb, 0x30,
	0x10, 0x26, 0xeb, 0x8d, 0x62, 0x4f, 0xb6, 0x90, 0x8a, 0xd2, 0xaa, 0xa1, 0x87, 0x65, 0xe9, 0xa1,
	0x94, 0x12, 0xca, 0xee, 0x23, 0x94, 0x1e, 0x7a, 0x2b, 0x86, 0x9e, 0x8d, 0x12, 0x4d, 0xd6, 0x82,
	0x75, 0x64, 0x24, 0x39, 0xd0, 0x07, 0xeb, 0x33, 0xf4, 0xb5, 0x3a, 0x23, 0xd9, 0x49, 0x7a, 0xc8,
	0x4d, 0xdf, 0x8f, 0x3c, 0x33, 0x9f, 0xc6, 0xb0, 0xea, 0x87, 0xd0, 0x36, 0x78, 0xc4, 0x43, 0xdc,
	0xf4, 0xde, 0x45, 0x27, 0xc5, 0xb3, 0x8d, 0xed, 0xb0, 0x5d, 0xc3, 0x10, 0xd0, 0x67, 0x6e, 0xfd,
	0x7a, 0xe7, 0xba, 0xce, 0xc6, 0xe6, 0x82, 0x5a, 0x79, 0xec, 0x5d, 0xb0, 0xd1, 0xf9, 0xdf, 0x99,
	0x79, 0xf8, 0x7b, 0x03, 0xf0, 0x93, 0xbe, 0xf6, 0x2d, 0x79, 0xe5, 0x0a, 0x8a, 0xd0, 0x6a, 0x35,
	0xbb, 0x9f, 0x7d, 0xaa, 0x6a, 0x3e, 0xca, 0x77, 0xb0, 0x88, 0x1e, 0xb1, 0xb1, 0x46, 0xdd, 0x24,
	0x56, 0x30, 0xfc, 0x61, 0xe4, 0x1a, 0x4a, 0x63, 0x43, 0xb4, 0x87, 0x5d, 0x54, 0x05, 0x29, 0x65,
	0x7d, 0xc2, 0x52, 0xc1, 0xa2, 0xc3, 0x10, 0xf4, 0x33, 0xaa, 0xdb, 0x74, 0x69, 0x82, 0xf2, 0x03,
	0x54, 0xd1, 0x12, 0x88, 0xba, 0xeb, 0xd5, 0x3c, 0x69, 0x67, 0x82, 0xcb, 0x0f, 0xfe, 0x45, 0x89,
	0x5c, 0x9e, 0x8e, 0xf2, 0x33, 0x08, 0x3d, 0xc4, 0xd6, 0x79, 0xb5, 0x20, 0x72, 0xf9, 0x28, 0x37,
	0x79, 0xd2, 0x4d, 0x6e, 0xf8, 0x17, 0xcd, 0x56, 0x8f, 0x0e, 0xf9, 0x15, 0xaa, 0x3c, 0x72, 0x44,
	0xaf, 0xca, 0xab, 0xf6, 0xb3, 0x49, 0xbe, 0x81, 0xb9, 0x36, 0x06, 0x8d, 0xaa, 0xee, 0x0b, 0xaa,
	0x98, 0x01, 0x77, 0xef, 0xb1, 0x73, 0x47, 0xe2, 0x21, 0xf1, 0x13, 0xe4, 0x99, 0x3b, 0x67, 0xec,
	0xde, 0x92, 0xb4, 0x4c, 0xd2, 0x09, 0x3f, 0xfc, 0x29, 0xa0, 0xe2, 0x24, 0xbf, 0xf3, 0xb3, 0xf0,
	0x24, 0x1e, 0xf7, 0x53, 0x90, 0x74, 0x94, 0x6f, 0x41, 0x6c, 0x71, 0xef, 0x3c, 0x4e, 0x39, 0x66,
	0x94, 0x7a, 0xd8, 0x73, 0xc7, 0x45, 0xa2, 0x33, 0xe0, 0x1e, 0x76, 0x1e, 0x75, 0xa4, 0x42, 0xb7,
	0x29, 0xdc, 0x09, 0xb2, 0x62, 0xf0, 0x05, 0x59, 0x99, 0x67, 0x65, 0x84, 0x5c, 0x81, 0xbe, 0xb8,
	0x23, 0x41, 0x24, 0x61, 0x44, 0xf2, 0x3d, 0x94, 0x5b, 0x1d, 0xb0, 0xe1, 0x86, 0x16, 0xf9, 0x39,
	0x18, 0xd7, 0xd4, 0x14, 0x97, 0x71, 0x5d, 0xaf, 0xa9, 0xab, 0x32, 0x2b, 0x23, 0x94, 0x5f, 0x92,
	0x42, 0x39, 0x85, 0x14, 0xce, 0x45, 0x94, 0xe7, 0x75, 0xa9, 0x27, 0x8b, 0x7c, 0x82, 0x65, 0x8b,
	0xda, 0x34, 0x19, 0x53, 0x6c, 0xb3, 0x2b, 0x37, 0x80, 0x6d, 0xe3, 0xb2, 0x3d, 0x02, 0x9c, 0xf7,
	0x91, 0xf2, 0xfc, 0xef, 0x4e, 0x7d, 0x52, 0xea, 0x0b, 0x17, 0xef, 0x03, 0x2f, 0x3f, 0xc5, 0x75,
	0x77, 0x7d, 0x1f, 0xb2, 0x43, 0x7e, 0x04, 0x11, 0xf0, 0x60, 0xc8, 0xfb, 0x2a, 0x79, 0xef, 0x26,
	0x6f, 0x76, 0x65, 0x6d, 0x2b, 0xd2, 0x8f, 0xf0, 0xf4, 0x2f, 0x00, 0x00, 0xff, 0xff, 0xa4, 0x2d,
	0x2b, 0xfb, 0x55, 0x03, 0x00, 0x00,
}
<|endoftext|>


<commit_before>package hdfs

import (
	"crypto/md5"
	"errors"
	"fmt"
	"io"
	"os"

	hdfs "github.com/colinmarc/hdfs/protocol/hadoop_hdfs"
	"github.com/colinmarc/hdfs/rpc"
	"github.com/golang/protobuf/proto"
)

// A FileReader represents an existing file or directory in HDFS. It implements
// io.Reader, io.ReaderAt, io.Seeker, and io.Closer, and can only be used for
// reads. For writes, see FileWriter and Client.Create.
type FileReader struct {
	client *Client
	name   string
	info   os.FileInfo

	blocks      []*hdfs.LocatedBlockProto
	blockReader *rpc.BlockReader
	offset      int64

	readdirLast string

	closed bool
}

// Open returns an FileReader which can be used for reading.
func (c *Client) Open(name string) (*FileReader, error) {
	info, err := c.getFileInfo(name)
	if err != nil {
		return nil, &os.PathError{"open", name, err}
	}

	return &FileReader{
		client: c,
		name:   name,
		info:   info,
		closed: false,
	}, nil
}

// Name returns the name of the file.
func (f *FileReader) Name() string {
	return f.info.Name()
}

// Stat returns the FileInfo structure describing file.
func (f *FileReader) Stat() os.FileInfo {
	return f.info
}

// Checksum returns HDFS's internal "MD5MD5CRC32C" checksum for a given file.
//
// Internally to HDFS, it works by calculating the MD5 of all the CRCs (which
// are stored alongside the data) for each block, and then calculating the MD5
// of all of those.
func (f *FileReader) Checksum() ([]byte, error) {
	if f.info.IsDir() {
		return nil, &os.PathError{
			"checksum",
			f.name,
			errors.New("is a directory"),
		}
	}

	if f.blocks == nil {
		err := f.getBlocks()
		if err != nil {
			return nil, err
		}
	}

	// The way the hadoop code calculates this, it writes all the checksums out to
	// a byte array, which is automatically padded with zeroes out to the next
	// power of 2 (with a minimum of 32)... and then takes the MD5 of that array,
	// including the zeroes. This is pretty shady business, but we want to track
	// the 'hadoop fs -checksum' behavior if possible.
	paddedLength := 32
	totalLength := 0
	checksum := md5.New()
	for _, block := range f.blocks {
		cr := rpc.NewChecksumReader(block)

		blockChecksum, err := cr.ReadChecksum()
		if err != nil {
			return nil, err
		}

		checksum.Write(blockChecksum)
		totalLength += len(blockChecksum)
		if paddedLength < totalLength {
			paddedLength *= 2
		}
	}

	checksum.Write(make([]byte, paddedLength-totalLength))
	return checksum.Sum(nil), nil
}

// Seek implements io.Seeker.
//
// The seek is virtual - it starts a new block read at the new position.
func (f *FileReader) Seek(offset int64, whence int) (int64, error) {
	if f.closed {
		return 0, io.ErrClosedPipe
	}

	var off int64
	if whence == 0 {
		off = offset
	} else if whence == 1 {
		off = f.offset + offset
	} else if whence == 2 {
		off = f.info.Size() + offset
	} else {
		return f.offset, fmt.Errorf("Invalid whence: %d", whence)
	}

	if off < 0 || off > f.info.Size() {
		return f.offset, fmt.Errorf("Invalid resulting offset: %d", off)
	}

	f.offset = off
	f.blockReader = nil

	return f.offset, nil
}

// Read implements io.Reader.
func (f *FileReader) Read(b []byte) (int, error) {
	if f.closed {
		return 0, io.ErrClosedPipe
	}

	if f.info.IsDir() {
		return 0, &os.PathError{
			"read",
			f.name,
			errors.New("is a directory"),
		}
	}

	if f.offset >= f.info.Size() {
		return 0, io.EOF
	}

	if f.blocks == nil {
		err := f.getBlocks()
		if err != nil {
			return 0, err
		}
	}

	if f.blockReader == nil {
		err := f.getNewBlockReader()
		if err != nil {
			return 0, err
		}
	}

	for {
		n, err := f.blockReader.Read(b)
		f.offset += int64(n)

		if err != nil && err != io.EOF {
			f.blockReader.Close()
			f.blockReader = nil
			return n, err
		} else if n > 0 {
			return n, nil
		} else {
			f.blockReader.Close()
			f.getNewBlockReader()
		}
	}
}

// ReadAt implements io.ReaderAt.
func (f *FileReader) ReadAt(b []byte, off int64) (int, error) {
	if f.closed {
		return 0, io.ErrClosedPipe
	}

	_, err := f.Seek(off, 0)
	if err != nil {
		return 0, err
	}

	return io.ReadFull(f, b)
}

// Readdir reads the contents of the directory associated with file and returns
// a slice of up to n os.FileInfo values, as would be returned by Stat, in
// directory order. Subsequent calls on the same file will yield further
// os.FileInfos.
//
// If n > 0, Readdir returns at most n os.FileInfo values. In this case, if
// Readdir returns an empty slice, it will return a non-nil error explaining
// why. At the end of a directory, the error is io.EOF.
//
// If n <= 0, Readdir returns all the os.FileInfo from the directory in a single
// slice. In this case, if Readdir succeeds (reads all the way to the end of
// the directory), it returns the slice and a nil error. If it encounters an
// error before the end of the directory, Readdir returns the os.FileInfo read
// until that point and a non-nil error.
func (f *FileReader) Readdir(n int) ([]os.FileInfo, error) {
	if f.closed {
		return nil, io.ErrClosedPipe
	}

	if !f.info.IsDir() {
		return nil, &os.PathError{
			"readdir",
			f.name,
			errors.New("the file is not a directory"),
		}
	}

	if n <= 0 {
		f.readdirLast = ""
	}

	res, err := f.client.getDirList(f.name, f.readdirLast, n)
	if err != nil {
		return res, err
	}

	if n > 0 {
		if len(res) == 0 {
			err = io.EOF
		} else {
			f.readdirLast = res[len(res)-1].Name()
		}
	}

	return res, err
}

// Readdirnames reads and returns a slice of names from the directory f.
//
// If n > 0, Readdirnames returns at most n names. In this case, if Readdirnames
// returns an empty slice, it will return a non-nil error explaining why. At the
// end of a directory, the error is io.EOF.
//
// If n <= 0, Readdirnames returns all the names from the directory in a single
// slice. In this case, if Readdirnames succeeds (reads all the way to the end
// of the directory), it returns the slice and a nil error. If it encounters an
// error before the end of the directory, Readdirnames returns the names read
// until that point and a non-nil error.
func (f *FileReader) Readdirnames(n int) ([]string, error) {
	if f.closed {
		return nil, io.ErrClosedPipe
	}

	fis, err := f.Readdir(n)
	if err != nil {
		return nil, err
	}

	names := make([]string, 0, len(fis))
	for _, fi := range fis {
		names = append(names, fi.Name())
	}

	return names, nil
}

// Close implements io.Closer.
func (f *FileReader) Close() error {
	f.closed = true

	if f.blockReader != nil {
		f.blockReader.Close()
	}

	return nil
}

func (f *FileReader) getBlocks() error {
	req := &hdfs.GetBlockLocationsRequestProto{
		Src:    proto.String(f.name),
		Offset: proto.Uint64(0),
		Length: proto.Uint64(uint64(f.info.Size())),
	}
	resp := &hdfs.GetBlockLocationsResponseProto{}

	err := f.client.namenode.Execute("getBlockLocations", req, resp)
	if err != nil {
		return err
	}

	f.blocks = resp.GetLocations().GetBlocks()
	return nil
}

func (f *FileReader) getNewBlockReader() error {
	off := uint64(f.offset)
	for _, block := range f.blocks {
		start := block.GetOffset()
		end := start + block.GetB().GetNumBytes()

		if start <= off && off < end {
			br := rpc.NewBlockReader(block, int64(off-start), f.client.namenode.ClientName())

			f.blockReader = br
			return nil
		}
	}

	return fmt.Errorf("Couldn't find block for offset: %d", off)
}
<commit_msg>fix for #28: properly closing BlockReader on Seek()<commit_after>package hdfs

import (
	"crypto/md5"
	"errors"
	"fmt"
	"io"
	"os"

	hdfs "github.com/colinmarc/hdfs/protocol/hadoop_hdfs"
	"github.com/colinmarc/hdfs/rpc"
	"github.com/golang/protobuf/proto"
)

// A FileReader represents an existing file or directory in HDFS. It implements
// io.Reader, io.ReaderAt, io.Seeker, and io.Closer, and can only be used for
// reads. For writes, see FileWriter and Client.Create.
type FileReader struct {
	client *Client
	name   string
	info   os.FileInfo

	blocks      []*hdfs.LocatedBlockProto
	blockReader *rpc.BlockReader
	offset      int64

	readdirLast string

	closed bool
}

// Open returns an FileReader which can be used for reading.
func (c *Client) Open(name string) (*FileReader, error) {
	info, err := c.getFileInfo(name)
	if err != nil {
		return nil, &os.PathError{"open", name, err}
	}

	return &FileReader{
		client: c,
		name:   name,
		info:   info,
		closed: false,
	}, nil
}

// Name returns the name of the file.
func (f *FileReader) Name() string {
	return f.info.Name()
}

// Stat returns the FileInfo structure describing file.
func (f *FileReader) Stat() os.FileInfo {
	return f.info
}

// Checksum returns HDFS's internal "MD5MD5CRC32C" checksum for a given file.
//
// Internally to HDFS, it works by calculating the MD5 of all the CRCs (which
// are stored alongside the data) for each block, and then calculating the MD5
// of all of those.
func (f *FileReader) Checksum() ([]byte, error) {
	if f.info.IsDir() {
		return nil, &os.PathError{
			"checksum",
			f.name,
			errors.New("is a directory"),
		}
	}

	if f.blocks == nil {
		err := f.getBlocks()
		if err != nil {
			return nil, err
		}
	}

	// The way the hadoop code calculates this, it writes all the checksums out to
	// a byte array, which is automatically padded with zeroes out to the next
	// power of 2 (with a minimum of 32)... and then takes the MD5 of that array,
	// including the zeroes. This is pretty shady business, but we want to track
	// the 'hadoop fs -checksum' behavior if possible.
	paddedLength := 32
	totalLength := 0
	checksum := md5.New()
	for _, block := range f.blocks {
		cr := rpc.NewChecksumReader(block)

		blockChecksum, err := cr.ReadChecksum()
		if err != nil {
			return nil, err
		}

		checksum.Write(blockChecksum)
		totalLength += len(blockChecksum)
		if paddedLength < totalLength {
			paddedLength *= 2
		}
	}

	checksum.Write(make([]byte, paddedLength-totalLength))
	return checksum.Sum(nil), nil
}

// Seek implements io.Seeker.
//
// The seek is virtual - it starts a new block read at the new position.
func (f *FileReader) Seek(offset int64, whence int) (int64, error) {
	if f.closed {
		return 0, io.ErrClosedPipe
	}

	var off int64
	if whence == 0 {
		off = offset
	} else if whence == 1 {
		off = f.offset + offset
	} else if whence == 2 {
		off = f.info.Size() + offset
	} else {
		return f.offset, fmt.Errorf("Invalid whence: %d", whence)
	}

	if off < 0 || off > f.info.Size() {
		return f.offset, fmt.Errorf("Invalid resulting offset: %d", off)
	}

	if f.offset != off {
		f.offset = off
		if f.blockReader != nil {
			f.blockReader.Close()
			f.blockReader = nil
		}
	}
	return f.offset, nil
}

// Read implements io.Reader.
func (f *FileReader) Read(b []byte) (int, error) {
	if f.closed {
		return 0, io.ErrClosedPipe
	}

	if f.info.IsDir() {
		return 0, &os.PathError{
			"read",
			f.name,
			errors.New("is a directory"),
		}
	}

	if f.offset >= f.info.Size() {
		return 0, io.EOF
	}

	if f.blocks == nil {
		err := f.getBlocks()
		if err != nil {
			return 0, err
		}
	}

	if f.blockReader == nil {
		err := f.getNewBlockReader()
		if err != nil {
			return 0, err
		}
	}

	for {
		n, err := f.blockReader.Read(b)
		f.offset += int64(n)

		if err != nil && err != io.EOF {
			f.blockReader.Close()
			f.blockReader = nil
			return n, err
		} else if n > 0 {
			return n, nil
		} else {
			f.blockReader.Close()
			f.getNewBlockReader()
		}
	}
}

// ReadAt implements io.ReaderAt.
func (f *FileReader) ReadAt(b []byte, off int64) (int, error) {
	if f.closed {
		return 0, io.ErrClosedPipe
	}

	_, err := f.Seek(off, 0)
	if err != nil {
		return 0, err
	}

	return io.ReadFull(f, b)
}

// Readdir reads the contents of the directory associated with file and returns
// a slice of up to n os.FileInfo values, as would be returned by Stat, in
// directory order. Subsequent calls on the same file will yield further
// os.FileInfos.
//
// If n > 0, Readdir returns at most n os.FileInfo values. In this case, if
// Readdir returns an empty slice, it will return a non-nil error explaining
// why. At the end of a directory, the error is io.EOF.
//
// If n <= 0, Readdir returns all the os.FileInfo from the directory in a single
// slice. In this case, if Readdir succeeds (reads all the way to the end of
// the directory), it returns the slice and a nil error. If it encounters an
// error before the end of the directory, Readdir returns the os.FileInfo read
// until that point and a non-nil error.
func (f *FileReader) Readdir(n int) ([]os.FileInfo, error) {
	if f.closed {
		return nil, io.ErrClosedPipe
	}

	if !f.info.IsDir() {
		return nil, &os.PathError{
			"readdir",
			f.name,
			errors.New("the file is not a directory"),
		}
	}

	if n <= 0 {
		f.readdirLast = ""
	}

	res, err := f.client.getDirList(f.name, f.readdirLast, n)
	if err != nil {
		return res, err
	}

	if n > 0 {
		if len(res) == 0 {
			err = io.EOF
		} else {
			f.readdirLast = res[len(res)-1].Name()
		}
	}

	return res, err
}

// Readdirnames reads and returns a slice of names from the directory f.
//
// If n > 0, Readdirnames returns at most n names. In this case, if Readdirnames
// returns an empty slice, it will return a non-nil error explaining why. At the
// end of a directory, the error is io.EOF.
//
// If n <= 0, Readdirnames returns all the names from the directory in a single
// slice. In this case, if Readdirnames succeeds (reads all the way to the end
// of the directory), it returns the slice and a nil error. If it encounters an
// error before the end of the directory, Readdirnames returns the names read
// until that point and a non-nil error.
func (f *FileReader) Readdirnames(n int) ([]string, error) {
	if f.closed {
		return nil, io.ErrClosedPipe
	}

	fis, err := f.Readdir(n)
	if err != nil {
		return nil, err
	}

	names := make([]string, 0, len(fis))
	for _, fi := range fis {
		names = append(names, fi.Name())
	}

	return names, nil
}

// Close implements io.Closer.
func (f *FileReader) Close() error {
	f.closed = true

	if f.blockReader != nil {
		f.blockReader.Close()
	}

	return nil
}

func (f *FileReader) getBlocks() error {
	req := &hdfs.GetBlockLocationsRequestProto{
		Src:    proto.String(f.name),
		Offset: proto.Uint64(0),
		Length: proto.Uint64(uint64(f.info.Size())),
	}
	resp := &hdfs.GetBlockLocationsResponseProto{}

	err := f.client.namenode.Execute("getBlockLocations", req, resp)
	if err != nil {
		return err
	}

	f.blocks = resp.GetLocations().GetBlocks()
	return nil
}

func (f *FileReader) getNewBlockReader() error {
	off := uint64(f.offset)
	for _, block := range f.blocks {
		start := block.GetOffset()
		end := start + block.GetB().GetNumBytes()

		if start <= off && off < end {
			br := rpc.NewBlockReader(block, int64(off-start), f.client.namenode.ClientName())

			f.blockReader = br
			return nil
		}
	}

	return fmt.Errorf("Couldn't find block for offset: %d", off)
}
<|endoftext|>


<commit_before>package finder

import (
	"context"
	"fmt"

	"github.com/lomik/graphite-clickhouse/config"
	"github.com/lomik/graphite-clickhouse/helper/clickhouse"
)

type DateFinder struct {
	*BaseFinder
	dateWhere string
}

func NewDateFinder(dateWhere string, ctx context.Context, config *config.Config) Finder {
	b := &BaseFinder{
		ctx:         ctx,
		url:         config.ClickHouse.Url,
		table:       config.ClickHouse.DateTreeTable,
		expandLimit: config.ClickHouse.MetricLimitWithExpand,
		timeout:     config.ClickHouse.TreeTimeout.Value(),
	}
	return &DateFinder{b, dateWhere}
}

func (b *DateFinder) Execute(query string) (err error) {
	where := b.where(query)

	// TODO add deleted field to table?
	b.body, err = clickhouse.Query(
		b.ctx,
		b.url,
		fmt.Sprintf(`
		SELECT distinct Path FROM %s PREWHERE (%s) WHERE (%s)
		`, b.table, b.dateWhere, where),
		b.timeout,
	)

	return
}
<commit_msg>cleanup<commit_after>package finder

import (
	"context"
	"fmt"

	"github.com/lomik/graphite-clickhouse/config"
	"github.com/lomik/graphite-clickhouse/helper/clickhouse"
)

type DateFinder struct {
	*BaseFinder
	dateWhere string
}

func NewDateFinder(dateWhere string, ctx context.Context, config *config.Config) Finder {
	b := &BaseFinder{
		ctx:     ctx,
		url:     config.ClickHouse.Url,
		table:   config.ClickHouse.DateTreeTable,
		timeout: config.ClickHouse.TreeTimeout.Value(),
	}
	return &DateFinder{b, dateWhere}
}

func (b *DateFinder) Execute(query string) (err error) {
	where := b.where(query)

	b.body, err = clickhouse.Query(
		b.ctx,
		b.url,
		fmt.Sprintf(`SELECT distinct Path FROM %s PREWHERE (%s) WHERE (%s)`, b.table, b.dateWhere, where),
		b.timeout,
	)

	return
}
<|endoftext|>


<commit_before>package finder

import (
	"fmt"
	"testing"
)

type testCase struct {
	finder *Finder
	paths  []string
}

// test runs a slice of test cases.
func test(t *testing.T, cases []testCase) {
	for _, tc := range cases {
		tc.Test(t)
	}
}

func TestIn(t *testing.T) {
	tests := []testCase{
		{
			New().In("test-fixtures"),
			[]string{
				"test-fixtures/1.log",
				"test-fixtures/1.txt",
				"test-fixtures/d1",
				"test-fixtures/d1/1",
			},
		},
		{
			New().In("test-fixtures/d1"),
			[]string{"test-fixtures/d1/1"},
		},
	}
	test(t, tests)
}

func TestPath(t *testing.T) {
	tests := []testCase{
		{
			New().In("test-fixtures").Path("d1"),
			[]string{
				"test-fixtures/d1",
				"test-fixtures/d1/1",
			},
		},
		{
			New().In("test-fixtures").Path("x"),
			[]string{},
		},
		{
			New().In("test-fixtures").Path("*"),
			[]string{
				"test-fixtures/1.log",
				"test-fixtures/1.txt",
				"test-fixtures/d1",
				"test-fixtures/d1/1",
			},
		},
	}
	test(t, tests)
}

func TestNotPath(t *testing.T) {
	tests := []testCase{
		{
			New().In("test-fixtures").NotPath("d1"),
			[]string{
				"test-fixtures/1.log",
				"test-fixtures/1.txt",
			},
		},
	}
	test(t, tests)
}

func TestName(t *testing.T) {
	tests := []testCase{
		{
			New().In("test-fixtures").Name("*.log"),
			[]string{"test-fixtures/1.log"},
		},
		{
			New().In("test-fixtures").Name("?.log"),
			[]string{"test-fixtures/1.log"},
		},
		{
			New().In("test-fixtures").Name("?.{log,txt}"),
			[]string{
				"test-fixtures/1.log",
				"test-fixtures/1.txt",
			},
		},
	}
	test(t, tests)
}

func TestNameRegex(t *testing.T) {
	tests := []testCase{
		{
			New().In("test-fixtures").NameRegex("\\.log$"),
			[]string{"test-fixtures/1.log"},
		},
	}
	test(t, tests)
}

func TestNotName(t *testing.T) {
	tests := []testCase{
		{
			New().In("test-fixtures").NotName("*.log").NotName("*.txt"),
			[]string{
				"test-fixtures/d1/1",
				"test-fixtures/d1",
			},
		},
	}
	test(t, tests)
}

func TestNotNameRegex(t *testing.T) {
	tests := []testCase{
		{
			New().In("test-fixtures").NotNameRegex("\\.(log|txt)$"),
			[]string{
				"test-fixtures/d1/1",
				"test-fixtures/d1",
			},
		},
	}
	test(t, tests)
}

func TestFiles(t *testing.T) {
	tests := []testCase{
		{
			New().In("test-fixtures").Files(),
			[]string{
				"test-fixtures/1.log",
				"test-fixtures/1.txt",
				"test-fixtures/d1/1",
			},
		},
	}
	test(t, tests)
}

func TestDirs(t *testing.T) {
	tests := []testCase{
		{
			New().In("test-fixtures").Dirs(),
			[]string{
				"test-fixtures/d1",
			},
		},
	}
	test(t, tests)
}

// Test runs a single test case.
func (tc testCase) Test(t *testing.T) {
	files, errs := tc.finder.ToSlice()
	if len(errs) > 0 {
		t.Fatal(errs)
	}
	err := matchFiles(t, tc.paths, files)
	if err != nil {
		t.Fatal(err)
	}
}

// matchFiles asserts that only the expected paths occur in the given files.
func matchFiles(t *testing.T, paths []string, files []Item) error {
	m := make(map[string]bool)
	for _, path := range paths {
		m[path] = false
	}
	for _, file := range files {
		_, exists := m[file.Path()]
		if !exists {
			return fmt.Errorf("unexpected path: %s", file.Path())
		}
		m[file.Path()] = true
	}
	for path, found := range m {
		if !found {
			return fmt.Errorf("missing path: %s", path)
		}
	}
	return nil
}
<commit_msg>Rearrange test helpers<commit_after>package finder

import (
	"fmt"
	"testing"
)

// test runs a slice of test cases.
func test(t *testing.T, cases []testCase) {
	for _, tc := range cases {
		tc.Test(t)
	}
}

type testCase struct {
	finder *Finder
	paths  []string
}

// Test runs a single test case.
func (tc testCase) Test(t *testing.T) {
	files, errs := tc.finder.ToSlice()
	if len(errs) > 0 {
		t.Fatal(errs)
	}
	err := matchFiles(t, tc.paths, files)
	if err != nil {
		t.Fatal(err)
	}
}

// matchFiles asserts that only the expected paths occur in the given files.
func matchFiles(t *testing.T, paths []string, files []Item) error {
	m := make(map[string]bool)
	for _, path := range paths {
		m[path] = false
	}
	for _, file := range files {
		_, exists := m[file.Path()]
		if !exists {
			return fmt.Errorf("unexpected path: %s", file.Path())
		}
		m[file.Path()] = true
	}
	for path, found := range m {
		if !found {
			return fmt.Errorf("missing path: %s", path)
		}
	}
	return nil
}

func TestIn(t *testing.T) {
	tests := []testCase{
		{
			New().In("test-fixtures"),
			[]string{
				"test-fixtures/1.log",
				"test-fixtures/1.txt",
				"test-fixtures/d1",
				"test-fixtures/d1/1",
			},
		},
		{
			New().In("test-fixtures/d1"),
			[]string{"test-fixtures/d1/1"},
		},
	}
	test(t, tests)
}

func TestPath(t *testing.T) {
	tests := []testCase{
		{
			New().In("test-fixtures").Path("d1"),
			[]string{
				"test-fixtures/d1",
				"test-fixtures/d1/1",
			},
		},
		{
			New().In("test-fixtures").Path("x"),
			[]string{},
		},
		{
			New().In("test-fixtures").Path("*"),
			[]string{
				"test-fixtures/1.log",
				"test-fixtures/1.txt",
				"test-fixtures/d1",
				"test-fixtures/d1/1",
			},
		},
	}
	test(t, tests)
}

func TestNotPath(t *testing.T) {
	tests := []testCase{
		{
			New().In("test-fixtures").NotPath("d1"),
			[]string{
				"test-fixtures/1.log",
				"test-fixtures/1.txt",
			},
		},
	}
	test(t, tests)
}

func TestName(t *testing.T) {
	tests := []testCase{
		{
			New().In("test-fixtures").Name("*.log"),
			[]string{"test-fixtures/1.log"},
		},
		{
			New().In("test-fixtures").Name("?.log"),
			[]string{"test-fixtures/1.log"},
		},
		{
			New().In("test-fixtures").Name("?.{log,txt}"),
			[]string{
				"test-fixtures/1.log",
				"test-fixtures/1.txt",
			},
		},
	}
	test(t, tests)
}

func TestNameRegex(t *testing.T) {
	tests := []testCase{
		{
			New().In("test-fixtures").NameRegex("\\.log$"),
			[]string{"test-fixtures/1.log"},
		},
	}
	test(t, tests)
}

func TestNotName(t *testing.T) {
	tests := []testCase{
		{
			New().In("test-fixtures").NotName("*.log").NotName("*.txt"),
			[]string{
				"test-fixtures/d1/1",
				"test-fixtures/d1",
			},
		},
	}
	test(t, tests)
}

func TestNotNameRegex(t *testing.T) {
	tests := []testCase{
		{
			New().In("test-fixtures").NotNameRegex("\\.(log|txt)$"),
			[]string{
				"test-fixtures/d1/1",
				"test-fixtures/d1",
			},
		},
	}
	test(t, tests)
}

func TestFiles(t *testing.T) {
	tests := []testCase{
		{
			New().In("test-fixtures").Files(),
			[]string{
				"test-fixtures/1.log",
				"test-fixtures/1.txt",
				"test-fixtures/d1/1",
			},
		},
	}
	test(t, tests)
}

func TestDirs(t *testing.T) {
	tests := []testCase{
		{
			New().In("test-fixtures").Dirs(),
			[]string{
				"test-fixtures/d1",
			},
		},
	}
	test(t, tests)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"strconv"

	"net/http"
	"path"
	"strings"
	"time"

	"github.com/boltdb/bolt"
	"github.com/gin-gonic/gin"
)

// Fingerprint is the prototypical information from the fingerprinting device
type Fingerprint struct {
	Group           string   `json:"group"`
	Username        string   `json:"username"`
	Location        string   `json:"location"`
	WifiFingerprint []Router `json:"wifi-fingerprint"`
}

// Router is the router information for each invdividual mac address
type Router struct {
	Mac  string `json:"mac"`
	Rssi int    `json:"rssi"`
}

var jsonExample = `{
	"group": "whatevergroup",
	"username": "iamauser",
	"location": null,
	"wififingerprint": [{
		"mac": "AA:AA:AA:AA:AA:AA",
		"rssi": -45
	}, {
		"mac": "BB:BB:BB:BB:BB:BB",
		"rssi": -55
	}]
}`

// compression 9 us -> 900 us
func dumpFingerprint(res Fingerprint) []byte {
	dumped, _ := res.MarshalJSON()
	return compressByte(dumped)
}

// compression 30 us -> 600 us
func loadFingerprint(jsonByte []byte) Fingerprint {
	res := Fingerprint{}
	res.UnmarshalJSON(decompressByte(jsonByte))
	return res
}

func cleanFingerprint(res *Fingerprint) {
	res.Group = strings.ToLower(res.Group)
	res.Location = strings.ToLower(res.Location)
	res.Username = strings.ToLower(res.Username)
	deleteIndex := -1
	for r := range res.WifiFingerprint {
		if res.WifiFingerprint[r].Rssi >= 0 { // https://stackoverflow.com/questions/15797920/how-to-convert-wifi-signal-strength-from-quality-percent-to-rssi-dbm
			res.WifiFingerprint[r].Rssi = int(res.WifiFingerprint[r].Rssi/2) - 100
		}
		if res.WifiFingerprint[r].Mac == "00:00:00:00:00:00" {
			deleteIndex = r
		}
	}
	if deleteIndex > -1 {
		res.WifiFingerprint[deleteIndex] = res.WifiFingerprint[len(res.WifiFingerprint)-1]
		res.WifiFingerprint = res.WifiFingerprint[:len(res.WifiFingerprint)-1]
	}
}

func putFingerprintIntoDatabase(res Fingerprint, database string) error {
	db, err := bolt.Open(path.Join(RuntimeArgs.SourcePath, res.Group+".db"), 0600, nil)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	err = db.Update(func(tx *bolt.Tx) error {
		bucket, err := tx.CreateBucketIfNotExists([]byte(database))
		if err != nil {
			return fmt.Errorf("create bucket: %s", err)
		}

		timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
		err = bucket.Put([]byte(timestamp), dumpFingerprint(res))
		if err != nil {
			return fmt.Errorf("could add to bucket: %s", err)
		}
		return err
	})
	return err
}

func handleFingerprint(c *gin.Context) {
	var jsonFingerprint Fingerprint
	if c.BindJSON(&jsonFingerprint) == nil {
		cleanFingerprint(&jsonFingerprint)
		if jsonFingerprint.Location != "" {
			go putFingerprintIntoDatabase(jsonFingerprint, "fingerprints")
			isLearning[strings.ToLower(jsonFingerprint.Group)] = true
			Debug.Println("Inserted fingerprint for " + jsonFingerprint.Username + " (" + jsonFingerprint.Group + ") at " + jsonFingerprint.Location)
			c.JSON(http.StatusOK, gin.H{"message": "Inserted " + strconv.Itoa(len(jsonFingerprint.WifiFingerprint)) + " fingerprints for " + jsonFingerprint.Username + " at " + jsonFingerprint.Location, "success": true})
		} else {
			trackFingerprint(c)
		}
	}
}

func trackFingerprint(c *gin.Context) {
	var jsonFingerprint Fingerprint
	if c.BindJSON(&jsonFingerprint) == nil {
		cleanFingerprint(&jsonFingerprint)
		if wasLearning, ok := isLearning[strings.ToLower(jsonFingerprint.Group)]; ok {
			if wasLearning {
				Debug.Println("Was learning, calculating priors")
				group := strings.ToLower(jsonFingerprint.Group)
				isLearning[group] = false
				optimizePriorsThreaded(group)
				if _, ok := usersCache[group]; ok {
					if len(usersCache[group]) == 0 {
						usersCache[group] = append([]string{}, strings.ToLower(jsonFingerprint.Username))
					}
				}
			}
		}
		locationGuess, bayes := calculatePosterior(jsonFingerprint, *NewFullParameters())
		jsonFingerprint.Location = locationGuess
		go putFingerprintIntoDatabase(jsonFingerprint, "fingerprints-track")
		positions := [][]string{}
		positions1 := []string{}
		positions2 := []string{}
		positions1 = append(positions1, locationGuess)
		positions2 = append(positions2, " ")
		positions = append(positions, positions1)
		positions = append(positions, positions2)
		var userJSON UserPositionJSON
		userJSON.Location = locationGuess
		userJSON.Bayes = bayes
		userJSON.Time = time.Now().String()
		userPositionCache[strings.ToLower(jsonFingerprint.Group)+strings.ToLower(jsonFingerprint.Username)] = userJSON
		Debug.Println("Tracking fingerprint for " + jsonFingerprint.Username + " (" + jsonFingerprint.Group + ") at " + jsonFingerprint.Location + " (guess)")
		c.JSON(http.StatusOK, gin.H{"message": "Calculated location: " + locationGuess, "success": true, "location": locationGuess})
	} else {
		c.JSON(http.StatusOK, gin.H{"message": "Something went wrong", "success": false})
	}
}
<commit_msg>Misnomer. One fingerprint contains several APs<commit_after>package main

import (
	"fmt"
	"log"
	"strconv"

	"net/http"
	"path"
	"strings"
	"time"

	"github.com/boltdb/bolt"
	"github.com/gin-gonic/gin"
)

// Fingerprint is the prototypical information from the fingerprinting device
type Fingerprint struct {
	Group           string   `json:"group"`
	Username        string   `json:"username"`
	Location        string   `json:"location"`
	WifiFingerprint []Router `json:"wifi-fingerprint"`
}

// Router is the router information for each invdividual mac address
type Router struct {
	Mac  string `json:"mac"`
	Rssi int    `json:"rssi"`
}

var jsonExample = `{
	"group": "whatevergroup",
	"username": "iamauser",
	"location": null,
	"wififingerprint": [{
		"mac": "AA:AA:AA:AA:AA:AA",
		"rssi": -45
	}, {
		"mac": "BB:BB:BB:BB:BB:BB",
		"rssi": -55
	}]
}`

// compression 9 us -> 900 us
func dumpFingerprint(res Fingerprint) []byte {
	dumped, _ := res.MarshalJSON()
	return compressByte(dumped)
}

// compression 30 us -> 600 us
func loadFingerprint(jsonByte []byte) Fingerprint {
	res := Fingerprint{}
	res.UnmarshalJSON(decompressByte(jsonByte))
	return res
}

func cleanFingerprint(res *Fingerprint) {
	res.Group = strings.ToLower(res.Group)
	res.Location = strings.ToLower(res.Location)
	res.Username = strings.ToLower(res.Username)
	deleteIndex := -1
	for r := range res.WifiFingerprint {
		if res.WifiFingerprint[r].Rssi >= 0 { // https://stackoverflow.com/questions/15797920/how-to-convert-wifi-signal-strength-from-quality-percent-to-rssi-dbm
			res.WifiFingerprint[r].Rssi = int(res.WifiFingerprint[r].Rssi/2) - 100
		}
		if res.WifiFingerprint[r].Mac == "00:00:00:00:00:00" {
			deleteIndex = r
		}
	}
	if deleteIndex > -1 {
		res.WifiFingerprint[deleteIndex] = res.WifiFingerprint[len(res.WifiFingerprint)-1]
		res.WifiFingerprint = res.WifiFingerprint[:len(res.WifiFingerprint)-1]
	}
}

func putFingerprintIntoDatabase(res Fingerprint, database string) error {
	db, err := bolt.Open(path.Join(RuntimeArgs.SourcePath, res.Group+".db"), 0600, nil)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	err = db.Update(func(tx *bolt.Tx) error {
		bucket, err := tx.CreateBucketIfNotExists([]byte(database))
		if err != nil {
			return fmt.Errorf("create bucket: %s", err)
		}

		timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
		err = bucket.Put([]byte(timestamp), dumpFingerprint(res))
		if err != nil {
			return fmt.Errorf("could add to bucket: %s", err)
		}
		return err
	})
	return err
}

func handleFingerprint(c *gin.Context) {
	var jsonFingerprint Fingerprint
	if c.BindJSON(&jsonFingerprint) == nil {
		cleanFingerprint(&jsonFingerprint)
		if jsonFingerprint.Location != "" {
			go putFingerprintIntoDatabase(jsonFingerprint, "fingerprints")
			isLearning[strings.ToLower(jsonFingerprint.Group)] = true
			Debug.Println("Inserted fingerprint for " + jsonFingerprint.Username + " (" + jsonFingerprint.Group + ") at " + jsonFingerprint.Location)
			c.JSON(http.StatusOK, gin.H{"message": "Inserted fingerprint containing " + strconv.Itoa(len(jsonFingerprint.WifiFingerprint)) + " APs for " + jsonFingerprint.Username + " at " + jsonFingerprint.Location, "success": true})
		} else {
			trackFingerprint(c)
		}
	}
}

func trackFingerprint(c *gin.Context) {
	var jsonFingerprint Fingerprint
	if c.BindJSON(&jsonFingerprint) == nil {
		cleanFingerprint(&jsonFingerprint)
		if wasLearning, ok := isLearning[strings.ToLower(jsonFingerprint.Group)]; ok {
			if wasLearning {
				Debug.Println("Was learning, calculating priors")
				group := strings.ToLower(jsonFingerprint.Group)
				isLearning[group] = false
				optimizePriorsThreaded(group)
				if _, ok := usersCache[group]; ok {
					if len(usersCache[group]) == 0 {
						usersCache[group] = append([]string{}, strings.ToLower(jsonFingerprint.Username))
					}
				}
			}
		}
		locationGuess, bayes := calculatePosterior(jsonFingerprint, *NewFullParameters())
		jsonFingerprint.Location = locationGuess
		go putFingerprintIntoDatabase(jsonFingerprint, "fingerprints-track")
		positions := [][]string{}
		positions1 := []string{}
		positions2 := []string{}
		positions1 = append(positions1, locationGuess)
		positions2 = append(positions2, " ")
		positions = append(positions, positions1)
		positions = append(positions, positions2)
		var userJSON UserPositionJSON
		userJSON.Location = locationGuess
		userJSON.Bayes = bayes
		userJSON.Time = time.Now().String()
		userPositionCache[strings.ToLower(jsonFingerprint.Group)+strings.ToLower(jsonFingerprint.Username)] = userJSON
		Debug.Println("Tracking fingerprint for " + jsonFingerprint.Username + " (" + jsonFingerprint.Group + ") at " + jsonFingerprint.Location + " (guess)")
		c.JSON(http.StatusOK, gin.H{"message": "Calculated location: " + locationGuess, "success": true, "location": locationGuess})
	} else {
		c.JSON(http.StatusOK, gin.H{"message": "Something went wrong", "success": false})
	}
}
<|endoftext|>


<commit_before>package goriak

import (
	"bytes"
	"errors"

	riak "github.com/basho/riak-go-client"
)

func NewSet() *Set {
	return &Set{}
}

type Set struct {
	path    []string    // Path to the counter (can be a map in a map in a map, etc..)
	name    string      // Name of the counter
	key     requestData // bucket information
	context []byte      // riak context

	value   [][]byte // The full content
	adds    [][]byte // Not-yet performed Add actions (performed locally but not to Riak)
	removes [][]byte // Same as adds, but for removal of items
}

func (s *Set) Value() [][]byte {
	r := make([][]byte, 0)

	// Remove empty items
	for _, v := range s.value {
		if len(v) != 0 {
			r = append(r, v)
		}
	}

	return r
}

func (s *Set) Strings() []string {
	all := s.Value()

	r := make([]string, len(all))

	for i, v := range all {
		r[i] = string(v)
	}

	return r
}

func (s *Set) Add(add []byte) *Set {
	// Make sure that our set doesn't already contain this value
	for _, item := range s.value {
		if bytes.Equal(item, add) {
			return s
		}
	}

	// Add to s.value
	s.value = append(s.value, add)

	// Add to s.adds (Riak actions not yet saved)
	s.adds = append(s.adds, add)

	// Remove from s.removes
	for i, item := range s.removes {
		if bytes.Equal(item, add) {

			// https://github.com/golang/go/wiki/SliceTricks
			s.removes[i] = s.removes[len(s.removes)-1]
			s.removes = s.removes[:len(s.removes)-1]
		}
	}

	return s
}

func (s *Set) Remove(remove []byte) *Set {

	// Remove from s.value
	for i, item := range s.value {
		if bytes.Equal(item, remove) {

			// https://github.com/golang/go/wiki/SliceTricks
			s.value[i] = s.value[len(s.value)-1]
			s.value = s.value[:len(s.value)-1]
		}
	}

	// Remove from s.adds
	for i, item := range s.adds {
		if bytes.Equal(item, remove) {

			// https://github.com/golang/go/wiki/SliceTricks
			s.adds[i] = s.adds[len(s.adds)-1]
			s.adds = s.adds[:len(s.adds)-1]
		}
	}

	// Add to s.removes
	s.removes = append(s.removes, remove)

	return s
}

func (s *Set) AddString(add string) *Set {
	return s.Add([]byte(add))
}

func (s *Set) RemoveString(remove string) *Set {
	return s.Remove([]byte(remove))
}

func (s *Set) Exec(client *Client) error {
	if s == nil {
		return errors.New("Nil Set")
	}

	if s.name == "" {
		return errors.New("Unknown path to Set. Retreive Set with GetMap before updating the Set")
	}

	op := &riak.MapOperation{}
	outerOp := op

	// Traverse c.path so that we increment the correct counter in nested maps
	for _, subMapName := range s.path {
		op = op.Map(subMapName)
	}

	// Perform Add actions
	for _, val := range s.adds {
		op.AddToSet(s.name, val)
	}

	// Perform Remove actions
	for _, val := range s.removes {
		op.RemoveFromSet(s.name, val)
	}

	cmd, err := riak.NewUpdateMapCommandBuilder().
		WithBucket(s.key.bucket).
		WithBucketType(s.key.bucketType).
		WithKey(s.key.key).
		WithMapOperation(outerOp).
		WithContext(s.context).
		Build()

	if err != nil {
		return err
	}

	err = client.riak.Execute(cmd)

	if err != nil {
		return err
	}

	res, ok := cmd.(*riak.UpdateMapCommand)

	if !ok {
		return errors.New("Could not convert")
	}

	if !res.Success() {
		return errors.New("Not successful")
	}

	return nil
}
<commit_msg>#12 - More comments<commit_after>package goriak

import (
	"bytes"
	"errors"

	riak "github.com/basho/riak-go-client"
)

// NewSet returnes a new and empty Set.
// Sets returned from NewSet() can not be used with Set.Exec()
func NewSet() *Set {
	return &Set{}
}

// Set is a special type to make it easier to work with Riak Sets in Go.
type Set struct {
	path    []string    // Path to the counter (can be a map in a map in a map, etc..)
	name    string      // Name of the counter
	key     requestData // bucket information
	context []byte      // riak context

	value   [][]byte // The full content
	adds    [][]byte // Not-yet performed Add actions (performed locally but not to Riak)
	removes [][]byte // Same as adds, but for removal of items
}

// Value returnes the raw values from the Set
func (s *Set) Value() [][]byte {
	r := make([][]byte, 0)

	// Remove empty items
	for _, v := range s.value {
		if len(v) != 0 {
			r = append(r, v)
		}
	}

	return r
}

// Strings returns the same data as Value(), but encoded as strings
func (s *Set) Strings() []string {
	all := s.Value()

	r := make([]string, len(all))

	for i, v := range all {
		r[i] = string(v)
	}

	return r
}

// Add adds an item to the direct value of the Set.
// Save the changes to Riak with Set.Exec() or SetMap().
func (s *Set) Add(add []byte) *Set {
	// Make sure that our set doesn't already contain this value
	for _, item := range s.value {
		if bytes.Equal(item, add) {
			return s
		}
	}

	// Add to s.value
	s.value = append(s.value, add)

	// Add to s.adds (Riak actions not yet saved)
	s.adds = append(s.adds, add)

	// Remove from s.removes
	for i, item := range s.removes {
		if bytes.Equal(item, add) {

			// https://github.com/golang/go/wiki/SliceTricks
			s.removes[i] = s.removes[len(s.removes)-1]
			s.removes = s.removes[:len(s.removes)-1]
		}
	}

	return s
}

// Remove deletes an item to the direct value of the Set.
// Save the changes to Riak with Set.Exec() or SetMap().
func (s *Set) Remove(remove []byte) *Set {

	// Remove from s.value
	for i, item := range s.value {
		if bytes.Equal(item, remove) {

			// https://github.com/golang/go/wiki/SliceTricks
			s.value[i] = s.value[len(s.value)-1]
			s.value = s.value[:len(s.value)-1]
		}
	}

	// Remove from s.adds
	for i, item := range s.adds {
		if bytes.Equal(item, remove) {

			// https://github.com/golang/go/wiki/SliceTricks
			s.adds[i] = s.adds[len(s.adds)-1]
			s.adds = s.adds[:len(s.adds)-1]
		}
	}

	// Add to s.removes
	s.removes = append(s.removes, remove)

	return s
}

// AddString is a shortcut to Add
func (s *Set) AddString(add string) *Set {
	return s.Add([]byte(add))
}

// RemoveString is a shortcut to Remove
func (s *Set) RemoveString(remove string) *Set {
	return s.Remove([]byte(remove))
}

// Exec executes the diff created by Add() and Remove(), and saves the data to Riak
func (s *Set) Exec(client *Client) error {
	if s == nil {
		return errors.New("Nil Set")
	}

	if s.name == "" {
		return errors.New("Unknown path to Set. Retreive Set with GetMap before updating the Set")
	}

	op := &riak.MapOperation{}
	outerOp := op

	// Traverse c.path so that we increment the correct counter in nested maps
	for _, subMapName := range s.path {
		op = op.Map(subMapName)
	}

	// Perform Add actions
	for _, val := range s.adds {
		op.AddToSet(s.name, val)
	}

	// Perform Remove actions
	for _, val := range s.removes {
		op.RemoveFromSet(s.name, val)
	}

	cmd, err := riak.NewUpdateMapCommandBuilder().
		WithBucket(s.key.bucket).
		WithBucketType(s.key.bucketType).
		WithKey(s.key.key).
		WithMapOperation(outerOp).
		WithContext(s.context).
		Build()

	if err != nil {
		return err
	}

	err = client.riak.Execute(cmd)

	if err != nil {
		return err
	}

	res, ok := cmd.(*riak.UpdateMapCommand)

	if !ok {
		return errors.New("Could not convert")
	}

	if !res.Success() {
		return errors.New("Not successful")
	}

	return nil
}
<|endoftext|>


<commit_before>package indexer

import (
	"fmt"
	"testing"

	"github.com/deepfabric/indexer/cql"
	"github.com/deepfabric/pilosa"
)

const (
	InitialNumDocs int = 100000 //insert some documents before benchmark
)

func newDocProt1() *cql.DocumentWithIdx {
	return &cql.DocumentWithIdx{
		Document: cql.Document{
			DocID: 0,
			UintProps: []cql.UintProp{
				cql.UintProp{
					Name:   "object",
					ValLen: 8,
					Val:    0,
				},
				cql.UintProp{
					Name:   "price",
					ValLen: 4,
					Val:    0,
				},
				cql.UintProp{
					Name:   "number",
					ValLen: 4,
					Val:    0,
				},
				cql.UintProp{
					Name:   "date",
					ValLen: 8,
					Val:    0,
				},
			},
			StrProps: []cql.StrProp{
				cql.StrProp{
					Name: "description",
					Val:  "",
				},
				cql.StrProp{
					Name: "note",
					Val:  "",
				},
			},
		},
		Index: "orders",
	}
}

func newDocProt2() *cql.DocumentWithIdx {
	return &cql.DocumentWithIdx{
		Document: cql.Document{
			DocID: 0,
			UintProps: []cql.UintProp{
				cql.UintProp{
					Name:   "ip",
					ValLen: 4,
					Val:    0,
				},
				cql.UintProp{
					Name:   "created",
					ValLen: 8,
					Val:    0,
				},
				cql.UintProp{
					Name:   "updated",
					ValLen: 8,
					Val:    0,
				},
			},
			StrProps: []cql.StrProp{
				cql.StrProp{
					Name: "description",
					Val:  "",
				},
				cql.StrProp{
					Name: "note",
					Val:  "",
				},
			},
		},
		Index: "addrs",
	}
}

//TESTCASE: normal operation sequence: new_empty, create, insert, close, new_existing, query, del, destroy
func TestIndexerNormal(t *testing.T) {
	var err error
	var docProt *cql.DocumentWithIdx
	var ir, ir2 *Indexer
	var found bool

	conf := Conf{
		T0mCap:   1000,
		LeafCap:  100,
		IntraCap: 4,
	}

	//create empty indexer
	if ir, err = NewIndexer("/tmp/indexer_test", &conf, true); err != nil {
		t.Fatalf("%+v", err)
	}

	//create index 1
	docProt = newDocProt1()
	if err = ir.CreateIndex(docProt); err != nil {
		return
	}

	//create index 2
	docProt = newDocProt2()
	if err = ir.CreateIndex(docProt); err != nil {
		return
	}

	//insert documents
	for i := 0; i < BkdCapTest; i++ {
		doc := newDocProt1()
		doc.DocID = uint64(i)
		for j := 0; j < len(doc.UintProps); j++ {
			doc.UintProps[j].Val = uint64(i * (j + 1))
		}
		if err = ir.Insert(doc); err != nil {
			t.Fatalf("%+v", err)
		}
	}

	//close indexer
	if err = ir.Close(); err != nil {
		t.Fatalf("%+v", err)
	}

	//create indexer with existing data
	if ir2, err = NewIndexer("/tmp/indexer_test", &conf, false); err != nil {
		t.Fatalf("%+v", err)
	}

	//query
	var rb *pilosa.Bitmap
	low := 30
	high := 600
	cs := &cql.CqlSelect{
		Index: "orders",
		UintPreds: map[string]cql.UintPred{
			"price": cql.UintPred{
				Name: "price",
				Low:  uint64(low),
				High: uint64(high),
			},
		},
	}
	if rb, err = ir2.Select(cs); err != nil {
		t.Fatalf("%+v", err)
	}
	fmt.Println(rb.Bits())

	//delete documents
	for i := 0; i < BkdCapTest; i++ {
		doc := newDocProt1()
		doc.DocID = uint64(i)
		for j := 0; j < len(doc.UintProps); j++ {
			doc.UintProps[j].Val = uint64(i * (j + 1))
		}
		if found, err = ir2.Del(doc); err != nil {
			t.Fatalf("%+v", err)
		} else if !found {
			t.Fatalf("document %v not found", doc)
		}
	}

	//destroy index 1
	if err = ir2.DestroyIndex("orders"); err != nil {
		t.Fatalf("%+v", err)
	}

	//destroy index 2
	if err = ir2.DestroyIndex("addrs"); err != nil {
		t.Fatalf("%+v", err)
	}
}

func TestIndexerOpenClose(t *testing.T) {
	var err error
	var ir *Indexer

	conf := Conf{
		T0mCap:   1000,
		LeafCap:  100,
		IntraCap: 4,
	}

	//create indexer
	if ir, err = NewIndexer("/tmp/indexer_test", &conf, true); err != nil {
		t.Fatalf("%+v", err)
	}

	//create index
	docProt := newDocProt1()
	if err = ir.CreateIndex(docProt); err != nil {
		return
	}

	//close indexer
	if err = ir.Close(); err != nil {
		t.Fatalf("%+v", err)
	}

	//open indexer
	if err = ir.Open(); err != nil {
		t.Fatalf("%+v", err)
	}

	//close indexer
	if err = ir.Close(); err != nil {
		t.Fatalf("%+v", err)
	}
}

func prepareIndexer(numDocs int, docProts []*cql.DocumentWithIdx) (ir *Indexer, err error) {
	conf := Conf{
		T0mCap:   1000,
		LeafCap:  100,
		IntraCap: 4,
	}

	//create indexer
	if ir, err = NewIndexer("/tmp/indexer_test", &conf, true); err != nil {
		return
	}

	//insert documents
	for _, docProt := range docProts {
		if err = ir.CreateIndex(docProt); err != nil {
			return
		}
		for i := 0; i < numDocs; i++ {
			docProt.DocID = uint64(i)
			for j := 0; j < len(docProt.UintProps); j++ {
				docProt.UintProps[j].Val = uint64(i * (j + 1))
			}
			for j := 0; j < len(docProt.StrProps); j++ {
				docProt.StrProps[j].Val = fmt.Sprintf("%d_%d ", i, j) + "Go's standard library does not have a function solely intended to check if a file exists or not (like Python's os.path.exists). What is the idiomatic way to do it?"
			}
			if err = ir.Insert(docProt); err != nil {
				return
			}
		}
	}
	return
}

func BenchmarkIndexer(b *testing.B) {
	var err error
	var ir *Indexer

	//https://golang.org/pkg/testing/#hdr-Subtests_and_Sub_benchmarks
	//"The Run methods of T and B allow defining subtests and sub-benchmarks, ...provides a way to share common setup and tear-down code"
	//"A subbenchmark is like any other benchmark. A benchmark that calls Run at least once will not be measured itself and will be called once with N=1."
	//prepareIndexer is expensive setup, so it's better to share among sub-benchmarks.
	ir, err = prepareIndexer(InitialNumDocs, []*cql.DocumentWithIdx{newDocProt1(), newDocProt2()})
	if err != nil {
		b.Fatalf("%+v", err)
	}

	b.Run("Insert", func(b *testing.B) {
		//insert documents
		doc := newDocProt1()
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			doc.DocID = uint64(InitialNumDocs + i)
			for j := 0; j < len(doc.UintProps); j++ {
				doc.UintProps[j].Val = doc.DocID * uint64(j+1)
			}
			for j := 0; j < len(doc.StrProps); j++ {
				doc.StrProps[j].Val = "Go's standard library does not have a function solely intended to check if a file exists or not (like Python's os.path.exists). What is the idiomatic way to do it?"
			}
			if err = ir.Insert(doc); err != nil {
				//document could already be there since the benchmark function is called "at least once".
				//b.Fatalf("%+v", err)
			}
		}
	})

	b.Run("Query", func(b *testing.B) {
		//query documents
		var rb *pilosa.Bitmap
		low := 30
		high := 600
		cs := &cql.CqlSelect{
			Index: "orders",
			UintPreds: map[string]cql.UintPred{
				"price": cql.UintPred{
					Name: "price",
					Low:  uint64(low),
					High: uint64(high),
				},
			},
			StrPreds: map[string]cql.StrPred{
				"note": cql.StrPred{
					Name:     "note",
					ContWord: "17_1",
				},
			},
		}
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			if rb, err = ir.Select(cs); err != nil {
				b.Fatalf("%+v", err)
			}
			if rb.Count() == 0 {
				b.Fatalf("rb.Count() is zero")
			}
		}
	})
	return
}
<commit_msg>added indexer parallel test<commit_after>package indexer

import (
	"fmt"
	"sync/atomic"
	"testing"
	"time"

	"github.com/deepfabric/indexer/cql"
	"github.com/deepfabric/pilosa"
)

const (
	InitialNumDocs int = 100000 //insert some documents before benchmark
)

func newDocProt1() *cql.DocumentWithIdx {
	return &cql.DocumentWithIdx{
		Document: cql.Document{
			DocID: 0,
			UintProps: []cql.UintProp{
				cql.UintProp{
					Name:   "object",
					ValLen: 8,
					Val:    0,
				},
				cql.UintProp{
					Name:   "price",
					ValLen: 4,
					Val:    0,
				},
				cql.UintProp{
					Name:   "number",
					ValLen: 4,
					Val:    0,
				},
				cql.UintProp{
					Name:   "date",
					ValLen: 8,
					Val:    0,
				},
			},
			StrProps: []cql.StrProp{
				cql.StrProp{
					Name: "description",
					Val:  "",
				},
				cql.StrProp{
					Name: "note",
					Val:  "",
				},
			},
		},
		Index: "orders",
	}
}

func newDocProt2() *cql.DocumentWithIdx {
	return &cql.DocumentWithIdx{
		Document: cql.Document{
			DocID: 0,
			UintProps: []cql.UintProp{
				cql.UintProp{
					Name:   "ip",
					ValLen: 4,
					Val:    0,
				},
				cql.UintProp{
					Name:   "created",
					ValLen: 8,
					Val:    0,
				},
				cql.UintProp{
					Name:   "updated",
					ValLen: 8,
					Val:    0,
				},
			},
			StrProps: []cql.StrProp{
				cql.StrProp{
					Name: "description",
					Val:  "",
				},
				cql.StrProp{
					Name: "note",
					Val:  "",
				},
			},
		},
		Index: "addrs",
	}
}

//TESTCASE: normal operation sequence: new_empty, create, insert, close, new_existing, query, del, destroy
func TestIndexerNormal(t *testing.T) {
	var err error
	var docProt *cql.DocumentWithIdx
	var ir, ir2 *Indexer
	var found bool

	conf := Conf{
		T0mCap:   1000,
		LeafCap:  100,
		IntraCap: 4,
	}

	//create empty indexer
	if ir, err = NewIndexer("/tmp/indexer_test", &conf, true); err != nil {
		t.Fatalf("%+v", err)
	}

	//create index 1
	docProt = newDocProt1()
	if err = ir.CreateIndex(docProt); err != nil {
		return
	}

	//create index 2
	docProt = newDocProt2()
	if err = ir.CreateIndex(docProt); err != nil {
		return
	}

	//insert documents
	for i := 0; i < BkdCapTest; i++ {
		doc := newDocProt1()
		doc.DocID = uint64(i)
		for j := 0; j < len(doc.UintProps); j++ {
			doc.UintProps[j].Val = uint64(i * (j + 1))
		}
		if err = ir.Insert(doc); err != nil {
			t.Fatalf("%+v", err)
		}
	}

	//close indexer
	if err = ir.Close(); err != nil {
		t.Fatalf("%+v", err)
	}

	//create indexer with existing data
	if ir2, err = NewIndexer("/tmp/indexer_test", &conf, false); err != nil {
		t.Fatalf("%+v", err)
	}

	//query
	var rb *pilosa.Bitmap
	low := 30
	high := 600
	cs := &cql.CqlSelect{
		Index: "orders",
		UintPreds: map[string]cql.UintPred{
			"price": cql.UintPred{
				Name: "price",
				Low:  uint64(low),
				High: uint64(high),
			},
		},
	}
	if rb, err = ir2.Select(cs); err != nil {
		t.Fatalf("%+v", err)
	}
	fmt.Println(rb.Bits())

	//delete documents
	for i := 0; i < BkdCapTest; i++ {
		doc := newDocProt1()
		doc.DocID = uint64(i)
		for j := 0; j < len(doc.UintProps); j++ {
			doc.UintProps[j].Val = uint64(i * (j + 1))
		}
		if found, err = ir2.Del(doc); err != nil {
			t.Fatalf("%+v", err)
		} else if !found {
			t.Fatalf("document %v not found", doc)
		}
	}

	//destroy index 1
	if err = ir2.DestroyIndex("orders"); err != nil {
		t.Fatalf("%+v", err)
	}

	//destroy index 2
	if err = ir2.DestroyIndex("addrs"); err != nil {
		t.Fatalf("%+v", err)
	}
}

func TestIndexerOpenClose(t *testing.T) {
	var err error
	var ir *Indexer

	conf := Conf{
		T0mCap:   1000,
		LeafCap:  100,
		IntraCap: 4,
	}

	//create indexer
	if ir, err = NewIndexer("/tmp/indexer_test", &conf, true); err != nil {
		t.Fatalf("%+v", err)
	}

	//create index
	docProt := newDocProt1()
	if err = ir.CreateIndex(docProt); err != nil {
		return
	}

	//close indexer
	if err = ir.Close(); err != nil {
		t.Fatalf("%+v", err)
	}

	//open indexer
	if err = ir.Open(); err != nil {
		t.Fatalf("%+v", err)
	}

	//close indexer
	if err = ir.Close(); err != nil {
		t.Fatalf("%+v", err)
	}
}

func prepareIndexer(numDocs int, docProts []*cql.DocumentWithIdx) (ir *Indexer, err error) {
	conf := Conf{
		T0mCap:   1000,
		LeafCap:  100,
		IntraCap: 4,
	}

	//create indexer
	if ir, err = NewIndexer("/tmp/indexer_test", &conf, true); err != nil {
		return
	}

	//insert documents
	for _, docProt := range docProts {
		if err = ir.CreateIndex(docProt); err != nil {
			return
		}
		for i := 0; i < numDocs; i++ {
			docProt.DocID = uint64(i)
			for j := 0; j < len(docProt.UintProps); j++ {
				docProt.UintProps[j].Val = uint64(i * (j + 1))
			}
			for j := 0; j < len(docProt.StrProps); j++ {
				docProt.StrProps[j].Val = fmt.Sprintf("%d_%d ", i, j) + "Go's standard library does not have a function solely intended to check if a file exists or not (like Python's os.path.exists). What is the idiomatic way to do it?"
			}
			if err = ir.Insert(docProt); err != nil {
				return
			}
		}
	}
	return
}

func TestIndexerParallel(t *testing.T) {
	var err error
	var ir *Indexer
	initialNumDocs := 10000
	parallelism := 5
	duration := 120 * time.Second
	durationIns := duration + 3*time.Second //insertion runs longger than deletion to ensure deletion succeed

	//https://golang.org/pkg/testing/#hdr-Subtests_and_Sub_benchmarks
	//"The Run methods of T and B allow defining subtests and sub-benchmarks, ...provides a way to share common setup and tear-down code"
	//"A subbenchmark is like any other benchmark. A benchmark that calls Run at least once will not be measured itself and will be called once with N=1."
	//prepareIndexer is expensive setup, so it's better to share among sub-benchmarks.
	ir, err = prepareIndexer(initialNumDocs, []*cql.DocumentWithIdx{newDocProt1(), newDocProt2()})
	if err != nil {
		t.Fatalf("%+v", err)
	}

	end := time.Now().Add(duration)
	endIns := end.Add(3 * time.Second)
	seqIns := []uint64{uint64(initialNumDocs - 1), uint64(initialNumDocs - 1)}
	seqDel := []uint64{0, 0}
	var errIns, errDel, errSel error
	//Concurrently insert to index 1 and 2
	for i := 0; i < parallelism; i++ {
		go func() {
			docs := []*cql.DocumentWithIdx{newDocProt1(), newDocProt2()}
			for {
				if now := time.Now(); now.After(endIns) {
					return
				}
				for d, doc := range docs {
					seq := atomic.AddUint64(&seqIns[d], 1)
					doc.DocID = seq
					for j := 0; j < len(doc.UintProps); j++ {
						doc.UintProps[j].Val = doc.DocID * uint64(j+1)
					}
					for j := 0; j < len(doc.StrProps); j++ {
						doc.StrProps[j].Val = "Go's standard library does not have a function solely intended to check if a file exists or not (like Python's os.path.exists). What is the idiomatic way to do it?"
					}
					if err = ir.Insert(doc); err != nil {
						errIns = err
						//t.Fatalf("%+v", err)
					}
				}
			}
		}()
	}
	//Concurrently delete to index 1 and 2
	for i := 0; i < parallelism; i++ {
		go func() {
			docs := []*cql.DocumentWithIdx{newDocProt1(), newDocProt2()}
			for {
				if now := time.Now(); now.After(end) {
					return
				}
				for d, doc := range docs {
					seq := atomic.AddUint64(&seqDel[d], 1)
					doc.DocID = seq
					for j := 0; j < len(doc.UintProps); j++ {
						doc.UintProps[j].Val = doc.DocID * uint64(j+1)
					}
					for j := 0; j < len(doc.StrProps); j++ {
						doc.StrProps[j].Val = "Go's standard library does not have a function solely intended to check if a file exists or not (like Python's os.path.exists). What is the idiomatic way to do it?"
					}
					if _, err = ir.Del(doc); err != nil {
						//deletion could be scheduled more often than insertion.
						errDel = err
						//t.Fatalf("%+v", err)
					}
				}
			}
		}()
	}
	//Concurrently query to index 1 and 2
	for i := 0; i < parallelism; i++ {
		go func() {
			low := 30
			high := 600
			queries := []*cql.CqlSelect{
				&cql.CqlSelect{
					Index: "orders",
					UintPreds: map[string]cql.UintPred{
						"price": cql.UintPred{
							Name: "price",
							Low:  uint64(low),
							High: uint64(high),
						},
					},
					StrPreds: map[string]cql.StrPred{
						"note": cql.StrPred{
							Name:     "note",
							ContWord: "17_1",
						},
					},
				},
				&cql.CqlSelect{
					Index: "addrs",
					StrPreds: map[string]cql.StrPred{
						"description": cql.StrPred{
							Name:     "description",
							ContWord: "17_0",
						},
					},
				},
			}
			for {
				if now := time.Now(); now.After(end) {
					return
				}
				for _, query := range queries {
					if _, err = ir.Select(query); err != nil {
						errSel = err
						//t.Fatalf("%+v", err)
					}
				}
			}
		}()
	}
	//wait goroutines to quit
	time.Sleep(durationIns)
	fmt.Printf("errIns %v, errDel %v, errSel %v\n", errIns, errDel, errSel)
	return
}

func BenchmarkIndexer(b *testing.B) {
	var err error
	var ir *Indexer

	//https://golang.org/pkg/testing/#hdr-Subtests_and_Sub_benchmarks
	//"The Run methods of T and B allow defining subtests and sub-benchmarks, ...provides a way to share common setup and tear-down code"
	//"A subbenchmark is like any other benchmark. A benchmark that calls Run at least once will not be measured itself and will be called once with N=1."
	//prepareIndexer is expensive setup, so it's better to share among sub-benchmarks.
	ir, err = prepareIndexer(InitialNumDocs, []*cql.DocumentWithIdx{newDocProt1(), newDocProt2()})
	if err != nil {
		b.Fatalf("%+v", err)
	}

	b.Run("Insert", func(b *testing.B) {
		//insert documents
		doc := newDocProt1()
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			doc.DocID = uint64(InitialNumDocs + i)
			for j := 0; j < len(doc.UintProps); j++ {
				doc.UintProps[j].Val = doc.DocID * uint64(j+1)
			}
			for j := 0; j < len(doc.StrProps); j++ {
				doc.StrProps[j].Val = "Go's standard library does not have a function solely intended to check if a file exists or not (like Python's os.path.exists). What is the idiomatic way to do it?"
			}
			if err = ir.Insert(doc); err != nil {
				//document could already be there since the benchmark function is called "at least once".
				//b.Fatalf("%+v", err)
			}
		}
	})

	b.Run("Query", func(b *testing.B) {
		//query documents
		var rb *pilosa.Bitmap
		low := 30
		high := 600
		cs := &cql.CqlSelect{
			Index: "orders",
			UintPreds: map[string]cql.UintPred{
				"price": cql.UintPred{
					Name: "price",
					Low:  uint64(low),
					High: uint64(high),
				},
			},
			StrPreds: map[string]cql.StrPred{
				"note": cql.StrPred{
					Name:     "note",
					ContWord: "17_1",
				},
			},
		}
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			if rb, err = ir.Select(cs); err != nil {
				b.Fatalf("%+v", err)
			}
			if rb.Count() == 0 {
				b.Fatalf("rb.Count() is zero")
			}
		}
	})
	return
}
<|endoftext|>


<commit_before>package clock

import (
	"reflect"
	"testing"
)

// Clock type API:
//
// Time(hour, minute int) Clock    // a "constructor"
// (Clock) String() string         // a "stringer"
// (Clock) Add(minutes int) Clock
//
// Add should also handle subtraction by accepting negative values.
// To satisfy the readme requirement about clocks being equal, values of
// your Clock type need to work with the == operator.
//
// It might help to study the time.Time type in the standard library
// (https://golang.org/pkg/time/#Time) as a model.  See how constructors there
// (Date and Now) return Time values rather than pointers.  Note also how
// most time.Time methods have value receivers rather that pointer recievers.
// For more background on this read
// https://github.com/golang/go/wiki/CodeReviewComments#receiver-type.

const testVersion = 2

// Retired testVersions
// (none) 79937f6d58e25ebafe12d1cb4a9f88f4de70cfd6
// 1      8d0cb8b617be2e36b2ca5ad2034e5f80f2372924

func TestCreateClock(t *testing.T) {
	if TestVersion != testVersion {
		t.Fatalf("Found TestVersion = %v, want %v", TestVersion, testVersion)
	}
	for _, n := range timeTests {
		if got := Time(n.h, n.m); got.String() != n.want {
			t.Fatalf("Time(%d, %d) = %q, want %q", n.h, n.m, got, n.want)
		}
	}
	t.Log(len(timeTests), "test cases")
}

func TestAddMinutes(t *testing.T) {
	for _, a := range addTests {
		if got := Time(a.h, a.m).Add(a.a); got.String() != a.want {
			t.Fatalf("Time(%d, %d).Add(%d) = %q, want %q",
				a.h, a.m, a.a, got, a.want)
		}
	}
	t.Log(len(addTests), "test cases")
}

func TestCompareClocks(t *testing.T) {
	for _, e := range eqTests {
		clock1 := Time(e.c1.h, e.c1.m)
		clock2 := Time(e.c2.h, e.c2.m)
		got := clock1 == clock2
		if got != e.want {
			t.Log("Clock1:", clock1)
			t.Log("Clock2:", clock2)
			t.Logf("Clock1 == Clock2 is %t, want %t", got, e.want)
			if reflect.DeepEqual(clock1, clock2) {
				t.Log("(Hint: see comments in clock_test.go.)")
			}
			t.FailNow()
		}
	}
	t.Log(len(eqTests), "test cases")
}
<commit_msg>Fix a few typos in the `clock_test` file<commit_after>package clock

import (
	"reflect"
	"testing"
)

// Clock type API:
//
// Time(hour, minute int) Clock    // a "constructor"
// (Clock) String() string         // a "stringer"
// (Clock) Add(minutes int) Clock
//
// The Add method should also handle subtraction by accepting negative values.
// To satisfy the readme requirement about clocks being equal, values of
// your Clock type need to work with the == operator.
//
// It might help to study the time.Time type in the standard library
// (https://golang.org/pkg/time/#Time) as a model.  See how constructors there
// (Date and Now) return Time values rather than pointers.  Note also how
// most time.Time methods have value receivers rather than pointer receivers.
// For more background on this read
// https://github.com/golang/go/wiki/CodeReviewComments#receiver-type.

const testVersion = 2

// Retired testVersions
// (none) 79937f6d58e25ebafe12d1cb4a9f88f4de70cfd6
// 1      8d0cb8b617be2e36b2ca5ad2034e5f80f2372924

func TestCreateClock(t *testing.T) {
	if TestVersion != testVersion {
		t.Fatalf("Found TestVersion = %v, want %v", TestVersion, testVersion)
	}
	for _, n := range timeTests {
		if got := Time(n.h, n.m); got.String() != n.want {
			t.Fatalf("Time(%d, %d) = %q, want %q", n.h, n.m, got, n.want)
		}
	}
	t.Log(len(timeTests), "test cases")
}

func TestAddMinutes(t *testing.T) {
	for _, a := range addTests {
		if got := Time(a.h, a.m).Add(a.a); got.String() != a.want {
			t.Fatalf("Time(%d, %d).Add(%d) = %q, want %q",
				a.h, a.m, a.a, got, a.want)
		}
	}
	t.Log(len(addTests), "test cases")
}

func TestCompareClocks(t *testing.T) {
	for _, e := range eqTests {
		clock1 := Time(e.c1.h, e.c1.m)
		clock2 := Time(e.c2.h, e.c2.m)
		got := clock1 == clock2
		if got != e.want {
			t.Log("Clock1:", clock1)
			t.Log("Clock2:", clock2)
			t.Logf("Clock1 == Clock2 is %t, want %t", got, e.want)
			if reflect.DeepEqual(clock1, clock2) {
				t.Log("(Hint: see comments in clock_test.go.)")
			}
			t.FailNow()
		}
	}
	t.Log(len(eqTests), "test cases")
}
<|endoftext|>


<commit_before>//go:generate go get github.com/blynn/nex
//go:generate ./bin/nex -o lexer.nn.go lexer.nn
//go:generate go fmt lexer.nn.go
//go:generate sed -i .tmp s:Lexer:lexer:g lexer.nn.go
//go:generate sed -i .tmp s:Newlexer:newLexer:g lexer.nn.go
//go:generate go tool yacc -o parser.y.go parser.y
//go:generate sed -i .tmp -f fixparser.sed parser.y.go

package edn

import (
	"errors"
	"fmt"
	"github.com/csm/go-edn/types"
	"io"
	"strings"
)

// ParseString is like ParseReader except it takes a string.
// See ParseReader for more details.
func ParseString(string string) (val types.Value, err error) {
	val, err = ParseReader(strings.NewReader(string))
	return
}

// ParseReader parses EDN from an io.Reader.
//
// Data is returned as a Value in the first return value.
// The second return value is nil on successful parses, and
// an error on unsuccessful parses (e.g. syntax error).
func ParseReader(reader io.Reader) (val types.Value, err error) {
	defer func() {
		// Nex's parser calls panic() on a lexing error
		if r := recover(); r != nil {
			if err == nil {
				err = errors.New(fmt.Sprintf("Error: %v", r))
			}
		}
	}()

	lexer := newLexer(reader)
	result := new(yySymType)
	if yyParse(lexer, result) == 0 {
		//fmt.Printf("result: v:%T:%+v\n", result.v, result.v)
		val = result.v
	} else {
		err = errors.New("Error: could not parse provided EDN")
	}

	return
}

// DumpString accepts any EDN value and will return the EDN string
// representation.
func DumpString(value types.Value) string {
	return value.String()
}
<commit_msg>Try install?<commit_after>//go:generate go get github.com/blynn/nex
//go:generate go install github.com/blynn/nex
//go:generate ./bin/nex -o lexer.nn.go lexer.nn
//go:generate go fmt lexer.nn.go
//go:generate sed -i .tmp s:Lexer:lexer:g lexer.nn.go
//go:generate sed -i .tmp s:Newlexer:newLexer:g lexer.nn.go
//go:generate go tool yacc -o parser.y.go parser.y
//go:generate sed -i .tmp -f fixparser.sed parser.y.go

package edn

import (
	"errors"
	"fmt"
	"github.com/csm/go-edn/types"
	"io"
	"strings"
)

// ParseString is like ParseReader except it takes a string.
// See ParseReader for more details.
func ParseString(string string) (val types.Value, err error) {
	val, err = ParseReader(strings.NewReader(string))
	return
}

// ParseReader parses EDN from an io.Reader.
//
// Data is returned as a Value in the first return value.
// The second return value is nil on successful parses, and
// an error on unsuccessful parses (e.g. syntax error).
func ParseReader(reader io.Reader) (val types.Value, err error) {
	defer func() {
		// Nex's parser calls panic() on a lexing error
		if r := recover(); r != nil {
			if err == nil {
				err = errors.New(fmt.Sprintf("Error: %v", r))
			}
		}
	}()

	lexer := newLexer(reader)
	result := new(yySymType)
	if yyParse(lexer, result) == 0 {
		//fmt.Printf("result: v:%T:%+v\n", result.v, result.v)
		val = result.v
	} else {
		err = errors.New("Error: could not parse provided EDN")
	}

	return
}

// DumpString accepts any EDN value and will return the EDN string
// representation.
func DumpString(value types.Value) string {
	return value.String()
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/exec"
	"os/signal"
	"syscall"
	"time"

	flag "github.com/heroku/busl/Godeps/_workspace/src/github.com/ogier/pflag"
)

const usage = "Usage: busltee <url> [-k|--insecure] [--connect-timeout N] -- <command>"

type config struct {
	insecure  bool
	timeout   float64
	retry     int
	logPrefix string
	logFile   string
}

func main() {
	conf := &config{}

	// Connection related flags
	flag.BoolVarP(&conf.insecure, "insecure", "k", false, "allows insecure SSL connections")
	flag.IntVar(&conf.retry, "retry", 5, "max retries for connect timeout errors")
	flag.Float64Var(&conf.timeout, "connect-timeout", 1, "max number of seconds to connect to busl URL")

	// Logging related flags
	flag.StringVar(&conf.logPrefix, "log-prefix", "", "log prefix")
	flag.StringVar(&conf.logFile, "log-file", "", "log file")

	if flag.Parse(); len(flag.Args()) < 2 {
		fmt.Println(usage)
		os.Exit(1)
	}

	// Setup logger
	out := getLogOutput(conf.logFile)
	log.SetPrefix(conf.logPrefix + " ")
	log.SetOutput(out)
	log.SetFlags(0)
	if f, ok := out.(io.Closer); ok {
		defer f.Close()
	}

	// Run command, piping output to stdout / stderr and to the URL.
	url := flag.Arg(0)
	args := flag.Args()[1:]

	if err := busltee(conf, url, args); err != nil {
		log.Printf("busltee.main.error count#busltee.main.error=1 error=%v", err.Error())
		os.Exit(exitStatus(err))
	}
}

func monitor(subject string, ts time.Time) {
	log.Printf("%s.time time=%f", subject, time.Now().Sub(ts).Seconds())
}

func busltee(conf *config, url string, args []string) error {
	defer monitor("busltee.busltee", time.Now())

	reader, writer := io.Pipe()
	uploaded := make(chan struct{})

	go func() {
		if err := stream(conf.retry, url, reader, conf.insecure, conf.timeout); err != nil {
			log.Printf("busltee.stream.error count#busltee.stream.error=1 error=%v", err.Error())
			// Prevent writes from blocking.
			io.Copy(ioutil.Discard, reader)
		} else {
			log.Printf("busltee.stream.success count#busltee.stream.success=1")
		}
		close(uploaded)
	}()

	err := run(args, writer, writer)
	<-uploaded

	return err
}

func stream(retry int, url string, stdin io.Reader, insecure bool, timeout float64) (err error) {
	for retries := retry; retries > 0; retries-- {
		err = streamNoRetry(url, stdin, insecure, timeout)

		if err == nil || !isTimeout(err) {
			return err
		}
		log.Printf("count#busltee.stream.retry")
	}
	return err
}

func streamNoRetry(url string, stdin io.Reader, insecure bool, timeout float64) error {
	defer monitor("busltee.stream", time.Now())

	if url == "" {
		log.Printf("count#busltee.stream.missingurl")
		return errors.New("Missing URL")
	}

	tr := &http.Transport{
		Dial: (&net.Dialer{
			Timeout:   time.Duration(timeout) * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
	}

	if insecure {
		tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
	}

	req, err := http.NewRequest("POST", url, ioutil.NopCloser(stdin))
	if err != nil {
		return err
	}

	res, err := tr.RoundTrip(req)
	if res != nil {
		defer res.Body.Close()
	}
	return err
}

func run(args []string, stdout, stderr io.WriteCloser) error {
	defer stdout.Close()
	defer stderr.Close()
	defer monitor("busltee.run", time.Now())

	cmd := exec.Command(args[0], args[1:]...)
	cmd.Stdout = io.MultiWriter(stdout, os.Stdout)
	cmd.Stderr = io.MultiWriter(stderr, os.Stderr)

	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc)
	go func() {
		s := <-sigc
		if cmd.Process == nil {
			log.Printf("count#busltee.run.error error=cmd.Process is nil")
		} else {
			cmd.Process.Signal(s)
		}
	}()

	return cmd.Run()
}

func isTimeout(err error) bool {
	e, ok := err.(net.Error)
	return ok && e.Timeout()
}

func exitStatus(err error) int {
	if exit, ok := err.(*exec.ExitError); ok {
		if status, ok := exit.Sys().(syscall.WaitStatus); ok {
			return status.ExitStatus()
		}
	}

	return 0
}

func getLogOutput(logFile string) io.Writer {
	if logFile == "" {
		return ioutil.Discard
	}
	if file, err := os.OpenFile(logFile, os.O_RDWR|os.O_APPEND, 0660); err != nil {
		return ioutil.Discard
	} else {
		return file
	}
}
<commit_msg>Simplify `stream` function error handling<commit_after>package main

import (
	"crypto/tls"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/exec"
	"os/signal"
	"syscall"
	"time"

	flag "github.com/heroku/busl/Godeps/_workspace/src/github.com/ogier/pflag"
)

const usage = "Usage: busltee <url> [-k|--insecure] [--connect-timeout N] -- <command>"

type config struct {
	insecure  bool
	timeout   float64
	retry     int
	logPrefix string
	logFile   string
}

func main() {
	conf := &config{}

	// Connection related flags
	flag.BoolVarP(&conf.insecure, "insecure", "k", false, "allows insecure SSL connections")
	flag.IntVar(&conf.retry, "retry", 5, "max retries for connect timeout errors")
	flag.Float64Var(&conf.timeout, "connect-timeout", 1, "max number of seconds to connect to busl URL")

	// Logging related flags
	flag.StringVar(&conf.logPrefix, "log-prefix", "", "log prefix")
	flag.StringVar(&conf.logFile, "log-file", "", "log file")

	if flag.Parse(); len(flag.Args()) < 2 {
		fmt.Println(usage)
		os.Exit(1)
	}

	// Setup logger
	out := getLogOutput(conf.logFile)
	log.SetPrefix(conf.logPrefix + " ")
	log.SetOutput(out)
	log.SetFlags(0)
	if f, ok := out.(io.Closer); ok {
		defer f.Close()
	}

	// Run command, piping output to stdout / stderr and to the URL.
	url := flag.Arg(0)
	args := flag.Args()[1:]

	if err := busltee(conf, url, args); err != nil {
		log.Printf("busltee.main.error count#busltee.main.error=1 error=%v", err.Error())
		os.Exit(exitStatus(err))
	}
}

func monitor(subject string, ts time.Time) {
	log.Printf("%s.time time=%f", subject, time.Now().Sub(ts).Seconds())
}

func busltee(conf *config, url string, args []string) error {
	defer monitor("busltee.busltee", time.Now())

	reader, writer := io.Pipe()
	uploaded := make(chan struct{})

	go func() {
		if err := stream(conf.retry, url, reader, conf.insecure, conf.timeout); err != nil {
			log.Printf("busltee.stream.error count#busltee.stream.error=1 error=%v", err.Error())
			// Prevent writes from blocking.
			io.Copy(ioutil.Discard, reader)
		} else {
			log.Printf("busltee.stream.success count#busltee.stream.success=1")
		}
		close(uploaded)
	}()

	err := run(args, writer, writer)
	<-uploaded

	return err
}

func stream(retry int, url string, stdin io.Reader, insecure bool, timeout float64) (err error) {
	for retries := retry; retries > 0; retries-- {
		if err = streamNoRetry(url, stdin, insecure, timeout); !isTimeout(err) {
			return err
		}
		log.Printf("count#busltee.stream.retry")
	}
	return err
}

func streamNoRetry(url string, stdin io.Reader, insecure bool, timeout float64) error {
	defer monitor("busltee.stream", time.Now())

	if url == "" {
		log.Printf("count#busltee.stream.missingurl")
		return errors.New("Missing URL")
	}

	tr := &http.Transport{
		Dial: (&net.Dialer{
			Timeout:   time.Duration(timeout) * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
	}

	if insecure {
		tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
	}

	req, err := http.NewRequest("POST", url, ioutil.NopCloser(stdin))
	if err != nil {
		return err
	}

	res, err := tr.RoundTrip(req)
	if res != nil {
		defer res.Body.Close()
	}
	return err
}

func run(args []string, stdout, stderr io.WriteCloser) error {
	defer stdout.Close()
	defer stderr.Close()
	defer monitor("busltee.run", time.Now())

	cmd := exec.Command(args[0], args[1:]...)
	cmd.Stdout = io.MultiWriter(stdout, os.Stdout)
	cmd.Stderr = io.MultiWriter(stderr, os.Stderr)

	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc)
	go func() {
		s := <-sigc
		if cmd.Process == nil {
			log.Printf("count#busltee.run.error error=cmd.Process is nil")
		} else {
			cmd.Process.Signal(s)
		}
	}()

	return cmd.Run()
}

func isTimeout(err error) bool {
	e, ok := err.(net.Error)
	return ok && e.Timeout()
}

func exitStatus(err error) int {
	if exit, ok := err.(*exec.ExitError); ok {
		if status, ok := exit.Sys().(syscall.WaitStatus); ok {
			return status.ExitStatus()
		}
	}

	return 0
}

func getLogOutput(logFile string) io.Writer {
	if logFile == "" {
		return ioutil.Discard
	}
	if file, err := os.OpenFile(logFile, os.O_RDWR|os.O_APPEND, 0660); err != nil {
		return ioutil.Discard
	} else {
		return file
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"compress/gzip"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/uluyol/fabbench/readers"
)

func main() {
	log.SetPrefix("fabcdfs: ")
	log.SetFlags(0)
	if len(os.Args) != 2 {
		fmt.Fprintln(os.Stderr, "usage: fabcdfs log.gz > cdfs.cs")
		fmt.Fprintln(os.Stderr, "\nOUTPUT FORMAT")
		fmt.Fprintln(os.Stderr, "\t#start StepNum=N NumSamples=K UnixTime=Sec,Nano")
		fmt.Fprintln(os.Stderr, "\tStepNum,Percenile,Micros")
		os.Exit(2)
	}

	f, err := os.Open(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	gr, err := gzip.NewReader(f)
	if err != nil {
		log.Fatal(err)
	}
	defer gr.Close()

	br := bufio.NewReader(gr)

	for i := 0; ; i++ {
		l, err := readers.ReadLatency(br)
		if err != nil {
			break
		}
		allVals := l.AllVals()
		fmt.Printf("#start StepNum=%d NumSamples=%d UnixTime=%d,%d\n", i, len(allVals), l.Start().Unix(), l.Start().Nanosecond())

		prev := readers.HistVal{Value: -1, Percentile: -1}

		for _, cur := range allVals {
			if cur.Value/time.Microsecond != prev.Value/time.Microsecond {
				if prev.Percentile >= 0 {
					fmt.Printf("%d,%f,%d\n", i, prev.Percentile, prev.Value/time.Microsecond)
				}
			}
			prev = cur
		}

		if prev.Percentile >= 0 {
			fmt.Printf("%d,%f,%d\n", i, prev.Percentile, prev.Value/time.Microsecond)
		}
	}
}
<commit_msg>cmd/fabcdfs: allow for filtering<commit_after>package main

import (
	"bufio"
	"compress/gzip"
	"flag"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/uluyol/fabbench/readers"
)

var (
	start = flag.Int64("start", 0, "filter from this start time (in unix seconds)")
	end   = flag.Int64("end", -1, "filter from this end time (in unix seconds)")
)

func usage() {
	fmt.Fprintln(os.Stderr, "usage: fabcdfs [flags] log.gz > cdfs.csv")
	fmt.Fprintln(os.Stderr, "\nOUTPUT FORMAT")
	fmt.Fprintln(os.Stderr, "\t#start StepNum=N NumSamples=K UnixTime=Sec,Nano")
	fmt.Fprintln(os.Stderr, "\tStepNum,Percenile,Micros")
	flag.PrintDefaults()
	os.Exit(2)
}

func main() {
	log.SetPrefix("fabcdfs: ")
	log.SetFlags(0)
	flag.Usage = usage
	flag.Parse()
	if flag.NArg() != 1 {
		usage()
	}

	f, err := os.Open(flag.Arg(0))
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	gr, err := gzip.NewReader(f)
	if err != nil {
		log.Fatal(err)
	}
	defer gr.Close()

	br := bufio.NewReader(gr)

	startTime := time.Unix(*start, 0)
	endTime := time.Unix(*end, 0)
	if *end == -1 {
		endTime = time.Date(2050, time.January, 1, 1, 1, 1, 0, time.UTC)
	}

	for i := 0; ; i++ {
		l, err := readers.ReadLatency(br)
		if err != nil {
			break
		}
		if l.Start().Before(startTime) || l.Start().After(endTime) {
			continue
		}
		allVals := l.AllVals()
		fmt.Printf("#start StepNum=%d NumSamples=%d UnixTime=%d,%d\n", i, len(allVals), l.Start().Unix(), l.Start().Nanosecond())

		prev := readers.HistVal{Value: -1, Percentile: -1}

		for _, cur := range allVals {
			if cur.Value/time.Microsecond != prev.Value/time.Microsecond {
				if prev.Percentile >= 0 {
					fmt.Printf("%d,%f,%d\n", i, prev.Percentile, prev.Value/time.Microsecond)
				}
			}
			prev = cur
		}

		if prev.Percentile >= 0 {
			fmt.Printf("%d,%f,%d\n", i, prev.Percentile, prev.Value/time.Microsecond)
		}
	}
}
<|endoftext|>