<commit_before>package github

import (
	"encoding/json"
	"github.com/google/go-github/github"
	"log"
	"time"
	"sort"
	"strings"
	"fmt"
)

//{"ref":"yaelna_120","ref_type":"branch","pusher_type":"user"}
type EventData struct {
	Ref        string `json:"ref"`
	RefType    string `json:"ref_type"`
	PusherType string `json:"pusher_type"`
}

type BranchOwnershipEvent struct {
	Owner, Repo string
	Login       string
	CreatedAt   time.Time
	Branch      string
	Type        string
	Id          string
}

type SortByDateBranchOwnershipEvents  []*BranchOwnershipEvent

func (s SortByDateBranchOwnershipEvents) Len() int {
	return len(s)
}
func (s SortByDateBranchOwnershipEvents) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}
func (s SortByDateBranchOwnershipEvents) Less(i, j int) bool {
	return s[i].CreatedAt.Before(s[j].CreatedAt)
}

func GetEvents(owner, repo string, lastEventId *string, client *github.Client) (firstEventId *string, eventsMap map[string]*BranchOwnershipEvent, deletedMap map[string]bool, err error) {
	var page int = 0
	eventsMap = make(map[string]*BranchOwnershipEvent)
	deletedMap = make(map[string]bool)
	ownershipEvents := make([]*BranchOwnershipEvent, 0)
	for {
		var events []*github.Event
		var resp *github.Response
		opt := &github.ListOptions{page, 100}
		events, resp, err = client.Activity.ListRepositoryEvents(owner, repo, opt)
		if err != nil {
			log.Printf("error geting repository events: %v\n", err)
			return
		}
		for _, event := range events {
			if (lastEventId != nil) && (*event.ID == *lastEventId) {
				page = resp.LastPage
				break
			}
			if firstEventId == nil {
				firstEventId = event.ID
			}
			if (*event.Type == "CreateEvent") || (*event.Type == "DeleteEvent") {
				var bytes []byte
				bytes, err = event.RawPayload.MarshalJSON()
				if err != nil {
					log.Printf("error geting parsing event data: %v\n", err)
					continue
				}
				data := EventData{}
				err = json.Unmarshal(bytes, &data)
				if err != nil {
					log.Printf("error geting parsing event data: %v\n", err)
					continue
				}
				if data.RefType == "branch" && data.PusherType == "user" {
					oe := &BranchOwnershipEvent{owner, repo, *event.Actor.Login, *event.CreatedAt, data.Ref, *event.Type, *event.ID}
					//log.Printf("[%v] Seeing event: %v\n", oe.CreatedAt, oe)
					ownershipEvents = append(ownershipEvents, oe)
				}
			}
		}
		if page < resp.LastPage {
			page = resp.NextPage
		} else {
			break;
		}
	}

	// sort events
	sort.Sort(sort.Reverse(SortByDateBranchOwnershipEvents(ownershipEvents)))
	seenMap := make(map[string]bool)

	// loop over events
	for _, event := range ownershipEvents {
		//log.Printf("[%v] Handling event: %v\n", event.CreatedAt, event)
		if seenMap[event.Branch] {
			continue
		}
		seenMap[event.Branch] = true
		if event.Type == "CreateEvent" {
			eventsMap[event.Branch] = event
		} else {
			deletedMap[event.Branch] = true
		}
	}
	return
}

func GetEventsForRepos(client *github.Client) (created chan map[string]map[string]*BranchOwnershipEvent) {

	type repoBranchRes struct {
		branches map[string]*BranchOwnershipEvent
		repo     string
	}
	perRepoBranches := make(chan repoBranchRes, len(ReposNames))
	created = make(chan map[string]map[string]*BranchOwnershipEvent)
	for _, repo := range ReposNames {
		sli := strings.Split(repo, "/")
		owner, repo := sli[0], sli[1]
		go func(owner, repo string) {
			_, eventsMap, _, _ := GetEvents(owner, repo, nil, client)
			perRepoBranches <- repoBranchRes{eventsMap, fmt.Sprintf("%s/%s", owner, repo)}
		}(owner, repo)
	}
	returnedMap := make(map[string]map[string]*BranchOwnershipEvent)
	go func() {
		for range ReposNames {
			repoBranchs := <-perRepoBranches
			returnedMap[repoBranchs.repo] = repoBranchs.branches

		}
		created <- returnedMap
	}()
	return created
}
<commit_msg>Fix worning.<commit_after>package github

import (
	"encoding/json"
	"github.com/google/go-github/github"
	"log"
	"time"
	"sort"
	"strings"
	"fmt"
)

//{"ref":"yaelna_120","ref_type":"branch","pusher_type":"user"}
type EventData struct {
	Ref        string `json:"ref"`
	RefType    string `json:"ref_type"`
	PusherType string `json:"pusher_type"`
}

type BranchOwnershipEvent struct {
	Owner, Repo string
	Login       string
	CreatedAt   time.Time
	Branch      string
	Type        string
	Id          string
}

type SortByDateBranchOwnershipEvents  []*BranchOwnershipEvent

func (s SortByDateBranchOwnershipEvents) Len() int {
	return len(s)
}
func (s SortByDateBranchOwnershipEvents) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}
func (s SortByDateBranchOwnershipEvents) Less(i, j int) bool {
	return s[i].CreatedAt.Before(s[j].CreatedAt)
}

func GetEvents(owner, repo string, lastEventId *string, client *github.Client) (firstEventId *string, eventsMap map[string]*BranchOwnershipEvent, deletedMap map[string]bool, err error) {
	var page int = 0
	eventsMap = make(map[string]*BranchOwnershipEvent)
	deletedMap = make(map[string]bool)
	ownershipEvents := make([]*BranchOwnershipEvent, 0)
	for {
		var events []*github.Event
		var resp *github.Response
		opt := &github.ListOptions{Page:page,  PerPage:100}
		events, resp, err = client.Activity.ListRepositoryEvents(owner, repo, opt)
		if err != nil {
			log.Printf("error geting repository events: %v\n", err)
			return
		}
		for _, event := range events {
			if (lastEventId != nil) && (*event.ID == *lastEventId) {
				page = resp.LastPage
				break
			}
			if firstEventId == nil {
				firstEventId = event.ID
			}
			if (*event.Type == "CreateEvent") || (*event.Type == "DeleteEvent") {
				var bytes []byte
				bytes, err = event.RawPayload.MarshalJSON()
				if err != nil {
					log.Printf("error geting parsing event data: %v\n", err)
					continue
				}
				data := EventData{}
				err = json.Unmarshal(bytes, &data)
				if err != nil {
					log.Printf("error geting parsing event data: %v\n", err)
					continue
				}
				if data.RefType == "branch" && data.PusherType == "user" {
					oe := &BranchOwnershipEvent{owner, repo, *event.Actor.Login, *event.CreatedAt, data.Ref, *event.Type, *event.ID}
					//log.Printf("[%v] Seeing event: %v\n", oe.CreatedAt, oe)
					ownershipEvents = append(ownershipEvents, oe)
				}
			}
		}
		if page < resp.LastPage {
			page = resp.NextPage
		} else {
			break;
		}
	}

	// sort events
	sort.Sort(sort.Reverse(SortByDateBranchOwnershipEvents(ownershipEvents)))
	seenMap := make(map[string]bool)

	// loop over events
	for _, event := range ownershipEvents {
		//log.Printf("[%v] Handling event: %v\n", event.CreatedAt, event)
		if seenMap[event.Branch] {
			continue
		}
		seenMap[event.Branch] = true
		if event.Type == "CreateEvent" {
			eventsMap[event.Branch] = event
		} else {
			deletedMap[event.Branch] = true
		}
	}
	return
}

func GetEventsForRepos(client *github.Client) (created chan map[string]map[string]*BranchOwnershipEvent) {

	type repoBranchRes struct {
		branches map[string]*BranchOwnershipEvent
		repo     string
	}
	perRepoBranches := make(chan repoBranchRes, len(ReposNames))
	created = make(chan map[string]map[string]*BranchOwnershipEvent)
	for _, repo := range ReposNames {
		sli := strings.Split(repo, "/")
		owner, repo := sli[0], sli[1]
		go func(owner, repo string) {
			_, eventsMap, _, _ := GetEvents(owner, repo, nil, client)
			perRepoBranches <- repoBranchRes{eventsMap, fmt.Sprintf("%s/%s", owner, repo)}
		}(owner, repo)
	}
	returnedMap := make(map[string]map[string]*BranchOwnershipEvent)
	go func() {
		for range ReposNames {
			repoBranchs := <-perRepoBranches
			returnedMap[repoBranchs.repo] = repoBranchs.branches

		}
		created <- returnedMap
	}()
	return created
}
<|endoftext|>


<commit_before>// Copyright 2013 The go-github AUTHORS. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package github

import (
	"context"
	"fmt"
	"strconv"

	qs "github.com/google/go-querystring/query"
)

// SearchService provides access to the search related functions
// in the GitHub API.
//
// Each method takes a query string defining the search keywords and any search qualifiers.
// For example, when searching issues, the query "gopher is:issue language:go" will search
// for issues containing the word "gopher" in Go repositories. The method call
//   opts :=  &github.SearchOptions{Sort: "created", Order: "asc"}
//   cl.Search.Issues(ctx, "gopher is:issue language:go", opts)
// will search for such issues, sorting by creation date in ascending order
// (i.e., oldest first).
//
// If query includes multiple conditions, it MUST NOT include "+" as the condition separator.
// You have to use " " as the separator instead.
// For example, querying with "language:c++" and "leveldb", then query should be
// "language:c++ leveldb" but not "language:c+++leveldb".
//
// GitHub API docs: https://docs.github.com/en/free-pro-team@latest/rest/reference/search/
type SearchService service

// SearchOptions specifies optional parameters to the SearchService methods.
type SearchOptions struct {
	// How to sort the search results. Possible values are:
	//   - for repositories: stars, fork, updated
	//   - for commits: author-date, committer-date
	//   - for code: indexed
	//   - for issues: comments, created, updated
	//   - for users: followers, repositories, joined
	//
	// Default is to sort by best match.
	Sort string `url:"sort,omitempty"`

	// Sort order if sort parameter is provided. Possible values are: asc,
	// desc. Default is desc.
	Order string `url:"order,omitempty"`

	// Whether to retrieve text match metadata with a query
	TextMatch bool `url:"-"`

	ListOptions
}

// Common search parameters.
type searchParameters struct {
	Query        string
	RepositoryID *int64 // Sent if non-nil.
}

// RepositoriesSearchResult represents the result of a repositories search.
type RepositoriesSearchResult struct {
	Total             *int          `json:"total_count,omitempty"`
	IncompleteResults *bool         `json:"incomplete_results,omitempty"`
	Repositories      []*Repository `json:"items,omitempty"`
}

// Repositories searches repositories via various criteria.
//
// GitHub API docs: https://docs.github.com/en/free-pro-team@latest/rest/reference/search/#search-repositories
func (s *SearchService) Repositories(ctx context.Context, query string, opts *SearchOptions) (*RepositoriesSearchResult, *Response, error) {
	result := new(RepositoriesSearchResult)
	resp, err := s.search(ctx, "repositories", &searchParameters{Query: query}, opts, result)
	return result, resp, err
}

// TopicsSearchResult represents the result of a topics search.
type TopicsSearchResult struct {
	Total             *int           `json:"total_count,omitempty"`
	IncompleteResults *bool          `json:"incomplete_results,omitempty"`
	Topics            []*TopicResult `json:"items,omitempty"`
}

type TopicResult struct {
	Name             *string    `json:"name,omitempty"`
	DisplayName      *string    `json:"display_name,omitempty"`
	ShortDescription *string    `json:"short_description,omitempty"`
	Description      *string    `json:"description,omitempty"`
	CreatedBy        *string    `json:"created_by,omitempty"`
	CreatedAt        *Timestamp `json:"created_at,omitempty"`
	UpdatedAt        *string    `json:"updated_at,omitempty"`
	Featured         *bool      `json:"featured,omitempty"`
	Curated          *bool      `json:"curated,omitempty"`
	Score            *float64   `json:"score,omitempty"`
}

// Topics finds topics via various criteria. Results are sorted by best match.
// Please see https://help.github.com/en/articles/searching-topics for more
// information about search qualifiers.
//
// GitHub API docs: https://docs.github.com/en/free-pro-team@latest/rest/reference/search/#search-topics
func (s *SearchService) Topics(ctx context.Context, query string, opts *SearchOptions) (*TopicsSearchResult, *Response, error) {
	result := new(TopicsSearchResult)
	resp, err := s.search(ctx, "topics", &searchParameters{Query: query}, opts, result)
	return result, resp, err
}

// CommitsSearchResult represents the result of a commits search.
type CommitsSearchResult struct {
	Total             *int            `json:"total_count,omitempty"`
	IncompleteResults *bool           `json:"incomplete_results,omitempty"`
	Commits           []*CommitResult `json:"items,omitempty"`
}

// CommitResult represents a commit object as returned in commit search endpoint response.
type CommitResult struct {
	SHA         *string   `json:"sha,omitempty"`
	Commit      *Commit   `json:"commit,omitempty"`
	Author      *User     `json:"author,omitempty"`
	Committer   *User     `json:"committer,omitempty"`
	Parents     []*Commit `json:"parents,omitempty"`
	HTMLURL     *string   `json:"html_url,omitempty"`
	URL         *string   `json:"url,omitempty"`
	CommentsURL *string   `json:"comments_url,omitempty"`

	Repository *Repository `json:"repository,omitempty"`
	Score      *float64    `json:"score,omitempty"`
}

// Commits searches commits via various criteria.
//
// GitHub API docs: https://docs.github.com/en/free-pro-team@latest/rest/reference/search/#search-commits
func (s *SearchService) Commits(ctx context.Context, query string, opts *SearchOptions) (*CommitsSearchResult, *Response, error) {
	result := new(CommitsSearchResult)
	resp, err := s.search(ctx, "commits", &searchParameters{Query: query}, opts, result)
	return result, resp, err
}

// IssuesSearchResult represents the result of an issues search.
type IssuesSearchResult struct {
	Total             *int     `json:"total_count,omitempty"`
	IncompleteResults *bool    `json:"incomplete_results,omitempty"`
	Issues            []*Issue `json:"items,omitempty"`
}

// Issues searches issues via various criteria.
//
// GitHub API docs: https://docs.github.com/en/free-pro-team@latest/rest/reference/search/#search-issues-and-pull-requests
func (s *SearchService) Issues(ctx context.Context, query string, opts *SearchOptions) (*IssuesSearchResult, *Response, error) {
	result := new(IssuesSearchResult)
	resp, err := s.search(ctx, "issues", &searchParameters{Query: query}, opts, result)
	return result, resp, err
}

// UsersSearchResult represents the result of a users search.
type UsersSearchResult struct {
	Total             *int    `json:"total_count,omitempty"`
	IncompleteResults *bool   `json:"incomplete_results,omitempty"`
	Users             []*User `json:"items,omitempty"`
}

// Users searches users via various criteria.
//
// GitHub API docs: https://docs.github.com/en/free-pro-team@latest/rest/reference/search/#search-users
func (s *SearchService) Users(ctx context.Context, query string, opts *SearchOptions) (*UsersSearchResult, *Response, error) {
	result := new(UsersSearchResult)
	resp, err := s.search(ctx, "users", &searchParameters{Query: query}, opts, result)
	return result, resp, err
}

// Match represents a single text match.
type Match struct {
	Text    *string `json:"text,omitempty"`
	Indices []int   `json:"indices,omitempty"`
}

// TextMatch represents a text match for a SearchResult
type TextMatch struct {
	ObjectURL  *string  `json:"object_url,omitempty"`
	ObjectType *string  `json:"object_type,omitempty"`
	Property   *string  `json:"property,omitempty"`
	Fragment   *string  `json:"fragment,omitempty"`
	Matches    []*Match `json:"matches,omitempty"`
}

func (tm TextMatch) String() string {
	return Stringify(tm)
}

// CodeSearchResult represents the result of a code search.
type CodeSearchResult struct {
	Total             *int          `json:"total_count,omitempty"`
	IncompleteResults *bool         `json:"incomplete_results,omitempty"`
	CodeResults       []*CodeResult `json:"items,omitempty"`
}

// CodeResult represents a single search result.
type CodeResult struct {
	Name        *string      `json:"name,omitempty"`
	Path        *string      `json:"path,omitempty"`
	SHA         *string      `json:"sha,omitempty"`
	HTMLURL     *string      `json:"html_url,omitempty"`
	Repository  *Repository  `json:"repository,omitempty"`
	TextMatches []*TextMatch `json:"text_matches,omitempty"`
}

func (c CodeResult) String() string {
	return Stringify(c)
}

// Code searches code via various criteria.
//
// GitHub API docs: https://docs.github.com/en/free-pro-team@latest/rest/reference/search/#search-code
func (s *SearchService) Code(ctx context.Context, query string, opts *SearchOptions) (*CodeSearchResult, *Response, error) {
	result := new(CodeSearchResult)
	resp, err := s.search(ctx, "code", &searchParameters{Query: query}, opts, result)
	return result, resp, err
}

// LabelsSearchResult represents the result of a code search.
type LabelsSearchResult struct {
	Total             *int           `json:"total_count,omitempty"`
	IncompleteResults *bool          `json:"incomplete_results,omitempty"`
	Labels            []*LabelResult `json:"items,omitempty"`
}

// LabelResult represents a single search result.
type LabelResult struct {
	ID          *int64   `json:"id,omitempty"`
	URL         *string  `json:"url,omitempty"`
	Name        *string  `json:"name,omitempty"`
	Color       *string  `json:"color,omitempty"`
	Default     *bool    `json:"default,omitempty"`
	Description *string  `json:"description,omitempty"`
	Score       *float64 `json:"score,omitempty"`
}

func (l LabelResult) String() string {
	return Stringify(l)
}

// Labels searches labels in the repository with ID repoID via various criteria.
//
// GitHub API docs: https://docs.github.com/en/free-pro-team@latest/rest/reference/search/#search-labels
func (s *SearchService) Labels(ctx context.Context, repoID int64, query string, opts *SearchOptions) (*LabelsSearchResult, *Response, error) {
	result := new(LabelsSearchResult)
	resp, err := s.search(ctx, "labels", &searchParameters{RepositoryID: &repoID, Query: query}, opts, result)
	return result, resp, err
}

// Helper function that executes search queries against different
// GitHub search types (repositories, commits, code, issues, users, labels)
//
// If searchParameters.Query includes multiple condition, it MUST NOT include "+" as condition separator.
// For example, querying with "language:c++" and "leveldb", then searchParameters.Query should be "language:c++ leveldb" but not "language:c+++leveldb".
func (s *SearchService) search(ctx context.Context, searchType string, parameters *searchParameters, opts *SearchOptions, result interface{}) (*Response, error) {
	params, err := qs.Values(opts)
	if err != nil {
		return nil, err
	}
	if parameters.RepositoryID != nil {
		params.Set("repository_id", strconv.FormatInt(*parameters.RepositoryID, 10))
	}
	params.Set("q", parameters.Query)
	u := fmt.Sprintf("search/%s?%s", searchType, params.Encode())

	req, err := s.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, err
	}

	switch {
	case searchType == "commits":
		// Accept header for search commits preview endpoint
		// TODO: remove custom Accept header when this API fully launches.
		req.Header.Set("Accept", mediaTypeCommitSearchPreview)
	case searchType == "topics":
		// Accept header for search repositories based on topics preview endpoint
		// TODO: remove custom Accept header when this API fully launches.
		req.Header.Set("Accept", mediaTypeTopicsPreview)
	case searchType == "repositories":
		// Accept header for search repositories based on topics preview endpoint
		// TODO: remove custom Accept header when this API fully launches.
		req.Header.Set("Accept", mediaTypeTopicsPreview)
	case opts != nil && opts.TextMatch:
		// Accept header defaults to "application/vnd.github.v3+json"
		// We change it here to fetch back text-match metadata
		req.Header.Set("Accept", "application/vnd.github.v3.text-match+json")
	}

	return s.client.Do(ctx, req, result)
}
<commit_msg>Add reactions preview to search (#1818)<commit_after>// Copyright 2013 The go-github AUTHORS. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package github

import (
	"context"
	"fmt"
	"strconv"

	qs "github.com/google/go-querystring/query"
)

// SearchService provides access to the search related functions
// in the GitHub API.
//
// Each method takes a query string defining the search keywords and any search qualifiers.
// For example, when searching issues, the query "gopher is:issue language:go" will search
// for issues containing the word "gopher" in Go repositories. The method call
//   opts :=  &github.SearchOptions{Sort: "created", Order: "asc"}
//   cl.Search.Issues(ctx, "gopher is:issue language:go", opts)
// will search for such issues, sorting by creation date in ascending order
// (i.e., oldest first).
//
// If query includes multiple conditions, it MUST NOT include "+" as the condition separator.
// You have to use " " as the separator instead.
// For example, querying with "language:c++" and "leveldb", then query should be
// "language:c++ leveldb" but not "language:c+++leveldb".
//
// GitHub API docs: https://docs.github.com/en/free-pro-team@latest/rest/reference/search/
type SearchService service

// SearchOptions specifies optional parameters to the SearchService methods.
type SearchOptions struct {
	// How to sort the search results. Possible values are:
	//   - for repositories: stars, fork, updated
	//   - for commits: author-date, committer-date
	//   - for code: indexed
	//   - for issues: comments, created, updated
	//   - for users: followers, repositories, joined
	//
	// Default is to sort by best match.
	Sort string `url:"sort,omitempty"`

	// Sort order if sort parameter is provided. Possible values are: asc,
	// desc. Default is desc.
	Order string `url:"order,omitempty"`

	// Whether to retrieve text match metadata with a query
	TextMatch bool `url:"-"`

	ListOptions
}

// Common search parameters.
type searchParameters struct {
	Query        string
	RepositoryID *int64 // Sent if non-nil.
}

// RepositoriesSearchResult represents the result of a repositories search.
type RepositoriesSearchResult struct {
	Total             *int          `json:"total_count,omitempty"`
	IncompleteResults *bool         `json:"incomplete_results,omitempty"`
	Repositories      []*Repository `json:"items,omitempty"`
}

// Repositories searches repositories via various criteria.
//
// GitHub API docs: https://docs.github.com/en/free-pro-team@latest/rest/reference/search/#search-repositories
func (s *SearchService) Repositories(ctx context.Context, query string, opts *SearchOptions) (*RepositoriesSearchResult, *Response, error) {
	result := new(RepositoriesSearchResult)
	resp, err := s.search(ctx, "repositories", &searchParameters{Query: query}, opts, result)
	return result, resp, err
}

// TopicsSearchResult represents the result of a topics search.
type TopicsSearchResult struct {
	Total             *int           `json:"total_count,omitempty"`
	IncompleteResults *bool          `json:"incomplete_results,omitempty"`
	Topics            []*TopicResult `json:"items,omitempty"`
}

type TopicResult struct {
	Name             *string    `json:"name,omitempty"`
	DisplayName      *string    `json:"display_name,omitempty"`
	ShortDescription *string    `json:"short_description,omitempty"`
	Description      *string    `json:"description,omitempty"`
	CreatedBy        *string    `json:"created_by,omitempty"`
	CreatedAt        *Timestamp `json:"created_at,omitempty"`
	UpdatedAt        *string    `json:"updated_at,omitempty"`
	Featured         *bool      `json:"featured,omitempty"`
	Curated          *bool      `json:"curated,omitempty"`
	Score            *float64   `json:"score,omitempty"`
}

// Topics finds topics via various criteria. Results are sorted by best match.
// Please see https://help.github.com/en/articles/searching-topics for more
// information about search qualifiers.
//
// GitHub API docs: https://docs.github.com/en/free-pro-team@latest/rest/reference/search/#search-topics
func (s *SearchService) Topics(ctx context.Context, query string, opts *SearchOptions) (*TopicsSearchResult, *Response, error) {
	result := new(TopicsSearchResult)
	resp, err := s.search(ctx, "topics", &searchParameters{Query: query}, opts, result)
	return result, resp, err
}

// CommitsSearchResult represents the result of a commits search.
type CommitsSearchResult struct {
	Total             *int            `json:"total_count,omitempty"`
	IncompleteResults *bool           `json:"incomplete_results,omitempty"`
	Commits           []*CommitResult `json:"items,omitempty"`
}

// CommitResult represents a commit object as returned in commit search endpoint response.
type CommitResult struct {
	SHA         *string   `json:"sha,omitempty"`
	Commit      *Commit   `json:"commit,omitempty"`
	Author      *User     `json:"author,omitempty"`
	Committer   *User     `json:"committer,omitempty"`
	Parents     []*Commit `json:"parents,omitempty"`
	HTMLURL     *string   `json:"html_url,omitempty"`
	URL         *string   `json:"url,omitempty"`
	CommentsURL *string   `json:"comments_url,omitempty"`

	Repository *Repository `json:"repository,omitempty"`
	Score      *float64    `json:"score,omitempty"`
}

// Commits searches commits via various criteria.
//
// GitHub API docs: https://docs.github.com/en/free-pro-team@latest/rest/reference/search/#search-commits
func (s *SearchService) Commits(ctx context.Context, query string, opts *SearchOptions) (*CommitsSearchResult, *Response, error) {
	result := new(CommitsSearchResult)
	resp, err := s.search(ctx, "commits", &searchParameters{Query: query}, opts, result)
	return result, resp, err
}

// IssuesSearchResult represents the result of an issues search.
type IssuesSearchResult struct {
	Total             *int     `json:"total_count,omitempty"`
	IncompleteResults *bool    `json:"incomplete_results,omitempty"`
	Issues            []*Issue `json:"items,omitempty"`
}

// Issues searches issues via various criteria.
//
// GitHub API docs: https://docs.github.com/en/free-pro-team@latest/rest/reference/search/#search-issues-and-pull-requests
func (s *SearchService) Issues(ctx context.Context, query string, opts *SearchOptions) (*IssuesSearchResult, *Response, error) {
	result := new(IssuesSearchResult)
	resp, err := s.search(ctx, "issues", &searchParameters{Query: query}, opts, result)
	return result, resp, err
}

// UsersSearchResult represents the result of a users search.
type UsersSearchResult struct {
	Total             *int    `json:"total_count,omitempty"`
	IncompleteResults *bool   `json:"incomplete_results,omitempty"`
	Users             []*User `json:"items,omitempty"`
}

// Users searches users via various criteria.
//
// GitHub API docs: https://docs.github.com/en/free-pro-team@latest/rest/reference/search/#search-users
func (s *SearchService) Users(ctx context.Context, query string, opts *SearchOptions) (*UsersSearchResult, *Response, error) {
	result := new(UsersSearchResult)
	resp, err := s.search(ctx, "users", &searchParameters{Query: query}, opts, result)
	return result, resp, err
}

// Match represents a single text match.
type Match struct {
	Text    *string `json:"text,omitempty"`
	Indices []int   `json:"indices,omitempty"`
}

// TextMatch represents a text match for a SearchResult
type TextMatch struct {
	ObjectURL  *string  `json:"object_url,omitempty"`
	ObjectType *string  `json:"object_type,omitempty"`
	Property   *string  `json:"property,omitempty"`
	Fragment   *string  `json:"fragment,omitempty"`
	Matches    []*Match `json:"matches,omitempty"`
}

func (tm TextMatch) String() string {
	return Stringify(tm)
}

// CodeSearchResult represents the result of a code search.
type CodeSearchResult struct {
	Total             *int          `json:"total_count,omitempty"`
	IncompleteResults *bool         `json:"incomplete_results,omitempty"`
	CodeResults       []*CodeResult `json:"items,omitempty"`
}

// CodeResult represents a single search result.
type CodeResult struct {
	Name        *string      `json:"name,omitempty"`
	Path        *string      `json:"path,omitempty"`
	SHA         *string      `json:"sha,omitempty"`
	HTMLURL     *string      `json:"html_url,omitempty"`
	Repository  *Repository  `json:"repository,omitempty"`
	TextMatches []*TextMatch `json:"text_matches,omitempty"`
}

func (c CodeResult) String() string {
	return Stringify(c)
}

// Code searches code via various criteria.
//
// GitHub API docs: https://docs.github.com/en/free-pro-team@latest/rest/reference/search/#search-code
func (s *SearchService) Code(ctx context.Context, query string, opts *SearchOptions) (*CodeSearchResult, *Response, error) {
	result := new(CodeSearchResult)
	resp, err := s.search(ctx, "code", &searchParameters{Query: query}, opts, result)
	return result, resp, err
}

// LabelsSearchResult represents the result of a code search.
type LabelsSearchResult struct {
	Total             *int           `json:"total_count,omitempty"`
	IncompleteResults *bool          `json:"incomplete_results,omitempty"`
	Labels            []*LabelResult `json:"items,omitempty"`
}

// LabelResult represents a single search result.
type LabelResult struct {
	ID          *int64   `json:"id,omitempty"`
	URL         *string  `json:"url,omitempty"`
	Name        *string  `json:"name,omitempty"`
	Color       *string  `json:"color,omitempty"`
	Default     *bool    `json:"default,omitempty"`
	Description *string  `json:"description,omitempty"`
	Score       *float64 `json:"score,omitempty"`
}

func (l LabelResult) String() string {
	return Stringify(l)
}

// Labels searches labels in the repository with ID repoID via various criteria.
//
// GitHub API docs: https://docs.github.com/en/free-pro-team@latest/rest/reference/search/#search-labels
func (s *SearchService) Labels(ctx context.Context, repoID int64, query string, opts *SearchOptions) (*LabelsSearchResult, *Response, error) {
	result := new(LabelsSearchResult)
	resp, err := s.search(ctx, "labels", &searchParameters{RepositoryID: &repoID, Query: query}, opts, result)
	return result, resp, err
}

// Helper function that executes search queries against different
// GitHub search types (repositories, commits, code, issues, users, labels)
//
// If searchParameters.Query includes multiple condition, it MUST NOT include "+" as condition separator.
// For example, querying with "language:c++" and "leveldb", then searchParameters.Query should be "language:c++ leveldb" but not "language:c+++leveldb".
func (s *SearchService) search(ctx context.Context, searchType string, parameters *searchParameters, opts *SearchOptions, result interface{}) (*Response, error) {
	params, err := qs.Values(opts)
	if err != nil {
		return nil, err
	}
	if parameters.RepositoryID != nil {
		params.Set("repository_id", strconv.FormatInt(*parameters.RepositoryID, 10))
	}
	params.Set("q", parameters.Query)
	u := fmt.Sprintf("search/%s?%s", searchType, params.Encode())

	req, err := s.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, err
	}

	switch {
	case searchType == "commits":
		// Accept header for search commits preview endpoint
		// TODO: remove custom Accept header when this API fully launches.
		req.Header.Set("Accept", mediaTypeCommitSearchPreview)
	case searchType == "topics":
		// Accept header for search repositories based on topics preview endpoint
		// TODO: remove custom Accept header when this API fully launches.
		req.Header.Set("Accept", mediaTypeTopicsPreview)
	case searchType == "repositories":
		// Accept header for search repositories based on topics preview endpoint
		// TODO: remove custom Accept header when this API fully launches.
		req.Header.Set("Accept", mediaTypeTopicsPreview)
	case searchType == "issues":
		// Accept header for search issues based on reactions preview endpoint
		// TODO: remove custom Accept header when this API fully launches.
		req.Header.Set("Accept", mediaTypeReactionsPreview)
	case opts != nil && opts.TextMatch:
		// Accept header defaults to "application/vnd.github.v3+json"
		// We change it here to fetch back text-match metadata
		req.Header.Set("Accept", "application/vnd.github.v3.text-match+json")
	}

	return s.client.Do(ctx, req, result)
}
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2015 Red Hat, Inc.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package http

import (
	"net/http"
	"strconv"
	"sync"
	"sync/atomic"
	"time"

	"github.com/gorilla/websocket"

	"github.com/skydive-project/skydive/common"
	"github.com/skydive-project/skydive/config"
	"github.com/skydive-project/skydive/logging"
)

const (
	maxMessages    = 1024
	maxMessageSize = 0
	writeWait      = 10 * time.Second
)

// Interface of a message to send over the wire
type Message interface {
	Bytes() []byte
}

// WSRawMessage represents a raw message (array of bytes)
type WSRawMessage []byte

// Return the string representation of the raw message
func (m WSRawMessage) Bytes() []byte {
	return m
}

// WebSocket client interface
type WSClient interface {
	GetHost() string
	GetAddrPort() (string, int)
	GetClientType() common.ServiceType
	IsConnected() bool
	Send(m Message)
	Connect()
	Disconnect()
	AddEventHandler(WSClientEventHandler)
}

// WSAsyncClient describes a WebSocket connection. WSAsyncClient is used
// by the client when connecting to a server, and also by the server when
// a client connects
type WSAsyncClient struct {
	sync.RWMutex
	Host              string
	ClientType        common.ServiceType
	Addr              string
	Port              int
	Path              string
	AuthClient        *AuthenticationClient
	headers           http.Header
	send              chan []byte
	read              chan []byte
	quit              chan bool
	wg                sync.WaitGroup
	wsConn            *websocket.Conn
	eventHandlers     []WSClientEventHandler
	eventHandlersLock sync.RWMutex
	connected         atomic.Value
	running           atomic.Value
	pongWait          time.Duration
	pingPeriod        time.Duration
	ticker            *time.Ticker
}

// Interface to be implement by the client events listeners
type WSClientEventHandler interface {
	OnMessage(c WSClient, m Message)
	OnConnected(c WSClient)
	OnDisconnected(c WSClient)
}

// DefaultWSClientEventHandler implements stubs for the WSClientEventHandler interface
type DefaultWSClientEventHandler struct {
}

// OnMessage is called when a message is received
func (d *DefaultWSClientEventHandler) OnMessage(c WSClient, m Message) {
}

// OnConnected is called when the connection is established
func (d *DefaultWSClientEventHandler) OnConnected(c WSClient) {
}

// OnDisconnected is called when the connection is closed or lost
func (d *DefaultWSClientEventHandler) OnDisconnected(c WSClient) {
}

// Return the hostname of the connection
func (c *WSAsyncClient) GetHost() string {
	return c.Host
}

// Return the address and the port of the remote end
func (c *WSAsyncClient) GetAddrPort() (string, int) {
	return c.Addr, c.Port
}

// Return the connection status
func (c *WSAsyncClient) IsConnected() bool {
	return c.connected.Load() == true
}

// Add a message to the send queue
func (c *WSAsyncClient) Send(m Message) {
	if c.running.Load() == false {
		return
	}

	c.send <- m.Bytes()
}

// Return the client type
func (c *WSAsyncClient) GetClientType() common.ServiceType {
	return c.ClientType
}

// Send a message directly over the wire
func (c *WSAsyncClient) SendMessage(msg []byte) error {
	c.wsConn.SetWriteDeadline(time.Now().Add(writeWait))

	w, err := c.wsConn.NextWriter(websocket.TextMessage)
	if err != nil {
		return err
	}

	if _, err = w.Write(msg); err != nil {
		return err
	}

	return w.Close()
}

// Return the URL scheme
func (c *WSAsyncClient) scheme() string {
	if config.IsTLSenabled() == true {
		return "wss://"
	}
	return "ws://"
}

// Connect to the server
func (c *WSAsyncClient) connect() {
	var err error
	host := c.Addr + ":" + strconv.FormatInt(int64(c.Port), 10)
	endpoint := c.scheme() + host + c.Path
	headers := http.Header{
		"X-Host-ID":             {c.Host},
		"Origin":                {endpoint},
		"X-Client-Type":         {c.ClientType.String()},
		"X-Websocket-Namespace": {WilcardNamespace},
	}

	if c.AuthClient != nil {
		if err = c.AuthClient.Authenticate(); err != nil {
			logging.GetLogger().Errorf("Unable to create a WebSocket connection %s : %s", endpoint, err.Error())
			return
		}
		c.AuthClient.SetHeaders(headers)
	}

	d := websocket.Dialer{
		Proxy:           http.ProxyFromEnvironment,
		ReadBufferSize:  1024,
		WriteBufferSize: 1024,
	}
	certPEM := config.GetConfig().GetString("agent.X509_cert")
	keyPEM := config.GetConfig().GetString("agent.X509_key")
	if certPEM != "" && keyPEM != "" {
		d.TLSClientConfig = common.SetupTLSClientConfig(certPEM, keyPEM)
		checkTLSConfig(d.TLSClientConfig)
	}
	c.wsConn, _, err = d.Dial(endpoint, headers)

	if err != nil {
		logging.GetLogger().Errorf("Unable to create a WebSocket connection %s : %s", endpoint, err.Error())
		return
	}
	defer c.wsConn.Close()
	c.wsConn.SetPingHandler(nil)

	c.connected.Store(true)
	defer c.connected.Store(false)

	logging.GetLogger().Infof("Connected to %s", endpoint)

	// notify connected
	c.RLock()
	for _, l := range c.eventHandlers {
		l.OnConnected(c)
	}
	c.RUnlock()

	c.wg.Add(1)
	c.run()
}

func (c *WSAsyncClient) start() {
	c.wg.Add(1)
	go c.run()
}

// Client main loop to read and send messages
func (c *WSAsyncClient) run() {
	defer c.wg.Done()

	go func() {
		for c.running.Load() == true {
			_, m, err := c.wsConn.ReadMessage()
			if err != nil {
				if c.running.Load() != false {
					c.quit <- true
				}
				break
			}

			c.read <- m
		}
	}()

	defer func() {
		c.connected.Store(false)
		c.RLock()
		for _, l := range c.eventHandlers {
			l.OnDisconnected(c)
		}
		c.RUnlock()
	}()

	defer c.ticker.Stop()

	for {
		select {
		case <-c.quit:
			return
		case m := <-c.send:
			err := c.SendMessage(m)
			if err != nil {
				logging.GetLogger().Errorf("Error while writing to the WebSocket: %s", err.Error())
			}
		case m := <-c.read:
			c.RLock()
			for _, l := range c.eventHandlers {
				l.OnMessage(c, WSRawMessage(m))
			}
			c.RUnlock()
		case <-c.ticker.C:
			c.SendPing()
		}
	}
}

func (c *WSAsyncClient) SendPing() {
	c.wsConn.SetWriteDeadline(time.Now().Add(writeWait))
	if err := c.wsConn.WriteMessage(websocket.PingMessage, []byte{}); err != nil {
		logging.GetLogger().Warningf("Error while sending ping to the websocket: %s", err.Error())
	}
}

// Connect to the server - and reconnect if necessary
func (c *WSAsyncClient) Connect() {
	go func() {
		for c.running.Load() == true {
			c.connect()
			time.Sleep(1 * time.Second)
		}
	}()
}

// Register a new event handler
func (c *WSAsyncClient) AddEventHandler(h WSClientEventHandler) {
	c.Lock()
	c.eventHandlers = append(c.eventHandlers, h)
	c.Unlock()
}

// Disconnect the client without waiting for termination
func (c *WSAsyncClient) Disconnect() {
	c.running.Store(false)
	if c.connected.Load() == true {
		c.quit <- true
		c.wsConn.Close()
		close(c.send)
		close(c.read)
	}
}

// NewWSAsyncClient returns a client with a new connection
func NewWSAsyncClient(host string, clientType common.ServiceType, addr string, port int, path string, authClient *AuthenticationClient) *WSAsyncClient {
	pongTimeout := time.Duration(config.GetConfig().GetInt("ws_pong_timeout")) * time.Second
	c := &WSAsyncClient{
		Host:       host,
		ClientType: clientType,
		Addr:       addr,
		Port:       port,
		Path:       path,
		AuthClient: authClient,
		send:       make(chan []byte, maxMessages),
		read:       make(chan []byte, maxMessages),
		quit:       make(chan bool, 2),
		pongWait:   pongTimeout,
		pingPeriod: pongTimeout * 8 / 10,
		ticker:     &time.Ticker{},
	}
	c.connected.Store(false)
	c.running.Store(true)
	return c
}

// NewWSAsyncClientFromConnection creates a client from an existing connection
func NewWSAsyncClientFromConnection(host string, clientType common.ServiceType, conn *websocket.Conn) *WSAsyncClient {
	svc, _ := common.ServiceAddressFromString(conn.RemoteAddr().String())
	c := NewWSAsyncClient(host, clientType, svc.Addr, svc.Port, "", nil)

	conn.SetReadLimit(maxMessageSize)
	conn.SetReadDeadline(time.Now().Add(100 * time.Second))
	conn.SetPongHandler(func(string) error {
		conn.SetReadDeadline(time.Now().Add(c.pongWait))
		return nil
	})
	c.wsConn = conn
	c.ticker = time.NewTicker(c.pingPeriod)

	// send a first ping to help firefox and some other client which wait for a
	// first ping before doing something
	c.SendPing()

	c.connected.Store(true)
	c.running.Store(true)
	return c
}

// NewWSAsyncClientFromConnection creates a client based on the configuration
func NewWSAsyncClientFromConfig(clientType common.ServiceType, addr string, port int, path string, authClient *AuthenticationClient) *WSAsyncClient {
	host := config.GetConfig().GetString("host_id")
	return NewWSAsyncClient(host, clientType, addr, port, path, authClient)
}
<commit_msg>websocket: ensure connected before sending msg<commit_after>/*
 * Copyright (C) 2015 Red Hat, Inc.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package http

import (
	"errors"
	"net/http"
	"strconv"
	"sync"
	"sync/atomic"
	"time"

	"github.com/gorilla/websocket"

	"github.com/skydive-project/skydive/common"
	"github.com/skydive-project/skydive/config"
	"github.com/skydive-project/skydive/logging"
)

const (
	maxMessages    = 1024
	maxMessageSize = 0
	writeWait      = 10 * time.Second
)

// Interface of a message to send over the wire
type Message interface {
	Bytes() []byte
}

// WSRawMessage represents a raw message (array of bytes)
type WSRawMessage []byte

// Return the string representation of the raw message
func (m WSRawMessage) Bytes() []byte {
	return m
}

// WebSocket client interface
type WSClient interface {
	GetHost() string
	GetAddrPort() (string, int)
	GetClientType() common.ServiceType
	IsConnected() bool
	Send(m Message)
	Connect()
	Disconnect()
	AddEventHandler(WSClientEventHandler)
}

// WSAsyncClient describes a WebSocket connection. WSAsyncClient is used
// by the client when connecting to a server, and also by the server when
// a client connects
type WSAsyncClient struct {
	sync.RWMutex
	Host              string
	ClientType        common.ServiceType
	Addr              string
	Port              int
	Path              string
	AuthClient        *AuthenticationClient
	headers           http.Header
	send              chan []byte
	read              chan []byte
	quit              chan bool
	wg                sync.WaitGroup
	wsConn            *websocket.Conn
	eventHandlers     []WSClientEventHandler
	eventHandlersLock sync.RWMutex
	connected         atomic.Value
	running           atomic.Value
	pongWait          time.Duration
	pingPeriod        time.Duration
	ticker            *time.Ticker
}

// Interface to be implement by the client events listeners
type WSClientEventHandler interface {
	OnMessage(c WSClient, m Message)
	OnConnected(c WSClient)
	OnDisconnected(c WSClient)
}

// DefaultWSClientEventHandler implements stubs for the WSClientEventHandler interface
type DefaultWSClientEventHandler struct {
}

// OnMessage is called when a message is received
func (d *DefaultWSClientEventHandler) OnMessage(c WSClient, m Message) {
}

// OnConnected is called when the connection is established
func (d *DefaultWSClientEventHandler) OnConnected(c WSClient) {
}

// OnDisconnected is called when the connection is closed or lost
func (d *DefaultWSClientEventHandler) OnDisconnected(c WSClient) {
}

// Return the hostname of the connection
func (c *WSAsyncClient) GetHost() string {
	return c.Host
}

// Return the address and the port of the remote end
func (c *WSAsyncClient) GetAddrPort() (string, int) {
	return c.Addr, c.Port
}

// Return the connection status
func (c *WSAsyncClient) IsConnected() bool {
	return c.connected.Load() == true
}

// Add a message to the send queue
func (c *WSAsyncClient) Send(m Message) {
	if c.running.Load() == false {
		return
	}

	c.send <- m.Bytes()
}

// Return the client type
func (c *WSAsyncClient) GetClientType() common.ServiceType {
	return c.ClientType
}

// Send a message directly over the wire
func (c *WSAsyncClient) SendMessage(msg []byte) error {
	if !c.IsConnected() {
		return errors.New("Not connected")
	}

	c.wsConn.SetWriteDeadline(time.Now().Add(writeWait))

	w, err := c.wsConn.NextWriter(websocket.TextMessage)
	if err != nil {
		return err
	}

	if _, err = w.Write(msg); err != nil {
		return err
	}

	return w.Close()
}

// Return the URL scheme
func (c *WSAsyncClient) scheme() string {
	if config.IsTLSenabled() == true {
		return "wss://"
	}
	return "ws://"
}

// Connect to the server
func (c *WSAsyncClient) connect() {
	var err error
	host := c.Addr + ":" + strconv.FormatInt(int64(c.Port), 10)
	endpoint := c.scheme() + host + c.Path
	headers := http.Header{
		"X-Host-ID":             {c.Host},
		"Origin":                {endpoint},
		"X-Client-Type":         {c.ClientType.String()},
		"X-Websocket-Namespace": {WilcardNamespace},
	}

	if c.AuthClient != nil {
		if err = c.AuthClient.Authenticate(); err != nil {
			logging.GetLogger().Errorf("Unable to create a WebSocket connection %s : %s", endpoint, err.Error())
			return
		}
		c.AuthClient.SetHeaders(headers)
	}

	d := websocket.Dialer{
		Proxy:           http.ProxyFromEnvironment,
		ReadBufferSize:  1024,
		WriteBufferSize: 1024,
	}
	certPEM := config.GetConfig().GetString("agent.X509_cert")
	keyPEM := config.GetConfig().GetString("agent.X509_key")
	if certPEM != "" && keyPEM != "" {
		d.TLSClientConfig = common.SetupTLSClientConfig(certPEM, keyPEM)
		checkTLSConfig(d.TLSClientConfig)
	}
	c.wsConn, _, err = d.Dial(endpoint, headers)

	if err != nil {
		logging.GetLogger().Errorf("Unable to create a WebSocket connection %s : %s", endpoint, err.Error())
		return
	}
	defer c.wsConn.Close()
	c.wsConn.SetPingHandler(nil)

	c.connected.Store(true)
	defer c.connected.Store(false)

	logging.GetLogger().Infof("Connected to %s", endpoint)

	// notify connected
	c.RLock()
	for _, l := range c.eventHandlers {
		l.OnConnected(c)
	}
	c.RUnlock()

	c.wg.Add(1)
	c.run()
}

func (c *WSAsyncClient) start() {
	c.wg.Add(1)
	go c.run()
}

// Client main loop to read and send messages
func (c *WSAsyncClient) run() {
	defer c.wg.Done()

	go func() {
		for c.running.Load() == true {
			_, m, err := c.wsConn.ReadMessage()
			if err != nil {
				if c.running.Load() != false {
					c.quit <- true
				}
				break
			}

			c.read <- m
		}
	}()

	defer func() {
		c.connected.Store(false)
		c.RLock()
		for _, l := range c.eventHandlers {
			l.OnDisconnected(c)
		}
		c.RUnlock()
	}()

	defer c.ticker.Stop()

	for {
		select {
		case <-c.quit:
			return
		case m := <-c.send:
			err := c.SendMessage(m)
			if err != nil {
				logging.GetLogger().Errorf("Error while writing to the WebSocket: %s", err.Error())
			}
		case m := <-c.read:
			c.RLock()
			for _, l := range c.eventHandlers {
				l.OnMessage(c, WSRawMessage(m))
			}
			c.RUnlock()
		case <-c.ticker.C:
			c.SendPing()
		}
	}
}

func (c *WSAsyncClient) SendPing() {
	c.wsConn.SetWriteDeadline(time.Now().Add(writeWait))
	if err := c.wsConn.WriteMessage(websocket.PingMessage, []byte{}); err != nil {
		logging.GetLogger().Warningf("Error while sending ping to the websocket: %s", err.Error())
	}
}

// Connect to the server - and reconnect if necessary
func (c *WSAsyncClient) Connect() {
	go func() {
		for c.running.Load() == true {
			c.connect()
			time.Sleep(1 * time.Second)
		}
	}()
}

// Register a new event handler
func (c *WSAsyncClient) AddEventHandler(h WSClientEventHandler) {
	c.Lock()
	c.eventHandlers = append(c.eventHandlers, h)
	c.Unlock()
}

// Disconnect the client without waiting for termination
func (c *WSAsyncClient) Disconnect() {
	c.running.Store(false)
	if c.connected.Load() == true {
		c.quit <- true
		c.wsConn.Close()
		close(c.send)
		close(c.read)
	}
}

// NewWSAsyncClient returns a client with a new connection
func NewWSAsyncClient(host string, clientType common.ServiceType, addr string, port int, path string, authClient *AuthenticationClient) *WSAsyncClient {
	pongTimeout := time.Duration(config.GetConfig().GetInt("ws_pong_timeout")) * time.Second
	c := &WSAsyncClient{
		Host:       host,
		ClientType: clientType,
		Addr:       addr,
		Port:       port,
		Path:       path,
		AuthClient: authClient,
		send:       make(chan []byte, maxMessages),
		read:       make(chan []byte, maxMessages),
		quit:       make(chan bool, 2),
		pongWait:   pongTimeout,
		pingPeriod: pongTimeout * 8 / 10,
		ticker:     &time.Ticker{},
	}
	c.connected.Store(false)
	c.running.Store(true)
	return c
}

// NewWSAsyncClientFromConnection creates a client from an existing connection
func NewWSAsyncClientFromConnection(host string, clientType common.ServiceType, conn *websocket.Conn) *WSAsyncClient {
	svc, _ := common.ServiceAddressFromString(conn.RemoteAddr().String())
	c := NewWSAsyncClient(host, clientType, svc.Addr, svc.Port, "", nil)

	conn.SetReadLimit(maxMessageSize)
	conn.SetReadDeadline(time.Now().Add(100 * time.Second))
	conn.SetPongHandler(func(string) error {
		conn.SetReadDeadline(time.Now().Add(c.pongWait))
		return nil
	})
	c.wsConn = conn
	c.ticker = time.NewTicker(c.pingPeriod)

	// send a first ping to help firefox and some other client which wait for a
	// first ping before doing something
	c.SendPing()

	c.connected.Store(true)
	c.running.Store(true)
	return c
}

// NewWSAsyncClientFromConnection creates a client based on the configuration
func NewWSAsyncClientFromConfig(clientType common.ServiceType, addr string, port int, path string, authClient *AuthenticationClient) *WSAsyncClient {
	host := config.GetConfig().GetString("host_id")
	return NewWSAsyncClient(host, clientType, addr, port, path, authClient)
}
<|endoftext|>


<commit_before>// Copyright 2016 Mender Software AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"strings"

	"github.com/mendersoftware/log"
	"github.com/pkg/errors"
)

const (
	identityDataHelper = "/usr/bin/mender-device-identity"
)

type IdentityDataGetter interface {
	// obtain identity data as a string or return an error
	Get() (string, error)
}

type IdentityDataRunner struct {
	Helper string
	cmdr   Commander
}

func NewIdentityDataGetter() IdentityDataGetter {
	return &IdentityDataRunner{
		identityDataHelper,
		&osCalls{},
	}
}

// Obtain identity data by calling a suitable helper tool
func (id IdentityDataRunner) Get() (string, error) {
	helper := identityDataHelper

	if id.Helper != "" {
		helper = id.Helper
	}

	cmd := id.cmdr.Command(helper)
	data, err := cmd.Output()

	if err != nil {
		return "", errors.Wrapf(err, "failed to call %s", helper)
	}

	idata, err := parseIdentityData(data)
	if err != nil {
		return "", errors.Wrapf(err, "failed to parse identity data")
	}

	encdata, err := json.Marshal(idata)
	if err != nil {
		return "", errors.Wrapf(err, "failed to encode identity data")
	}

	return string(encdata), nil
}

// device identity data content
type IdentityData map[string]string

func parseIdentityData(data []byte) (interface{}, error) {
	idata := make(IdentityData)

	in := bufio.NewScanner(bytes.NewBuffer(data))
	for in.Scan() {
		line := in.Text()

		if len(line) == 0 {
			continue
		}

		val := strings.SplitN(line, "=", 2)

		if len(val) < 2 {
			return nil, errors.Errorf("incorrect line '%s'", line)
		}

		if _, ok := idata[val[0]]; ok {
			log.Warningf("attribute %v already present in identity data", val[0])
		}
		idata[val[0]] = val[1]
	}

	if len(idata) == 0 {
		return nil, errors.Errorf("no data found")
	}

	return &idata, nil
}
<commit_msg>identity_data: make helper name a variable<commit_after>// Copyright 2016 Mender Software AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"strings"

	"github.com/mendersoftware/log"
	"github.com/pkg/errors"
)

var (
	identityDataHelper = "/usr/bin/mender-device-identity"
)

type IdentityDataGetter interface {
	// obtain identity data as a string or return an error
	Get() (string, error)
}

type IdentityDataRunner struct {
	Helper string
	cmdr   Commander
}

func NewIdentityDataGetter() IdentityDataGetter {
	return &IdentityDataRunner{
		identityDataHelper,
		&osCalls{},
	}
}

// Obtain identity data by calling a suitable helper tool
func (id IdentityDataRunner) Get() (string, error) {
	helper := identityDataHelper

	if id.Helper != "" {
		helper = id.Helper
	}

	cmd := id.cmdr.Command(helper)
	data, err := cmd.Output()

	if err != nil {
		return "", errors.Wrapf(err, "failed to call %s", helper)
	}

	idata, err := parseIdentityData(data)
	if err != nil {
		return "", errors.Wrapf(err, "failed to parse identity data")
	}

	encdata, err := json.Marshal(idata)
	if err != nil {
		return "", errors.Wrapf(err, "failed to encode identity data")
	}

	return string(encdata), nil
}

// device identity data content
type IdentityData map[string]string

func parseIdentityData(data []byte) (interface{}, error) {
	idata := make(IdentityData)

	in := bufio.NewScanner(bytes.NewBuffer(data))
	for in.Scan() {
		line := in.Text()

		if len(line) == 0 {
			continue
		}

		val := strings.SplitN(line, "=", 2)

		if len(val) < 2 {
			return nil, errors.Errorf("incorrect line '%s'", line)
		}

		if _, ok := idata[val[0]]; ok {
			log.Warningf("attribute %v already present in identity data", val[0])
		}
		idata[val[0]] = val[1]
	}

	if len(idata) == 0 {
		return nil, errors.Errorf("no data found")
	}

	return &idata, nil
}
<|endoftext|>


<commit_before>package html

import (
	"code.google.com/p/go.net/html"
	"errors"
	"github.com/slyrz/newscat/util"
	"io"
	"unicode"
)

const (
	// We remember a few special node types when descending into their
	// children.
	AncestorArticle = 1 << iota
	AncestorAside
	AncestorBlockquote
	AncestorList
)

type Document struct {
	Title  *util.Text // the <title>...</title> text.
	Chunks []*Chunk   // list of all chunks found in this document

	// Unexported fields.
	html *html.Node // the ... part
	head *html.Node // the ... part
	body *html.Node // the ... part

	// State variables used when collectiong chunks.
	ancestors int // bitmask which stores ancestor of the current node

	// Number of non-space characters inside link tags / normal tags
	// per html.ElementNode.
	linkText map[*html.Node]int // length of text inside  tags
	normText map[*html.Node]int // length of text outside  tags
}

func NewDocument(r io.Reader) (*Document, error) {
	doc := new(Document)
	doc.Chunks = make([]*Chunk, 0, 512)
	if err := doc.Parse(r); err != nil {
		return nil, err
	}
	return doc, nil
}

func (doc *Document) Parse(r io.Reader) error {
	root, err := html.Parse(r)
	if err != nil {
		return err
	}
	// Assign the fields html, head and body from the HTML page.
	doc.setNodes(root)

	// Check if ,  and  nodes were found.
	if doc.html == nil || doc.head == nil || doc.body == nil {
		return errors.New("Document missing ,  or .")
	}

	doc.parseHead(doc.head)

	// If no title was found, detecting the main heading might fail.
	if doc.Title == nil {
		doc.Title = util.NewText()
	}
	doc.linkText = make(map[*html.Node]int)
	doc.normText = make(map[*html.Node]int)

	doc.cleanBody(doc.body, 0)
	doc.countText(doc.body, false)
	doc.parseBody(doc.body)

	// Now link the chunks.
	for i := range doc.Chunks {
		if i > 0 {
			doc.Chunks[i].Prev = doc.Chunks[i-1]
		}
		if i < len(doc.Chunks)-1 {
			doc.Chunks[i].Next = doc.Chunks[i+1]
		}
	}
	return nil
}

// Assign the struct fields html, head and body from the HTML tree of node n.
//  doc.html ->
//  doc.head ->
//  doc.body ->
func (doc *Document) setNodes(n *html.Node) {
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		switch c.Data {
		case "html":
			doc.html = c
			doc.setNodes(c)
		case "body":
			doc.body = c
		case "head":
			doc.head = c
		}
	}
}

// parseHead parses the ... part of the HTML page. Right now it
// only detects the <title>...</title>.
func (doc *Document) parseHead(n *html.Node) {
	if n.Type == html.ElementNode && n.Data == "title" {
		if chunk, err := NewChunk(doc, n); err == nil {
			doc.Title = chunk.Text
		}
	}
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		doc.parseHead(c)
	}
}

// countText counts the link text and the normal text per html.Node.
// "Link text" is text inside  tags and "normal text" is text inside
// anything but  tags. Of course, counting is done cumulative, so the
// numbers of a parent node include the numbers of it's child nodes.
func (doc *Document) countText(n *html.Node, insideLink bool) (linkText int, normText int) {
	linkText = 0
	normText = 0
	if n.Type == html.ElementNode && n.Data == "a" {
		insideLink = true
	}
	for s := n.FirstChild; s != nil; s = s.NextSibling {
		linkTextChild, normTextChild := doc.countText(s, insideLink)
		linkText += linkTextChild
		normText += normTextChild
	}
	if n.Type == html.TextNode {
		count := 0
		for _, rune := range n.Data {
			if unicode.IsLetter(rune) {
				count += 1
			}
		}
		if insideLink {
			linkText += count
		} else {
			normText += count
		}
	}
	doc.linkText[n] = linkText
	doc.normText[n] = normText
	return
}

// cleanBody removes unwanted HTML elements from the HTML body.
func (doc *Document) cleanBody(n *html.Node, level int) {

	// removeNode returns true if a node should be removed from HTML document.
	removeNode := func(c *html.Node, level int) bool {
		switch c.Data {
		// Elements save to ignore.
		case "address", "audio", "button", "canvas", "caption", "fieldset",
			"figcaption", "figure", "footer", "form", "frame", "iframe",
			"map", "menu", "nav", "noscript", "object", "option", "output",
			"script", "select", "style", "svg", "textarea", "video":
			return true
		// High-level tables might be used to layout the document, so we better
		// not ignore them.
		case "table":
			return level > 5
		}
		return false
	}

	var curr *html.Node = n.FirstChild
	var next *html.Node = nil
	for ; curr != nil; curr = next {
		// We have to remember the next sibling here becase calling RemoveChild
		// sets curr's NextSibling pointer to nil and we would quit the loop
		// prematurely.
		next = curr.NextSibling
		if curr.Type == html.ElementNode {
			if removeNode(curr, level) {
				n.RemoveChild(curr)
			} else {
				doc.cleanBody(curr, level+1)
			}
		}
	}
}

var (
	ignoreNames = util.NewRegexFromWords(
		"breadcrumb",
		"caption",
		"comment",
		"community",
		"credit",
		"description",
		"foot",
		"gallery",
		"hide",
		"related",
		"story-feature",
		"storybar",
	)
	ignoreStyle = util.NewRegex(`(?i)display:\s*none`)
)

// parseBody parses the ... part of the HTML page. It creates
// Chunks for every html.TextNode found in the body.
func (doc *Document) parseBody(n *html.Node) {
	switch n.Type {
	case html.ElementNode:
		// We ignore the node if it has some nasty classes/ids/itemprobs or if
		// it contains "display: none" in its style attribute.
		for _, attr := range n.Attr {
			switch attr.Key {
			case "id", "class", "itemprop":
				if ignoreNames.In(attr.Val) {
					return
				}
			case "style":
				if ignoreStyle.In(attr.Val) {
					return
				}
			}
		}
		ancestorMask := 0
		switch n.Data {
		// We convert headings and links to text immediately. This is easier
		// and feasible because headings and links don't contain many children.
		// Descending into these children and handling every TextNode separately
		// would make things unnecessary complicated and our results noisy.
		case "h1", "h2", "h3", "h4", "h5", "h6", "a":
			if chunk, err := NewChunk(doc, n); err == nil {
				doc.Chunks = append(doc.Chunks, chunk)
			}
			return
		// Now mask the element type, but only if it isn't already set.
		// If we mask a bit which was already set by one of our callers, we'd also
		// clear it at the end of this function, though it actually should be cleared
		// by the caller.
		case "article":
			ancestorMask = AncestorArticle &^ doc.ancestors
		case "aside":
			ancestorMask = AncestorAside &^ doc.ancestors
		case "blockquote":
			ancestorMask = AncestorBlockquote &^ doc.ancestors
		case "ul", "ol":
			ancestorMask = AncestorList &^ doc.ancestors
		}
		// Add our mask to the ancestor bitmask.
		doc.ancestors |= ancestorMask
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			doc.parseBody(c)
		}
		// Remove our mask from the ancestor bitmask.
		doc.ancestors &^= ancestorMask
	case html.TextNode:
		if chunk, err := NewChunk(doc, n); err == nil {
			doc.Chunks = append(doc.Chunks, chunk)
		}
	}
}

type TextStat struct {
	Words     int
	Sentences int
	Count     int
}

// GetClassStats groups the document chunks by their classes (defined by the
// class attribute of HTML nodes) and calculates TextStats for each class.
func (doc *Document) GetClassStats() map[string]*TextStat {
	result := make(map[string]*TextStat)
	for _, chunk := range doc.Chunks {
		for _, class := range chunk.Classes {
			if stat, ok := result[class]; ok {
				stat.Words += chunk.Text.Words
				stat.Sentences += chunk.Text.Sentences
				stat.Count += 1
			} else {
				result[class] = &TextStat{chunk.Text.Words, chunk.Text.Sentences, 1}
			}
		}
	}
	return result
}

// GetClusterStats groups the document chunks by common ancestors and
// calculates TextStats for each group of chunks.
func (doc *Document) GetClusterStats() map[*Chunk]*TextStat {
	// Don't ascend further than this constant.
	const maxAncestors = 3

	// Count TextStats for Chunk ancestors.
	ancestorStat := make(map[*html.Node]*TextStat)
	for _, chunk := range doc.Chunks {
		node, count := chunk.Block, 0
		for node != nil && count < maxAncestors {
			if stat, ok := ancestorStat[node]; ok {
				stat.Words += chunk.Text.Words
				stat.Sentences += chunk.Text.Sentences
				stat.Count += 1
			} else {
				ancestorStat[node] = &TextStat{chunk.Text.Words, chunk.Text.Sentences, 1}
			}
			node, count = node.Parent, count+1
		}
	}

	// Generate result.
	result := make(map[*Chunk]*TextStat)
	for _, chunk := range doc.Chunks {
		node := chunk.Block
		if node == nil {
			continue
		}
		// Start with the parent's TextStat. Then ascend and check if the
		// current chunk has an ancestor with better stats. Use the best stat
		// as result.
		stat := ancestorStat[node]
		for {
			if node = node.Parent; node == nil {
				break
			}
			if statPrev, ok := ancestorStat[node]; ok {
				if stat.Count < statPrev.Count {
					stat = statPrev
				}
			} else {
				break
			}
		}
		result[chunk] = stat
	}
	return result
}
<commit_msg>pimped ignored class names<commit_after>package html

import (
	"code.google.com/p/go.net/html"
	"errors"
	"github.com/slyrz/newscat/util"
	"io"
	"unicode"
)

const (
	// We remember a few special node types when descending into their
	// children.
	AncestorArticle = 1 << iota
	AncestorAside
	AncestorBlockquote
	AncestorList
)

type Document struct {
	Title  *util.Text // the <title>...</title> text.
	Chunks []*Chunk   // list of all chunks found in this document

	// Unexported fields.
	html *html.Node // the ... part
	head *html.Node // the ... part
	body *html.Node // the ... part

	// State variables used when collectiong chunks.
	ancestors int // bitmask which stores ancestor of the current node

	// Number of non-space characters inside link tags / normal tags
	// per html.ElementNode.
	linkText map[*html.Node]int // length of text inside  tags
	normText map[*html.Node]int // length of text outside  tags
}

func NewDocument(r io.Reader) (*Document, error) {
	doc := new(Document)
	doc.Chunks = make([]*Chunk, 0, 512)
	if err := doc.Parse(r); err != nil {
		return nil, err
	}
	return doc, nil
}

func (doc *Document) Parse(r io.Reader) error {
	root, err := html.Parse(r)
	if err != nil {
		return err
	}
	// Assign the fields html, head and body from the HTML page.
	doc.setNodes(root)

	// Check if ,  and  nodes were found.
	if doc.html == nil || doc.head == nil || doc.body == nil {
		return errors.New("Document missing ,  or .")
	}

	doc.parseHead(doc.head)

	// If no title was found, detecting the main heading might fail.
	if doc.Title == nil {
		doc.Title = util.NewText()
	}
	doc.linkText = make(map[*html.Node]int)
	doc.normText = make(map[*html.Node]int)

	doc.cleanBody(doc.body, 0)
	doc.countText(doc.body, false)
	doc.parseBody(doc.body)

	// Now link the chunks.
	for i := range doc.Chunks {
		if i > 0 {
			doc.Chunks[i].Prev = doc.Chunks[i-1]
		}
		if i < len(doc.Chunks)-1 {
			doc.Chunks[i].Next = doc.Chunks[i+1]
		}
	}
	return nil
}

// Assign the struct fields html, head and body from the HTML tree of node n.
//  doc.html ->
//  doc.head ->
//  doc.body ->
func (doc *Document) setNodes(n *html.Node) {
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		switch c.Data {
		case "html":
			doc.html = c
			doc.setNodes(c)
		case "body":
			doc.body = c
		case "head":
			doc.head = c
		}
	}
}

// parseHead parses the ... part of the HTML page. Right now it
// only detects the <title>...</title>.
func (doc *Document) parseHead(n *html.Node) {
	if n.Type == html.ElementNode && n.Data == "title" {
		if chunk, err := NewChunk(doc, n); err == nil {
			doc.Title = chunk.Text
		}
	}
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		doc.parseHead(c)
	}
}

// countText counts the link text and the normal text per html.Node.
// "Link text" is text inside  tags and "normal text" is text inside
// anything but  tags. Of course, counting is done cumulative, so the
// numbers of a parent node include the numbers of it's child nodes.
func (doc *Document) countText(n *html.Node, insideLink bool) (linkText int, normText int) {
	linkText = 0
	normText = 0
	if n.Type == html.ElementNode && n.Data == "a" {
		insideLink = true
	}
	for s := n.FirstChild; s != nil; s = s.NextSibling {
		linkTextChild, normTextChild := doc.countText(s, insideLink)
		linkText += linkTextChild
		normText += normTextChild
	}
	if n.Type == html.TextNode {
		count := 0
		for _, rune := range n.Data {
			if unicode.IsLetter(rune) {
				count += 1
			}
		}
		if insideLink {
			linkText += count
		} else {
			normText += count
		}
	}
	doc.linkText[n] = linkText
	doc.normText[n] = normText
	return
}

// cleanBody removes unwanted HTML elements from the HTML body.
func (doc *Document) cleanBody(n *html.Node, level int) {

	// removeNode returns true if a node should be removed from HTML document.
	removeNode := func(c *html.Node, level int) bool {
		switch c.Data {
		// Elements save to ignore.
		case "address", "audio", "button", "canvas", "caption", "fieldset",
			"figcaption", "figure", "footer", "form", "frame", "iframe",
			"map", "menu", "nav", "noscript", "object", "option", "output",
			"script", "select", "style", "svg", "textarea", "video":
			return true
		// High-level tables might be used to layout the document, so we better
		// not ignore them.
		case "table":
			return level > 5
		}
		return false
	}

	var curr *html.Node = n.FirstChild
	var next *html.Node = nil
	for ; curr != nil; curr = next {
		// We have to remember the next sibling here becase calling RemoveChild
		// sets curr's NextSibling pointer to nil and we would quit the loop
		// prematurely.
		next = curr.NextSibling
		if curr.Type == html.ElementNode {
			if removeNode(curr, level) {
				n.RemoveChild(curr)
			} else {
				doc.cleanBody(curr, level+1)
			}
		}
	}
}

var (
	ignoreNames = util.NewRegexFromWords(
		"breadcrumb",
		"byline",
		"caption",
		"comment",
		"community",
		"credit",
		"description",
		"email",
		"foot",
		"gallery",
		"hide",
		"related",
		"photo",
		"story[-_]?bar",
		"story[-_]?feature",
	)
	ignoreStyle = util.NewRegex(`(?i)display:\s*none`)
)

// parseBody parses the ... part of the HTML page. It creates
// Chunks for every html.TextNode found in the body.
func (doc *Document) parseBody(n *html.Node) {
	switch n.Type {
	case html.ElementNode:
		// We ignore the node if it has some nasty classes/ids/itemprobs or if
		// it contains "display: none" in its style attribute.
		for _, attr := range n.Attr {
			switch attr.Key {
			case "id", "class", "itemprop":
				if ignoreNames.In(attr.Val) {
					return
				}
			case "style":
				if ignoreStyle.In(attr.Val) {
					return
				}
			}
		}
		ancestorMask := 0
		switch n.Data {
		// We convert headings and links to text immediately. This is easier
		// and feasible because headings and links don't contain many children.
		// Descending into these children and handling every TextNode separately
		// would make things unnecessary complicated and our results noisy.
		case "h1", "h2", "h3", "h4", "h5", "h6", "a":
			if chunk, err := NewChunk(doc, n); err == nil {
				doc.Chunks = append(doc.Chunks, chunk)
			}
			return
		// Now mask the element type, but only if it isn't already set.
		// If we mask a bit which was already set by one of our callers, we'd also
		// clear it at the end of this function, though it actually should be cleared
		// by the caller.
		case "article":
			ancestorMask = AncestorArticle &^ doc.ancestors
		case "aside":
			ancestorMask = AncestorAside &^ doc.ancestors
		case "blockquote":
			ancestorMask = AncestorBlockquote &^ doc.ancestors
		case "ul", "ol":
			ancestorMask = AncestorList &^ doc.ancestors
		}
		// Add our mask to the ancestor bitmask.
		doc.ancestors |= ancestorMask
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			doc.parseBody(c)
		}
		// Remove our mask from the ancestor bitmask.
		doc.ancestors &^= ancestorMask
	case html.TextNode:
		if chunk, err := NewChunk(doc, n); err == nil {
			doc.Chunks = append(doc.Chunks, chunk)
		}
	}
}

type TextStat struct {
	Words     int
	Sentences int
	Count     int
}

// GetClassStats groups the document chunks by their classes (defined by the
// class attribute of HTML nodes) and calculates TextStats for each class.
func (doc *Document) GetClassStats() map[string]*TextStat {
	result := make(map[string]*TextStat)
	for _, chunk := range doc.Chunks {
		for _, class := range chunk.Classes {
			if stat, ok := result[class]; ok {
				stat.Words += chunk.Text.Words
				stat.Sentences += chunk.Text.Sentences
				stat.Count += 1
			} else {
				result[class] = &TextStat{chunk.Text.Words, chunk.Text.Sentences, 1}
			}
		}
	}
	return result
}

// GetClusterStats groups the document chunks by common ancestors and
// calculates TextStats for each group of chunks.
func (doc *Document) GetClusterStats() map[*Chunk]*TextStat {
	// Don't ascend further than this constant.
	const maxAncestors = 3

	// Count TextStats for Chunk ancestors.
	ancestorStat := make(map[*html.Node]*TextStat)
	for _, chunk := range doc.Chunks {
		node, count := chunk.Block, 0
		for node != nil && count < maxAncestors {
			if stat, ok := ancestorStat[node]; ok {
				stat.Words += chunk.Text.Words
				stat.Sentences += chunk.Text.Sentences
				stat.Count += 1
			} else {
				ancestorStat[node] = &TextStat{chunk.Text.Words, chunk.Text.Sentences, 1}
			}
			node, count = node.Parent, count+1
		}
	}

	// Generate result.
	result := make(map[*Chunk]*TextStat)
	for _, chunk := range doc.Chunks {
		node := chunk.Block
		if node == nil {
			continue
		}
		// Start with the parent's TextStat. Then ascend and check if the
		// current chunk has an ancestor with better stats. Use the best stat
		// as result.
		stat := ancestorStat[node]
		for {
			if node = node.Parent; node == nil {
				break
			}
			if statPrev, ok := ancestorStat[node]; ok {
				if stat.Count < statPrev.Count {
					stat = statPrev
				}
			} else {
				break
			}
		}
		result[chunk] = stat
	}
	return result
}
<|endoftext|>


<commit_before>package gitioapi

import (
	"errors"
	"net/url"
	"os"
	"testing"
)

type errorTestCase struct { //Test case for Error
	status string
	err    error
	errMsg string
}

var errorTests []errorTestCase //Test cases for Error

type parmsTestCase struct { //Test case for Param
	url    string
	code   string
	res    string
	errMsg string
}

var parmsTests []parmsTestCase //Test cases for Param
var parmsTests2 []parmsTestCase //Test cases for Param

func TestMain(m *testing.M) {
	//Test cases for Error
	err1 := errors.New("error 1 !")
	err2 := errors.New("error 2 !")
	errorTests = []errorTestCase{
		{status: "status", err: err1, errMsg: "error 1 ! (status)"},
		{status: "", err: err2, errMsg: "error 2 !"},
		{status: "status", err: nil, errMsg: ""},
		{status: "", err: nil, errMsg: ""},
	}
	//Test cases for Param
	parmsTests = []parmsTestCase{
		{url: "https://github.com/spiegel-im-spiegel", code: "", res: "http://git.io/vOj52", errMsg: ""},
		{url: "https://github.com/technoweenie", code: "t", res: "http://git.io/t", errMsg: ""},
		{url: "http://www.baldanders.info/", code: "", res: "", errMsg: "invalid argument (Must be a GitHub.com URL.)"},
		{url: "noturl", code: "t", res: "", errMsg: "invalid argument (Invalid url: noturl)"},
		{url: "noturl", code: "", res: "", errMsg: "invalid argument (Invalid url: noturl)"},
		{url: "", code: "t", res: "", errMsg: "invalid argument (Invalid url: )"},
		{url: "", code: "", res: "", errMsg: "invalid argument (Invalid url: )"},
	}
	parmsTests2 = []parmsTestCase{
		{url: "http://git.io/vOj52", code: "", res: "https://github.com/spiegel-im-spiegel", errMsg: ""},
		{url: "http://git.io/t", code: "", res: "https://github.com/technoweenie", errMsg: ""},
		{url: "http://git.io/t", code: "t", res: "https://github.com/technoweenie", errMsg: ""},
		{url: "http://git.io/", code: "", res: "http://git.io/", errMsg: ""},
		{url: "http://git.io", code: "", res: "http://git.io", errMsg: ""},
		{url: "http://git.is", code: "", res: "http://git.is", errMsg: ""},
		{url: "noturl", code: "", res: "noturl", errMsg: ""},
		{url: "", code: "", res: "", errMsg: ""},
	}

	//start test
	code := m.Run()

	//termination
	os.Exit(code)
}

func TestError(t *testing.T) {
	for _, testCase := range errorTests {
		err := NewApiError(testCase.status, testCase.err)
		if err == nil {
			if len(testCase.errMsg) > 0 {
				t.Error("Error Status  = 'false', want 'true'.")
			}
		} else {
			msg := err.Error()
			if msg != testCase.errMsg {
				t.Errorf("Status  = %v, want %v.", msg, testCase.errMsg)
			}
		}
	}
}

func TestParams(t *testing.T) {
	var prm Param
	var values url.Values
	for _, testCase := range parmsTests {
		//case x-1
		prm = Param{Url: testCase.url, Code: testCase.code}
		values = prm.GetUrlValuse()
		if values.Get("url") != testCase.url {
			t.Errorf("values[\"url\"]  = %v, want %v.", values.Get("url"), testCase.url)
		}
		if values.Get("code") != testCase.code {
			t.Errorf("values[\"code\"]  = %v, want %v.", values.Get("code"), testCase.code)
		}
		//case x-2
		prm = Param{Url: testCase.url}
		values = prm.GetUrlValuse()
		if values.Get("url") != testCase.url {
			t.Errorf("values[\"url\"]  = %v, want %v.", values["url"], testCase.url)
		}
		if len(values.Get("code")) > 0 {
			t.Errorf("values[\"code\"]  = %v, want empty.", values.Get("code"))
		}
		//case x-3
		prm = Param{Code: testCase.code}
		values = prm.GetUrlValuse()
		if len(values.Get("url")) > 0 {
			t.Errorf("values[\"url\"]  = %v, want empty.", values.Get("url"))
		}
		if values.Get("code") != testCase.code {
			t.Errorf("values[\"code\"]  = %v, want %v.", values.Get("code"), testCase.code)
		}
	}
}

func TestEncode(t *testing.T) {
	for _, testCase := range parmsTests {
		prm := Param{Url: testCase.url, Code: testCase.code}
		result, err := Encode(&prm)
		if result != testCase.res {
			t.Errorf("Encode()  = %v, want %v.", result, testCase.res)
		}
		if err == nil {
			if len(testCase.errMsg) > 0 {
				t.Error("Status of Encode() = false, want true.")
			}
		} else {
			if err.Error() != testCase.errMsg {
				t.Errorf("Status of Encode() = %v, want %v.", err.Error(), testCase.errMsg)
			}
		}
	}
}

func TestDecode(t *testing.T) {
	for _, testCase := range parmsTests2 {
		prm := Param{Url: testCase.url, Code: testCase.code}
		result, err := Decode(&prm)
		if result != testCase.res {
			t.Errorf("Decode() = %v, want %v.", result, testCase.res)
		}
		if err == nil {
			if len(testCase.errMsg) > 0 {
				t.Error("Status of Decode() = false, want true.")
			}
		//} else {
		//	if err.Error() != testCase.errMsg {
		//		t.Errorf("Status of Encode() = %v, want %v.", err.Error(), testCase.errMsg)
		//	}
		}
	}
}
<commit_msg>update test.<commit_after>package gitioapi

import (
	"errors"
	"net/url"
	"os"
	"testing"
)

type errorTestCase struct { //Test case for Error
	status string
	err    error
	errMsg string
}

var errorTests []errorTestCase //Test cases for Error

type parmsTestCase struct { //Test case for Param
	url    string
	code   string
	res    string
	errMsg string
}

var parmsTests []parmsTestCase //Test cases for Param
var parmsTests2 []parmsTestCase //Test cases for Param

func TestMain(m *testing.M) {
	//Test cases for Error
	err1 := errors.New("error 1 !")
	err2 := errors.New("error 2 !")
	errorTests = []errorTestCase{
		{status: "status", err: err1, errMsg: "error 1 ! (status)"},
		{status: "", err: err2, errMsg: "error 2 !"},
		{status: "status", err: nil, errMsg: ""},
		{status: "", err: nil, errMsg: ""},
	}
	//Test cases for Param
	parmsTests = []parmsTestCase{
		{url: "https://github.com/spiegel-im-spiegel", code: "", res: "https://git.io/vOj52", errMsg: ""},
		{url: "https://github.com/technoweenie", code: "t", res: "https://git.io/t", errMsg: ""},
		{url: "http://www.baldanders.info/", code: "", res: "", errMsg: "invalid argument (Must be a GitHub.com URL.)"},
		{url: "noturl", code: "t", res: "", errMsg: "invalid argument (Invalid url: noturl)"},
		{url: "noturl", code: "", res: "", errMsg: "invalid argument (Invalid url: noturl)"},
		{url: "", code: "t", res: "", errMsg: "invalid argument (Invalid url: )"},
		{url: "", code: "", res: "", errMsg: "invalid argument (Invalid url: )"},
	}
	parmsTests2 = []parmsTestCase{
		{url: "https://git.io/vOj52", code: "", res: "https://github.com/spiegel-im-spiegel", errMsg: ""},
		{url: "https://git.io/t", code: "", res: "https://github.com/technoweenie", errMsg: ""},
		{url: "https://git.io/t", code: "t", res: "https://github.com/technoweenie", errMsg: ""},
		{url: "https://git.io/", code: "", res: "https://git.io/", errMsg: ""},
		{url: "https://git.io", code: "", res: "https://git.io", errMsg: ""},
		{url: "https://git.is", code: "", res: "https://git.is", errMsg: ""},
		{url: "noturl", code: "", res: "noturl", errMsg: ""},
		{url: "", code: "", res: "", errMsg: ""},
	}

	//start test
	code := m.Run()

	//termination
	os.Exit(code)
}

func TestError(t *testing.T) {
	for _, testCase := range errorTests {
		err := NewApiError(testCase.status, testCase.err)
		if err == nil {
			if len(testCase.errMsg) > 0 {
				t.Error("Error Status  = 'false', want 'true'.")
			}
		} else {
			msg := err.Error()
			if msg != testCase.errMsg {
				t.Errorf("Status  = %v, want %v.", msg, testCase.errMsg)
			}
		}
	}
}

func TestParams(t *testing.T) {
	var prm Param
	var values url.Values
	for _, testCase := range parmsTests {
		//case x-1
		prm = Param{Url: testCase.url, Code: testCase.code}
		values = prm.GetUrlValuse()
		if values.Get("url") != testCase.url {
			t.Errorf("values[\"url\"]  = %v, want %v.", values.Get("url"), testCase.url)
		}
		if values.Get("code") != testCase.code {
			t.Errorf("values[\"code\"]  = %v, want %v.", values.Get("code"), testCase.code)
		}
		//case x-2
		prm = Param{Url: testCase.url}
		values = prm.GetUrlValuse()
		if values.Get("url") != testCase.url {
			t.Errorf("values[\"url\"]  = %v, want %v.", values["url"], testCase.url)
		}
		if len(values.Get("code")) > 0 {
			t.Errorf("values[\"code\"]  = %v, want empty.", values.Get("code"))
		}
		//case x-3
		prm = Param{Code: testCase.code}
		values = prm.GetUrlValuse()
		if len(values.Get("url")) > 0 {
			t.Errorf("values[\"url\"]  = %v, want empty.", values.Get("url"))
		}
		if values.Get("code") != testCase.code {
			t.Errorf("values[\"code\"]  = %v, want %v.", values.Get("code"), testCase.code)
		}
	}
}

func TestEncode(t *testing.T) {
	for _, testCase := range parmsTests {
		prm := Param{Url: testCase.url, Code: testCase.code}
		result, err := Encode(&prm)
		if result != testCase.res {
			t.Errorf("Encode()  = %v, want %v.", result, testCase.res)
		}
		if err == nil {
			if len(testCase.errMsg) > 0 {
				t.Error("Status of Encode() = false, want true.")
			}
		} else {
			if err.Error() != testCase.errMsg {
				t.Errorf("Status of Encode() = %v, want %v.", err.Error(), testCase.errMsg)
			}
		}
	}
}

func TestDecode(t *testing.T) {
	for _, testCase := range parmsTests2 {
		prm := Param{Url: testCase.url, Code: testCase.code}
		result, err := Decode(&prm)
		if result != testCase.res {
			t.Errorf("Decode() = %v, want %v.", result, testCase.res)
		}
		if err == nil {
			if len(testCase.errMsg) > 0 {
				t.Error("Status of Decode() = false, want true.")
			}
		//} else {
		//	if err.Error() != testCase.errMsg {
		//		t.Errorf("Status of Encode() = %v, want %v.", err.Error(), testCase.errMsg)
		//	}
		}
	}
}
<|endoftext|>


<commit_before>package nodefs

import (
	"log"

	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/go-fuse/raw"
)

type connectorDir struct {
	node       Node
	stream     []fuse.DirEntry
	lastOffset uint64
	rawFS      fuse.RawFileSystem
	lookups    []raw.EntryOut
}

func (d *connectorDir) ReadDir(list *fuse.DirEntryList, input *raw.ReadIn, context *fuse.Context) (code fuse.Status) {
	if d.stream == nil {
		return fuse.OK
	}
	// rewinddir() should be as if reopening directory.
	// TODO - test this.
	if d.lastOffset > 0 && input.Offset == 0 {
		d.stream, code = d.node.OpenDir(context)
		if !code.Ok() {
			return code
		}
	}

	todo := d.stream[input.Offset:]
	for _, e := range todo {
		if e.Name == "" {
			log.Printf("got emtpy directory entry, mode %o.", e.Mode)
			continue
		}
		if !list.AddDirEntry(e) {
			break
		}
	}
	d.lastOffset = list.Offset
	return fuse.OK
}

func (d *connectorDir) ReadDirPlus(list *fuse.DirEntryList, input *raw.ReadIn, context *fuse.Context) (code fuse.Status) {
	if d.stream == nil {
		return fuse.OK
	}

	// rewinddir() should be as if reopening directory.
	if d.lastOffset > 0 && input.Offset == 0 {
		d.stream, code = d.node.OpenDir(context)
		if !code.Ok() {
			return code
		}
		d.lookups = nil
	}

	if d.lookups == nil {
		d.lookups = make([]raw.EntryOut, len(d.stream))
		for i, n := range d.stream {
			if n.Name == "." || n.Name == ".." {
				continue
			}
			// We ignore the return value
			code := d.rawFS.Lookup(&d.lookups[i], context, n.Name)
			if !code.Ok() {
				d.lookups[i] = raw.EntryOut{}
			}
		}
	}

	todo := d.stream[input.Offset:]
	for i, e := range todo {
		if e.Name == "" {
			log.Printf("got empty directory entry, mode %o.", e.Mode)
			continue
		}
		if !list.AddDirLookupEntry(e, &d.lookups[i]) {
			break
		}
	}
	d.lastOffset = list.Offset
	return fuse.OK

}

// Read everything so we make goroutines exit.
func (d *connectorDir) Release() {
}

type rawDir interface {
	ReadDir(out *fuse.DirEntryList, input *raw.ReadIn, c *fuse.Context) fuse.Status
	ReadDirPlus(out *fuse.DirEntryList, input *raw.ReadIn, c *fuse.Context) fuse.Status
	Release()
}
<commit_msg>fuse: fix offset of readdirplus lookup entries<commit_after>package nodefs

import (
	"log"

	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/go-fuse/raw"
)

type connectorDir struct {
	node       Node
	stream     []fuse.DirEntry
	lastOffset uint64
	rawFS      fuse.RawFileSystem
	lookups    []raw.EntryOut
}

func (d *connectorDir) ReadDir(list *fuse.DirEntryList, input *raw.ReadIn, context *fuse.Context) (code fuse.Status) {
	if d.stream == nil {
		return fuse.OK
	}
	// rewinddir() should be as if reopening directory.
	// TODO - test this.
	if d.lastOffset > 0 && input.Offset == 0 {
		d.stream, code = d.node.OpenDir(context)
		if !code.Ok() {
			return code
		}
	}

	todo := d.stream[input.Offset:]
	for _, e := range todo {
		if e.Name == "" {
			log.Printf("got emtpy directory entry, mode %o.", e.Mode)
			continue
		}
		if !list.AddDirEntry(e) {
			break
		}
	}
	d.lastOffset = list.Offset
	return fuse.OK
}

func (d *connectorDir) ReadDirPlus(list *fuse.DirEntryList, input *raw.ReadIn, context *fuse.Context) (code fuse.Status) {
	if d.stream == nil {
		return fuse.OK
	}

	// rewinddir() should be as if reopening directory.
	if d.lastOffset > 0 && input.Offset == 0 {
		d.stream, code = d.node.OpenDir(context)
		if !code.Ok() {
			return code
		}
		d.lookups = nil
	}

	if d.lookups == nil {
		d.lookups = make([]raw.EntryOut, len(d.stream))
		for i, n := range d.stream {
			if n.Name == "." || n.Name == ".." {
				continue
			}
			// We ignore the return value
			code := d.rawFS.Lookup(&d.lookups[i], context, n.Name)
			if !code.Ok() {
				d.lookups[i] = raw.EntryOut{}
			}
		}
	}

	todo := d.stream[input.Offset:]
	for i, e := range todo {
		if e.Name == "" {
			log.Printf("got empty directory entry, mode %o.", e.Mode)
			continue
		}
		if !list.AddDirLookupEntry(e, &d.lookups[input.Offset+uint64(i)]) {
			break
		}
	}
	d.lastOffset = list.Offset
	return fuse.OK

}

// Read everything so we make goroutines exit.
func (d *connectorDir) Release() {
}

type rawDir interface {
	ReadDir(out *fuse.DirEntryList, input *raw.ReadIn, c *fuse.Context) fuse.Status
	ReadDirPlus(out *fuse.DirEntryList, input *raw.ReadIn, c *fuse.Context) fuse.Status
	Release()
}
<|endoftext|>