<commit_before>package v1alpha1

import (
	"context"
	"fmt"
	"strings"

	"github.com/caicloud/nirvana/log"
	"github.com/caicloud/nirvana/service"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/rand"

	"github.com/caicloud/cyclone/pkg/apis/cyclone/v1alpha1"
	"github.com/caicloud/cyclone/pkg/meta"
	api "github.com/caicloud/cyclone/pkg/server/apis/v1alpha1"
	"github.com/caicloud/cyclone/pkg/server/biz/accelerator"
	"github.com/caicloud/cyclone/pkg/server/biz/hook"
	"github.com/caicloud/cyclone/pkg/server/biz/scm"
	"github.com/caicloud/cyclone/pkg/server/biz/scm/bitbucket"
	"github.com/caicloud/cyclone/pkg/server/biz/scm/github"
	"github.com/caicloud/cyclone/pkg/server/biz/scm/gitlab"
	"github.com/caicloud/cyclone/pkg/server/biz/scm/svn"
	"github.com/caicloud/cyclone/pkg/server/common"
	"github.com/caicloud/cyclone/pkg/server/handler"
	"github.com/caicloud/cyclone/pkg/util/cerr"
)

const (
	succeededMsg = "Successfully triggered"

	ignoredMsg = "Is ignored"
)

func newWebhookResponse(msg string) api.WebhookResponse {
	return api.WebhookResponse{
		Message: msg,
	}
}

// HandleWebhook handles webhooks from integrated systems.
func HandleWebhook(ctx context.Context, tenant, eventType, integration string) (api.WebhookResponse, error) {
	if eventType != string(v1alpha1.TriggerTypeSCM) {
		err := fmt.Errorf("eventType %s unsupported, support SCM for now", eventType)
		return newWebhookResponse(err.Error()), err
	}
	request := service.HTTPContextFrom(ctx).Request()

	var data *scm.EventData

	if request.Header.Get(github.EventTypeHeader) != "" {
		in, err := getIntegration(common.TenantNamespace(tenant), integration)
		if err != nil {
			return newWebhookResponse(err.Error()), err
		}
		data = github.ParseEvent(in.Spec.SCM, request)
	}

	if request.Header.Get(gitlab.EventTypeHeader) != "" {
		data = gitlab.ParseEvent(request)
	}

	if request.Header.Get(bitbucket.EventTypeHeader) != "" {
		in, err := getIntegration(common.TenantNamespace(tenant), integration)
		if err != nil {
			return newWebhookResponse(err.Error()), err
		}
		data = bitbucket.ParseEvent(in.Spec.SCM, request)
	}

	if request.Header.Get(svn.EventTypeHeader) != "" {
		data = svn.ParseEvent(request)
	}

	if data == nil {
		return newWebhookResponse(ignoredMsg), nil
	}

	wfts, err := hook.ListSCMWfts(tenant, data.Repo, integration)
	if err != nil {
		return newWebhookResponse(err.Error()), err
	}

	sftName := make([]string, 0)
	for _, wft := range wfts.Items {
		log.Infof("Trigger workflow trigger %s", wft.Name)
		sftName = append(sftName, wft.Name)
		if err = createWorkflowRun(tenant, wft, data); err != nil {
			log.Errorf("wft %s create workflow run error:%v", wft.Name, err)
		}
	}
	if len(sftName) > 0 {
		return newWebhookResponse(fmt.Sprintf("%s: %s", succeededMsg, sftName)), nil
	}

	return newWebhookResponse(ignoredMsg), nil
}

func createWorkflowRun(tenant string, wft v1alpha1.WorkflowTrigger, data *scm.EventData) error {
	ns := wft.Namespace
	var err error
	var project string
	if wft.Labels != nil {
		project = wft.Labels[meta.LabelProjectName]
	}
	if project == "" {
		return fmt.Errorf("failed to get project from workflowtrigger labels")
	}

	wfName := wft.Spec.WorkflowRef.Name
	if wfName == "" {
		return fmt.Errorf("workflow reference of workflowtrigger is empty")
	}

	trigger := false
	var tag string
	st := wft.Spec.SCM
	switch data.Type {
	case scm.TagReleaseEventType:
		if st.TagRelease.Enabled {
			trigger = true
			tag = data.Ref
			splitTags := strings.Split(data.Ref, "/")
			if len(splitTags) == 3 {
				tag = splitTags[2]
			}
		}
	case scm.PushEventType:
		trimmedBranch := data.Branch
		if index := strings.LastIndex(data.Branch, "/"); index >= 0 {
			trimmedBranch = trimmedBranch[index+1:]
		}
		for _, branch := range st.Push.Branches {
			if branch == trimmedBranch {
				trigger = true
				break
			}
		}
	case scm.PullRequestEventType:
		if st.PullRequest.Enabled {
			trigger = true
		}
	case scm.PullRequestCommentEventType:
		for _, comment := range st.PullRequestComment.Comments {
			if comment == data.Comment {
				trigger = true
			}
		}
	case scm.PostCommitEventType:
		if st.PostCommit.Enabled {
			trigger = true
		}
	}

	if !trigger {
		return nil
	}

	log.Infof("Trigger wft %s with event data: %v", wft.Name, data)

	name := fmt.Sprintf("%s-%s", wfName, rand.String(5))

	// Create workflowrun.
	wfr := &v1alpha1.WorkflowRun{
		ObjectMeta: metav1.ObjectMeta{
			Name: name,
			Annotations: map[string]string{
				meta.AnnotationWorkflowRunTrigger: string(data.Type),
				meta.AnnotationAlias:              name,
			},
			Labels: map[string]string{
				meta.LabelProjectName:             project,
				meta.LabelWorkflowName:            wfName,
				meta.LabelWorkflowRunAcceleration: wft.Labels[meta.LabelWorkflowRunAcceleration],
			},
		},
		Spec: wft.Spec.WorkflowRunSpec,
	}

	wfr.Annotations, err = setSCMEventData(wfr.Annotations, data)
	if err != nil {
		return err
	}

	// Set "Tag" and "SCM_REVISION" for all resource configs.
	for _, r := range wft.Spec.WorkflowRunSpec.Resources {
		for i, p := range r.Parameters {
			if p.Name == "TAG" && tag != "" {
				r.Parameters[i].Value = &tag
			}

			if p.Name == "SCM_REVISION" && data.Ref != "" {
				r.Parameters[i].Value = &data.Ref
			}
		}
	}

	// Set "Tag" for all stage configs.
	for _, s := range wft.Spec.WorkflowRunSpec.Stages {
		for i, p := range s.Parameters {
			if p.Name == "tag" && tag != "" {
				s.Parameters[i].Value = &tag
			}
		}
	}

	accelerator.NewAccelerator(tenant, project, wfr).Accelerate()
	_, err = handler.K8sClient.CycloneV1alpha1().WorkflowRuns(ns).Create(wfr)
	if err != nil {
		return cerr.ConvertK8sError(err)
	}

	return nil
}
<commit_msg>fix: init pull request status when wfr start (#1245)<commit_after>package v1alpha1

import (
	"context"
	"fmt"
	"strings"

	"github.com/caicloud/nirvana/log"
	"github.com/caicloud/nirvana/service"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/rand"

	"github.com/caicloud/cyclone/pkg/apis/cyclone/v1alpha1"
	"github.com/caicloud/cyclone/pkg/meta"
	api "github.com/caicloud/cyclone/pkg/server/apis/v1alpha1"
	"github.com/caicloud/cyclone/pkg/server/biz/accelerator"
	"github.com/caicloud/cyclone/pkg/server/biz/hook"
	"github.com/caicloud/cyclone/pkg/server/biz/scm"
	"github.com/caicloud/cyclone/pkg/server/biz/scm/bitbucket"
	"github.com/caicloud/cyclone/pkg/server/biz/scm/github"
	"github.com/caicloud/cyclone/pkg/server/biz/scm/gitlab"
	"github.com/caicloud/cyclone/pkg/server/biz/scm/svn"
	"github.com/caicloud/cyclone/pkg/server/common"
	"github.com/caicloud/cyclone/pkg/server/handler"
	"github.com/caicloud/cyclone/pkg/util/cerr"
)

const (
	succeededMsg = "Successfully triggered"

	ignoredMsg = "Is ignored"
)

func newWebhookResponse(msg string) api.WebhookResponse {
	return api.WebhookResponse{
		Message: msg,
	}
}

// HandleWebhook handles webhooks from integrated systems.
func HandleWebhook(ctx context.Context, tenant, eventType, integration string) (api.WebhookResponse, error) {
	if eventType != string(v1alpha1.TriggerTypeSCM) {
		err := fmt.Errorf("eventType %s unsupported, support SCM for now", eventType)
		return newWebhookResponse(err.Error()), err
	}
	request := service.HTTPContextFrom(ctx).Request()

	var data *scm.EventData

	if request.Header.Get(github.EventTypeHeader) != "" {
		in, err := getIntegration(common.TenantNamespace(tenant), integration)
		if err != nil {
			return newWebhookResponse(err.Error()), err
		}
		data = github.ParseEvent(in.Spec.SCM, request)
	}

	if request.Header.Get(gitlab.EventTypeHeader) != "" {
		data = gitlab.ParseEvent(request)
	}

	if request.Header.Get(bitbucket.EventTypeHeader) != "" {
		in, err := getIntegration(common.TenantNamespace(tenant), integration)
		if err != nil {
			return newWebhookResponse(err.Error()), err
		}
		data = bitbucket.ParseEvent(in.Spec.SCM, request)
	}

	if request.Header.Get(svn.EventTypeHeader) != "" {
		data = svn.ParseEvent(request)
	}

	if data == nil {
		return newWebhookResponse(ignoredMsg), nil
	}

	wfts, err := hook.ListSCMWfts(tenant, data.Repo, integration)
	if err != nil {
		return newWebhookResponse(err.Error()), err
	}

	sftName := make([]string, 0)
	for _, wft := range wfts.Items {
		log.Infof("Trigger workflow trigger %s", wft.Name)
		sftName = append(sftName, wft.Name)
		if err = createWorkflowRun(tenant, wft, data); err != nil {
			log.Errorf("wft %s create workflow run error:%v", wft.Name, err)
		}
	}
	if len(sftName) > 0 {
		return newWebhookResponse(fmt.Sprintf("%s: %s", succeededMsg, sftName)), nil
	}

	return newWebhookResponse(ignoredMsg), nil
}

func createWorkflowRun(tenant string, wft v1alpha1.WorkflowTrigger, data *scm.EventData) error {
	ns := wft.Namespace
	var err error
	var project string
	if wft.Labels != nil {
		project = wft.Labels[meta.LabelProjectName]
	}
	if project == "" {
		return fmt.Errorf("failed to get project from workflowtrigger labels")
	}

	wfName := wft.Spec.WorkflowRef.Name
	if wfName == "" {
		return fmt.Errorf("workflow reference of workflowtrigger is empty")
	}

	trigger := false
	var tag string
	st := wft.Spec.SCM
	switch data.Type {
	case scm.TagReleaseEventType:
		if st.TagRelease.Enabled {
			trigger = true
			tag = data.Ref
			splitTags := strings.Split(data.Ref, "/")
			if len(splitTags) == 3 {
				tag = splitTags[2]
			}
		}
	case scm.PushEventType:
		trimmedBranch := data.Branch
		if index := strings.LastIndex(data.Branch, "/"); index >= 0 {
			trimmedBranch = trimmedBranch[index+1:]
		}
		for _, branch := range st.Push.Branches {
			if branch == trimmedBranch {
				trigger = true
				break
			}
		}
	case scm.PullRequestEventType:
		if st.PullRequest.Enabled {
			trigger = true
		}
	case scm.PullRequestCommentEventType:
		for _, comment := range st.PullRequestComment.Comments {
			if comment == data.Comment {
				trigger = true
			}
		}
	case scm.PostCommitEventType:
		if st.PostCommit.Enabled {
			trigger = true
		}
	}

	if !trigger {
		return nil
	}

	log.Infof("Trigger wft %s with event data: %v", wft.Name, data)

	name := fmt.Sprintf("%s-%s", wfName, rand.String(5))

	// Create workflowrun.
	wfr := &v1alpha1.WorkflowRun{
		ObjectMeta: metav1.ObjectMeta{
			Name: name,
			Annotations: map[string]string{
				meta.AnnotationWorkflowRunTrigger: string(data.Type),
				meta.AnnotationAlias:              name,
			},
			Labels: map[string]string{
				meta.LabelProjectName:             project,
				meta.LabelWorkflowName:            wfName,
				meta.LabelWorkflowRunAcceleration: wft.Labels[meta.LabelWorkflowRunAcceleration],
			},
		},
		Spec: wft.Spec.WorkflowRunSpec,
	}

	wfr.Annotations, err = setSCMEventData(wfr.Annotations, data)
	if err != nil {
		return err
	}

	// Set "Tag" and "SCM_REVISION" for all resource configs.
	for _, r := range wft.Spec.WorkflowRunSpec.Resources {
		for i, p := range r.Parameters {
			if p.Name == "TAG" && tag != "" {
				r.Parameters[i].Value = &tag
			}

			if p.Name == "SCM_REVISION" && data.Ref != "" {
				r.Parameters[i].Value = &data.Ref
			}
		}
	}

	// Set "Tag" for all stage configs.
	for _, s := range wft.Spec.WorkflowRunSpec.Stages {
		for i, p := range s.Parameters {
			if p.Name == "tag" && tag != "" {
				s.Parameters[i].Value = &tag
			}
		}
	}

	accelerator.NewAccelerator(tenant, project, wfr).Accelerate()
	_, err = handler.K8sClient.CycloneV1alpha1().WorkflowRuns(ns).Create(wfr)
	if err != nil {
		return cerr.ConvertK8sError(err)
	}

	// Init pull-request status to pending
	wfrCopy := wfr.DeepCopy()
	wfrCopy.Status.Overall.Phase = v1alpha1.StatusRunning
	err = updatePullRequestStatus(wfrCopy)
	if err != nil {
		log.Warningf("Init pull request status for %s error: %v", wfr.Name, err)
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"

	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
)

const (
	projectOwner = "golang"
	projectRepo  = "go"
)

var githubClient *github.Client

// GitHub personal access token, from https://github.com/settings/applications.
var githubAuthToken string

func loadGithubAuth() {
	const short = ".github-issue-token"
	filename := filepath.Clean(os.Getenv("HOME") + "/" + short)
	shortFilename := filepath.Clean("$HOME/" + short)
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Fatal("reading token: ", err, "\n\n"+
			"Please create a personal access token at https://github.com/settings/tokens/new\n"+
			"and write it to ", shortFilename, " to use this program.\n"+
			"The token only needs the repo scope, or private_repo if you want to\n"+
			"view or edit issues for private repositories.\n"+
			"The benefit of using a personal access token over using your GitHub\n"+
			"password directly is that you can limit its use and revoke it at any time.\n\n")
	}
	fi, err := os.Stat(filename)
	if fi.Mode()&0077 != 0 {
		log.Fatalf("reading token: %s mode is %#o, want %#o", shortFilename, fi.Mode()&0777, fi.Mode()&0700)
	}
	githubAuthToken = strings.TrimSpace(string(data))
	t := &oauth2.Transport{
		Source: &tokenSource{AccessToken: githubAuthToken},
	}
	githubClient = github.NewClient(&http.Client{Transport: t})
}

// releaseStatusTitle returns the title of the release status issue
// for the given milestone.
// If you change this function, releasebot will not be able to find an
// existing tracking issue using the old name and will create a new one.
func releaseStatusTitle(m *github.Milestone) string {
	return "all: " + strings.Replace(m.GetTitle(), "Go", "Go ", -1) + " release status"
}

type tokenSource oauth2.Token

func (t *tokenSource) Token() (*oauth2.Token, error) {
	return (*oauth2.Token)(t), nil
}

func loadMilestones() ([]*github.Milestone, error) {
	// NOTE(rsc): There appears to be no paging possible.
	all, _, err := githubClient.Issues.ListMilestones(context.TODO(), projectOwner, projectRepo, &github.MilestoneListOptions{
		State: "open",
	})
	if err != nil {
		return nil, err
	}
	if all == nil {
		all = []*github.Milestone{}
	}
	return all, nil
}

// findIssues finds all the issues for the given milestone and
// categorizes them into approved cherry-picks (w.Picks)
// and other issues (w.OtherIssues).
// It also finds the release summary issue (w.ReleaseIssue).
func (w *Work) findIssues() {
	issues, err := listRepoIssues(github.IssueListByRepoOptions{
		Milestone: fmt.Sprint(w.Milestone.GetNumber()),
	})
	if err != nil {
		w.log.Panic(err)
	}

	for _, issue := range issues {
		if issue.GetTitle() == releaseStatusTitle(w.Milestone) {
			if w.ReleaseIssue != nil {
				w.log.Printf("**warning**: multiple release issues: #%d and #%d\n", w.ReleaseIssue.GetNumber(), issue.GetNumber())
				continue
			}
			w.ReleaseIssue = issue
			continue
		}
		if hasLabel(issue, "cherry-pick-approved") {
			w.Picks = append(w.Picks, issue)
			continue
		}
		w.OtherIssues = append(w.OtherIssues, issue)
	}
	sort.Slice(w.Picks, func(i, j int) bool { return w.Picks[i].GetNumber() < w.Picks[j].GetNumber() })

	if w.ReleaseIssue == nil {
		title := releaseStatusTitle(w.Milestone)
		body := wrapStatus(w.Milestone, "Nothing yet.")
		req := &github.IssueRequest{
			Title:     &title,
			Body:      &body,
			Milestone: w.Milestone.Number,
		}
		issue, _, err := githubClient.Issues.Create(context.TODO(), projectOwner, projectRepo, req)
		if err != nil {
			w.log.Panic(err)
		}
		w.ReleaseIssue = issue
	}
}

// listRepoIssues wraps Issues.ListByRepo to deal with paging.
func listRepoIssues(opt github.IssueListByRepoOptions) ([]*github.Issue, error) {
	var all []*github.Issue
	for page := 1; ; {
		xopt := opt
		xopt.ListOptions = github.ListOptions{
			Page:    page,
			PerPage: 100,
		}
		list, resp, err := githubClient.Issues.ListByRepo(context.TODO(), projectOwner, projectRepo, &xopt)
		all = append(all, list...)
		if err != nil {
			return all, err
		}
		if resp.NextPage < page {
			break
		}
		page = resp.NextPage
	}
	return all, nil
}

// hasLabel reports whether issue has the given label.
func hasLabel(issue *github.Issue, label string) bool {
	for _, l := range issue.Labels {
		if l.GetName() == label {
			return true
		}
	}
	return false
}

var clOK = regexp.MustCompile(`(?i)^CL (\d+) OK(( for Go \d+\.\d+\.\d+)?.*)`)
var afterCL = regexp.MustCompile(`(?i)after CL (\d+)`)

// listIssueComments wraps Issues.ListComments to deal with paging.
func listIssueComments(number int) ([]*github.IssueComment, error) {
	var all []*github.IssueComment
	for page := 1; ; {
		list, resp, err := githubClient.Issues.ListComments(context.TODO(), projectOwner, projectRepo, number, &github.IssueListCommentsOptions{
			ListOptions: github.ListOptions{
				Page:    page,
				PerPage: 100,
			},
		})
		all = append(all, list...)
		if err != nil {
			return all, err
		}
		if resp.NextPage < page {
			break
		}
		page = resp.NextPage
	}
	return all, nil
}

func (w *Work) findCLs() {
	// Preload all CLs in parallel.
	type comments struct {
		list []*github.IssueComment
		err  error
	}
	preload := make([]comments, len(w.Picks))
	var wg sync.WaitGroup
	for i, pick := range w.Picks {
		i := i
		number := pick.GetNumber()
		wg.Add(1)
		go func() {
			defer wg.Done()
			list, err := listIssueComments(number)
			preload[i] = comments{list, err}
		}()
	}
	wg.Wait()

	var cls []*CL
	for i, pick := range w.Picks {
		number := pick.GetNumber()
		fmt.Printf("load #%d\n", number)
		found := false
		list, err := preload[i].list, preload[i].err
		if err != nil {
			w.log.Panic(err)
		}
		var last *CL
		for _, com := range list {
			user := com.User.GetLogin()
			text := com.GetBody()
			for _, line := range strings.Split(text, "\n") {
				if m := clOK.FindStringSubmatch(line); m != nil {
					if m[3] != " for Go "+strings.TrimPrefix(w.Milestone.GetTitle(), "Go") {
						w.log.Printf("#%d: %s: wrong milestone: %s\n", number, user, line)
						continue
					}
					if !githubCherryPickApprovers[user] {
						w.log.Printf("#%d: %s: not an approver: %s\n", number, user, line)
						continue
					}
					n, err := strconv.Atoi(m[1])
					if err != nil {
						w.log.Printf("#%d: %s: invalid CL number: %s\n", number, user, line)
						continue
					}
					cl := &CL{Num: n, Approver: user, Issues: []int{number}}
					if last != nil {
						cl.Prereq = []int{last.Num}
					}
					for _, am := range afterCL.FindAllStringSubmatch(m[2], -1) {
						n, err := strconv.Atoi(am[1])
						if err != nil {
							w.log.Printf("#%d: %s: invalid after CL number: %s\n", number, user, line)
							continue
						}
						cl.Prereq = append(cl.Prereq, n)
					}
					cls = append(cls, cl)
					found = true
					last = cl
				}
			}
		}
		if !found {
			log.Printf("#%d: has cherry-pick-approved label but no approvals found", number)
		}
	}

	sort.Slice(cls, func(i, j int) bool {
		return cls[i].Num < cls[j].Num || cls[i].Num == cls[j].Num && cls[i].Approver < cls[j].Approver
	})

	out := cls[:0]
	var last CL
	for _, cl := range cls {
		if cl.Num == last.Num {
			end := out[len(out)-1]
			if cl.Approver != last.Approver {
				end.Approver += "," + cl.Approver
			}
			end.Issues = append(end.Issues, cl.Issues...)
			end.Prereq = append(end.Prereq, cl.Prereq...)
		} else {
			out = append(out, cl)
		}
		last = *cl
	}
	w.CLs = out
}

func (w *Work) closeIssues() {
	all := append(w.Picks[:len(w.Picks):len(w.Picks)], w.ReleaseIssue)
	for _, issue := range all {
		if issue.GetState() == "closed" {
			continue
		}
		number := issue.GetNumber()
		var md bytes.Buffer
		fmt.Fprintf(&md, "%s has been packaged and includes:\n\n", w.Version)
		for _, cl := range w.CLs {
			match := issue == w.ReleaseIssue
			for _, n := range cl.Issues {
				if n == number {
					match = true
					break
				}
			}
			if match {
				fmt.Fprintf(&md, "  - %s %s\n", mdChangeLink(cl.Num), mdEscape(cl.Title))
			}
		}
		fmt.Fprintf(&md, "\nThe release is posted at [golang.org/dl](https://golang.org/dl).\n")
		md.WriteString(signature())
		postGithubComment(number, md.String())
		closed := "closed"
		_, _, err := githubClient.Issues.Edit(context.TODO(), projectOwner, projectRepo, number, &github.IssueRequest{
			State: &closed,
		})
		if err != nil {
			w.logError(nil, fmt.Sprintf("closing #%d: %v", number, err))
		}
	}
}

func (w *Work) closeMilestone() {
	closed := "closed"
	_, _, err := githubClient.Issues.EditMilestone(context.TODO(), projectOwner, projectRepo, w.Milestone.GetNumber(), &github.Milestone{
		State: &closed,
	})
	if err != nil {
		w.logError(nil, fmt.Sprintf("closing milestone: %v", err))
	}

}

func findGithubComment(number int, prefix string) *github.IssueComment {
	list, _ := listIssueComments(number)
	for _, com := range list {
		if strings.HasPrefix(com.GetBody(), prefix) {
			return com
		}
	}
	return nil
}

func updateGithubComment(number int, com *github.IssueComment, body string) error {
	_, _, err := githubClient.Issues.EditComment(context.TODO(), projectOwner, projectRepo, number, &github.IssueComment{
		ID:   com.ID,
		Body: &body,
	})
	return err
}

func postGithubComment(number int, body string) error {
	_, _, err := githubClient.Issues.CreateComment(context.TODO(), projectOwner, projectRepo, number, &github.IssueComment{
		Body: &body,
	})
	return err
}
<commit_msg>cmd/releasebot: fix usage of GitHub client EditComment<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"

	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
)

const (
	projectOwner = "golang"
	projectRepo  = "go"
)

var githubClient *github.Client

// GitHub personal access token, from https://github.com/settings/applications.
var githubAuthToken string

func loadGithubAuth() {
	const short = ".github-issue-token"
	filename := filepath.Clean(os.Getenv("HOME") + "/" + short)
	shortFilename := filepath.Clean("$HOME/" + short)
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Fatal("reading token: ", err, "\n\n"+
			"Please create a personal access token at https://github.com/settings/tokens/new\n"+
			"and write it to ", shortFilename, " to use this program.\n"+
			"The token only needs the repo scope, or private_repo if you want to\n"+
			"view or edit issues for private repositories.\n"+
			"The benefit of using a personal access token over using your GitHub\n"+
			"password directly is that you can limit its use and revoke it at any time.\n\n")
	}
	fi, err := os.Stat(filename)
	if fi.Mode()&0077 != 0 {
		log.Fatalf("reading token: %s mode is %#o, want %#o", shortFilename, fi.Mode()&0777, fi.Mode()&0700)
	}
	githubAuthToken = strings.TrimSpace(string(data))
	t := &oauth2.Transport{
		Source: &tokenSource{AccessToken: githubAuthToken},
	}
	githubClient = github.NewClient(&http.Client{Transport: t})
}

// releaseStatusTitle returns the title of the release status issue
// for the given milestone.
// If you change this function, releasebot will not be able to find an
// existing tracking issue using the old name and will create a new one.
func releaseStatusTitle(m *github.Milestone) string {
	return "all: " + strings.Replace(m.GetTitle(), "Go", "Go ", -1) + " release status"
}

type tokenSource oauth2.Token

func (t *tokenSource) Token() (*oauth2.Token, error) {
	return (*oauth2.Token)(t), nil
}

func loadMilestones() ([]*github.Milestone, error) {
	// NOTE(rsc): There appears to be no paging possible.
	all, _, err := githubClient.Issues.ListMilestones(context.TODO(), projectOwner, projectRepo, &github.MilestoneListOptions{
		State: "open",
	})
	if err != nil {
		return nil, err
	}
	if all == nil {
		all = []*github.Milestone{}
	}
	return all, nil
}

// findIssues finds all the issues for the given milestone and
// categorizes them into approved cherry-picks (w.Picks)
// and other issues (w.OtherIssues).
// It also finds the release summary issue (w.ReleaseIssue).
func (w *Work) findIssues() {
	issues, err := listRepoIssues(github.IssueListByRepoOptions{
		Milestone: fmt.Sprint(w.Milestone.GetNumber()),
	})
	if err != nil {
		w.log.Panic(err)
	}

	for _, issue := range issues {
		if issue.GetTitle() == releaseStatusTitle(w.Milestone) {
			if w.ReleaseIssue != nil {
				w.log.Printf("**warning**: multiple release issues: #%d and #%d\n", w.ReleaseIssue.GetNumber(), issue.GetNumber())
				continue
			}
			w.ReleaseIssue = issue
			continue
		}
		if hasLabel(issue, "cherry-pick-approved") {
			w.Picks = append(w.Picks, issue)
			continue
		}
		w.OtherIssues = append(w.OtherIssues, issue)
	}
	sort.Slice(w.Picks, func(i, j int) bool { return w.Picks[i].GetNumber() < w.Picks[j].GetNumber() })

	if w.ReleaseIssue == nil {
		title := releaseStatusTitle(w.Milestone)
		body := wrapStatus(w.Milestone, "Nothing yet.")
		req := &github.IssueRequest{
			Title:     &title,
			Body:      &body,
			Milestone: w.Milestone.Number,
		}
		issue, _, err := githubClient.Issues.Create(context.TODO(), projectOwner, projectRepo, req)
		if err != nil {
			w.log.Panic(err)
		}
		w.ReleaseIssue = issue
	}
}

// listRepoIssues wraps Issues.ListByRepo to deal with paging.
func listRepoIssues(opt github.IssueListByRepoOptions) ([]*github.Issue, error) {
	var all []*github.Issue
	for page := 1; ; {
		xopt := opt
		xopt.ListOptions = github.ListOptions{
			Page:    page,
			PerPage: 100,
		}
		list, resp, err := githubClient.Issues.ListByRepo(context.TODO(), projectOwner, projectRepo, &xopt)
		all = append(all, list...)
		if err != nil {
			return all, err
		}
		if resp.NextPage < page {
			break
		}
		page = resp.NextPage
	}
	return all, nil
}

// hasLabel reports whether issue has the given label.
func hasLabel(issue *github.Issue, label string) bool {
	for _, l := range issue.Labels {
		if l.GetName() == label {
			return true
		}
	}
	return false
}

var clOK = regexp.MustCompile(`(?i)^CL (\d+) OK(( for Go \d+\.\d+\.\d+)?.*)`)
var afterCL = regexp.MustCompile(`(?i)after CL (\d+)`)

// listIssueComments wraps Issues.ListComments to deal with paging.
func listIssueComments(number int) ([]*github.IssueComment, error) {
	var all []*github.IssueComment
	for page := 1; ; {
		list, resp, err := githubClient.Issues.ListComments(context.TODO(), projectOwner, projectRepo, number, &github.IssueListCommentsOptions{
			ListOptions: github.ListOptions{
				Page:    page,
				PerPage: 100,
			},
		})
		all = append(all, list...)
		if err != nil {
			return all, err
		}
		if resp.NextPage < page {
			break
		}
		page = resp.NextPage
	}
	return all, nil
}

func (w *Work) findCLs() {
	// Preload all CLs in parallel.
	type comments struct {
		list []*github.IssueComment
		err  error
	}
	preload := make([]comments, len(w.Picks))
	var wg sync.WaitGroup
	for i, pick := range w.Picks {
		i := i
		number := pick.GetNumber()
		wg.Add(1)
		go func() {
			defer wg.Done()
			list, err := listIssueComments(number)
			preload[i] = comments{list, err}
		}()
	}
	wg.Wait()

	var cls []*CL
	for i, pick := range w.Picks {
		number := pick.GetNumber()
		fmt.Printf("load #%d\n", number)
		found := false
		list, err := preload[i].list, preload[i].err
		if err != nil {
			w.log.Panic(err)
		}
		var last *CL
		for _, com := range list {
			user := com.User.GetLogin()
			text := com.GetBody()
			for _, line := range strings.Split(text, "\n") {
				if m := clOK.FindStringSubmatch(line); m != nil {
					if m[3] != " for Go "+strings.TrimPrefix(w.Milestone.GetTitle(), "Go") {
						w.log.Printf("#%d: %s: wrong milestone: %s\n", number, user, line)
						continue
					}
					if !githubCherryPickApprovers[user] {
						w.log.Printf("#%d: %s: not an approver: %s\n", number, user, line)
						continue
					}
					n, err := strconv.Atoi(m[1])
					if err != nil {
						w.log.Printf("#%d: %s: invalid CL number: %s\n", number, user, line)
						continue
					}
					cl := &CL{Num: n, Approver: user, Issues: []int{number}}
					if last != nil {
						cl.Prereq = []int{last.Num}
					}
					for _, am := range afterCL.FindAllStringSubmatch(m[2], -1) {
						n, err := strconv.Atoi(am[1])
						if err != nil {
							w.log.Printf("#%d: %s: invalid after CL number: %s\n", number, user, line)
							continue
						}
						cl.Prereq = append(cl.Prereq, n)
					}
					cls = append(cls, cl)
					found = true
					last = cl
				}
			}
		}
		if !found {
			log.Printf("#%d: has cherry-pick-approved label but no approvals found", number)
		}
	}

	sort.Slice(cls, func(i, j int) bool {
		return cls[i].Num < cls[j].Num || cls[i].Num == cls[j].Num && cls[i].Approver < cls[j].Approver
	})

	out := cls[:0]
	var last CL
	for _, cl := range cls {
		if cl.Num == last.Num {
			end := out[len(out)-1]
			if cl.Approver != last.Approver {
				end.Approver += "," + cl.Approver
			}
			end.Issues = append(end.Issues, cl.Issues...)
			end.Prereq = append(end.Prereq, cl.Prereq...)
		} else {
			out = append(out, cl)
		}
		last = *cl
	}
	w.CLs = out
}

func (w *Work) closeIssues() {
	all := append(w.Picks[:len(w.Picks):len(w.Picks)], w.ReleaseIssue)
	for _, issue := range all {
		if issue.GetState() == "closed" {
			continue
		}
		number := issue.GetNumber()
		var md bytes.Buffer
		fmt.Fprintf(&md, "%s has been packaged and includes:\n\n", w.Version)
		for _, cl := range w.CLs {
			match := issue == w.ReleaseIssue
			for _, n := range cl.Issues {
				if n == number {
					match = true
					break
				}
			}
			if match {
				fmt.Fprintf(&md, "  - %s %s\n", mdChangeLink(cl.Num), mdEscape(cl.Title))
			}
		}
		fmt.Fprintf(&md, "\nThe release is posted at [golang.org/dl](https://golang.org/dl).\n")
		md.WriteString(signature())
		postGithubComment(number, md.String())
		closed := "closed"
		_, _, err := githubClient.Issues.Edit(context.TODO(), projectOwner, projectRepo, number, &github.IssueRequest{
			State: &closed,
		})
		if err != nil {
			w.logError(nil, fmt.Sprintf("closing #%d: %v", number, err))
		}
	}
}

func (w *Work) closeMilestone() {
	closed := "closed"
	_, _, err := githubClient.Issues.EditMilestone(context.TODO(), projectOwner, projectRepo, w.Milestone.GetNumber(), &github.Milestone{
		State: &closed,
	})
	if err != nil {
		w.logError(nil, fmt.Sprintf("closing milestone: %v", err))
	}

}

func findGithubComment(number int, prefix string) *github.IssueComment {
	list, _ := listIssueComments(number)
	for _, com := range list {
		if strings.HasPrefix(com.GetBody(), prefix) {
			return com
		}
	}
	return nil
}

func updateGithubComment(number int, com *github.IssueComment, body string) error {
	_, _, err := githubClient.Issues.EditComment(context.TODO(), projectOwner, projectRepo, int64(number), &github.IssueComment{
		ID:   com.ID,
		Body: &body,
	})
	return err
}

func postGithubComment(number int, body string) error {
	_, _, err := githubClient.Issues.CreateComment(context.TODO(), projectOwner, projectRepo, number, &github.IssueComment{
		Body: &body,
	})
	return err
}
<|endoftext|>


<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"
	"strconv"
)

var port = flag.Int("port", 8000, "port")

func main() {
	fmt.Fprintf(os.Stderr, "Hello on stderr\n")
	fmt.Fprintf(os.Stdout, "Hello on stdout\n")
	http.HandleFunc("/crash", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(os.Stderr, "crashing")
		os.Exit(2)
	})
	http.ListenAndServe(":"+strconv.Itoa(*port), nil)
}
<commit_msg>crash and status handlers<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strconv"
	"time"
)

var port = flag.Int("port", 8000, "port")

func crashHandler(w http.ResponseWriter, r *http.Request) {
	status := 2
	if st := r.FormValue("status"); st != "" {
		status, _ = strconv.Atoi(st)
	}
	fmt.Fprintf(os.Stderr, "crashing with status %d", status)
	os.Exit(status)
}

func statusHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	fmt.Fprintf(w, "pid=%d\n", os.Getpid())
	cwd, _ := os.Getwd()
	fmt.Fprintf(w, "cwd=%s\n", cwd)
	fmt.Fprintf(w, "uid=%d\n", os.Getuid())
	fmt.Fprintf(w, "euid=%d\n", os.Geteuid())
	fmt.Fprintf(w, "gid=%d\n", os.Getgid())

	for _, env := range os.Environ() {
		fmt.Fprintf(w, "%s\n", env)
	}

}

func logNoise() {
	for {
		log.Printf("some log noise")
		time.Sleep(1 * time.Second)
	}
}

func main() {
	fmt.Fprintf(os.Stdout, "Hello on stdout; listening on port %d\n", *port)
	fmt.Fprintf(os.Stderr, "Hello on stderr\n")
	go logNoise()
	http.HandleFunc("/crash", crashHandler)
	http.HandleFunc("/", statusHandler)
	http.ListenAndServe(":"+strconv.Itoa(*port), nil)
}
<|endoftext|>


<commit_before>package atom

import (
	"bytes"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"strings"
	"time"

	"golang.org/x/net/html"
)

// CategoryはAtom文書におけるCategory要素をあらわす。
type Category struct {
	Term   string `xml:"term,attr"`
	Scheme string `xml:"scheme,attr,omitempty"`
	Label  string `xml:"label,attr,omitempty"`
}

// TextはAtom文書におけるTextコンストラクトをあらわす。
type Text struct {
	Type    string `xml:"type,attr"`
	Content string `xml:",chardata"`
}

// IsZeroはtが空だった場合にtrueを返す。
func (t Text) IsZero() bool {
	return t.Content == ""
}

func buildPlain(n *html.Node) (s string, err error) {
	buf := new(bytes.Buffer)
	err = html.Render(buf, n)
	if err != nil {
		return
	}
	s = buf.String()
	return
}

func (t Text) Plain() (s string, err error) {
	switch t.Type {
	case "html", "xhtml":
		err = errors.New("not implement")
	case "text":
		s = t.Content
	default:
		s = t.Content
	}
	return
}

func (t Text) HTML() (s string, err error) {
	switch t.Type {
	case "html":
		t := html.UnescapeString(t.Content)
		s = fmt.Sprintf("
%s
", t)
	case "xhtml":
		r := strings.NewReader(t.Content)
		tokenizer := html.NewTokenizer(r)
		err = nextToken(tokenizer)
		if err != nil {
			return
		}
		s, err = buildHTML(tokenizer)
	case "text":
		s = fmt.Sprintf("
%s
", t.Content)
	default:
		s = fmt.Sprintf("
%s
", t.Content)
	}
	return
}

func nextToken(tokenizer *html.Tokenizer) error {
	if t := tokenizer.Next(); t == html.ErrorToken {
		return tokenizer.Err()
	}
	return nil
}

func buildHTML(tokenizer *html.Tokenizer) (s string, err error) {
	buf := new(bytes.Buffer)

	bp := 0
	if tag, _ := tokenizer.TagName(); string(tag) == "div" {
		div := tokenizer.Raw()
		buf.Write(div)
		bp = len(div)
		err = nextToken(tokenizer)
	}

	ep := bp
	for err != io.EOF {
		if err != nil && err != io.EOF {
			return
		}
		ep = buf.Len()
		b := tokenizer.Raw()
		if _, err := buf.Write(b); err != nil {
			return "", err
		}
		err = nextToken(tokenizer)
	}
	b := buf.Bytes()
	if bp > 0 {
		b = b[bp:ep]
	}
	return string(b), nil
}

// PersonはAtom文書におけるPersonコンストラクトをあらわす。
type Person struct {
	Name  string `xml:"name"`
	URL   string `xml:"uri,omitempty"`
	Email string `xml:"email,omitempty"`
}

// LinkはAtom文書におけるLinkコンストラクトをあらわす。
type Link struct {
	Rel  string `xml:"rel,attr,omitempty"`
	Type string `xml:"type,attr,omitempty"`
	URL  string `xml:"href,attr"`
}

// FeedはAtom文書におけるFeed要素をあらわす。
type Feed struct {
	XMLName xml.Name `xml:"feed"`

	//Version string `xml:"version,attr"`
	//Lang string `xml:"lang,attr,omitempty"`

	//Contributors []Person `xml:"contributor,omitempty"`
	//Generator Generator `xml:"generator,omitempty"`
	//Icon string `xml:"icon,omitempty"`
	//Logo string `xml:"logo,omitempty"`

	Title      Text       `xml:"title"`
	Subtitle   Text       `xml:"subtitle,omitempty"`
	Links      []Link     `xml:"link"`
	Authors    []Person   `xml:"author"`
	ID         string     `xml:"id"`
	Rights     Text       `xml:"rights,omitempty"`
	Updated    time.Time  `xml:"updated"`
	Summary    string     `xml:"summary,omitempty"`
	Categories []Category `xml:"category,omitempty"`
	Entries    []*Entry   `xml:"entry"`
}

func (feed *Feed) AlternateURL() string {
	return alternateURL(feed.Links)
}

// EntryはAtom文書におけるEntry要素をあらわす。
type Entry struct {
	//Contributors []Person `xml:"contributor,omitempty"`
	//Created time.Time `xml:"created,omitempty"?
	//Source Link?

	Title      Text       `xml:"title"`
	Links      []Link     `xml:"link,omitempty"`
	Authors    []Person   `xml:"author,omitempty"`
	Categories []Category `xml:"category,omitempty"`
	ID         string     `xml:"id"`
	Updated    time.Time  `xml:"updated"`
	Published  time.Time  `xml:"published,omitempty"`
	Rights     Text       `xml:"rights,omitempty"`
	Summary    Text       `xml:"summary,omitempty"`
	Content    Text       `xml:"content,omitempty"`

	// atom 0.3 compatibility
	Modified time.Time `xml:"modified,omitempty"`
	Issued time.Time `xml:"issued,omitempty"`
}

func (entry *Entry) Article() string {
	return ""
}

func (entry *Entry) AlternateURL() string {
	return alternateURL(entry.Links)
}

func (entry *Entry) PublishedTime() time.Time {
	if !entry.Published.IsZero() {
		return entry.Published
	}
	return entry.Issued
}

func (entry *Entry) UpdatedTime() time.Time {
	if !entry.Updated.IsZero() {
		return entry.Updated
	}
	return entry.Modified
}

func alternateURL(links []Link) string {
	for _, link := range links {
		if link.Rel == "alternate" {
			return link.URL
		}
	}
	return ""
}

func Parse(r io.Reader) (feed *Feed, err error) {
	var x Feed
	d := xml.NewDecoder(r)
	err = d.Decode(&x)
	if err != nil {
		return
	}
	feed = &x
	return
}
<commit_msg>Fix format<commit_after>package atom

import (
	"bytes"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"strings"
	"time"

	"golang.org/x/net/html"
)

// CategoryはAtom文書におけるCategory要素をあらわす。
type Category struct {
	Term   string `xml:"term,attr"`
	Scheme string `xml:"scheme,attr,omitempty"`
	Label  string `xml:"label,attr,omitempty"`
}

// TextはAtom文書におけるTextコンストラクトをあらわす。
type Text struct {
	Type    string `xml:"type,attr"`
	Content string `xml:",chardata"`
}

// IsZeroはtが空だった場合にtrueを返す。
func (t Text) IsZero() bool {
	return t.Content == ""
}

func buildPlain(n *html.Node) (s string, err error) {
	buf := new(bytes.Buffer)
	err = html.Render(buf, n)
	if err != nil {
		return
	}
	s = buf.String()
	return
}

func (t Text) Plain() (s string, err error) {
	switch t.Type {
	case "html", "xhtml":
		err = errors.New("not implement")
	case "text":
		s = t.Content
	default:
		s = t.Content
	}
	return
}

func (t Text) HTML() (s string, err error) {
	switch t.Type {
	case "html":
		t := html.UnescapeString(t.Content)
		s = fmt.Sprintf("
%s
", t)
	case "xhtml":
		r := strings.NewReader(t.Content)
		tokenizer := html.NewTokenizer(r)
		err = nextToken(tokenizer)
		if err != nil {
			return
		}
		s, err = buildHTML(tokenizer)
	case "text":
		s = fmt.Sprintf("
%s
", t.Content)
	default:
		s = fmt.Sprintf("
%s
", t.Content)
	}
	return
}

func nextToken(tokenizer *html.Tokenizer) error {
	if t := tokenizer.Next(); t == html.ErrorToken {
		return tokenizer.Err()
	}
	return nil
}

func buildHTML(tokenizer *html.Tokenizer) (s string, err error) {
	buf := new(bytes.Buffer)

	bp := 0
	if tag, _ := tokenizer.TagName(); string(tag) == "div" {
		div := tokenizer.Raw()
		buf.Write(div)
		bp = len(div)
		err = nextToken(tokenizer)
	}

	ep := bp
	for err != io.EOF {
		if err != nil && err != io.EOF {
			return
		}
		ep = buf.Len()
		b := tokenizer.Raw()
		if _, err := buf.Write(b); err != nil {
			return "", err
		}
		err = nextToken(tokenizer)
	}
	b := buf.Bytes()
	if bp > 0 {
		b = b[bp:ep]
	}
	return string(b), nil
}

// PersonはAtom文書におけるPersonコンストラクトをあらわす。
type Person struct {
	Name  string `xml:"name"`
	URL   string `xml:"uri,omitempty"`
	Email string `xml:"email,omitempty"`
}

// LinkはAtom文書におけるLinkコンストラクトをあらわす。
type Link struct {
	Rel  string `xml:"rel,attr,omitempty"`
	Type string `xml:"type,attr,omitempty"`
	URL  string `xml:"href,attr"`
}

// FeedはAtom文書におけるFeed要素をあらわす。
type Feed struct {
	XMLName xml.Name `xml:"feed"`

	//Version string `xml:"version,attr"`
	//Lang string `xml:"lang,attr,omitempty"`

	//Contributors []Person `xml:"contributor,omitempty"`
	//Generator Generator `xml:"generator,omitempty"`
	//Icon string `xml:"icon,omitempty"`
	//Logo string `xml:"logo,omitempty"`

	Title      Text       `xml:"title"`
	Subtitle   Text       `xml:"subtitle,omitempty"`
	Links      []Link     `xml:"link"`
	Authors    []Person   `xml:"author"`
	ID         string     `xml:"id"`
	Rights     Text       `xml:"rights,omitempty"`
	Updated    time.Time  `xml:"updated"`
	Summary    string     `xml:"summary,omitempty"`
	Categories []Category `xml:"category,omitempty"`
	Entries    []*Entry   `xml:"entry"`
}

func (feed *Feed) AlternateURL() string {
	return alternateURL(feed.Links)
}

// EntryはAtom文書におけるEntry要素をあらわす。
type Entry struct {
	//Contributors []Person `xml:"contributor,omitempty"`
	//Created time.Time `xml:"created,omitempty"?
	//Source Link?

	Title      Text       `xml:"title"`
	Links      []Link     `xml:"link,omitempty"`
	Authors    []Person   `xml:"author,omitempty"`
	Categories []Category `xml:"category,omitempty"`
	ID         string     `xml:"id"`
	Updated    time.Time  `xml:"updated"`
	Published  time.Time  `xml:"published,omitempty"`
	Rights     Text       `xml:"rights,omitempty"`
	Summary    Text       `xml:"summary,omitempty"`
	Content    Text       `xml:"content,omitempty"`

	// atom 0.3 compatibility
	Modified time.Time `xml:"modified,omitempty"`
	Issued   time.Time `xml:"issued,omitempty"`
}

func (entry *Entry) Article() string {
	return ""
}

func (entry *Entry) AlternateURL() string {
	return alternateURL(entry.Links)
}

func (entry *Entry) PublishedTime() time.Time {
	if !entry.Published.IsZero() {
		return entry.Published
	}
	return entry.Issued
}

func (entry *Entry) UpdatedTime() time.Time {
	if !entry.Updated.IsZero() {
		return entry.Updated
	}
	return entry.Modified
}

func alternateURL(links []Link) string {
	for _, link := range links {
		if link.Rel == "alternate" {
			return link.URL
		}
	}
	return ""
}

func Parse(r io.Reader) (feed *Feed, err error) {
	var x Feed
	d := xml.NewDecoder(r)
	err = d.Decode(&x)
	if err != nil {
		return
	}
	feed = &x
	return
}
<|endoftext|>


<commit_before>package cluster

import (
	"engine"
	"errors"
	"fmt"
	"parser"
	"protocol"
	"sort"
	"time"
	"wal"
)

// A shard imements an interface for writing and querying data.
// It can be copied to multiple servers or the local datastore.
// Shard contains data from [startTime, endTime)
// Ids are unique across the cluster
type Shard interface {
	Id() uint32
	StartTime() time.Time
	EndTime() time.Time
	Write(*protocol.Request) error
	Query(querySpec *parser.QuerySpec, response chan *protocol.Response) error
	IsMicrosecondInRange(t int64) bool
}

// Passed to a shard (local datastore or whatever) that gets yielded points from series.
type QueryProcessor interface {
	// This method returns true if the query should continue. If the query should be stopped,
	// like maybe the limit was hit, it should return false
	YieldPoint(seriesName *string, columnNames []string, point *protocol.Point) bool
	Close()
}

type NewShardData struct {
	Id            uint32 `json:",omitempty"`
	StartTime     time.Time
	EndTime       time.Time
	ServerIds     []uint32
	Type          ShardType
	DurationSplit bool `json:",omitempty"`
}

type ShardType int

const (
	LONG_TERM ShardType = iota
	SHORT_TERM
)

type ShardData struct {
	id              uint32
	startTime       time.Time
	startMicro      int64
	endMicro        int64
	endTime         time.Time
	wal             WAL
	servers         []wal.Server
	clusterServers  []*ClusterServer
	store           LocalShardStore
	localShard      LocalShardDb
	serverIds       []uint32
	shardType       ShardType
	durationIsSplit bool
	shardDuration   time.Duration
	localServerId   uint32
}

func NewShard(id uint32, startTime, endTime time.Time, shardType ShardType, durationIsSplit bool, wal WAL) *ShardData {
	return &ShardData{
		id:              id,
		startTime:       startTime,
		endTime:         endTime,
		wal:             wal,
		startMicro:      startTime.Unix() * int64(1000*1000),
		endMicro:        endTime.Unix() * int64(1000*1000),
		serverIds:       make([]uint32, 0),
		shardType:       shardType,
		durationIsSplit: durationIsSplit,
		shardDuration:   endTime.Sub(startTime),
	}
}

const (
	PER_SERVER_BUFFER_SIZE  = 10
	LOCAL_WRITE_BUFFER_SIZE = 10
)

var (
	queryResponse       = protocol.Response_QUERY
	endStreamResponse   = protocol.Response_END_STREAM
	queryRequest        = protocol.Request_QUERY
	dropDatabaseRequest = protocol.Request_DROP_DATABASE
)

type LocalShardDb interface {
	Write(database string, series *protocol.Series) error
	Query(*parser.QuerySpec, QueryProcessor) error
	DropDatabase(database string) error
}

type LocalShardStore interface {
	Write(request *protocol.Request) error
	SetWriteBuffer(writeBuffer *WriteBuffer)
	BufferWrite(request *protocol.Request)
	GetOrCreateShard(id uint32) (LocalShardDb, error)
	DeleteShard(shardId uint32) error
}

func (self *ShardData) Id() uint32 {
	return self.id
}

func (self *ShardData) StartTime() time.Time {
	return self.startTime
}

func (self *ShardData) EndTime() time.Time {
	return self.endTime
}

func (self *ShardData) IsMicrosecondInRange(t int64) bool {
	return t >= self.startMicro && t < self.endMicro
}

func (self *ShardData) SetServers(servers []*ClusterServer) {
	self.clusterServers = servers
	self.servers = make([]wal.Server, len(servers), len(servers))
	for i, server := range servers {
		self.serverIds = append(self.serverIds, server.Id)
		self.servers[i] = server
	}
	self.sortServerIds()
}

func (self *ShardData) SetLocalStore(store LocalShardStore, localServerId uint32) error {
	self.serverIds = append(self.serverIds, localServerId)
	self.localServerId = localServerId
	self.sortServerIds()

	self.store = store
	shard, err := self.store.GetOrCreateShard(self.id)
	if err != nil {
		return err
	}
	self.localShard = shard

	return nil
}

func (self *ShardData) IsLocal() bool {
	return self.store != nil
}

func (self *ShardData) ServerIds() []uint32 {
	return self.serverIds
}

func (self *ShardData) Write(request *protocol.Request) error {
	fmt.Println("SHARD Write: ", self.id, request)
	request.ShardId = &self.id
	requestNumber, err := self.wal.AssignSequenceNumbersAndLog(request, self)
	if err != nil {
		return err
	}
	request.RequestNumber = &requestNumber
	if self.store != nil {
		self.store.BufferWrite(request)
	}
	for _, server := range self.clusterServers {
		server.BufferWrite(request)
	}
	return nil
}

func (self *ShardData) WriteLocalOnly(request *protocol.Request) error {
	requestNumber, err := self.wal.AssignSequenceNumbersAndLog(request, self)
	if err != nil {
		return err
	}
	request.RequestNumber = &requestNumber
	self.store.BufferWrite(request)
	return nil
}

func (self *ShardData) Query(querySpec *parser.QuerySpec, response chan *protocol.Response) error {
	// This is only for queries that are deletes or drops. They need to be sent everywhere as opposed to just the local or one of the remote shards.
	// But this boolean should only be set to true on the server that receives the initial query.
	if querySpec.RunAgainstAllServersInShard {
		if querySpec.IsDeleteFromSeriesQuery() {
			return self.logAndHandleDeleteQuery(querySpec, response)
		} else if querySpec.IsDropSeriesQuery() {
			return self.logAndHandleDropSeriesQuery(querySpec, response)
		}
	}

	if self.localShard != nil {
		var processor QueryProcessor
		if querySpec.IsListSeriesQuery() {
			processor = engine.NewListSeriesEngine(response)
		} else if querySpec.IsDeleteFromSeriesQuery() || querySpec.IsDropSeriesQuery() {
			maxDeleteResults := 10000
			processor = engine.NewPassthroughEngine(response, maxDeleteResults)
		} else {
			if self.ShouldAggregateLocally(querySpec) {
				fmt.Println("SHARD: query aggregate locally", self.id)
				processor = engine.NewQueryEngine(querySpec.SelectQuery(), response)
			} else {
				fmt.Println("SHARD: query passthrough", self.id)
				maxPointsToBufferBeforeSending := 1000
				processor = engine.NewPassthroughEngine(response, maxPointsToBufferBeforeSending)
			}
		}
		fmt.Println("SHARD query local: ", self.id)
		err := self.localShard.Query(querySpec, processor)
		fmt.Println("SHARD: processor.Close()", self.id)
		processor.Close()
		return err
	}

	healthyServers := make([]*ClusterServer, 0, len(self.clusterServers))
	for _, s := range self.clusterServers {
		if !s.IsUp() {
			continue
		}
		healthyServers = append(healthyServers, s)
	}
	healthyCount := len(healthyServers)
	if healthyCount == 0 {
		message := fmt.Sprintf("No servers up to query shard %d", self.id)
		response <- &protocol.Response{Type: &endStreamResponse, ErrorMessage: &message}
		return errors.New(message)
	}
	randServerIndex := int(time.Now().UnixNano() % int64(healthyCount))
	server := healthyServers[randServerIndex]
	request := self.createRequest(querySpec)

	return server.MakeRequest(request, response)
}

func (self *ShardData) DropDatabase(database string, sendToServers bool) {
	if self.localShard != nil {
		fmt.Println("SHARD DropDatabase: ", database)
		self.localShard.DropDatabase(database)
	}

	if !sendToServers {
		return
	}

	responses := make([]chan *protocol.Response, len(self.clusterServers), len(self.clusterServers))
	for i, server := range self.clusterServers {
		responseChan := make(chan *protocol.Response, 1)
		responses[i] = responseChan
		request := &protocol.Request{Type: &dropDatabaseRequest, Database: &database, ShardId: &self.id}
		go server.MakeRequest(request, responseChan)
	}
	for _, responseChan := range responses {
		// TODO: handle error responses
		<-responseChan
	}
}

func (self *ShardData) ShouldAggregateLocally(querySpec *parser.QuerySpec) bool {
	if self.durationIsSplit && querySpec.ReadsFromMultipleSeries() {
		return false
	}
	groupByInterval := querySpec.GetGroupByInterval()
	if groupByInterval == nil {
		if querySpec.HasAggregates() {
			return false
		}
		return true
	}
	if self.shardDuration%*groupByInterval == 0 {
		return true
	}
	return false
}

func (self *ShardData) logAndHandleDeleteQuery(querySpec *parser.QuerySpec, response chan *protocol.Response) error {
	queryString := querySpec.GetQueryStringWithTimeCondition()
	request := self.createRequest(querySpec)
	request.Query = &queryString
	return self.LogAndHandleDestructiveQuery(querySpec, request, response, false)
}

func (self *ShardData) logAndHandleDropSeriesQuery(querySpec *parser.QuerySpec, response chan *protocol.Response) error {
	return self.LogAndHandleDestructiveQuery(querySpec, self.createRequest(querySpec), response, false)
}

func (self *ShardData) LogAndHandleDestructiveQuery(querySpec *parser.QuerySpec, request *protocol.Request, response chan *protocol.Response, runLocalOnly bool) error {
	fmt.Println("logAndHandleDestructiveQuery")
	requestNumber, err := self.wal.AssignSequenceNumbersAndLog(request, self)
	if err != nil {
		return err
	}
	var localResponses chan *protocol.Response
	if self.localShard != nil {
		localResponses = make(chan *protocol.Response, 1)

		// this doesn't really apply at this point since destructive queries don't output anything, but it may later
		maxPointsFromDestructiveQuery := 1000
		processor := engine.NewPassthroughEngine(localResponses, maxPointsFromDestructiveQuery)
		err := self.localShard.Query(querySpec, processor)
		processor.Close()
		if err != nil {
			return err
		}
	}
	if !runLocalOnly {
		responses := make([]chan *protocol.Response, len(self.clusterServers), len(self.clusterServers))
		for i, server := range self.clusterServers {
			fmt.Println("SHARD: requesting to server: ", server.Id)
			responseChan := make(chan *protocol.Response, 1)
			responses[i] = responseChan
			// do this so that a new id will get assigned
			request.Id = nil
			server.MakeRequest(request, responseChan)
		}
		for i, responseChan := range responses {
			for {
				res := <-responseChan
				if *res.Type == endStreamResponse {
					self.wal.Commit(requestNumber, self.clusterServers[i].Id)
					break
				}
				response <- res
			}
		}
	}

	if localResponses != nil {
		for {
			res := <-localResponses
			if *res.Type == endStreamResponse {
				self.wal.Commit(requestNumber, self.localServerId)
				break
			}
			response <- res
		}
	}

	response <- &protocol.Response{Type: &endStreamResponse}
	return nil
}

func (self *ShardData) createRequest(querySpec *parser.QuerySpec) *protocol.Request {
	queryString := querySpec.GetQueryString()
	user := querySpec.User()
	userName := user.GetName()
	database := querySpec.Database()
	isDbUser := !user.IsClusterAdmin()

	return &protocol.Request{
		Type:     &queryRequest,
		ShardId:  &self.id,
		Query:    &queryString,
		UserName: &userName,
		Database: &database,
		IsDbUser: &isDbUser,
	}
}

// used to serialize shards when sending around in raft or when snapshotting in the log
func (self *ShardData) ToNewShardData() *NewShardData {
	return &NewShardData{
		Id:        self.id,
		StartTime: self.startTime,
		EndTime:   self.endTime,
		Type:      self.shardType,
		ServerIds: self.serverIds,
	}
}

// server ids should always be returned in sorted order
func (self *ShardData) sortServerIds() {
	serverIdInts := make([]int, len(self.serverIds), len(self.serverIds))
	for i, id := range self.serverIds {
		serverIdInts[i] = int(id)
	}
	sort.Ints(serverIdInts)
	for i, id := range serverIdInts {
		self.serverIds[i] = uint32(id)
	}
}

func SortShardsByTimeAscending(shards []*ShardData) {
	sort.Sort(ByShardTimeAsc{shards})
}

func SortShardsByTimeDescending(shards []*ShardData) {
	sort.Sort(ByShardTimeDesc{shards})
}

type ShardCollection []*ShardData

func (s ShardCollection) Len() int      { return len(s) }
func (s ShardCollection) Swap(i, j int) { s[i], s[j] = s[j], s[i] }

type ByShardTimeDesc struct{ ShardCollection }
type ByShardTimeAsc struct{ ShardCollection }

func (s ByShardTimeAsc) Less(i, j int) bool {
	if s.ShardCollection[i] != nil && s.ShardCollection[j] != nil {
		iStartTime := s.ShardCollection[i].StartTime().Unix()
		jStartTime := s.ShardCollection[j].StartTime().Unix()
		if iStartTime == jStartTime {
			return s.ShardCollection[i].Id() < s.ShardCollection[j].Id()
		}
		return iStartTime < jStartTime
	}
	return false
}
func (s ByShardTimeDesc) Less(i, j int) bool {
	if s.ShardCollection[i] != nil && s.ShardCollection[j] != nil {
		iStartTime := s.ShardCollection[i].StartTime().Unix()
		jStartTime := s.ShardCollection[j].StartTime().Unix()
		if iStartTime == jStartTime {
			return s.ShardCollection[i].Id() < s.ShardCollection[j].Id()
		}
		return iStartTime > jStartTime
	}
	return false
}
<commit_msg>remove println<commit_after>package cluster

import (
	"engine"
	"errors"
	"fmt"
	"parser"
	"protocol"
	"sort"
	"time"
	"wal"
)

// A shard imements an interface for writing and querying data.
// It can be copied to multiple servers or the local datastore.
// Shard contains data from [startTime, endTime)
// Ids are unique across the cluster
type Shard interface {
	Id() uint32
	StartTime() time.Time
	EndTime() time.Time
	Write(*protocol.Request) error
	Query(querySpec *parser.QuerySpec, response chan *protocol.Response) error
	IsMicrosecondInRange(t int64) bool
}

// Passed to a shard (local datastore or whatever) that gets yielded points from series.
type QueryProcessor interface {
	// This method returns true if the query should continue. If the query should be stopped,
	// like maybe the limit was hit, it should return false
	YieldPoint(seriesName *string, columnNames []string, point *protocol.Point) bool
	Close()
}

type NewShardData struct {
	Id            uint32 `json:",omitempty"`
	StartTime     time.Time
	EndTime       time.Time
	ServerIds     []uint32
	Type          ShardType
	DurationSplit bool `json:",omitempty"`
}

type ShardType int

const (
	LONG_TERM ShardType = iota
	SHORT_TERM
)

type ShardData struct {
	id              uint32
	startTime       time.Time
	startMicro      int64
	endMicro        int64
	endTime         time.Time
	wal             WAL
	servers         []wal.Server
	clusterServers  []*ClusterServer
	store           LocalShardStore
	localShard      LocalShardDb
	serverIds       []uint32
	shardType       ShardType
	durationIsSplit bool
	shardDuration   time.Duration
	localServerId   uint32
}

func NewShard(id uint32, startTime, endTime time.Time, shardType ShardType, durationIsSplit bool, wal WAL) *ShardData {
	return &ShardData{
		id:              id,
		startTime:       startTime,
		endTime:         endTime,
		wal:             wal,
		startMicro:      startTime.Unix() * int64(1000*1000),
		endMicro:        endTime.Unix() * int64(1000*1000),
		serverIds:       make([]uint32, 0),
		shardType:       shardType,
		durationIsSplit: durationIsSplit,
		shardDuration:   endTime.Sub(startTime),
	}
}

const (
	PER_SERVER_BUFFER_SIZE  = 10
	LOCAL_WRITE_BUFFER_SIZE = 10
)

var (
	queryResponse       = protocol.Response_QUERY
	endStreamResponse   = protocol.Response_END_STREAM
	queryRequest        = protocol.Request_QUERY
	dropDatabaseRequest = protocol.Request_DROP_DATABASE
)

type LocalShardDb interface {
	Write(database string, series *protocol.Series) error
	Query(*parser.QuerySpec, QueryProcessor) error
	DropDatabase(database string) error
}

type LocalShardStore interface {
	Write(request *protocol.Request) error
	SetWriteBuffer(writeBuffer *WriteBuffer)
	BufferWrite(request *protocol.Request)
	GetOrCreateShard(id uint32) (LocalShardDb, error)
	DeleteShard(shardId uint32) error
}

func (self *ShardData) Id() uint32 {
	return self.id
}

func (self *ShardData) StartTime() time.Time {
	return self.startTime
}

func (self *ShardData) EndTime() time.Time {
	return self.endTime
}

func (self *ShardData) IsMicrosecondInRange(t int64) bool {
	return t >= self.startMicro && t < self.endMicro
}

func (self *ShardData) SetServers(servers []*ClusterServer) {
	self.clusterServers = servers
	self.servers = make([]wal.Server, len(servers), len(servers))
	for i, server := range servers {
		self.serverIds = append(self.serverIds, server.Id)
		self.servers[i] = server
	}
	self.sortServerIds()
}

func (self *ShardData) SetLocalStore(store LocalShardStore, localServerId uint32) error {
	self.serverIds = append(self.serverIds, localServerId)
	self.localServerId = localServerId
	self.sortServerIds()

	self.store = store
	shard, err := self.store.GetOrCreateShard(self.id)
	if err != nil {
		return err
	}
	self.localShard = shard

	return nil
}

func (self *ShardData) IsLocal() bool {
	return self.store != nil
}

func (self *ShardData) ServerIds() []uint32 {
	return self.serverIds
}

func (self *ShardData) Write(request *protocol.Request) error {
	request.ShardId = &self.id
	requestNumber, err := self.wal.AssignSequenceNumbersAndLog(request, self)
	if err != nil {
		return err
	}
	request.RequestNumber = &requestNumber
	if self.store != nil {
		self.store.BufferWrite(request)
	}
	for _, server := range self.clusterServers {
		server.BufferWrite(request)
	}
	return nil
}

func (self *ShardData) WriteLocalOnly(request *protocol.Request) error {
	requestNumber, err := self.wal.AssignSequenceNumbersAndLog(request, self)
	if err != nil {
		return err
	}
	request.RequestNumber = &requestNumber
	self.store.BufferWrite(request)
	return nil
}

func (self *ShardData) Query(querySpec *parser.QuerySpec, response chan *protocol.Response) error {
	// This is only for queries that are deletes or drops. They need to be sent everywhere as opposed to just the local or one of the remote shards.
	// But this boolean should only be set to true on the server that receives the initial query.
	if querySpec.RunAgainstAllServersInShard {
		if querySpec.IsDeleteFromSeriesQuery() {
			return self.logAndHandleDeleteQuery(querySpec, response)
		} else if querySpec.IsDropSeriesQuery() {
			return self.logAndHandleDropSeriesQuery(querySpec, response)
		}
	}

	if self.localShard != nil {
		var processor QueryProcessor
		if querySpec.IsListSeriesQuery() {
			processor = engine.NewListSeriesEngine(response)
		} else if querySpec.IsDeleteFromSeriesQuery() || querySpec.IsDropSeriesQuery() {
			maxDeleteResults := 10000
			processor = engine.NewPassthroughEngine(response, maxDeleteResults)
		} else {
			if self.ShouldAggregateLocally(querySpec) {
				fmt.Println("SHARD: query aggregate locally", self.id)
				processor = engine.NewQueryEngine(querySpec.SelectQuery(), response)
			} else {
				fmt.Println("SHARD: query passthrough", self.id)
				maxPointsToBufferBeforeSending := 1000
				processor = engine.NewPassthroughEngine(response, maxPointsToBufferBeforeSending)
			}
		}
		fmt.Println("SHARD query local: ", self.id)
		err := self.localShard.Query(querySpec, processor)
		fmt.Println("SHARD: processor.Close()", self.id)
		processor.Close()
		return err
	}

	healthyServers := make([]*ClusterServer, 0, len(self.clusterServers))
	for _, s := range self.clusterServers {
		if !s.IsUp() {
			continue
		}
		healthyServers = append(healthyServers, s)
	}
	healthyCount := len(healthyServers)
	if healthyCount == 0 {
		message := fmt.Sprintf("No servers up to query shard %d", self.id)
		response <- &protocol.Response{Type: &endStreamResponse, ErrorMessage: &message}
		return errors.New(message)
	}
	randServerIndex := int(time.Now().UnixNano() % int64(healthyCount))
	server := healthyServers[randServerIndex]
	request := self.createRequest(querySpec)

	return server.MakeRequest(request, response)
}

func (self *ShardData) DropDatabase(database string, sendToServers bool) {
	if self.localShard != nil {
		fmt.Println("SHARD DropDatabase: ", database)
		self.localShard.DropDatabase(database)
	}

	if !sendToServers {
		return
	}

	responses := make([]chan *protocol.Response, len(self.clusterServers), len(self.clusterServers))
	for i, server := range self.clusterServers {
		responseChan := make(chan *protocol.Response, 1)
		responses[i] = responseChan
		request := &protocol.Request{Type: &dropDatabaseRequest, Database: &database, ShardId: &self.id}
		go server.MakeRequest(request, responseChan)
	}
	for _, responseChan := range responses {
		// TODO: handle error responses
		<-responseChan
	}
}

func (self *ShardData) ShouldAggregateLocally(querySpec *parser.QuerySpec) bool {
	if self.durationIsSplit && querySpec.ReadsFromMultipleSeries() {
		return false
	}
	groupByInterval := querySpec.GetGroupByInterval()
	if groupByInterval == nil {
		if querySpec.HasAggregates() {
			return false
		}
		return true
	}
	if self.shardDuration%*groupByInterval == 0 {
		return true
	}
	return false
}

func (self *ShardData) logAndHandleDeleteQuery(querySpec *parser.QuerySpec, response chan *protocol.Response) error {
	queryString := querySpec.GetQueryStringWithTimeCondition()
	request := self.createRequest(querySpec)
	request.Query = &queryString
	return self.LogAndHandleDestructiveQuery(querySpec, request, response, false)
}

func (self *ShardData) logAndHandleDropSeriesQuery(querySpec *parser.QuerySpec, response chan *protocol.Response) error {
	return self.LogAndHandleDestructiveQuery(querySpec, self.createRequest(querySpec), response, false)
}

func (self *ShardData) LogAndHandleDestructiveQuery(querySpec *parser.QuerySpec, request *protocol.Request, response chan *protocol.Response, runLocalOnly bool) error {
	fmt.Println("logAndHandleDestructiveQuery")
	requestNumber, err := self.wal.AssignSequenceNumbersAndLog(request, self)
	if err != nil {
		return err
	}
	var localResponses chan *protocol.Response
	if self.localShard != nil {
		localResponses = make(chan *protocol.Response, 1)

		// this doesn't really apply at this point since destructive queries don't output anything, but it may later
		maxPointsFromDestructiveQuery := 1000
		processor := engine.NewPassthroughEngine(localResponses, maxPointsFromDestructiveQuery)
		err := self.localShard.Query(querySpec, processor)
		processor.Close()
		if err != nil {
			return err
		}
	}
	if !runLocalOnly {
		responses := make([]chan *protocol.Response, len(self.clusterServers), len(self.clusterServers))
		for i, server := range self.clusterServers {
			fmt.Println("SHARD: requesting to server: ", server.Id)
			responseChan := make(chan *protocol.Response, 1)
			responses[i] = responseChan
			// do this so that a new id will get assigned
			request.Id = nil
			server.MakeRequest(request, responseChan)
		}
		for i, responseChan := range responses {
			for {
				res := <-responseChan
				if *res.Type == endStreamResponse {
					self.wal.Commit(requestNumber, self.clusterServers[i].Id)
					break
				}
				response <- res
			}
		}
	}

	if localResponses != nil {
		for {
			res := <-localResponses
			if *res.Type == endStreamResponse {
				self.wal.Commit(requestNumber, self.localServerId)
				break
			}
			response <- res
		}
	}

	response <- &protocol.Response{Type: &endStreamResponse}
	return nil
}

func (self *ShardData) createRequest(querySpec *parser.QuerySpec) *protocol.Request {
	queryString := querySpec.GetQueryString()
	user := querySpec.User()
	userName := user.GetName()
	database := querySpec.Database()
	isDbUser := !user.IsClusterAdmin()

	return &protocol.Request{
		Type:     &queryRequest,
		ShardId:  &self.id,
		Query:    &queryString,
		UserName: &userName,
		Database: &database,
		IsDbUser: &isDbUser,
	}
}

// used to serialize shards when sending around in raft or when snapshotting in the log
func (self *ShardData) ToNewShardData() *NewShardData {
	return &NewShardData{
		Id:        self.id,
		StartTime: self.startTime,
		EndTime:   self.endTime,
		Type:      self.shardType,
		ServerIds: self.serverIds,
	}
}

// server ids should always be returned in sorted order
func (self *ShardData) sortServerIds() {
	serverIdInts := make([]int, len(self.serverIds), len(self.serverIds))
	for i, id := range self.serverIds {
		serverIdInts[i] = int(id)
	}
	sort.Ints(serverIdInts)
	for i, id := range serverIdInts {
		self.serverIds[i] = uint32(id)
	}
}

func SortShardsByTimeAscending(shards []*ShardData) {
	sort.Sort(ByShardTimeAsc{shards})
}

func SortShardsByTimeDescending(shards []*ShardData) {
	sort.Sort(ByShardTimeDesc{shards})
}

type ShardCollection []*ShardData

func (s ShardCollection) Len() int      { return len(s) }
func (s ShardCollection) Swap(i, j int) { s[i], s[j] = s[j], s[i] }

type ByShardTimeDesc struct{ ShardCollection }
type ByShardTimeAsc struct{ ShardCollection }

func (s ByShardTimeAsc) Less(i, j int) bool {
	if s.ShardCollection[i] != nil && s.ShardCollection[j] != nil {
		iStartTime := s.ShardCollection[i].StartTime().Unix()
		jStartTime := s.ShardCollection[j].StartTime().Unix()
		if iStartTime == jStartTime {
			return s.ShardCollection[i].Id() < s.ShardCollection[j].Id()
		}
		return iStartTime < jStartTime
	}
	return false
}
func (s ByShardTimeDesc) Less(i, j int) bool {
	if s.ShardCollection[i] != nil && s.ShardCollection[j] != nil {
		iStartTime := s.ShardCollection[i].StartTime().Unix()
		jStartTime := s.ShardCollection[j].StartTime().Unix()
		if iStartTime == jStartTime {
			return s.ShardCollection[i].Id() < s.ShardCollection[j].Id()
		}
		return iStartTime > jStartTime
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*

Godoc extracts and generates documentation for Go programs.

It has two modes.

Without the -http flag, it runs in command-line mode and prints plain text
documentation to standard output and exits. If both a library package and
a command with the same name exists, using the prefix cmd/ will force
documentation on the command rather than the library package. If the -src
flag is specified, godoc prints the exported interface of a package in Go
source form, or the implementation of a specific exported language entity:

	godoc fmt                # documentation for package fmt
	godoc fmt Printf         # documentation for fmt.Printf
	godoc cmd/go             # force documentation for the go command
	godoc -src fmt           # fmt package interface in Go source form
	godoc -src fmt Printf    # implementation of fmt.Printf

In command-line mode, the -q flag enables search queries against a godoc running
as a webserver. If no explicit server address is specified with the -server flag,
godoc first tries localhost:6060 and then http://golang.org.

	godoc -q Reader Writer
	godoc -q math.Sin
	godoc -server=:6060 -q sin

With the -http flag, it runs as a web server and presents the documentation as a
web page.

	godoc -http=:6060

Usage:
	godoc [flag] package [name ...]

The flags are:
	-v
		verbose mode
	-q
		arguments are considered search queries: a legal query is a
		single identifier (such as ToLower) or a qualified identifier
		(such as math.Sin).
	-src
		print (exported) source in command-line mode
	-tabwidth=4
		width of tabs in units of spaces
	-timestamps=true
		show timestamps with directory listings
	-index
		enable identifier and full text search index
		(no search box is shown if -index is not set)
	-index_files=""
		glob pattern specifying index files; if not empty,
		the index is read from these files in sorted order
	-index_throttle=0.75
		index throttle value; a value of 0 means no time is allocated
		to the indexer (the indexer will never finish), a value of 1.0
		means that index creation is running at full throttle (other
		goroutines may get no time while the index is built)
	-write_index=false
		write index to a file; the file name must be specified with
		-index_files
	-maxresults=10000
		maximum number of full text search results shown
		(no full text index is built if maxresults <= 0)
	-path=""
		additional package directories (colon-separated)
	-html
		print HTML in command-line mode
	-goroot=$GOROOT
		Go root directory
	-http=addr
		HTTP service address (e.g., '127.0.0.1:6060' or just ':6060')
	-server=addr
		webserver address for command line searches
	-sync="command"
		if this and -sync_minutes are set, run the argument as a
		command every sync_minutes; it is intended to update the
		repository holding the source files.
	-sync_minutes=0
		sync interval in minutes; sync is disabled if <= 0
	-templates=""
		directory containing alternate template files; if set,
		the directory may provide alternative template files
		for the files in $GOROOT/lib/godoc
	-filter=""
		filter file containing permitted package directory paths
	-filter_minutes=0
		filter file update interval in minutes; update is disabled if <= 0
	-zip=""
		zip file providing the file system to serve; disabled if empty

The -path flag accepts a list of colon-separated paths; unrooted paths are relative
to the current working directory. Each path is considered as an additional root for
packages in order of appearance. The last (absolute) path element is the prefix for
the package path. For instance, given the flag value:

	path=".:/home/bar:/public"

for a godoc started in /home/user/godoc, absolute paths are mapped to package paths
as follows:

	/home/user/godoc/x -> godoc/x
	/home/bar/x        -> bar/x
	/public/x          -> public/x

Paths provided via -path may point to very large file systems that contain
non-Go files. Creating the subtree of directories with Go packages may take
a long amount of time. A file containing newline-separated directory paths
may be provided with the -filter flag; if it exists, only directories
on those paths are considered. If -filter_minutes is set, the filter_file is
updated regularly by walking the entire directory tree.

When godoc runs as a web server and -index is set, a search index is maintained.
The index is created at startup and is automatically updated every time the
-sync command terminates with exit status 0, indicating that files have changed.

If the sync exit status is 1, godoc assumes that it succeeded without errors
but that no files changed; the index is not updated in this case.

In all other cases, sync is assumed to have failed and godoc backs off running
sync exponentially (up to 1 day). As soon as sync succeeds again (exit status 0
or 1), the normal sync rhythm is re-established.

The index contains both identifier and full text search information (searchable
via regular expressions). The maximum number of full text search results shown
can be set with the -maxresults flag; if set to 0, no full text results are
shown, and only an identifier index but no full text search index is created.

The presentation mode of web pages served by godoc can be controlled with the
"m" URL parameter; it accepts a comma-separated list of flag names as value:

	all	show documentation for all declarations, not just the exported ones
	methods	show all embedded methods, not just those of unexported anonymous fields
	src	show the original source code rather then the extracted documentation
	text	present the page in textual (command-line) form rather than HTML
	flat	present flat (not indented) directory listings using full paths

For instance, http://golang.org/pkg/math/big/?m=all,text shows the documentation
for all (not just the exported) declarations of package big, in textual form (as
it would appear when using godoc from the command line: "godoc -src math/big .*").

By default, godoc serves files from the file system of the underlying OS.
Instead, a .zip file may be provided via the -zip flag, which contains
the file system to serve. The file paths stored in the .zip file must use
slash ('/') as path separator; and they must be unrooted. $GOROOT (or -goroot)
must be set to the .zip file directory path containing the Go root directory.
For instance, for a .zip file created by the command:

	zip go.zip $HOME/go

one may run godoc as follows:

	godoc -http=:6060 -zip=go.zip -goroot=$HOME/go

See "Godoc: documenting Go code" for how to write good comments for godoc:
http://blog.golang.org/2011/03/godoc-documenting-go-code.html
*/
package documentation
<commit_msg>godoc: update documentation<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*

Godoc extracts and generates documentation for Go programs.

It has two modes.

Without the -http flag, it runs in command-line mode and prints plain text
documentation to standard output and exits. If both a library package and
a command with the same name exists, using the prefix cmd/ will force
documentation on the command rather than the library package. If the -src
flag is specified, godoc prints the exported interface of a package in Go
source form, or the implementation of a specific exported language entity:

	godoc fmt                # documentation for package fmt
	godoc fmt Printf         # documentation for fmt.Printf
	godoc cmd/go             # force documentation for the go command
	godoc -src fmt           # fmt package interface in Go source form
	godoc -src fmt Printf    # implementation of fmt.Printf

In command-line mode, the -q flag enables search queries against a godoc running
as a webserver. If no explicit server address is specified with the -server flag,
godoc first tries localhost:6060 and then http://golang.org.

	godoc -q Reader
	godoc -q math.Sin
	godoc -server=:6060 -q sin

With the -http flag, it runs as a web server and presents the documentation as a
web page.

	godoc -http=:6060

Usage:
	godoc [flag] package [name ...]

The flags are:
	-v
		verbose mode
	-q
		arguments are considered search queries: a legal query is a
		single identifier (such as ToLower) or a qualified identifier
		(such as math.Sin).
	-src
		print (exported) source in command-line mode
	-tabwidth=4
		width of tabs in units of spaces
	-timestamps=true
		show timestamps with directory listings
	-index
		enable identifier and full text search index
		(no search box is shown if -index is not set)
	-index_files=""
		glob pattern specifying index files; if not empty,
		the index is read from these files in sorted order
	-index_throttle=0.75
		index throttle value; a value of 0 means no time is allocated
		to the indexer (the indexer will never finish), a value of 1.0
		means that index creation is running at full throttle (other
		goroutines may get no time while the index is built)
	-write_index=false
		write index to a file; the file name must be specified with
		-index_files
	-maxresults=10000
		maximum number of full text search results shown
		(no full text index is built if maxresults <= 0)
	-path=""
		additional package directories (colon-separated)
	-html
		print HTML in command-line mode
	-goroot=$GOROOT
		Go root directory
	-http=addr
		HTTP service address (e.g., '127.0.0.1:6060' or just ':6060')
	-server=addr
		webserver address for command line searches
	-sync="command"
		if this and -sync_minutes are set, run the argument as a
		command every sync_minutes; it is intended to update the
		repository holding the source files.
	-sync_minutes=0
		sync interval in minutes; sync is disabled if <= 0
	-templates=""
		directory containing alternate template files; if set,
		the directory may provide alternative template files
		for the files in $GOROOT/lib/godoc
	-zip=""
		zip file providing the file system to serve; disabled if empty

By default, godoc looks at the packages it finds via $GOROOT and $GOPATH (if set).
Additional directories may be specified via the -path flag which accepts a list
of colon-separated paths; unrooted paths are relative to the current working
directory. Each path is considered as an additional root for packages in order
of appearance. The last (absolute) path element is the prefix for the package
path. For instance, given the flag value:

	path=".:/home/bar:/public"

for a godoc started in /home/user/godoc, absolute paths are mapped to package paths
as follows:

	/home/user/godoc/x -> godoc/x
	/home/bar/x        -> bar/x
	/public/x          -> public/x

When godoc runs as a web server and -index is set, a search index is maintained.
The index is created at startup and is automatically updated every time the
-sync command terminates with exit status 0, indicating that files have changed.

If the sync exit status is 1, godoc assumes that it succeeded without errors
but that no files changed; the index is not updated in this case.

In all other cases, sync is assumed to have failed and godoc backs off running
sync exponentially (up to 1 day). As soon as sync succeeds again (exit status 0
or 1), the normal sync rhythm is re-established.

The index contains both identifier and full text search information (searchable
via regular expressions). The maximum number of full text search results shown
can be set with the -maxresults flag; if set to 0, no full text results are
shown, and only an identifier index but no full text search index is created.

The presentation mode of web pages served by godoc can be controlled with the
"m" URL parameter; it accepts a comma-separated list of flag names as value:

	all	show documentation for all declarations, not just the exported ones
	methods	show all embedded methods, not just those of unexported anonymous fields
	src	show the original source code rather then the extracted documentation
	text	present the page in textual (command-line) form rather than HTML
	flat	present flat (not indented) directory listings using full paths

For instance, http://golang.org/pkg/math/big/?m=all,text shows the documentation
for all (not just the exported) declarations of package big, in textual form (as
it would appear when using godoc from the command line: "godoc -src math/big .*").

By default, godoc serves files from the file system of the underlying OS.
Instead, a .zip file may be provided via the -zip flag, which contains
the file system to serve. The file paths stored in the .zip file must use
slash ('/') as path separator; and they must be unrooted. $GOROOT (or -goroot)
must be set to the .zip file directory path containing the Go root directory.
For instance, for a .zip file created by the command:

	zip go.zip $HOME/go

one may run godoc as follows:

	godoc -http=:6060 -zip=go.zip -goroot=$HOME/go

See "Godoc: documenting Go code" for how to write good comments for godoc:
http://blog.golang.org/2011/03/godoc-documenting-go-code.html

*/
package documentation
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"regexp"
	"strconv"
)

// dataStoreSize limits the storage used by the data store. If exceeded, the
// data store will start garbage collection old files until enough storage is
// available again.
const dataStoreSize = 1024 * 1024 * 1024

// fileRoute matches /files/<id>. Go seems to use \r to terminate header
// values, so to ease bash scripting, the route ignores a trailing \r in the
// route. Better ideas are welcome.
var fileRoute = regexp.MustCompile("^/files/([^/\r\n]+)\r?$")

var filesRoute = regexp.MustCompile("^/files/?$")
var dataStore *DataStore

func init() {
	wd, err := os.Getwd()
	if err != nil {
		panic(err)
	}

	dataDir := path.Join(wd, "tus_data")
	if err := os.MkdirAll(dataDir, 0777); err != nil {
		panic(err)
	}
	dataStore = NewDataStore(dataDir, dataStoreSize)
}

func serveHttp() error {
	http.HandleFunc("/", route)

	addr := ":1080"
	if port := os.Getenv("TUSD_PORT"); port != "" {
		addr = ":" + port
	}
	log.Printf("serving clients at %s", addr)

	return http.ListenAndServe(addr, nil)
}

func route(w http.ResponseWriter, r *http.Request) {
	log.Printf("request: %s %s", r.Method, r.URL.RequestURI())

	w.Header().Set("Server", "tusd")

	// Allow CORS for almost everything. This needs to be revisted / limited to
	// routes and methods that need it.
	w.Header().Add("Access-Control-Allow-Origin", "*")
	w.Header().Add("Access-Control-Allow-Methods", "HEAD,GET,PUT,POST,DELETE")
	w.Header().Add("Access-Control-Allow-Headers", "Origin, x-requested-with, content-type, accept, Content-Range, Content-Disposition")
	w.Header().Add("Access-Control-Expose-Headers", "Location, Range, Content-Disposition")

	if r.Method == "OPTIONS" {
		reply(w, http.StatusOK, "")
		return
	}

	if r.Method == "POST" && filesRoute.Match([]byte(r.URL.Path)) {
		postFiles(w, r)
	} else if match := fileRoute.FindStringSubmatch(r.URL.Path); match != nil {
		id := match[1]
		switch r.Method {
		case "HEAD":
			headFile(w, r, id)
		case "GET":
			getFile(w, r, id)
		case "PUT":
			putFile(w, r, id)
		default:
			reply(w, http.StatusMethodNotAllowed, "Invalid http method")
		}
	} else {
		reply(w, http.StatusNotFound, "No matching route")
	}
}

func reply(w http.ResponseWriter, code int, message string) {
	w.WriteHeader(code)
	fmt.Fprintf(w, "%d - %s: %s\n", code, http.StatusText(code), message)
}

func postFiles(w http.ResponseWriter, r *http.Request) {
	contentRange, err := parseContentRange(r.Header.Get("Content-Range"))
	if err != nil {
		reply(w, http.StatusBadRequest, err.Error())
		return
	}

	if contentRange.Size == -1 {
		reply(w, http.StatusBadRequest, "Content-Range must indicate total file size.")
		return
	}

	contentType := r.Header.Get("Content-Type")
	if contentType == "" {
		contentType = "application/octet-stream"
	}

	contentDisposition := r.Header.Get("Content-Disposition")

	id := uid()
	if err := dataStore.CreateFile(id, contentRange.Size, contentType, contentDisposition); err != nil {
		reply(w, http.StatusInternalServerError, err.Error())
		return
	}

	if contentRange.End != -1 {
		if err := dataStore.WriteFileChunk(id, contentRange.Start, contentRange.End, r.Body); err != nil {
			// @TODO: Could be a 404 as well
			reply(w, http.StatusInternalServerError, err.Error())
			return
		}
	}

	w.Header().Set("Location", "/files/"+id)
	setFileHeaders(w, id)
	w.WriteHeader(http.StatusCreated)
}

func headFile(w http.ResponseWriter, r *http.Request, fileId string) {
	// Work around a bug in Go that would cause HEAD responses to hang. Should be
	// fixed in future release, see:
	// http://code.google.com/p/go/issues/detail?id=4126
	w.Header().Set("Content-Length", "0")
	setFileHeaders(w, fileId)
}

func getFile(w http.ResponseWriter, r *http.Request, fileId string) {
	meta, err := dataStore.GetFileMeta(fileId)
	if err != nil {
		// @TODO: Could be a 404 as well
		reply(w, http.StatusInternalServerError, err.Error())
		return
	}

	data, err := dataStore.ReadFile(fileId)
	if err != nil {
		// @TODO: Could be a 404 as well
		reply(w, http.StatusInternalServerError, err.Error())
		return
	}
	defer data.Close()

	setFileHeaders(w, fileId)
	w.Header().Set("Content-Length", strconv.FormatInt(meta.Size, 10))

	if _, err := io.CopyN(w, data, meta.Size); err != nil {
		log.Printf("getFile: CopyN failed with: %s", err.Error())
		return
	}
}

func putFile(w http.ResponseWriter, r *http.Request, fileId string) {
	var start int64 = 0
	var end int64 = 0

	contentRange, err := parseContentRange(r.Header.Get("Content-Range"))
	if err != nil {
		contentLength := r.Header.Get("Content-Length")
		end, err = strconv.ParseInt(contentLength, 10, 64)
		if err != nil {
			reply(w, http.StatusBadRequest, "Invalid content length provided")
		}

		// we are zero-indexed
		end = end - 1

		// @TODO: Make sure contentLength matches the content length of the initial
		// POST request
	} else {

		// @TODO: Make sure contentRange.Size matches file size

		start = contentRange.Start
		end = contentRange.End
	}

	// @TODO: Check that file exists

	if err := dataStore.WriteFileChunk(fileId, start, end, r.Body); err != nil {
		// @TODO: Could be a 404 as well
		reply(w, http.StatusInternalServerError, err.Error())
		return
	}

	setFileHeaders(w, fileId)
}

func setFileHeaders(w http.ResponseWriter, fileId string) {
	meta, err := dataStore.GetFileMeta(fileId)
	if os.IsNotExist(err) {
		reply(w, http.StatusNotFound, err.Error())
		return
	} else if err != nil {
		reply(w, http.StatusInternalServerError, err.Error())
		return
	}

	rangeHeader := ""
	for i, chunk := range meta.Chunks {
		rangeHeader += fmt.Sprintf("%d-%d", chunk.Start, chunk.End)
		if i+1 < len(meta.Chunks) {
			rangeHeader += ","
		}
	}

	if rangeHeader != "" {
		w.Header().Set("Range", "bytes="+rangeHeader)
	}

	w.Header().Set("Content-Type", meta.ContentType)
	w.Header().Set("Content-Disposition", meta.ContentDisposition)
}
<commit_msg>make datastore_dir and datastore_max_size configurable env vars, fixes #2<commit_after>package main

import (
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"regexp"
	"strconv"
)

// fileRoute matches /files/<id>. Go seems to use \r to terminate header
// values, so to ease bash scripting, the route ignores a trailing \r in the
// route. Better ideas are welcome.
var fileRoute = regexp.MustCompile("^/files/([^/\r\n]+)\r?$")

var filesRoute = regexp.MustCompile("^/files/?$")
var dataStore *DataStore

func init() {
	wd, err := os.Getwd()
	if err != nil {
		panic(err)
	}

	dataDir := path.Join(wd, "tus_data")
	if configDir := os.Getenv("TUSD_DATA_DIR"); configDir != "" {
		dataDir = configDir
	}

	// dataStoreSize limits the storage used by the data store. If exceeded, the
	// data store will start garbage collection old files until enough storage is
	// available again.
	var dataStoreSize int64
	dataStoreSize = 1024 * 1024 * 1024
	if configStoreSize := os.Getenv("TUSD_DATA_STORE_MAXSIZE"); configStoreSize != "" {
		parsed, err := strconv.ParseInt(configStoreSize, 10, 64)
		if err != nil {
			panic(errors.New("Invalid data store max size configured"))
		}
		dataStoreSize = parsed
	}

	log.Print("Datastore directory: ", dataDir)
	log.Print("Datastore max size: ", dataStoreSize)

	if err := os.MkdirAll(dataDir, 0777); err != nil {
		panic(err)
	}
	dataStore = NewDataStore(dataDir, dataStoreSize)
}

func serveHttp() error {
	http.HandleFunc("/", route)

	addr := ":1080"
	if port := os.Getenv("TUSD_PORT"); port != "" {
		addr = ":" + port
	}
	log.Printf("serving clients at %s", addr)

	return http.ListenAndServe(addr, nil)
}

func route(w http.ResponseWriter, r *http.Request) {
	log.Printf("request: %s %s", r.Method, r.URL.RequestURI())

	w.Header().Set("Server", "tusd")

	// Allow CORS for almost everything. This needs to be revisted / limited to
	// routes and methods that need it.
	w.Header().Add("Access-Control-Allow-Origin", "*")
	w.Header().Add("Access-Control-Allow-Methods", "HEAD,GET,PUT,POST,DELETE")
	w.Header().Add("Access-Control-Allow-Headers", "Origin, x-requested-with, content-type, accept, Content-Range, Content-Disposition")
	w.Header().Add("Access-Control-Expose-Headers", "Location, Range, Content-Disposition")

	if r.Method == "OPTIONS" {
		reply(w, http.StatusOK, "")
		return
	}

	if r.Method == "POST" && filesRoute.Match([]byte(r.URL.Path)) {
		postFiles(w, r)
	} else if match := fileRoute.FindStringSubmatch(r.URL.Path); match != nil {
		id := match[1]
		switch r.Method {
		case "HEAD":
			headFile(w, r, id)
		case "GET":
			getFile(w, r, id)
		case "PUT":
			putFile(w, r, id)
		default:
			reply(w, http.StatusMethodNotAllowed, "Invalid http method")
		}
	} else {
		reply(w, http.StatusNotFound, "No matching route")
	}
}

func reply(w http.ResponseWriter, code int, message string) {
	w.WriteHeader(code)
	fmt.Fprintf(w, "%d - %s: %s\n", code, http.StatusText(code), message)
}

func postFiles(w http.ResponseWriter, r *http.Request) {
	contentRange, err := parseContentRange(r.Header.Get("Content-Range"))
	if err != nil {
		reply(w, http.StatusBadRequest, err.Error())
		return
	}

	if contentRange.Size == -1 {
		reply(w, http.StatusBadRequest, "Content-Range must indicate total file size.")
		return
	}

	contentType := r.Header.Get("Content-Type")
	if contentType == "" {
		contentType = "application/octet-stream"
	}

	contentDisposition := r.Header.Get("Content-Disposition")

	id := uid()
	if err := dataStore.CreateFile(id, contentRange.Size, contentType, contentDisposition); err != nil {
		reply(w, http.StatusInternalServerError, err.Error())
		return
	}

	if contentRange.End != -1 {
		if err := dataStore.WriteFileChunk(id, contentRange.Start, contentRange.End, r.Body); err != nil {
			// @TODO: Could be a 404 as well
			reply(w, http.StatusInternalServerError, err.Error())
			return
		}
	}

	w.Header().Set("Location", "/files/"+id)
	setFileHeaders(w, id)
	w.WriteHeader(http.StatusCreated)
}

func headFile(w http.ResponseWriter, r *http.Request, fileId string) {
	// Work around a bug in Go that would cause HEAD responses to hang. Should be
	// fixed in future release, see:
	// http://code.google.com/p/go/issues/detail?id=4126
	w.Header().Set("Content-Length", "0")
	setFileHeaders(w, fileId)
}

func getFile(w http.ResponseWriter, r *http.Request, fileId string) {
	meta, err := dataStore.GetFileMeta(fileId)
	if err != nil {
		// @TODO: Could be a 404 as well
		reply(w, http.StatusInternalServerError, err.Error())
		return
	}

	data, err := dataStore.ReadFile(fileId)
	if err != nil {
		// @TODO: Could be a 404 as well
		reply(w, http.StatusInternalServerError, err.Error())
		return
	}
	defer data.Close()

	setFileHeaders(w, fileId)
	w.Header().Set("Content-Length", strconv.FormatInt(meta.Size, 10))

	if _, err := io.CopyN(w, data, meta.Size); err != nil {
		log.Printf("getFile: CopyN failed with: %s", err.Error())
		return
	}
}

func putFile(w http.ResponseWriter, r *http.Request, fileId string) {
	var start int64 = 0
	var end int64 = 0

	contentRange, err := parseContentRange(r.Header.Get("Content-Range"))
	if err != nil {
		contentLength := r.Header.Get("Content-Length")
		end, err = strconv.ParseInt(contentLength, 10, 64)
		if err != nil {
			reply(w, http.StatusBadRequest, "Invalid content length provided")
		}

		// we are zero-indexed
		end = end - 1

		// @TODO: Make sure contentLength matches the content length of the initial
		// POST request
	} else {

		// @TODO: Make sure contentRange.Size matches file size

		start = contentRange.Start
		end = contentRange.End
	}

	// @TODO: Check that file exists

	if err := dataStore.WriteFileChunk(fileId, start, end, r.Body); err != nil {
		// @TODO: Could be a 404 as well
		reply(w, http.StatusInternalServerError, err.Error())
		return
	}

	setFileHeaders(w, fileId)
}

func setFileHeaders(w http.ResponseWriter, fileId string) {
	meta, err := dataStore.GetFileMeta(fileId)
	if os.IsNotExist(err) {
		reply(w, http.StatusNotFound, err.Error())
		return
	} else if err != nil {
		reply(w, http.StatusInternalServerError, err.Error())
		return
	}

	rangeHeader := ""
	for i, chunk := range meta.Chunks {
		rangeHeader += fmt.Sprintf("%d-%d", chunk.Start, chunk.End)
		if i+1 < len(meta.Chunks) {
			rangeHeader += ","
		}
	}

	if rangeHeader != "" {
		w.Header().Set("Range", "bytes="+rangeHeader)
	}

	w.Header().Set("Content-Type", meta.ContentType)
	w.Header().Set("Content-Disposition", meta.ContentDisposition)
}
<|endoftext|>


<commit_before>package auth

import (
	"net/http"
	"strings"
)

func BadRequestHanlder(rw http.ResponseWriter, req *http.Request) {
	rw.Header().Set("Content-Type", "application/json; charset=utf-8")
	rw.WriteHeader(http.StatusBadRequest)
	rw.Write([]byte("{error:\"Badd Request\"}"))
}

func ForbiddenHanlder(rw http.ResponseWriter, req *http.Request) {
	rw.Header().Set("Content-Type", "application/json; charset=utf-8")
	rw.WriteHeader(http.StatusForbidden)
	rw.Write([]byte("{error:\"" + ErrNotLogged.Error() + "\"}"))
}

func InternalServerErrorHanlder(rw http.ResponseWriter, req *http.Request) {
	rw.Header().Set("Content-Type", "application/json; charset=utf-8")
	rw.WriteHeader(http.StatusForbidden)
	rw.Write([]byte("{error:\"Internal Server Error\"}"))
}

func OAuthHandleWrapper(handler http.HandlerFunc,
	pri map[string]bool) http.HandlerFunc {
	return func(rw http.ResponseWriter, req *http.Request) {
		userMngr, err := Provider().OpenUserMngr()
		if err != nil {
			InternalServerErrorHanlder(rw, req)
		}
		defer userMngr.Close()

		token := strings.TrimPrefix(req.Header.Get("Authorization"), "Bearer ")
		user, err := userMngr.GetUser(token)
		if err != nil {
			if err == ErrNotLogged {
				ForbiddenHanlder(rw, req)
				return
			}
			InternalServerErrorHanlder(rw, req)
			return
		}

		for key, val := range pri {
			if val {
				if !userMngr.Can(user, key) {
					ForbiddenHanlder(rw, req)
					return
				}
			}
		}

		// run user defined handler
		handler(rw, req)
	}
}
<commit_msg>OAuthhandleWraper now recive an slice of string instead of map<commit_after>package auth

import (
	"net/http"
	"strings"
)

func BadRequestHanlder(rw http.ResponseWriter, req *http.Request) {
	rw.Header().Set("Content-Type", "application/json; charset=utf-8")
	rw.WriteHeader(http.StatusBadRequest)
	rw.Write([]byte(`{"error":"Badd Request"}`))
}

func ForbiddenHanlder(rw http.ResponseWriter, req *http.Request) {
	rw.Header().Set("Content-Type", "application/json; charset=utf-8")
	rw.WriteHeader(http.StatusForbidden)
	rw.Write([]byte(`{"error":"` + ErrNotLogged.Error() + `"}`))
}

func InternalServerErrorHanlder(rw http.ResponseWriter, req *http.Request) {
	rw.Header().Set("Content-Type", "application/json; charset=utf-8")
	rw.WriteHeader(http.StatusInternalServerError)
	rw.Write([]byte(`{"error":"Internal Server Error"}`))
}

func OAuthHandleWrapper(handler http.HandlerFunc, pri ...string) http.HandlerFunc {
	return func(rw http.ResponseWriter, req *http.Request) {
		userMngr, err := Provider().OpenUserMngr()
		if err != nil {
			InternalServerErrorHanlder(rw, req)
		}
		defer userMngr.Close()

		token := strings.TrimPrefix(req.Header.Get("Authorization"), "Bearer ")
		user, err := userMngr.GetUser(token)
		if err != nil {
			if err == ErrNotLogged {
				ForbiddenHanlder(rw, req)
				return
			}
			InternalServerErrorHanlder(rw, req)
			return
		}

		for _, val := range pri {
			if !userMngr.Can(user, val) {
				ForbiddenHanlder(rw, req)
				return
			}
		}

		// run user defined handler
		handler(rw, req)
	}
}
<|endoftext|>