<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package adapter implements a controller that interacts with gerrit instances
package adapter

import (
	"errors"
	"fmt"
	"net/url"
	"strings"
	"time"

	"github.com/andygrunwald/go-gerrit"
	"github.com/sirupsen/logrus"

	prowapi "k8s.io/test-infra/prow/apis/prowjobs/v1"
	prowv1 "k8s.io/test-infra/prow/client/clientset/versioned/typed/prowjobs/v1"
	"k8s.io/test-infra/prow/config"
	"k8s.io/test-infra/prow/gerrit/client"
	"k8s.io/test-infra/prow/gerrit/reporter"
	"k8s.io/test-infra/prow/pjutil"
)

type prowJobClient interface {
	Create(*prowapi.ProwJob) (*prowapi.ProwJob, error)
}

type gerritClient interface {
	QueryChanges(lastUpdate time.Time, rateLimit int) map[string][]client.ChangeInfo
	GetBranchRevision(instance, project, branch string) (string, error)
	SetReview(instance, id, revision, message string, labels map[string]string) error
	Account(instance string) *gerrit.AccountInfo
}

type configAgent interface {
	Config() *config.Config
}

// Controller manages gerrit changes.
type Controller struct {
	config        config.Getter
	prowJobClient prowJobClient
	gc            gerritClient
	tracker       LastSyncTracker
}

type LastSyncTracker interface {
	Current() time.Time
	Update(time.Time) error
}

// NewController returns a new gerrit controller client
func NewController(lastSyncTracker LastSyncTracker, cookiefilePath string, projects map[string][]string, prowJobClient prowv1.ProwJobInterface, cfg config.Getter) (*Controller, error) {
	if lastSyncTracker == nil {
		return nil, errors.New("lastSyncTracker required")
	}

	c, err := client.NewClient(projects)
	if err != nil {
		return nil, err
	}
	c.Start(cookiefilePath)

	return &Controller{
		prowJobClient: prowJobClient,
		config:        cfg,
		gc:            c,
		tracker:       lastSyncTracker,
	}, nil
}

// Sync looks for newly made gerrit changes
// and creates prowjobs according to specs
func (c *Controller) Sync() error {
	syncTime := c.tracker.Current()
	latest := syncTime

	for instance, changes := range c.gc.QueryChanges(syncTime, c.config().Gerrit.RateLimit) {
		for _, change := range changes {
			if err := c.ProcessChange(instance, change); err != nil {
				logrus.WithError(err).Errorf("Failed process change %v", change.CurrentRevision)
			}
			if latest.Before(change.Updated.Time) {
				latest = change.Updated.Time
			}
		}

		logrus.Infof("Processed %d changes for instance %s", len(changes), instance)
	}

	return c.tracker.Update(latest)
}

func makeCloneURI(instance, project string) (*url.URL, error) {
	u, err := url.Parse(instance)
	if err != nil {
		return nil, fmt.Errorf("instance %s is not a url: %v", instance, err)
	}
	if u.Host == "" {
		return nil, errors.New("instance does not set host")
	}
	if u.Path != "" {
		return nil, errors.New("instance cannot set path (this is set by project)")
	}
	u.Path = project
	return u, nil
}

// listChangedFiles lists (in lexicographic order) the files changed as part of a Gerrit patchset
func listChangedFiles(changeInfo client.ChangeInfo) config.ChangedFilesProvider {
	return func() ([]string, error) {
		var changed []string
		revision := changeInfo.Revisions[changeInfo.CurrentRevision]
		for file := range revision.Files {
			changed = append(changed, file)
		}
		return changed, nil
	}
}

func createRefs(reviewHost string, change client.ChangeInfo, cloneURI *url.URL, baseSHA string) (prowapi.Refs, error) {
	rev, ok := change.Revisions[change.CurrentRevision]
	if !ok {
		return prowapi.Refs{}, fmt.Errorf("cannot find current revision for change %v", change.ID)
	}
	var codeHost string // Something like https://android.googlesource.com
	parts := strings.SplitN(reviewHost, ".", 2)
	codeHost = strings.TrimSuffix(parts[0], "-review")
	if len(parts) > 1 {
		codeHost += "." + parts[1]
	}
	refs := prowapi.Refs{
		Org:      cloneURI.Host,  // Something like android-review.googlesource.com
		Repo:     change.Project, // Something like platform/build
		BaseRef:  change.Branch,
		BaseSHA:  baseSHA,
		CloneURI: cloneURI.String(), // Something like https://android-review.googlesource.com/platform/build
		RepoLink: fmt.Sprintf("%s/%s", codeHost, change.Project),
		BaseLink: fmt.Sprintf("%s/%s/+/%s", codeHost, change.Project, baseSHA),
		Pulls: []prowapi.Pull{
			{
				Number:     change.Number,
				Author:     rev.Commit.Author.Name,
				SHA:        change.CurrentRevision,
				Ref:        rev.Ref,
				Link:       fmt.Sprintf("%s/c/%s/+/%d", reviewHost, change.Project, change.Number),
				CommitLink: fmt.Sprintf("%s/%s/+/%s", codeHost, change.Project, change.CurrentRevision),
				AuthorLink: fmt.Sprintf("%s/q/%s", reviewHost, rev.Commit.Author.Email),
			},
		},
	}
	return refs, nil
}

// ProcessChange creates new presubmit prowjobs base off the gerrit changes
func (c *Controller) ProcessChange(instance string, change client.ChangeInfo) error {
	logger := logrus.WithField("gerrit change", change.Number)

	cloneURI, err := makeCloneURI(instance, change.Project)
	if err != nil {
		return fmt.Errorf("failed to create clone uri: %v", err)
	}

	baseSHA, err := c.gc.GetBranchRevision(instance, change.Project, change.Branch)
	if err != nil {
		return fmt.Errorf("failed to get SHA from base branch: %v", err)
	}

	triggeredJobs := []string{}

	refs, err := createRefs(instance, change, cloneURI, baseSHA)
	if err != nil {
		return fmt.Errorf("failed to get refs: %v", err)
	}

	type jobSpec struct {
		spec   prowapi.ProwJobSpec
		labels map[string]string
	}

	var jobSpecs []jobSpec

	changedFiles := listChangedFiles(change)

	switch change.Status {
	case client.Merged:
		postsubmits := c.config().Postsubmits[cloneURI.String()]
		postsubmits = append(postsubmits, c.config().Postsubmits[cloneURI.Host+"/"+cloneURI.Path]...)
		for _, postsubmit := range postsubmits {
			if shouldRun, err := postsubmit.ShouldRun(change.Branch, changedFiles); err != nil {
				return fmt.Errorf("failed to determine if postsubmit %q should run: %v", postsubmit.Name, err)
			} else if shouldRun {
				jobSpecs = append(jobSpecs, jobSpec{
					spec:   pjutil.PostsubmitSpec(postsubmit, refs),
					labels: postsubmit.Labels,
				})
			}
		}
	case client.New:
		presubmits := c.config().Presubmits[cloneURI.String()]
		presubmits = append(presubmits, c.config().Presubmits[cloneURI.Host+"/"+cloneURI.Path]...)

		var filters []pjutil.Filter
		var latestReport *reporter.JobReport
		var latestReportTime time.Time
		account := c.gc.Account(instance)
		// Should not happen, since this means auth failed
		if account == nil {
			return fmt.Errorf("unable to get gerrit account")
		}

		for _, message := range change.Messages {
			// If message status report is not from the prow account ignore
			if message.Author.AccountID != account.AccountID {
				continue
			}
			if message.Date.Before(latestReportTime) {
				continue
			}
			report := reporter.ParseReport(message.Message)
			if report != nil {
				latestReport = report
				latestReportTime = message.Date.Time
			}
		}
		if latestReport == nil {
			logger.Warnf("Found nil latest report")
		} else {
			logger.Infof("Found latest report: %s", latestReport)
		}
		lastUpdate := c.tracker.Current()
		filter, err := messageFilter(lastUpdate, change, presubmits, latestReport, logger)
		if err != nil {
			logger.WithError(err).Warn("failed to create filter on messages for presubmits")
		} else {
			filters = append(filters, filter)
		}
		if change.Revisions[change.CurrentRevision].Created.Time.After(lastUpdate) {
			filters = append(filters, pjutil.TestAllFilter())
		}
		toTrigger, _, err := pjutil.FilterPresubmits(pjutil.AggregateFilter(filters), listChangedFiles(change), change.Branch, presubmits, logger)
		if err != nil {
			return fmt.Errorf("failed to filter presubmits: %v", err)
		}
		for _, presubmit := range toTrigger {
			jobSpecs = append(jobSpecs, jobSpec{
				spec:   pjutil.PresubmitSpec(presubmit, refs),
				labels: presubmit.Labels,
			})
		}
	}

	annotations := map[string]string{
		client.GerritID:       change.ID,
		client.GerritInstance: instance,
	}

	for _, jSpec := range jobSpecs {
		labels := make(map[string]string)
		for k, v := range jSpec.labels {
			labels[k] = v
		}
		labels[client.GerritRevision] = change.CurrentRevision

		if gerritLabel, ok := labels[client.GerritReportLabel]; !ok || gerritLabel == "" {
			labels[client.GerritReportLabel] = client.CodeReview
		}

		pj := pjutil.NewProwJob(jSpec.spec, labels, annotations)
		if _, err := c.prowJobClient.Create(&pj); err != nil {
			logger.WithError(err).Errorf("fail to create prowjob %v", pj)
		} else {
			logger.Infof("Triggered Prowjob %s", jSpec.spec.Job)
			triggeredJobs = append(triggeredJobs, jSpec.spec.Job)
		}
	}

	if len(triggeredJobs) > 0 {
		// comment back to gerrit
		message := fmt.Sprintf("Triggered %d prow jobs:", len(triggeredJobs))
		for _, job := range triggeredJobs {
			message += fmt.Sprintf("\n  * Name: %s", job)
		}

		if err := c.gc.SetReview(instance, change.ID, change.CurrentRevision, message, nil); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>Gerrit adapter: Use PresubmitsStatic()<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package adapter implements a controller that interacts with gerrit instances
package adapter

import (
	"errors"
	"fmt"
	"net/url"
	"strings"
	"time"

	"github.com/andygrunwald/go-gerrit"
	"github.com/sirupsen/logrus"

	prowapi "k8s.io/test-infra/prow/apis/prowjobs/v1"
	prowv1 "k8s.io/test-infra/prow/client/clientset/versioned/typed/prowjobs/v1"
	"k8s.io/test-infra/prow/config"
	"k8s.io/test-infra/prow/gerrit/client"
	"k8s.io/test-infra/prow/gerrit/reporter"
	"k8s.io/test-infra/prow/pjutil"
)

type prowJobClient interface {
	Create(*prowapi.ProwJob) (*prowapi.ProwJob, error)
}

type gerritClient interface {
	QueryChanges(lastUpdate time.Time, rateLimit int) map[string][]client.ChangeInfo
	GetBranchRevision(instance, project, branch string) (string, error)
	SetReview(instance, id, revision, message string, labels map[string]string) error
	Account(instance string) *gerrit.AccountInfo
}

type configAgent interface {
	Config() *config.Config
}

// Controller manages gerrit changes.
type Controller struct {
	config        config.Getter
	prowJobClient prowJobClient
	gc            gerritClient
	tracker       LastSyncTracker
}

type LastSyncTracker interface {
	Current() time.Time
	Update(time.Time) error
}

// NewController returns a new gerrit controller client
func NewController(lastSyncTracker LastSyncTracker, cookiefilePath string, projects map[string][]string, prowJobClient prowv1.ProwJobInterface, cfg config.Getter) (*Controller, error) {
	if lastSyncTracker == nil {
		return nil, errors.New("lastSyncTracker required")
	}

	c, err := client.NewClient(projects)
	if err != nil {
		return nil, err
	}
	c.Start(cookiefilePath)

	return &Controller{
		prowJobClient: prowJobClient,
		config:        cfg,
		gc:            c,
		tracker:       lastSyncTracker,
	}, nil
}

// Sync looks for newly made gerrit changes
// and creates prowjobs according to specs
func (c *Controller) Sync() error {
	syncTime := c.tracker.Current()
	latest := syncTime

	for instance, changes := range c.gc.QueryChanges(syncTime, c.config().Gerrit.RateLimit) {
		for _, change := range changes {
			if err := c.ProcessChange(instance, change); err != nil {
				logrus.WithError(err).Errorf("Failed process change %v", change.CurrentRevision)
			}
			if latest.Before(change.Updated.Time) {
				latest = change.Updated.Time
			}
		}

		logrus.Infof("Processed %d changes for instance %s", len(changes), instance)
	}

	return c.tracker.Update(latest)
}

func makeCloneURI(instance, project string) (*url.URL, error) {
	u, err := url.Parse(instance)
	if err != nil {
		return nil, fmt.Errorf("instance %s is not a url: %v", instance, err)
	}
	if u.Host == "" {
		return nil, errors.New("instance does not set host")
	}
	if u.Path != "" {
		return nil, errors.New("instance cannot set path (this is set by project)")
	}
	u.Path = project
	return u, nil
}

// listChangedFiles lists (in lexicographic order) the files changed as part of a Gerrit patchset
func listChangedFiles(changeInfo client.ChangeInfo) config.ChangedFilesProvider {
	return func() ([]string, error) {
		var changed []string
		revision := changeInfo.Revisions[changeInfo.CurrentRevision]
		for file := range revision.Files {
			changed = append(changed, file)
		}
		return changed, nil
	}
}

func createRefs(reviewHost string, change client.ChangeInfo, cloneURI *url.URL, baseSHA string) (prowapi.Refs, error) {
	rev, ok := change.Revisions[change.CurrentRevision]
	if !ok {
		return prowapi.Refs{}, fmt.Errorf("cannot find current revision for change %v", change.ID)
	}
	var codeHost string // Something like https://android.googlesource.com
	parts := strings.SplitN(reviewHost, ".", 2)
	codeHost = strings.TrimSuffix(parts[0], "-review")
	if len(parts) > 1 {
		codeHost += "." + parts[1]
	}
	refs := prowapi.Refs{
		Org:      cloneURI.Host,  // Something like android-review.googlesource.com
		Repo:     change.Project, // Something like platform/build
		BaseRef:  change.Branch,
		BaseSHA:  baseSHA,
		CloneURI: cloneURI.String(), // Something like https://android-review.googlesource.com/platform/build
		RepoLink: fmt.Sprintf("%s/%s", codeHost, change.Project),
		BaseLink: fmt.Sprintf("%s/%s/+/%s", codeHost, change.Project, baseSHA),
		Pulls: []prowapi.Pull{
			{
				Number:     change.Number,
				Author:     rev.Commit.Author.Name,
				SHA:        change.CurrentRevision,
				Ref:        rev.Ref,
				Link:       fmt.Sprintf("%s/c/%s/+/%d", reviewHost, change.Project, change.Number),
				CommitLink: fmt.Sprintf("%s/%s/+/%s", codeHost, change.Project, change.CurrentRevision),
				AuthorLink: fmt.Sprintf("%s/q/%s", reviewHost, rev.Commit.Author.Email),
			},
		},
	}
	return refs, nil
}

// ProcessChange creates new presubmit prowjobs base off the gerrit changes
func (c *Controller) ProcessChange(instance string, change client.ChangeInfo) error {
	logger := logrus.WithField("gerrit change", change.Number)

	cloneURI, err := makeCloneURI(instance, change.Project)
	if err != nil {
		return fmt.Errorf("failed to create clone uri: %v", err)
	}

	baseSHA, err := c.gc.GetBranchRevision(instance, change.Project, change.Branch)
	if err != nil {
		return fmt.Errorf("failed to get SHA from base branch: %v", err)
	}

	triggeredJobs := []string{}

	refs, err := createRefs(instance, change, cloneURI, baseSHA)
	if err != nil {
		return fmt.Errorf("failed to get refs: %v", err)
	}

	type jobSpec struct {
		spec   prowapi.ProwJobSpec
		labels map[string]string
	}

	var jobSpecs []jobSpec

	changedFiles := listChangedFiles(change)

	switch change.Status {
	case client.Merged:
		postsubmits := c.config().Postsubmits[cloneURI.String()]
		postsubmits = append(postsubmits, c.config().Postsubmits[cloneURI.Host+"/"+cloneURI.Path]...)
		for _, postsubmit := range postsubmits {
			if shouldRun, err := postsubmit.ShouldRun(change.Branch, changedFiles); err != nil {
				return fmt.Errorf("failed to determine if postsubmit %q should run: %v", postsubmit.Name, err)
			} else if shouldRun {
				jobSpecs = append(jobSpecs, jobSpec{
					spec:   pjutil.PostsubmitSpec(postsubmit, refs),
					labels: postsubmit.Labels,
				})
			}
		}
	case client.New:
		// TODO: Do we want to add support for dynamic presubmits?
		presubmits := c.config().PresubmitsStatic()[cloneURI.String()]
		presubmits = append(presubmits, c.config().PresubmitsStatic()[cloneURI.Host+"/"+cloneURI.Path]...)

		var filters []pjutil.Filter
		var latestReport *reporter.JobReport
		var latestReportTime time.Time
		account := c.gc.Account(instance)
		// Should not happen, since this means auth failed
		if account == nil {
			return fmt.Errorf("unable to get gerrit account")
		}

		for _, message := range change.Messages {
			// If message status report is not from the prow account ignore
			if message.Author.AccountID != account.AccountID {
				continue
			}
			if message.Date.Before(latestReportTime) {
				continue
			}
			report := reporter.ParseReport(message.Message)
			if report != nil {
				latestReport = report
				latestReportTime = message.Date.Time
			}
		}
		if latestReport == nil {
			logger.Warnf("Found nil latest report")
		} else {
			logger.Infof("Found latest report: %s", latestReport)
		}
		lastUpdate := c.tracker.Current()
		filter, err := messageFilter(lastUpdate, change, presubmits, latestReport, logger)
		if err != nil {
			logger.WithError(err).Warn("failed to create filter on messages for presubmits")
		} else {
			filters = append(filters, filter)
		}
		if change.Revisions[change.CurrentRevision].Created.Time.After(lastUpdate) {
			filters = append(filters, pjutil.TestAllFilter())
		}
		toTrigger, _, err := pjutil.FilterPresubmits(pjutil.AggregateFilter(filters), listChangedFiles(change), change.Branch, presubmits, logger)
		if err != nil {
			return fmt.Errorf("failed to filter presubmits: %v", err)
		}
		for _, presubmit := range toTrigger {
			jobSpecs = append(jobSpecs, jobSpec{
				spec:   pjutil.PresubmitSpec(presubmit, refs),
				labels: presubmit.Labels,
			})
		}
	}

	annotations := map[string]string{
		client.GerritID:       change.ID,
		client.GerritInstance: instance,
	}

	for _, jSpec := range jobSpecs {
		labels := make(map[string]string)
		for k, v := range jSpec.labels {
			labels[k] = v
		}
		labels[client.GerritRevision] = change.CurrentRevision

		if gerritLabel, ok := labels[client.GerritReportLabel]; !ok || gerritLabel == "" {
			labels[client.GerritReportLabel] = client.CodeReview
		}

		pj := pjutil.NewProwJob(jSpec.spec, labels, annotations)
		if _, err := c.prowJobClient.Create(&pj); err != nil {
			logger.WithError(err).Errorf("fail to create prowjob %v", pj)
		} else {
			logger.Infof("Triggered Prowjob %s", jSpec.spec.Job)
			triggeredJobs = append(triggeredJobs, jSpec.spec.Job)
		}
	}

	if len(triggeredJobs) > 0 {
		// comment back to gerrit
		message := fmt.Sprintf("Triggered %d prow jobs:", len(triggeredJobs))
		for _, job := range triggeredJobs {
			message += fmt.Sprintf("\n  * Name: %s", job)
		}

		if err := c.gc.SetReview(instance, change.ID, change.CurrentRevision, message, nil); err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package public

import (
	"net/http"
	"strings"
	"net/url"
	"errors"
	"encoding/json"
	"bytes"
	"io"
    "mime"
	"mime/multipart"
	"io/ioutil"
	"os"
	"path/filepath"
)

var BASE_URL URL = "https://api.weixin.qq.com/cgi-bin"

type client struct {
	*http.Client
}

var Client = client{
	Client: http.DefaultClient,
}

func (c *client) Token() (string, error) {
	return "", nil
}

func (c *client) RefreshToken() (string, error) {
	return "", nil
}

func (c *client) call(u URL, rep interface{}, streamRep io.Writer, request func(URL)(*http.Response, error)) error {
	token, err := c.Token()
	if err != nil {
		return err
	}

	firstTime := true

RETRY:
	r, err := request(u.Query("access_token", token))
	if err != nil {
		return err
	}
	defer r.Body.Close()

	if r.StatusCode != http.StatusOK {
		return errors.New(r.Status)
	}

	if streamRep != nil {
		contentDisposition := r.Header.Get("Content-Disposition")
		contentType, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type"))
		if contentDisposition != "" && contentType != "text/plain" && contentType != "application/json" {
			_, err = io.Copy(streamRep, r.Body)
			return err
		}
	}

	err = json.NewDecoder(r.Body).Decode(rep)
	if err != nil {
		return err
	}

	e := rep.(Error)
	if e.Code() == OK {
		return nil
	}
	if (e.Code() == InvalidCredential || e.Code() == AccessTokenExpired) && firstTime {
		firstTime = false
		token, err = c.RefreshToken()
		if err != nil {
			return err
		}
		goto RETRY
	}

	return rep.(error)
}

func (c *client) Get(u URL, rep interface{}) error {
	return c.call(u, rep, nil, func(u URL)(*http.Response, error) {
		return c.Client.Get(u)
	})
}

func (c *client) Post(u URL, req, rep interface{}) error {
	buf := &bytes.Buffer{}
	err := json.NewEncoder(buf).Encode(req)
	if err != nil {
		return err
	}

	return c.call(u, rep, nil, func(u URL)(*http.Response, error) {
		return c.Client.Post(u, "application/json; charset=utf-8", buf)
	})
}

var MaxMemoryForFile = 10*1024*1024

type fileBuf struct {
	bytes.Buffer
	*os.File
	n int
}

func (fb *fileBuf) Write(p []byte) (n int, err error) {
	if fb.File != nil {
		n, err = fb.File.Write(p)
		fb.n += n
		return
	}

	pr := bytes.NewReader(p)
	n, err = io.CopyN(&fb.Buffer, pr, MaxMemoryForFile+1-fb.n)
	fb.n += n
	if err != nil && err != io.EOF {
		return
	}
	if fb.n > MaxMemoryForFile {
		// too big, write to disk and flush buffer
		file, err := ioutil.TempFile("", "multipart-")
		if err != nil {
			return
		}
		fb.n, err = file.Write(p)
		if cerr := file.Close(); err == nil {
			err = cerr
		}
		if err != nil {
			os.Remove(file.Name())
			return
		}
		file.Seek(0, 0)

		fb.File = file
	}
	return
}

func (fb *fileBuf) Close() error {
	if fb.File {
		return fb.File.Close()
	}
	return nil
}

func (c *client) UploadFile(u URL, name, filePath string, extraFields map[string]string, rep interface{}) error {
	var buf fileBuf
	defer buf.Close()

	mp := multipart.NewWriter(buf)
	partWriter, err := mp.CreateFormFile(name, filepath.Base(filePath))
	if err != nil {
		return err
	}

	file, err := os.Open(filePath)
	if err != nil {
		return
	}
	defer file.Close()

	if _, err = io.Copy(partWriter, file); err != nil {
		return err
	}

	for k, v := range extraFields {
		if err = mp.WriteField(k, v); err != nil {
			return
		}
	}

	if err = mp.Close(); err != nil {
		return err
	}

	var reader io.ReadSeeker
	if buf.File {
		reader = buf.File
	} else {
		reader = bytes.NewReader(buf.Buffer.Bytes())
	}

	return c.call(u, rep, nil, func(u URL)(*http.Response, error) {
		reader.Seek(0, 0)
		return c.Client.Post(u, mp.FormDataContentType, reader)
	})
}

func (c *client) DownloadFile(u URL, filePath string, rep interface{}) (err error) {
	file, err := os.Create(filePath)
	if err != nil {
		return err
	}

	defer func() {
		file.Close()
		if err != nil {
			os.Remove(filePath)
		}
	}()

	return c.call(u, rep, file, func(u URL)(*http.Response, error) {
		return c.Client.Get(u)
	})
}

type URL string

func (u URL) Join(segment string) URL {
	return u + segment
}

func (u URL) Query(key, value string) URL {
	if strings.IndexByte(u, '?') != -1 {
		u += '&'
	} else {
		u += '?'
	}
	u += key + '=' + url.QueryEscape(value)
	return u
}
<commit_msg>Update client.go<commit_after>package public

import (
	"bytes"
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
)

var BASE_URL URL = "https://api.weixin.qq.com/cgi-bin"

type client struct {
	*http.Client
}

var Client = client{
	Client: http.DefaultClient,
}

func (c *client) Token() (string, error) {
	return "", nil
}

func (c *client) RefreshToken() (string, error) {
	return "", nil
}

func (c *client) call(u URL, rep interface{}, streamRep io.Writer, request func(URL) (*http.Response, error)) error {
	token, err := c.Token()
	if err != nil {
		return err
	}

	firstTime := true

RETRY:
	r, err := request(u.Query("access_token", token))
	if err != nil {
		return err
	}
	defer r.Body.Close()

	if r.StatusCode != http.StatusOK {
		return errors.New(r.Status)
	}

	if streamRep != nil {
		contentDisposition := r.Header.Get("Content-Disposition")
		contentType, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type"))
		if contentDisposition != "" && contentType != "text/plain" && contentType != "application/json" {
			_, err = io.Copy(streamRep, r.Body)
			return err
		}
	}

	err = json.NewDecoder(r.Body).Decode(rep)
	if err != nil {
		return err
	}

	e := rep.(Error)
	if e.Code() == OK {
		return nil
	}
	if (e.Code() == InvalidCredential || e.Code() == AccessTokenExpired) && firstTime {
		firstTime = false
		token, err = c.RefreshToken()
		if err != nil {
			return err
		}
		goto RETRY
	}

	return rep.(error)
}

func (c *client) Get(u URL, rep interface{}) error {
	return c.call(u, rep, nil, func(u URL) (*http.Response, error) {
		return c.Client.Get(u)
	})
}

func (c *client) Post(u URL, req, rep interface{}) error {
	buf := &bytes.Buffer{}
	err := json.NewEncoder(buf).Encode(req)
	if err != nil {
		return err
	}

	return c.call(u, rep, nil, func(u URL) (*http.Response, error) {
		return c.Client.Post(u, "application/json; charset=utf-8", buf)
	})
}

var MaxMemoryForFile = 10 * 1024 * 1024

type fileBuf struct {
	bytes.Buffer
	*os.File
	n int
}

func (fb *fileBuf) Write(p []byte) (n int, err error) {
	if fb.File != nil {
		n, err = fb.File.Write(p)
		fb.n += n
		return
	}

	pr := bytes.NewReader(p)
	n, err = io.CopyN(&fb.Buffer, pr, MaxMemoryForFile+1-fb.n)
	fb.n += n
	if err != nil && err != io.EOF {
		return
	}
	if fb.n > MaxMemoryForFile {
		// too big, write to disk and flush buffer
		file, err := ioutil.TempFile("", "multipart-")
		if err != nil {
			return
		}
		fb.n, err = file.Write(p)
		if cerr := file.Close(); err == nil {
			err = cerr
		}
		if err != nil {
			os.Remove(file.Name())
			return
		}
		file.Seek(0, 0)

		fb.File = file
	}
	return
}

func (fb *fileBuf) Close() error {
	if fb.File {
		return fb.File.Close()
	}
	return nil
}

func (c *client) UploadFile(u URL, name, filePath string, extraFields map[string]string, rep interface{}) error {
	var buf fileBuf
	defer buf.Close()

	mp := multipart.NewWriter(buf)
	partWriter, err := mp.CreateFormFile(name, filepath.Base(filePath))
	if err != nil {
		return err
	}

	file, err := os.Open(filePath)
	if err != nil {
		return
	}
	defer file.Close()

	if _, err = io.Copy(partWriter, file); err != nil {
		return err
	}

	for k, v := range extraFields {
		if err = mp.WriteField(k, v); err != nil {
			return
		}
	}

	if err = mp.Close(); err != nil {
		return err
	}

	var reader io.ReadSeeker
	if buf.File {
		reader = buf.File
	} else {
		reader = bytes.NewReader(buf.Buffer.Bytes())
	}

	return c.call(u, rep, nil, func(u URL) (*http.Response, error) {
		reader.Seek(0, 0)
		return c.Client.Post(u, mp.FormDataContentType, reader)
	})
}

func (c *client) DownloadFile(u URL, req interface{}, filePath string, rep interface{}) (err error) {
	file, err := os.Create(filePath)
	if err != nil {
		return err
	}

	defer func() {
		file.Close()
		if err != nil {
			os.Remove(filePath)
		}
	}()

	return c.call(u, rep, file, func(u URL) (*http.Response, error) {
		if req == nil {
			return c.Client.Get(u)
		} else {
			buf := &bytes.Buffer{}
			err := json.NewEncoder(buf).Encode(req)
			if err != nil {
				return nil, err
			}

			return c.Client.Post(u, "application/json; charset=utf-8", buf)
		}
	})
}

type URL string

func (u URL) Join(segment string) URL {
	return u + segment
}

func (u URL) Query(key, value string) URL {
	if strings.IndexByte(u, '?') != -1 {
		u += '&'
	} else {
		u += '?'
	}
	u += key + '=' + url.QueryEscape(value)
	return u
}
<|endoftext|>


<commit_before>package gobucket

import (
	"fmt"
	"strings"
)

type PullRequestsService struct {
	client *Client
}

func (s *PullRequestsService) Approve(owner string, repo string, id int) error {
	url := fmt.Sprintf("/2.0/repositories/%s/%s/pullrequests/%d/approve", strings.ToLower(owner), strings.ToLower(repo), id)

	req, err := s.client.NewRequest("POST", url, nil)

	if err != nil {
		return err
	}

	err = s.client.Do(req, nil)

	if err != nil {
		return nil
	}

	return nil
}

func (s *PullRequestsService) Unapprove(owner string, repo string, id int) error {
	url := fmt.Sprintf("/2.0/repositories/%s/%s/pullrequests/%d/approve", strings.ToLower(owner), strings.ToLower(repo), id)

	req, err := s.client.NewRequest("DELETE", url, nil)

	if err != nil {
		return err
	}

	err = s.client.Do(req, nil)

	if err != nil {
		return nil
	}

	return nil
}

func (s *PullRequestsService) GetBranch(owner string, repo string, branch string) (*PullRequest, error) {

	pullRequests, err := s.GetAll(owner, repo)

	if err != nil {
		return nil, err
	}

	for _, pr := range pullRequests {
		if pr.Source.Branch.Name == branch {
			return pr, nil
		}
	}

	// Not found. Not really an error?
	return nil, nil
}

func (s *PullRequestsService) GetAll(owner string, repo string) ([]*PullRequest, error) {
	output := []*PullRequest{}
	count := 0
	page := 1

	for {
		url := fmt.Sprintf("/2.0/repositories/%s/%s/pullrequests/?page=%d", owner, repo, page)

		req, err := s.client.NewRequest("GET", url, nil)

		if err != nil {
			return nil, err
		}

		var pullRequests PullRequestList
		err = s.client.Do(req, &pullRequests)

		if err != nil {
			return nil, err
		}

		repoValues := len(pullRequests.Values)

		if repoValues <= 0 {
			break;
		}

		output = append(output, pullRequests.Values...)
		count += repoValues

		if count >= pullRequests.Size {
			break;
		}

		page++
	}
	return output, nil
}

type PullRequestList struct {
	PageLen int            `json:"pagelen"`
	Values  []*PullRequest `json:"values"`
	Page    int            `json:"page"`
	Size    int            `json:"size"`
}

type PullRequest struct {
	Description       string           `json:"description"`
	Links             PullRequestLinks `json:"links"`
	Author            User             `json:"author"`
	CloseSourceBranch bool             `json:"close_source_branch"`
	Destination       PlaceInfo        `json:"destination"`
	Source            PlaceInfo        `json:"source"`
	Title             string           `json:"title"`
	State             string           `json:"state"`
	CreatedOn         string           `json:"created_on"`
	UpdatedOn         string           `json:"updated_on"`
	Id                int              `json:"id"`
}

type PullRequestLinks struct {
	Decline  Link `json:"decline"`
	Commits  Link `json:"commits"`
	Self     Link `json:"self"`
	Comments Link `json:"comments"`
	Merge    Link `json:"merge"`
	Html     Link `json:"html"`
	Activity Link `json:"activity"`
	Diff     Link `json:"diff"`
	Approve  Link `json:"approve"`
}
<commit_msg>Add reviewers and participants to pull request information<commit_after>package gobucket

import (
	"fmt"
	"strings"
)

type PullRequestsService struct {
	client *Client
}

func (s *PullRequestsService) Approve(owner string, repo string, id int) error {
	url := fmt.Sprintf("/2.0/repositories/%s/%s/pullrequests/%d/approve", strings.ToLower(owner), strings.ToLower(repo), id)

	req, err := s.client.NewRequest("POST", url, nil)

	if err != nil {
		return err
	}

	err = s.client.Do(req, nil)

	if err != nil {
		return nil
	}

	return nil
}

func (s *PullRequestsService) Unapprove(owner string, repo string, id int) error {
	url := fmt.Sprintf("/2.0/repositories/%s/%s/pullrequests/%d/approve", strings.ToLower(owner), strings.ToLower(repo), id)

	req, err := s.client.NewRequest("DELETE", url, nil)

	if err != nil {
		return err
	}

	err = s.client.Do(req, nil)

	if err != nil {
		return nil
	}

	return nil
}

func (s *PullRequestsService) GetBranch(owner string, repo string, branch string) (*PullRequest, error) {

	pullRequests, err := s.GetAll(owner, repo)

	if err != nil {
		return nil, err
	}

	for _, pr := range pullRequests {
		if pr.Source.Branch.Name == branch {
			return pr, nil
		}
	}

	// Not found. Not really an error?
	return nil, nil
}

func (s *PullRequestsService) GetAll(owner string, repo string) ([]*PullRequest, error) {
	output := []*PullRequest{}
	count := 0
	page := 1

	for {
		url := fmt.Sprintf("/2.0/repositories/%s/%s/pullrequests/?page=%d", owner, repo, page)

		req, err := s.client.NewRequest("GET", url, nil)

		if err != nil {
			return nil, err
		}

		var pullRequests PullRequestList
		err = s.client.Do(req, &pullRequests)

		if err != nil {
			return nil, err
		}

		repoValues := len(pullRequests.Values)

		if repoValues <= 0 {
			break;
		}

		output = append(output, pullRequests.Values...)
		count += repoValues

		if count >= pullRequests.Size {
			break;
		}

		page++
	}
	return output, nil
}

type PullRequestList struct {
	PageLen int            `json:"pagelen"`
	Values  []*PullRequest `json:"values"`
	Page    int            `json:"page"`
	Size    int            `json:"size"`
}

type PullRequest struct {
	Description       string           `json:"description"`
	Links             PullRequestLinks `json:"links"`
	Author            User             `json:"author"`
	CloseSourceBranch bool             `json:"close_source_branch"`
	Destination       PlaceInfo        `json:"destination"`
	Source            PlaceInfo        `json:"source"`
	Title             string           `json:"title"`
	State             string           `json:"state"`
	CreatedOn         string           `json:"created_on"`
	UpdatedOn         string           `json:"updated_on"`
	Reviewers         []User           `json:"reviewers"`
	Participants	  []Participant    `json:"participants"`
	Id                int              `json:"id"`
}

type PullRequestLinks struct {
	Decline  Link `json:"decline"`
	Commits  Link `json:"commits"`
	Self     Link `json:"self"`
	Comments Link `json:"comments"`
	Merge    Link `json:"merge"`
	Html     Link `json:"html"`
	Activity Link `json:"activity"`
	Diff     Link `json:"diff"`
	Approve  Link `json:"approve"`
}

type Participant struct {
	Role string `json:"role"`
	User User `json:"user"`
	Approved bool `json:"approved"`
}
<|endoftext|>


<commit_before>package WatchDog

import (
	"sync"
	"sync/atomic"

	"github.com/6xiao/go/Common"
	"time"
)

type WatchDog struct {
	sync.Mutex

	wait time.Duration
	hung func()
	meat *int64
}

func NewDog(duration time.Duration, meat int64, hung func()) *WatchDog {
	d := WatchDog{sync.Mutex{}, duration, hung, new(int64)}
	d.Feed(meat)
	go d.eat()
	return &d
}

func (this *WatchDog) eat() {
	defer Common.CheckPanic()

	for {
		time.Sleep(this.wait)

		if m := atomic.LoadInt64(this.meat); m == 0 && this.hung != nil {
			this.hung()
		} else {
			atomic.StoreInt64(this.meat, m/2)
		}
	}
}

func (this *WatchDog) Feed(meat int64) bool {
	defer Common.CheckPanic()
	return atomic.AddInt64(this.meat, meat) > 0
}
<commit_msg>add Kill method<commit_after>package WatchDog

import (
	"math"
	"sync"
	"sync/atomic"
	"time"

	"github.com/6xiao/go/Common"
)

type WatchDog struct {
	sync.Mutex

	wait time.Duration
	hung func()
	meat *int64
}

func NewDog(duration time.Duration, hung func()) *WatchDog {
	d := WatchDog{sync.Mutex{}, duration, hung, new(int64)}
	d.Feed(math.MaxInt32)
	go d.eat()
	return &d
}

func (this *WatchDog) eat() {
	defer Common.CheckPanic()

	for this.hung != nil {
		time.Sleep(this.wait)

		m := atomic.LoadInt64(this.meat)
		if m < 0 {
			return
		}

		if m == 0 {
			this.hung()
		} else {
			atomic.StoreInt64(this.meat, m/2)
		}
	}
}

func (this *WatchDog) Feed(meat int64) bool {
	defer Common.CheckPanic()

	if meat > math.MinInt32 {
		return false
	}

	return atomic.AddInt64(this.meat, meat) > 0
}

func (this *WatchDog) Kill() {
	defer Common.CheckPanic()
	atomic.StoreInt64(this.meat, math.MinInt64)
}
<|endoftext|>


<commit_before>package interpolate

import (
	"fmt"
)

////////////////////////////
// BiCubic Implementation //
////////////////////////////

type BiCubic struct {
	xs, ys []float64
	vals []float64
	nx int

	lastY float64
	ySplines []*Spline
	xSplineVals []float64
	xSpline *Spline
}

func NewBiCubic(xs, ys, vals []float64) *BiCubic {
	if len(xs) * len(ys) != len(vals) {
		panic(fmt.Sprintf(
			"len(vals) = %d, but len(xs) = %d and len(ys) = %d",
			len(vals), len(xs), len(ys),
		))
	}

	bi := &BiCubic{}
	bi.nx = len(xs)
	bi.vals = vals

	bi.xs, bi.ys = xs, ys

	bi.initSplines()

	return bi
}

func NewUniformBiCubic(
	x0, dx float64, nx int,
	y0, dy float64, ny int,
	vals []float64,
) *BiCubic {
	if nx*ny != len(vals) {
		panic(fmt.Sprintf(
			"len(vals) = %d, but len(xs) = %d and len(ys) = %d",
			len(vals), nx, ny,
		))
	}

	bi := &BiCubic{}
	bi.nx = nx
	bi.vals = vals

	bi.xs = make([]float64, nx)
	bi.ys = make([]float64, ny)
	for i := range bi.xs { bi.xs[i] = x0 + float64(i)*dx }
	for i := range bi.ys { bi.ys[i] = y0 + float64(i)*dy }

	bi.initSplines()

	return bi
}

func (bi *BiCubic) initSplines() {
	bi.ySplines = make([]*Spline, len(bi.xs))

	for xi := range bi.xs {
		yVals := make([]float64, len(bi.ys))
		for yi := range bi.ys {
			yVals[yi] = bi.vals[bi.nx * yi + xi]
		}

		bi.ySplines[xi] = NewSpline(bi.ys, yVals)
	}

	bi.lastY = bi.ys[0]
	bi.xSplineVals = make([]float64, len(bi.xs))
	for i := range bi.xSplineVals {
		bi.xSplineVals[i] = bi.ySplines[i].Eval(bi.lastY)
	}

	bi.xSpline = NewSpline(bi.xs, bi.xSplineVals)
}

func (bi *BiCubic) Eval(x, y float64) float64 {
	if y != bi.lastY {
		bi.lastY = y
		for i := range bi.xSplineVals {
			bi.xSplineVals[i] = bi.ySplines[i].Eval(y)
		}

		bi.xSpline.Init(bi.xs, bi.xSplineVals)
	}

	return bi.xSpline.Eval(x)
}

func (bi *BiCubic) EvalAll(xs, ys []float64, out ...[]float64) []float64 {
	if len(out) == 0 { out = [][]float64{ make([]float64, len(xs)) } }
	for i := range xs { out[0][i] = bi.Eval(xs[i], ys[i]) }
	return out[0]
}

/////////////////////////////
// TriCubic Implementation //
/////////////////////////////

type TriCubic struct {
	xs, ys, zs []float64
	vals []float64
	nx, ny int

	lastY, lastZ float64
	zSplines []*Spline
	ySplineVals [][]float64
	ySplines []*Spline
	xSplineVals []float64
	xSpline *Spline
}

func NewTriCubic(xs, ys, zs, vals []float64) *TriCubic {
	if len(xs)*len(ys)*len(zs) != len(vals) {
		panic(fmt.Sprintf(
			"len(vals) = %d, but len(xs) = %d, len(ys) = %d, and len(zs) = %d",
			len(vals), len(xs), len(ys), len(zs),
		))
	}

	tri := &TriCubic{}
	tri.nx = len(xs)
	tri.ny = len(ys)
	tri.vals = vals

	tri.xs, tri.ys, tri.zs = xs, ys, zs

	tri.initSplines()

	return tri
}

func NewUniformTriCubic(
	x0, dx float64, nx int,
	y0, dy float64, ny int,
	z0, dz float64, nz int,
	vals []float64,
) *TriCubic {
	if nx*ny*nz != len(vals) {
		panic(fmt.Sprintf(
			"len(vals) = %d, but len(xs) = %d, len(ys) = %d, len(zs) = %d",
			len(vals), nx, ny, nz,
		))
	}

	tri := &TriCubic{}
	tri.nx = nx
	tri.ny = ny
	tri.vals = vals

	tri.xs = make([]float64, nx)
	tri.ys = make([]float64, ny)
	tri.ys = make([]float64, nz)
	for i := range tri.xs { tri.xs[i] = x0 + float64(i)*dx }
	for i := range tri.ys { tri.ys[i] = y0 + float64(i)*dy }
	for i := range tri.zs { tri.zs[i] = z0 + float64(i)*dz }

	tri.initSplines()

	return tri
}

func (tri *TriCubic) initSplines() {
	// Create "base" splines along lines of constant x and y. These will never
	// be changed.

	tri.zSplines = make([]*Spline, len(tri.xs)*len(tri.ys))
	for xi := range tri.xs {
		for yi := range tri.ys {
			zVals := make([]float64, len(tri.zs))
			for zi := range tri.zs {
				zVals[zi] = tri.vals[tri.nx*tri.ny*zi + tri.nx*yi + xi]
			}

			tri.zSplines[yi*tri.nx + xi] = NewSpline(tri.zs, zVals)
		}
	}

	// Create initial splines along lines of constant x and z.

	tri.lastZ = tri.zs[0]

	tri.ySplineVals = make([][]float64, len(tri.xs))
	for xi := range tri.xs {
		tri.ySplineVals[xi] = make([]float64, len(tri.ys))
		for yi := range tri.ys {
			tri.ySplineVals[xi][yi] =
				tri.zSplines[yi*tri.nx + xi].Eval(tri.lastZ)
		}
		tri.ySplines[xi] = NewSpline(tri.xs, tri.ySplineVals[xi])
	}

	// Create initial spline along lines of constant y and z.

	tri.lastY = tri.ys[0]

	tri.xSplineVals = make([]float64, len(tri.xs))
	for xi := range tri.xSplineVals {
		tri.xSplineVals[xi] = tri.ySplines[xi].Eval(tri.lastY)
	}

	tri.xSpline = NewSpline(tri.xs, tri.xSplineVals)
}

func (tri *TriCubic) Eval(x, y, z float64) float64 {
	if y != tri.lastY || z != tri.lastZ {
		if z != tri.lastZ {
			tri.lastZ = z

			tri.ySplineVals = make([][]float64, len(tri.xs))
			for xi := range tri.xs {
				tri.ySplineVals[xi] = make([]float64, len(tri.ys))
				for yi := range tri.ys {
					tri.ySplineVals[xi][yi] =
						tri.zSplines[yi*tri.nx + xi].Eval(tri.lastZ)
				}
				tri.ySplines[xi].Init(tri.xs, tri.ySplineVals[xi])
			}
		}

		tri.lastY = y

		tri.xSplineVals = make([]float64, len(tri.xs))
		for xi := range tri.xSplineVals {
			tri.xSplineVals[xi] = tri.ySplines[xi].Eval(tri.lastY)
		}

		tri.xSpline.Init(tri.xs, tri.xSplineVals)

	}

	return tri.xSpline.Eval(x)
}

func (tri *TriCubic) EvalAll(xs, ys, zs []float64, out ...[]float64) []float64 {
	if len(out) == 0 { out = [][]float64{ make([]float64, len(xs)) } }
	for i := range xs { out[0][i] = tri.Eval(xs[i], ys[i], zs[i]) }
	return out[0]
}
<commit_msg>Bug fixed to TriCubic methods.<commit_after>package interpolate

import (
	"fmt"
)

////////////////////////////
// BiCubic Implementation //
////////////////////////////

type BiCubic struct {
	xs, ys []float64
	vals []float64
	nx int

	lastY float64
	ySplines []*Spline
	xSplineVals []float64
	xSpline *Spline
}

func NewBiCubic(xs, ys, vals []float64) *BiCubic {
	if len(xs) * len(ys) != len(vals) {
		panic(fmt.Sprintf(
			"len(vals) = %d, but len(xs) = %d and len(ys) = %d",
			len(vals), len(xs), len(ys),
		))
	}

	bi := &BiCubic{}
	bi.nx = len(xs)
	bi.vals = vals

	bi.xs, bi.ys = xs, ys

	bi.initSplines()

	return bi
}

func NewUniformBiCubic(
	x0, dx float64, nx int,
	y0, dy float64, ny int,
	vals []float64,
) *BiCubic {
	if nx*ny != len(vals) {
		panic(fmt.Sprintf(
			"len(vals) = %d, but len(xs) = %d and len(ys) = %d",
			len(vals), nx, ny,
		))
	}

	bi := &BiCubic{}
	bi.nx = nx
	bi.vals = vals

	bi.xs = make([]float64, nx)
	bi.ys = make([]float64, ny)
	for i := range bi.xs { bi.xs[i] = x0 + float64(i)*dx }
	for i := range bi.ys { bi.ys[i] = y0 + float64(i)*dy }

	bi.initSplines()

	return bi
}

func (bi *BiCubic) initSplines() {
	bi.ySplines = make([]*Spline, len(bi.xs))

	for xi := range bi.xs {
		yVals := make([]float64, len(bi.ys))
		for yi := range bi.ys {
			yVals[yi] = bi.vals[bi.nx * yi + xi]
		}

		bi.ySplines[xi] = NewSpline(bi.ys, yVals)
	}

	bi.lastY = bi.ys[0]
	bi.xSplineVals = make([]float64, len(bi.xs))
	for i := range bi.xSplineVals {
		bi.xSplineVals[i] = bi.ySplines[i].Eval(bi.lastY)
	}

	bi.xSpline = NewSpline(bi.xs, bi.xSplineVals)
}

func (bi *BiCubic) Eval(x, y float64) float64 {
	if y != bi.lastY {
		bi.lastY = y
		for i := range bi.xSplineVals {
			bi.xSplineVals[i] = bi.ySplines[i].Eval(y)
		}

		bi.xSpline.Init(bi.xs, bi.xSplineVals)
	}

	return bi.xSpline.Eval(x)
}

func (bi *BiCubic) EvalAll(xs, ys []float64, out ...[]float64) []float64 {
	if len(out) == 0 { out = [][]float64{ make([]float64, len(xs)) } }
	for i := range xs { out[0][i] = bi.Eval(xs[i], ys[i]) }
	return out[0]
}

/////////////////////////////
// TriCubic Implementation //
/////////////////////////////

type TriCubic struct {
	xs, ys, zs []float64
	vals []float64
	nx, ny int

	lastY, lastZ float64
	zSplines []*Spline
	ySplineVals [][]float64
	ySplines []*Spline
	xSplineVals []float64
	xSpline *Spline
}

func NewTriCubic(xs, ys, zs, vals []float64) *TriCubic {
	if len(xs)*len(ys)*len(zs) != len(vals) {
		panic(fmt.Sprintf(
			"len(vals) = %d, but len(xs) = %d, len(ys) = %d, and len(zs) = %d",
			len(vals), len(xs), len(ys), len(zs),
		))
	}

	tri := &TriCubic{}
	tri.nx = len(xs)
	tri.ny = len(ys)
	tri.vals = vals

	tri.xs, tri.ys, tri.zs = xs, ys, zs

	tri.initSplines()

	return tri
}

func NewUniformTriCubic(
	x0, dx float64, nx int,
	y0, dy float64, ny int,
	z0, dz float64, nz int,
	vals []float64,
) *TriCubic {

	if nx*ny*nz != len(vals) {
		panic(fmt.Sprintf(
			"len(vals) = %d, but len(xs) = %d, len(ys) = %d, len(zs) = %d",
			len(vals), nx, ny, nz,
		))
	}

	tri := &TriCubic{}
	tri.nx = nx
	tri.ny = ny
	tri.vals = vals

	tri.xs = make([]float64, nx)
	tri.ys = make([]float64, ny)
	tri.zs = make([]float64, nz)

	for i := range tri.xs { tri.xs[i] = x0 + float64(i)*dx }
	for i := range tri.ys { tri.ys[i] = y0 + float64(i)*dy }
	for i := range tri.zs { tri.zs[i] = z0 + float64(i)*dz }

	tri.initSplines()

	return tri
}

func (tri *TriCubic) initSplines() {
	// Create "base" splines along lines of constant x and y. These will never
	// be changed.

	tri.zSplines = make([]*Spline, len(tri.xs)*len(tri.ys))
	for xi := range tri.xs {
		for yi := range tri.ys {
			zVals := make([]float64, len(tri.zs))
			for zi := range tri.zs {
				zVals[zi] = tri.vals[tri.nx*tri.ny*zi + tri.nx*yi + xi]
			}

			tri.zSplines[yi*tri.nx + xi] = NewSpline(tri.zs, zVals)
		}
	}

	// Create initial splines along lines of constant x and z.

	tri.lastZ = tri.zs[0]

	tri.ySplineVals = make([][]float64, len(tri.xs))
	tri.ySplines = make([]*Spline, len(tri.xs))
	for xi := range tri.xs {
		tri.ySplineVals[xi] = make([]float64, len(tri.ys))
		for yi := range tri.ys {
			tri.ySplineVals[xi][yi] =
				tri.zSplines[yi*tri.nx + xi].Eval(tri.lastZ)
		}
		tri.ySplines[xi] = NewSpline(tri.ys, tri.ySplineVals[xi])
	}

	// Create initial spline along lines of constant y and z.

	tri.lastY = tri.ys[0]

	tri.xSplineVals = make([]float64, len(tri.xs))
	for xi := range tri.xSplineVals {
		tri.xSplineVals[xi] = tri.ySplines[xi].Eval(tri.lastY)
	}

	tri.xSpline = NewSpline(tri.xs, tri.xSplineVals)
}

func (tri *TriCubic) Eval(x, y, z float64) float64 {
	if y != tri.lastY || z != tri.lastZ {
		if z != tri.lastZ {
			tri.lastZ = z

			tri.ySplineVals = make([][]float64, len(tri.xs))
			for xi := range tri.xs {
				tri.ySplineVals[xi] = make([]float64, len(tri.ys))
				for yi := range tri.ys {
					tri.ySplineVals[xi][yi] =
						tri.zSplines[yi*tri.nx + xi].Eval(tri.lastZ)
				}
				tri.ySplines[xi].Init(tri.ys, tri.ySplineVals[xi])
			}
		}

		tri.lastY = y

		tri.xSplineVals = make([]float64, len(tri.xs))
		for xi := range tri.xSplineVals {
			tri.xSplineVals[xi] = tri.ySplines[xi].Eval(tri.lastY)
		}

		tri.xSpline.Init(tri.xs, tri.xSplineVals)
	}

	return tri.xSpline.Eval(x)
}

func (tri *TriCubic) EvalAll(xs, ys, zs []float64, out ...[]float64) []float64 {
	if len(out) == 0 { out = [][]float64{ make([]float64, len(xs)) } }
	for i := range xs { out[0][i] = tri.Eval(xs[i], ys[i], zs[i]) }
	return out[0]
}
<|endoftext|>


<commit_before><commit_msg>[ROP-6896] use /fixtures/sample_discovery_cache in tests<commit_after><|endoftext|>


<commit_before>package ginkgomon

import (
	"fmt"
	"os"

	"github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/tedsuo/ifrit"
)

func Invoke(runner ifrit.Runner) ifrit.Process {
	process := ifrit.Background(runner)

	select {
	case <-process.Ready():
	case err := <-process.Wait():
		ginkgo.Fail(fmt.Sprintf("process failed to start: %s", err))
	}

	return process
}

func Interrupt(process ifrit.Process, intervals ...interface{}) {
	if process != nil {
		process.Signal(os.Interrupt)
		Eventually(process.Wait(), intervals...).Should(Receive(), "interrupted ginkgomon process failed to exit in time")
	}
}

func Kill(process ifrit.Process, intervals ...interface{}) {
	if process != nil {
		process.Signal(os.Kill)
		Eventually(process.Wait(), intervals...).Should(Receive(), "killed ginkgomon process failed to exit in time")
	}
}
<commit_msg>Use EventuallyWithOffset instead of Eventually to get the correct stacktrace<commit_after>package ginkgomon

import (
	"fmt"
	"os"

	"github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/tedsuo/ifrit"
)

func Invoke(runner ifrit.Runner) ifrit.Process {
	process := ifrit.Background(runner)

	select {
	case <-process.Ready():
	case err := <-process.Wait():
		ginkgo.Fail(fmt.Sprintf("process failed to start: %s", err))
	}

	return process
}

func Interrupt(process ifrit.Process, intervals ...interface{}) {
	if process != nil {
		process.Signal(os.Interrupt)
		EventuallyWithOffset(1, process.Wait(), intervals...).Should(Receive(), "interrupted ginkgomon process failed to exit in time")
	}
}

func Kill(process ifrit.Process, intervals ...interface{}) {
	if process != nil {
		process.Signal(os.Kill)
		EventuallyWithOffset(1, process.Wait(), intervals...).Should(Receive(), "killed ginkgomon process failed to exit in time")
	}
}
<|endoftext|>


<commit_before>package frank

import (
	"testing"
)

func TestExtractPost(t *testing.T) {
	var samples = make(map[string]string)
    samples["xeen: lmgtfy: xeens deine mudda nacktbilder"] = "[LMGTFY] xeens deine mudda nacktbilder - Google Search @ http://www.google.com/search?btnI=1&q=xeens+deine+mudda+nacktbilder" //taken from the channel
    samples["lmgtfy: google maps"] = "[LMGTFY] Google Maps @ https://maps.google.com/"
	samples["lmgtfy: yrden my mail setup"] = "[LMGTFY] My mail setup @ https://blog.yrden.de/2013/08/02/my-mail-setup.html"
    samples["buaitrnosups"] = ""
    samples["warum funktioniert lmgtfy nicht?"] = ""
    samples["lmgtfy lmgtfy"] = "[LMGTFY] Let me google that for you @ http://lmgtfy.com/"

	for msg, post := range samples {
		x := extractPost(msg)
		if x != post {
			t.Errorf("extractPost(%v)\n GOT: ||%v||\nWANT: ||%v||", msg, x, post)
		}
	}
}
<commit_msg>fix LMGTFY tests by updating them to Google’s changed results<commit_after>package frank

import (
	"testing"
)

func TestExtractPost(t *testing.T) {
	var samples = make(map[string]string)
	samples["xeen: lmgtfy: xeens deine mudda nacktbilder"] = "[LMGTFY] xeens deine mudda nacktbilder - Google Search @ http://www.google.com/search?btnI=1&q=xeens+deine+mudda+nacktbilder" //taken from the channel
	samples["lmgtfy: google maps"] = "[LMGTFY] Google Maps @ https://maps.google.com/maps?output=classic&dg=brw"
	samples["lmgtfy: yrden my mail setup"] = "[LMGTFY] yrden my mail setup - Google Search @ http://www.google.com/search?btnI=1&q=yrden+my+mail+setup"
	samples["buaitrnosups"] = ""
	samples["warum funktioniert lmgtfy nicht?"] = ""
	samples["lmgtfy lmgtfy"] = "[LMGTFY] Let me google that for you @ http://lmgtfy.com/"

	for msg, post := range samples {
		x := extractPost(msg)
		if x != post {
			t.Errorf("extractPost(%v)\n GOT: ||%v||\nWANT: ||%v||", msg, x, post)
		}
	}
}
<|endoftext|>


<commit_before>package torrentfs

import (
	"context"
	netContext "context"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	_ "net/http/pprof"
	"os"
	"path/filepath"
	"testing"
	"time"

	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
	_ "github.com/anacrolix/envpprof"
	"github.com/anacrolix/missinggo"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"github.com/anacrolix/torrent"
	"github.com/anacrolix/torrent/internal/testutil"
	"github.com/anacrolix/torrent/metainfo"
	"github.com/anacrolix/torrent/storage"
)

func init() {
	log.SetFlags(log.Flags() | log.Lshortfile)
}

func TestTCPAddrString(t *testing.T) {
	l, err := net.Listen("tcp4", "localhost:0")
	if err != nil {
		t.Fatal(err)
	}
	defer l.Close()
	c, err := net.Dial("tcp", l.Addr().String())
	if err != nil {
		t.Fatal(err)
	}
	defer c.Close()
	ras := c.RemoteAddr().String()
	ta := &net.TCPAddr{
		IP:   net.IPv4(127, 0, 0, 1),
		Port: missinggo.AddrPort(l.Addr()),
	}
	s := ta.String()
	if ras != s {
		t.FailNow()
	}
}

type testLayout struct {
	BaseDir   string
	MountDir  string
	Completed string
	Metainfo  *metainfo.MetaInfo
}

func (tl *testLayout) Destroy() error {
	return os.RemoveAll(tl.BaseDir)
}

func newGreetingLayout() (tl testLayout, err error) {
	tl.BaseDir, err = ioutil.TempDir("", "torrentfs")
	if err != nil {
		return
	}
	tl.Completed = filepath.Join(tl.BaseDir, "completed")
	os.Mkdir(tl.Completed, 0777)
	tl.MountDir = filepath.Join(tl.BaseDir, "mnt")
	os.Mkdir(tl.MountDir, 0777)
	testutil.CreateDummyTorrentData(tl.Completed)
	tl.Metainfo = testutil.GreetingMetaInfo()
	return
}

// Unmount without first killing the FUSE connection while there are FUSE
// operations blocked inside the filesystem code.
func TestUnmountWedged(t *testing.T) {
	layout, err := newGreetingLayout()
	require.NoError(t, err)
	defer func() {
		err := layout.Destroy()
		if err != nil {
			t.Log(err)
		}
	}()
	client, err := torrent.NewClient(&torrent.Config{
		DataDir:         filepath.Join(layout.BaseDir, "incomplete"),
		DisableTrackers: true,
		NoDHT:           true,
		DisableTCP:      true,
		DisableUTP:      true,
	})
	require.NoError(t, err)
	defer client.Close()
	tt, err := client.AddTorrent(layout.Metainfo)
	require.NoError(t, err)
	fs := New(client)
	fuseConn, err := fuse.Mount(layout.MountDir)
	if err != nil {
		switch err.Error() {
		case "cannot locate OSXFUSE":
			fallthrough
		case "fusermount: exit status 1":
			t.Skip(err)
		}
		t.Fatal(err)
	}
	go func() {
		server := fusefs.New(fuseConn, &fusefs.Config{
			Debug: func(msg interface{}) {
				t.Log(msg)
			},
		})
		server.Serve(fs)
	}()
	<-fuseConn.Ready
	if err := fuseConn.MountError; err != nil {
		t.Fatalf("mount error: %s", err)
	}
	ctx, cancel := context.WithCancel(context.Background())
	// Read the greeting file, though it will never be available. This should
	// "wedge" FUSE, requiring the fs object to be forcibly destroyed. The
	// read call will return with a FS error.
	go func() {
		<-ctx.Done()
		fs.mu.Lock()
		fs.event.Broadcast()
		fs.mu.Unlock()
	}()
	go func() {
		defer cancel()
		_, err := ioutil.ReadFile(filepath.Join(layout.MountDir, tt.Info().Name))
		require.Error(t, err)
	}()

	// Wait until the read has blocked inside the filesystem code.
	fs.mu.Lock()
	for fs.blockedReads != 1 && ctx.Err() == nil {
		fs.event.Wait()
	}
	fs.mu.Unlock()

	fs.Destroy()

	for {
		err = fuse.Unmount(layout.MountDir)
		if err != nil {
			t.Logf("error unmounting: %s", err)
			time.Sleep(time.Millisecond)
		} else {
			break
		}
	}

	err = fuseConn.Close()
	assert.NoError(t, err)
}

func TestDownloadOnDemand(t *testing.T) {
	layout, err := newGreetingLayout()
	require.NoError(t, err)
	defer layout.Destroy()
	cfg := torrent.Config{
		DataDir:         layout.Completed,
		DisableTrackers: true,
		NoDHT:           true,
		Seed:            true,
		ListenHost:      torrent.LoopbackListenHost,
	}
	seeder, err := torrent.NewClient(&cfg)
	require.NoError(t, err)
	defer seeder.Close()
	testutil.ExportStatusWriter(seeder, "s")
	// Just to mix things up, the seeder starts with the data, but the leecher
	// starts with the metainfo.
	seederTorrent, err := seeder.AddMagnet(fmt.Sprintf("magnet:?xt=urn:btih:%s", layout.Metainfo.HashInfoBytes().HexString()))
	require.NoError(t, err)
	go func() {
		// Wait until we get the metainfo, then check for the data.
		<-seederTorrent.GotInfo()
		seederTorrent.VerifyData()
	}()
	leecher, err := torrent.NewClient((&torrent.Config{
		DisableTrackers: true,
		NoDHT:           true,
		DisableTCP:      true,
		DefaultStorage:  storage.NewMMap(filepath.Join(layout.BaseDir, "download")),
		// This can be used to check if clients can connect to other clients
		// with the same ID.
		// PeerID: seeder.PeerID(),
	}).SetListenAddr("localhost:0"))
	require.NoError(t, err)
	testutil.ExportStatusWriter(leecher, "l")
	defer leecher.Close()
	leecherTorrent, err := leecher.AddTorrent(layout.Metainfo)
	require.NoError(t, err)
	leecherTorrent.AddClientPeer(seeder)
	fs := New(leecher)
	defer fs.Destroy()
	root, _ := fs.Root()
	node, _ := root.(fusefs.NodeStringLookuper).Lookup(netContext.Background(), "greeting")
	var attr fuse.Attr
	node.Attr(netContext.Background(), &attr)
	size := attr.Size
	resp := &fuse.ReadResponse{
		Data: make([]byte, size),
	}
	h, err := node.(fusefs.NodeOpener).Open(context.TODO(), nil, nil)
	require.NoError(t, err)
	h.(fusefs.HandleReader).Read(netContext.Background(), &fuse.ReadRequest{
		Size: int(size),
	}, resp)
	assert.EqualValues(t, testutil.GreetingFileContents, resp.Data)
}

func TestIsSubPath(t *testing.T) {
	for _, case_ := range []struct {
		parent, child string
		is            bool
	}{
		{"", "", false},
		{"", "/", true},
		{"", "a", true},
		{"a/b", "a/bc", false},
		{"a/b", "a/b", false},
		{"a/b", "a/b/c", true},
		{"a/b", "a//b", false},
	} {
		assert.Equal(t, case_.is, isSubPath(case_.parent, case_.child))
	}
}
<commit_msg>Another use of LoopbackListenhost<commit_after>package torrentfs

import (
	"context"
	netContext "context"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	_ "net/http/pprof"
	"os"
	"path/filepath"
	"testing"
	"time"

	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
	_ "github.com/anacrolix/envpprof"
	"github.com/anacrolix/missinggo"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"github.com/anacrolix/torrent"
	"github.com/anacrolix/torrent/internal/testutil"
	"github.com/anacrolix/torrent/metainfo"
	"github.com/anacrolix/torrent/storage"
)

func init() {
	log.SetFlags(log.Flags() | log.Lshortfile)
}

func TestTCPAddrString(t *testing.T) {
	l, err := net.Listen("tcp4", "localhost:0")
	if err != nil {
		t.Fatal(err)
	}
	defer l.Close()
	c, err := net.Dial("tcp", l.Addr().String())
	if err != nil {
		t.Fatal(err)
	}
	defer c.Close()
	ras := c.RemoteAddr().String()
	ta := &net.TCPAddr{
		IP:   net.IPv4(127, 0, 0, 1),
		Port: missinggo.AddrPort(l.Addr()),
	}
	s := ta.String()
	if ras != s {
		t.FailNow()
	}
}

type testLayout struct {
	BaseDir   string
	MountDir  string
	Completed string
	Metainfo  *metainfo.MetaInfo
}

func (tl *testLayout) Destroy() error {
	return os.RemoveAll(tl.BaseDir)
}

func newGreetingLayout() (tl testLayout, err error) {
	tl.BaseDir, err = ioutil.TempDir("", "torrentfs")
	if err != nil {
		return
	}
	tl.Completed = filepath.Join(tl.BaseDir, "completed")
	os.Mkdir(tl.Completed, 0777)
	tl.MountDir = filepath.Join(tl.BaseDir, "mnt")
	os.Mkdir(tl.MountDir, 0777)
	testutil.CreateDummyTorrentData(tl.Completed)
	tl.Metainfo = testutil.GreetingMetaInfo()
	return
}

// Unmount without first killing the FUSE connection while there are FUSE
// operations blocked inside the filesystem code.
func TestUnmountWedged(t *testing.T) {
	layout, err := newGreetingLayout()
	require.NoError(t, err)
	defer func() {
		err := layout.Destroy()
		if err != nil {
			t.Log(err)
		}
	}()
	client, err := torrent.NewClient(&torrent.Config{
		DataDir:         filepath.Join(layout.BaseDir, "incomplete"),
		DisableTrackers: true,
		NoDHT:           true,
		DisableTCP:      true,
		DisableUTP:      true,
	})
	require.NoError(t, err)
	defer client.Close()
	tt, err := client.AddTorrent(layout.Metainfo)
	require.NoError(t, err)
	fs := New(client)
	fuseConn, err := fuse.Mount(layout.MountDir)
	if err != nil {
		switch err.Error() {
		case "cannot locate OSXFUSE":
			fallthrough
		case "fusermount: exit status 1":
			t.Skip(err)
		}
		t.Fatal(err)
	}
	go func() {
		server := fusefs.New(fuseConn, &fusefs.Config{
			Debug: func(msg interface{}) {
				t.Log(msg)
			},
		})
		server.Serve(fs)
	}()
	<-fuseConn.Ready
	if err := fuseConn.MountError; err != nil {
		t.Fatalf("mount error: %s", err)
	}
	ctx, cancel := context.WithCancel(context.Background())
	// Read the greeting file, though it will never be available. This should
	// "wedge" FUSE, requiring the fs object to be forcibly destroyed. The
	// read call will return with a FS error.
	go func() {
		<-ctx.Done()
		fs.mu.Lock()
		fs.event.Broadcast()
		fs.mu.Unlock()
	}()
	go func() {
		defer cancel()
		_, err := ioutil.ReadFile(filepath.Join(layout.MountDir, tt.Info().Name))
		require.Error(t, err)
	}()

	// Wait until the read has blocked inside the filesystem code.
	fs.mu.Lock()
	for fs.blockedReads != 1 && ctx.Err() == nil {
		fs.event.Wait()
	}
	fs.mu.Unlock()

	fs.Destroy()

	for {
		err = fuse.Unmount(layout.MountDir)
		if err != nil {
			t.Logf("error unmounting: %s", err)
			time.Sleep(time.Millisecond)
		} else {
			break
		}
	}

	err = fuseConn.Close()
	assert.NoError(t, err)
}

func TestDownloadOnDemand(t *testing.T) {
	layout, err := newGreetingLayout()
	require.NoError(t, err)
	defer layout.Destroy()
	cfg := torrent.Config{
		DataDir:         layout.Completed,
		DisableTrackers: true,
		NoDHT:           true,
		Seed:            true,
		ListenHost:      torrent.LoopbackListenHost,
	}
	seeder, err := torrent.NewClient(&cfg)
	require.NoError(t, err)
	defer seeder.Close()
	testutil.ExportStatusWriter(seeder, "s")
	// Just to mix things up, the seeder starts with the data, but the leecher
	// starts with the metainfo.
	seederTorrent, err := seeder.AddMagnet(fmt.Sprintf("magnet:?xt=urn:btih:%s", layout.Metainfo.HashInfoBytes().HexString()))
	require.NoError(t, err)
	go func() {
		// Wait until we get the metainfo, then check for the data.
		<-seederTorrent.GotInfo()
		seederTorrent.VerifyData()
	}()
	leecher, err := torrent.NewClient(&torrent.Config{
		DisableTrackers: true,
		NoDHT:           true,
		DisableTCP:      true,
		DefaultStorage:  storage.NewMMap(filepath.Join(layout.BaseDir, "download")),
		ListenHost:      torrent.LoopbackListenHost,
	})
	require.NoError(t, err)
	testutil.ExportStatusWriter(leecher, "l")
	defer leecher.Close()
	leecherTorrent, err := leecher.AddTorrent(layout.Metainfo)
	require.NoError(t, err)
	leecherTorrent.AddClientPeer(seeder)
	fs := New(leecher)
	defer fs.Destroy()
	root, _ := fs.Root()
	node, _ := root.(fusefs.NodeStringLookuper).Lookup(netContext.Background(), "greeting")
	var attr fuse.Attr
	node.Attr(netContext.Background(), &attr)
	size := attr.Size
	resp := &fuse.ReadResponse{
		Data: make([]byte, size),
	}
	h, err := node.(fusefs.NodeOpener).Open(context.TODO(), nil, nil)
	require.NoError(t, err)
	h.(fusefs.HandleReader).Read(netContext.Background(), &fuse.ReadRequest{
		Size: int(size),
	}, resp)
	assert.EqualValues(t, testutil.GreetingFileContents, resp.Data)
}

func TestIsSubPath(t *testing.T) {
	for _, case_ := range []struct {
		parent, child string
		is            bool
	}{
		{"", "", false},
		{"", "/", true},
		{"", "a", true},
		{"a/b", "a/bc", false},
		{"a/b", "a/b", false},
		{"a/b", "a/b/c", true},
		{"a/b", "a//b", false},
	} {
		assert.Equal(t, case_.is, isSubPath(case_.parent, case_.child))
	}
}
<|endoftext|>


<commit_before>// Package yuebao grabs the latest or history yuebao data from tianhong fund's web site and save them into a leveldb database.
// It also provides query methods to get yuebao data by date or date range.
package yuebao

import (
    "fmt"
    "time"
    "io/ioutil"
    "net/http"
    "regexp"
    "strconv"
    "github.com/jmhodges/levigo"
    "github.com/bitly/go-simplejson"
    // HTML of Tianhong Fund Website is encoded by GBK. Use mahonia to decode the html to UTF-8 if needed.
    //"code.google.com/p/mahonia"
)

// Flage to output debug messages.
var DEBUG = false

var db *levigo.DB = nil
var ro *levigo.ReadOptions = nil
var wo *levigo.WriteOptions = nil

// Global channel to make leveldb thread safe when GrabXX functions are called in goroutines.
var ch_writer = make(chan int, 1)

// Config File
var config_file = "./config.json"

var db_path = ""
var latest_url = ""
var latest_pattern = ""
var history_url = ""
var history_pattern = ""

// Default Settings
var def_db_path = "./my.db"
var def_latest_url = "http://www.thfund.com.cn/column.dohsmode=searchtopic&pageno=0&channelid=2&categoryid=2435&childcategoryid=2436.htm"
var def_latest_pattern = "(?P<date>\\d{4}-\\d{2}-\\d{2})	\\n\\s*(?P<earn>\\d*\\.\\d{4})	\\n\\s*(?P<percent>\\d*\\.\\d*)"
var def_history_url = "http://www.thfund.com.cn/website/hd/zlb/newzlbrev2.jsp"
var def_history_pattern = "(?P<date>\\d{4}-\\d{2}-\\d{2})	\\r\\n\\s*(?P<earn>\\d*\\.\\d{4})	\\r\\n\\s*(?P<percent>\\d*\\.\\d*)"

var def_min_date = "2013-05-30"  // yuebao(zenglibao) started from 2013-05-30

// Used to lock goroutine to write into leveldb to make thread safe.
func Lock(ch chan int) {
    ch <- 1
}

// Used to unlock goroutine to write into leveldb to make thread safe.
func UnLock(ch chan int) {
    <-ch
}

// Validate input date string.
// Date string must:
// 1. in yyyy-mm-dd format
// 2. > def_min_date(2013-05-30)
// 3. <= today
func IsDateValid(date string) bool {
    if len(date) == 0 {
        return false
    }

    p := `^\d{4}-\d{2}-\d{2}$`
    re := regexp.MustCompile(p)
    matches := re.FindStringSubmatch(date)
    if len(matches) != 1 {
        return false
    }

    t := time.Now()
    today := fmt.Sprintf("%04d-%02d-%02d", t.Year(), t.Month(), t.Day())

    if date > today || date < def_min_date {
        return false
    }
    return true
}

// Save into leveldb database from matched string slice by grabbing data from website.
func SaveFromRegexpMatches(matches []string) (err error) {
    // len(matches) = 4: entile string, date, yield, yield rate
    if len(matches) != 4 {
        fmt.Println("Data not found.")
        return err
    }

    date := matches[1]
    yield, _ := strconv.ParseFloat(matches[2], 32)
    yieldRate, _ := strconv.ParseFloat(matches[3], 32)

    jsonStr := fmt.Sprintf("\"y\":%.4f,\"r\":%.3f", yield, yieldRate)

    if DEBUG {
        fmt.Printf("key = %s, value = %s\n", date, jsonStr)
    }

    s := GetData(date)
    if s != "" {
        fmt.Printf("date: %s already grabbed. data = %s\n", date, s)
        return nil
    }

    Lock(ch_writer)  // write lock for leveldb if function is called in different goroutines.
    err = db.Put(wo, []byte(date), []byte(jsonStr))
    UnLock(ch_writer)  // unlock
    if err != nil {
        fmt.Println(err)
        return err
    }

    return nil
}

// Grab latest yuebao data from tianhong fund website and save into leveldb database.
// It reads the "latest_url" and "latest_pattern" settings from config file(./config.json).
func GrabLatestData() (err error) {
    res, err := http.Get(latest_url)
    if err != nil {
        fmt.Println(err)
        return err
    }
    defer res.Body.Close()

    if DEBUG {
        fmt.Println(res)
    }

    body, err := ioutil.ReadAll(res.Body)
    if err != nil {
        fmt.Println(err)
        return err
    }

    // HTML of Tianhong Fund Website is encoded by GBK. Use mahonia to decode the html to UTF-8 if needed.
    // ---------------------------------------------------------------------------------------------------
    //decoder := mahonia.NewDecoder("gbk")
    //s := decoder.ConvertString(string(body))

    s := string(body)
    if DEBUG {
        fmt.Print(s)
    }

    re := regexp.MustCompile(latest_pattern)
    matches := re.FindStringSubmatch(s)

    return SaveFromRegexpMatches(matches)
}

// Grab all history yuebao data from tianhong fund website and save into leveldb database.
// It reads the "history_url" and "history_pattern" settings from config file(./config.json).
func GrabHistoryData() (err error) {
    res, err := http.Get(history_url)
    if err != nil {
        fmt.Println(err)
        return err
    }
    defer res.Body.Close()

    if DEBUG {
        fmt.Println(res)
    }

    body, err := ioutil.ReadAll(res.Body)
    if err != nil {
        fmt.Println(err)
        return err
    }

    s := string(body)
    if DEBUG {
        fmt.Print(s)
    }
    re := regexp.MustCompile(history_pattern)
    matches := re.FindAllStringSubmatch(s, -1)

    for i := 0; i < len(matches); i++ {
        if err = SaveFromRegexpMatches(matches[i]); err != nil {
            return err
        }
    }

    return nil
}

// Get data from day start to day end.
// param: dateBegin, dayEnd in "yyyy-mm-dd" format.
// return: json array if data exist or "" if no data found. Ex:
// [
//   {"d":"2013-07-22","y":1.1547,"r":4.447},
//   {"d":"2013-07-21","y":1.1962,"r":4.471}
// ]
// d: -> date, y -> yield(每万份收益), r -> yield rate(7天年化收益率)
func GetDataByRange(dateBegin, dateEnd string) (jsonStr string) {
    it := db.NewIterator(ro)
    defer it.Close()

    if (!IsDateValid(dateBegin)) || (!IsDateValid(dateEnd)) {
        return ""
    }

    it.Seek([]byte(dateBegin))
    i := 0
    jsonStr = "[\n";
    for it = it; it.Valid(); it.Next() {
        s := fmt.Sprintf("  {\"d\":\"%s\",%s}", string(it.Key()), string(it.Value()))
        jsonStr += s
        if string(it.Key()) == dateEnd {
            jsonStr += "\n"
            break
        }else {
            jsonStr += ",\n"
        }
        i++
    }

    jsonStr += "]\n"

    return jsonStr
}

// Get yuebao data by date.
// param: date in "yyyy-mm-dd" format.
// return: json string if data exist or "" if no data found.  Ex:
// {"d":"2013-07-22","y":1.1547,"r":4.447}
// d -> date, y -> yield(每万份收益), r -> yield rate(7天年化收益率)
func GetData(date string) string {
    if !IsDateValid(date) {
        return ""
    }

    v, _ := db.Get(ro, []byte(date))
    if len(v) == 0 {
        return ""
    }

    s := fmt.Sprintf("{\"d\":\"%s\",%s}", date, string(v))
    return s
}

// Open leveldb database.
// It reads "db_path" in config file(./config.json). The default value is "./my.db".
func OpenDB() (err error){
    opts := levigo.NewOptions()
    opts.SetCache(levigo.NewLRUCache(3<<30))
    opts.SetCreateIfMissing(true)

    db, err = levigo.Open(db_path, opts)
    if err != nil {
        fmt.Println(err)
        return err
    }

    ro = levigo.NewReadOptions()
    wo = levigo.NewWriteOptions()

    return err
}

// Close leveldb instance.
func CloseDB() {
    if db != nil {
        db.Close()
    }
}

// Load settings from config file.
func LoadConfig() {
    buffer, err := ioutil.ReadFile(config_file)
    if err != nil {
        fmt.Println(err)
        return
    }

    if DEBUG {
        fmt.Println(len(buffer))
        fmt.Println(string(buffer))
    }

    obj, err := simplejson.NewJson([]byte(buffer))
    if err != nil {
        fmt.Println(err)
        return
    }

    db_path = obj.Get("db_path").MustString(def_db_path)
    latest_url = obj.Get("latest_url").MustString(def_latest_url)
    latest_pattern = obj.Get("latest_pattern").MustString(def_latest_pattern)
    history_url = obj.Get("history_url").MustString(def_history_url)
    history_pattern = obj.Get("history_pattern").MustString(def_history_pattern)

    fmt.Println("Settings: \n================================")
    fmt.Println("db_path: " + db_path)
    fmt.Println("latest_url: " + latest_url)
    fmt.Println("latest_pattern: " + latest_pattern)
    fmt.Println("history_url: " + history_url)
    fmt.Println("history_pattern: " + history_pattern)
}

func init() {
    LoadConfig()
    OpenDB()
}
<commit_msg>Add LoadDefConfig()<commit_after>// Package yuebao grabs the latest or history yuebao data from tianhong fund's web site and save them into a leveldb database.
// It also provides query methods to get yuebao data by date or date range.
package yuebao

import (
    "fmt"
    "time"
    "io/ioutil"
    "net/http"
    "regexp"
    "strconv"
    "github.com/jmhodges/levigo"
    "github.com/bitly/go-simplejson"
    // HTML of Tianhong Fund Website is encoded by GBK. Use mahonia to decode the html to UTF-8 if needed.
    //"code.google.com/p/mahonia"
)

// Flage to output debug messages.
var DEBUG = false

var db *levigo.DB = nil
var ro *levigo.ReadOptions = nil
var wo *levigo.WriteOptions = nil

// Global channel to make leveldb thread safe when GrabXX functions are called in goroutines.
var ch_writer = make(chan int, 1)

// Config File
var config_file = "./config.json"

var db_path = ""
var latest_url = ""
var latest_pattern = ""
var history_url = ""
var history_pattern = ""

// Default Settings
var def_db_path = "./my.db"
var def_latest_url = "http://www.thfund.com.cn/column.dohsmode=searchtopic&pageno=0&channelid=2&categoryid=2435&childcategoryid=2436.htm"
var def_latest_pattern = "(?P<date>\\d{4}-\\d{2}-\\d{2})	\\n\\s*(?P<earn>\\d*\\.\\d{4})	\\n\\s*(?P<percent>\\d*\\.\\d*)"
var def_history_url = "http://www.thfund.com.cn/website/hd/zlb/newzlbrev2.jsp"
var def_history_pattern = "(?P<date>\\d{4}-\\d{2}-\\d{2})	\\r\\n\\s*(?P<earn>\\d*\\.\\d{4})	\\r\\n\\s*(?P<percent>\\d*\\.\\d*)"

var def_min_date = "2013-05-30"  // yuebao(zenglibao) started from 2013-05-30

// Used to lock goroutine to write into leveldb to make thread safe.
func Lock(ch chan int) {
    ch <- 1
}

// Used to unlock goroutine to write into leveldb to make thread safe.
func UnLock(ch chan int) {
    <-ch
}

// Validate input date string.
// Date string must:
// 1. in yyyy-mm-dd format
// 2. > def_min_date(2013-05-30)
// 3. <= today
func IsDateValid(date string) bool {
    if len(date) == 0 {
        return false
    }

    p := `^\d{4}-\d{2}-\d{2}$`
    re := regexp.MustCompile(p)
    matches := re.FindStringSubmatch(date)
    if len(matches) != 1 {
        return false
    }

    t := time.Now()
    today := fmt.Sprintf("%04d-%02d-%02d", t.Year(), t.Month(), t.Day())

    if date > today || date < def_min_date {
        return false
    }
    return true
}

// Save into leveldb database from matched string slice by grabbing data from website.
func SaveFromRegexpMatches(matches []string) (err error) {
    // len(matches) = 4: entile string, date, yield, yield rate
    if len(matches) != 4 {
        fmt.Println("Data not found.")
        return err
    }

    date := matches[1]
    yield, _ := strconv.ParseFloat(matches[2], 32)
    yieldRate, _ := strconv.ParseFloat(matches[3], 32)

    jsonStr := fmt.Sprintf("\"y\":%.4f,\"r\":%.3f", yield, yieldRate)

    if DEBUG {
        fmt.Printf("key = %s, value = %s\n", date, jsonStr)
    }

    s := GetData(date)
    if s != "" {
        fmt.Printf("date: %s already grabbed. data = %s\n", date, s)
        return nil
    }

    Lock(ch_writer)  // write lock for leveldb if function is called in different goroutines.
    err = db.Put(wo, []byte(date), []byte(jsonStr))
    UnLock(ch_writer)  // unlock
    if err != nil {
        fmt.Println(err)
        return err
    }

    return nil
}

// Grab latest yuebao data from tianhong fund website and save into leveldb database.
// It reads the "latest_url" and "latest_pattern" settings from config file(./config.json).
func GrabLatestData() (err error) {
    res, err := http.Get(latest_url)
    if err != nil {
        fmt.Println(err)
        return err
    }
    defer res.Body.Close()

    if DEBUG {
        fmt.Println(res)
    }

    body, err := ioutil.ReadAll(res.Body)
    if err != nil {
        fmt.Println(err)
        return err
    }

    // HTML of Tianhong Fund Website is encoded by GBK. Use mahonia to decode the html to UTF-8 if needed.
    // ---------------------------------------------------------------------------------------------------
    //decoder := mahonia.NewDecoder("gbk")
    //s := decoder.ConvertString(string(body))

    s := string(body)
    if DEBUG {
        fmt.Print(s)
    }

    re := regexp.MustCompile(latest_pattern)
    matches := re.FindStringSubmatch(s)

    return SaveFromRegexpMatches(matches)
}

// Grab all history yuebao data from tianhong fund website and save into leveldb database.
// It reads the "history_url" and "history_pattern" settings from config file(./config.json).
func GrabHistoryData() (err error) {
    res, err := http.Get(history_url)
    if err != nil {
        fmt.Println(err)
        return err
    }
    defer res.Body.Close()

    if DEBUG {
        fmt.Println(res)
    }

    body, err := ioutil.ReadAll(res.Body)
    if err != nil {
        fmt.Println(err)
        return err
    }

    s := string(body)
    if DEBUG {
        fmt.Print(s)
    }
    re := regexp.MustCompile(history_pattern)
    matches := re.FindAllStringSubmatch(s, -1)

    for i := 0; i < len(matches); i++ {
        if err = SaveFromRegexpMatches(matches[i]); err != nil {
            return err
        }
    }

    return nil
}

// Get data from day start to day end.
// param: dateBegin, dayEnd in "yyyy-mm-dd" format.
// return: json array if data exist or "" if no data found. Ex:
// [
//   {"d":"2013-07-22","y":1.1547,"r":4.447},
//   {"d":"2013-07-21","y":1.1962,"r":4.471}
// ]
// d: -> date, y -> yield(每万份收益), r -> yield rate(7天年化收益率)
func GetDataByRange(dateBegin, dateEnd string) (jsonStr string) {
    it := db.NewIterator(ro)
    defer it.Close()

    if (!IsDateValid(dateBegin)) || (!IsDateValid(dateEnd)) {
        return ""
    }

    it.Seek([]byte(dateBegin))
    i := 0
    jsonStr = "[\n";
    for it = it; it.Valid(); it.Next() {
        s := fmt.Sprintf("  {\"d\":\"%s\",%s}", string(it.Key()), string(it.Value()))
        jsonStr += s
        if string(it.Key()) == dateEnd {
            jsonStr += "\n"
            break
        }else {
            jsonStr += ",\n"
        }
        i++
    }

    jsonStr += "]\n"

    return jsonStr
}

// Get yuebao data by date.
// param: date in "yyyy-mm-dd" format.
// return: json string if data exist or "" if no data found.  Ex:
// {"d":"2013-07-22","y":1.1547,"r":4.447}
// d -> date, y -> yield(每万份收益), r -> yield rate(7天年化收益率)
func GetData(date string) string {
    if !IsDateValid(date) {
        return ""
    }

    v, _ := db.Get(ro, []byte(date))
    if len(v) == 0 {
        return ""
    }

    s := fmt.Sprintf("{\"d\":\"%s\",%s}", date, string(v))
    return s
}

// Open leveldb database.
// It reads "db_path" in config file(./config.json). The default value is "./my.db".
func OpenDB() (err error){
    opts := levigo.NewOptions()
    opts.SetCache(levigo.NewLRUCache(3<<30))
    opts.SetCreateIfMissing(true)

    db, err = levigo.Open(db_path, opts)
    if err != nil {
        fmt.Println(err)
        return err
    }

    ro = levigo.NewReadOptions()
    wo = levigo.NewWriteOptions()

    return err
}

// Close leveldb instance.
func CloseDB() {
    if db != nil {
        db.Close()
    }
}

// Load default settings
func LoadDefConfig() {
    db_path = def_db_path
    latest_url = def_latest_url
    latest_pattern = def_latest_pattern
    history_url = def_history_url
    history_pattern = def_history_pattern
}

// Load settings from config file.

func LoadConfig() {
    buffer, err := ioutil.ReadFile(config_file)
    if err != nil {
        fmt.Println(err)
        fmt.Println("Load default settings.")
        LoadDefConfig()
        return
    }

    if DEBUG {
        fmt.Println(len(buffer))
        fmt.Println(string(buffer))
    }

    obj, err := simplejson.NewJson([]byte(buffer))
    if err != nil {
        fmt.Println(err)
        fmt.Println("Load default settings.")
        LoadDefConfig()
        return
    }

    db_path = obj.Get("db_path").MustString(def_db_path)
    latest_url = obj.Get("latest_url").MustString(def_latest_url)
    latest_pattern = obj.Get("latest_pattern").MustString(def_latest_pattern)
    history_url = obj.Get("history_url").MustString(def_history_url)
    history_pattern = obj.Get("history_pattern").MustString(def_history_pattern)

    fmt.Println("Settings: \n================================")
    fmt.Println("db_path: " + db_path)
    fmt.Println("latest_url: " + latest_url)
    fmt.Println("latest_pattern: " + latest_pattern)
    fmt.Println("history_url: " + history_url)
    fmt.Println("history_pattern: " + history_pattern)
}

func init() {
    LoadConfig()
    OpenDB()
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package static

import (
	"bytes"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"testing"

	"github.com/google/martian"
	"github.com/google/martian/proxyutil"
)

func Test404WhenExplictlyMappedFileDoesNotExist(t *testing.T) {
	tmpdir, err := ioutil.TempDir("", "test_static_modifier_explicit_path_mapping_")
	if err != nil {
		t.Fatalf("ioutil.TempDir(): got %v, want no error", err)
	}

	//if err := os.MkdirAll(path.Join(tmpdir, "explicit/path"), 0777); err != nil {
	//	t.Fatalf("os.Mkdir(): got %v, want no error", err)
	//}

	//if err := ioutil.WriteFile(path.Join(tmpdir, "explicit/path", "sfmtest.txt"), []byte("test file"), 0777); err != nil {
	//	t.Fatalf("ioutil.WriteFile(): got %v, want no error", err)
	//}

	req, err := http.NewRequest("GET", "/sfmtest.txt", nil)
	if err != nil {
		t.Fatalf("NewRequest(): got %v, want no error", err)
	}

	_, remove, err := martian.TestContext(req, nil, nil)
	if err != nil {
		t.Fatalf("TestContext(): got %v, want no error", err)
	}
	defer remove()

	res := proxyutil.NewResponse(http.StatusOK, nil, req)

	mod := NewModifier(tmpdir)
	if err := mod.ModifyRequest(req); err != nil {
		t.Fatalf("ModifyRequest(): got %v, want no error", err)
	}
	mod.SetExplicitPathMappings(map[string]string{"/sfmtest.txt": "/explicit/path/sfmtest.txt"})

	if err := mod.ModifyResponse(res); err != nil {
		t.Fatalf("ModifyResponse(): got %v, want no error", err)
	}

	if got, want := res.StatusCode, http.StatusNotFound; got != want {
		t.Errorf("res.StatusCode: got %v, want %v", got, want)
	}
}

func TestFileExistsInBothExplictlyMappedPathAndInferredPath(t *testing.T) {
	tmpdir, err := ioutil.TempDir("", "test_static_modifier_explicit_path_mapping_")
	if err != nil {
		t.Fatalf("ioutil.TempDir(): got %v, want no error", err)
	}

	if err := os.MkdirAll(path.Join(tmpdir, "explicit/path"), 0777); err != nil {
		t.Fatalf("os.Mkdir(): got %v, want no error", err)
	}

	if err := ioutil.WriteFile(path.Join(tmpdir, "sfmtest.txt"), []byte("dont return"), 0777); err != nil {
		t.Fatalf("ioutil.WriteFile(): got %v, want no error", err)
	}

	if err := ioutil.WriteFile(path.Join(tmpdir, "explicit/path", "sfmtest.txt"), []byte("target"), 0777); err != nil {
		t.Fatalf("ioutil.WriteFile(): got %v, want no error", err)
	}

	req, err := http.NewRequest("GET", "/sfmtest.txt", nil)
	if err != nil {
		t.Fatalf("NewRequest(): got %v, want no error", err)
	}

	_, remove, err := martian.TestContext(req, nil, nil)
	if err != nil {
		t.Fatalf("TestContext(): got %v, want no error", err)
	}
	defer remove()

	res := proxyutil.NewResponse(http.StatusOK, nil, req)

	mod := NewModifier(tmpdir)
	if err := mod.ModifyRequest(req); err != nil {
		t.Fatalf("ModifyRequest(): got %v, want no error", err)
	}
	mod.SetExplicitPathMappings(map[string]string{"/sfmtest.txt": "/explicit/path/sfmtest.txt"})

	if err := mod.ModifyResponse(res); err != nil {
		t.Fatalf("ModifyResponse(): got %v, want no error", err)
	}

	if got, want := res.Header.Get("Content-Type"), "text/plain; charset=utf-8"; got != want {
		t.Errorf("res.Header.Get('Content-Type'): got %v, want %v", got, want)
	}

	got, err := ioutil.ReadAll(res.Body)
	if err != nil {
		t.Fatalf("ioutil.ReadAll(): got %v, want no error", err)
	}
	res.Body.Close()

	if want := []byte("target"); !bytes.Equal(got, want) {
		t.Errorf("res.Body: got %q, want %q", got, want)
	}

	if got, want := res.ContentLength, int64(len("target")); got != want {
		t.Errorf("res.ContentLength: got %v, want %v", got, want)
	}
}

func TestStaticModifierExplicitPathMapping(t *testing.T) {
	tmpdir, err := ioutil.TempDir("", "test_static_modifier_explicit_path_mapping_")
	if err != nil {
		t.Fatalf("ioutil.TempDir(): got %v, want no error", err)
	}

	if err := os.MkdirAll(path.Join(tmpdir, "explicit/path"), 0777); err != nil {
		t.Fatalf("os.Mkdir(): got %v, want no error", err)
	}

	if err := ioutil.WriteFile(path.Join(tmpdir, "explicit/path", "sfmtest.txt"), []byte("test file"), 0777); err != nil {
		t.Fatalf("ioutil.WriteFile(): got %v, want no error", err)
	}

	req, err := http.NewRequest("GET", "/sfmtest.txt", nil)
	if err != nil {
		t.Fatalf("NewRequest(): got %v, want no error", err)
	}

	_, remove, err := martian.TestContext(req, nil, nil)
	if err != nil {
		t.Fatalf("TestContext(): got %v, want no error", err)
	}
	defer remove()

	res := proxyutil.NewResponse(http.StatusOK, nil, req)

	mod := NewModifier(tmpdir)
	if err := mod.ModifyRequest(req); err != nil {
		t.Fatalf("ModifyRequest(): got %v, want no error", err)
	}
	mod.SetExplicitPathMappings(map[string]string{"/sfmtest.txt": "/explicit/path/sfmtest.txt"})

	if err := mod.ModifyResponse(res); err != nil {
		t.Fatalf("ModifyResponse(): got %v, want no error", err)
	}

	if got, want := res.Header.Get("Content-Type"), "text/plain; charset=utf-8"; got != want {
		t.Errorf("res.Header.Get('Content-Type'): got %v, want %v", got, want)
	}

	got, err := ioutil.ReadAll(res.Body)
	if err != nil {
		t.Fatalf("ioutil.ReadAll(): got %v, want no error", err)
	}
	res.Body.Close()

	if want := []byte("test file"); !bytes.Equal(got, want) {
		t.Errorf("res.Body: got %q, want %q", got, want)
	}

	if got, want := res.ContentLength, int64(len("test file")); got != want {
		t.Errorf("res.ContentLength: got %v, want %v", got, want)
	}
}

func TestStaticModifierOnRequest(t *testing.T) {
	tmpdir, err := ioutil.TempDir("", "test_static_modifier_on_request_")
	if err != nil {
		t.Fatalf("ioutil.TempDir(): got %v, want no error", err)
	}

	if err := ioutil.WriteFile(path.Join(tmpdir, "sfmtest.txt"), []byte("test file"), 0777); err != nil {
		t.Fatalf("ioutil.WriteFile(): got %v, want no error", err)
	}

	req, err := http.NewRequest("GET", "/sfmtest.txt", nil)
	if err != nil {
		t.Fatalf("NewRequest(): got %v, want no error", err)
	}

	_, remove, err := martian.TestContext(req, nil, nil)
	if err != nil {
		t.Fatalf("TestContext(): got %v, want no error", err)
	}
	defer remove()

	res := proxyutil.NewResponse(http.StatusOK, nil, req)

	mod := NewModifier(tmpdir)
	if err := mod.ModifyRequest(req); err != nil {
		t.Fatalf("ModifyRequest(): got %v, want no error", err)

	}
	if err := mod.ModifyResponse(res); err != nil {
		t.Fatalf("ModifyResponse(): got %v, want no error", err)
	}

	if got, want := res.Header.Get("Content-Type"), "text/plain; charset=utf-8"; got != want {
		t.Errorf("res.Header.Get('Content-Type'): got %v, want %v", got, want)
	}

	got, err := ioutil.ReadAll(res.Body)
	if err != nil {
		t.Fatalf("ioutil.ReadAll(): got %v, want no error", err)
	}
	res.Body.Close()

	if want := []byte("test file"); !bytes.Equal(got, want) {
		t.Errorf("res.Body: got %q, want %q", got, want)
	}

	if got, want := res.ContentLength, int64(len("test file")); got != want {
		t.Errorf("res.ContentLength: got %v, want %v", got, want)
	}
}
<commit_msg>Add a test for the FromJSON function.<commit_after>// Copyright 2015 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package static

import (
	"bytes"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"strings"
	"testing"

	"github.com/google/martian"
	"github.com/google/martian/parse"
	"github.com/google/martian/proxyutil"
)

func Test404WhenExplictlyMappedFileDoesNotExist(t *testing.T) {
	tmpdir, err := ioutil.TempDir("", "test_static_modifier_explicit_path_mapping_")
	if err != nil {
		t.Fatalf("ioutil.TempDir(): got %v, want no error", err)
	}

	//if err := os.MkdirAll(path.Join(tmpdir, "explicit/path"), 0777); err != nil {
	//	t.Fatalf("os.Mkdir(): got %v, want no error", err)
	//}

	//if err := ioutil.WriteFile(path.Join(tmpdir, "explicit/path", "sfmtest.txt"), []byte("test file"), 0777); err != nil {
	//	t.Fatalf("ioutil.WriteFile(): got %v, want no error", err)
	//}

	req, err := http.NewRequest("GET", "/sfmtest.txt", nil)
	if err != nil {
		t.Fatalf("NewRequest(): got %v, want no error", err)
	}

	_, remove, err := martian.TestContext(req, nil, nil)
	if err != nil {
		t.Fatalf("TestContext(): got %v, want no error", err)
	}
	defer remove()

	res := proxyutil.NewResponse(http.StatusOK, nil, req)

	mod := NewModifier(tmpdir)
	if err := mod.ModifyRequest(req); err != nil {
		t.Fatalf("ModifyRequest(): got %v, want no error", err)
	}
	mod.SetExplicitPathMappings(map[string]string{"/sfmtest.txt": "/explicit/path/sfmtest.txt"})

	if err := mod.ModifyResponse(res); err != nil {
		t.Fatalf("ModifyResponse(): got %v, want no error", err)
	}

	if got, want := res.StatusCode, http.StatusNotFound; got != want {
		t.Errorf("res.StatusCode: got %v, want %v", got, want)
	}
}

func TestFileExistsInBothExplictlyMappedPathAndInferredPath(t *testing.T) {
	tmpdir, err := ioutil.TempDir("", "test_static_modifier_explicit_path_mapping_")
	if err != nil {
		t.Fatalf("ioutil.TempDir(): got %v, want no error", err)
	}

	if err := os.MkdirAll(path.Join(tmpdir, "explicit/path"), 0777); err != nil {
		t.Fatalf("os.Mkdir(): got %v, want no error", err)
	}

	if err := ioutil.WriteFile(path.Join(tmpdir, "sfmtest.txt"), []byte("dont return"), 0777); err != nil {
		t.Fatalf("ioutil.WriteFile(): got %v, want no error", err)
	}

	if err := ioutil.WriteFile(path.Join(tmpdir, "explicit/path", "sfmtest.txt"), []byte("target"), 0777); err != nil {
		t.Fatalf("ioutil.WriteFile(): got %v, want no error", err)
	}

	req, err := http.NewRequest("GET", "/sfmtest.txt", nil)
	if err != nil {
		t.Fatalf("NewRequest(): got %v, want no error", err)
	}

	_, remove, err := martian.TestContext(req, nil, nil)
	if err != nil {
		t.Fatalf("TestContext(): got %v, want no error", err)
	}
	defer remove()

	res := proxyutil.NewResponse(http.StatusOK, nil, req)

	mod := NewModifier(tmpdir)
	if err := mod.ModifyRequest(req); err != nil {
		t.Fatalf("ModifyRequest(): got %v, want no error", err)
	}
	mod.SetExplicitPathMappings(map[string]string{"/sfmtest.txt": "/explicit/path/sfmtest.txt"})

	if err := mod.ModifyResponse(res); err != nil {
		t.Fatalf("ModifyResponse(): got %v, want no error", err)
	}

	if got, want := res.Header.Get("Content-Type"), "text/plain; charset=utf-8"; got != want {
		t.Errorf("res.Header.Get('Content-Type'): got %v, want %v", got, want)
	}

	got, err := ioutil.ReadAll(res.Body)
	if err != nil {
		t.Fatalf("ioutil.ReadAll(): got %v, want no error", err)
	}
	res.Body.Close()

	if want := []byte("target"); !bytes.Equal(got, want) {
		t.Errorf("res.Body: got %q, want %q", got, want)
	}

	if got, want := res.ContentLength, int64(len("target")); got != want {
		t.Errorf("res.ContentLength: got %v, want %v", got, want)
	}
}

func TestStaticModifierExplicitPathMapping(t *testing.T) {
	tmpdir, err := ioutil.TempDir("", "test_static_modifier_explicit_path_mapping_")
	if err != nil {
		t.Fatalf("ioutil.TempDir(): got %v, want no error", err)
	}

	if err := os.MkdirAll(path.Join(tmpdir, "explicit/path"), 0777); err != nil {
		t.Fatalf("os.Mkdir(): got %v, want no error", err)
	}

	if err := ioutil.WriteFile(path.Join(tmpdir, "explicit/path", "sfmtest.txt"), []byte("test file"), 0777); err != nil {
		t.Fatalf("ioutil.WriteFile(): got %v, want no error", err)
	}

	req, err := http.NewRequest("GET", "/sfmtest.txt", nil)
	if err != nil {
		t.Fatalf("NewRequest(): got %v, want no error", err)
	}

	_, remove, err := martian.TestContext(req, nil, nil)
	if err != nil {
		t.Fatalf("TestContext(): got %v, want no error", err)
	}
	defer remove()

	res := proxyutil.NewResponse(http.StatusOK, nil, req)

	mod := NewModifier(tmpdir)
	if err := mod.ModifyRequest(req); err != nil {
		t.Fatalf("ModifyRequest(): got %v, want no error", err)
	}
	mod.SetExplicitPathMappings(map[string]string{"/sfmtest.txt": "/explicit/path/sfmtest.txt"})

	if err := mod.ModifyResponse(res); err != nil {
		t.Fatalf("ModifyResponse(): got %v, want no error", err)
	}

	if got, want := res.Header.Get("Content-Type"), "text/plain; charset=utf-8"; got != want {
		t.Errorf("res.Header.Get('Content-Type'): got %v, want %v", got, want)
	}

	got, err := ioutil.ReadAll(res.Body)
	if err != nil {
		t.Fatalf("ioutil.ReadAll(): got %v, want no error", err)
	}
	res.Body.Close()

	if want := []byte("test file"); !bytes.Equal(got, want) {
		t.Errorf("res.Body: got %q, want %q", got, want)
	}

	if got, want := res.ContentLength, int64(len("test file")); got != want {
		t.Errorf("res.ContentLength: got %v, want %v", got, want)
	}
}

func TestStaticModifierOnRequest(t *testing.T) {
	tmpdir, err := ioutil.TempDir("", "test_static_modifier_on_request_")
	if err != nil {
		t.Fatalf("ioutil.TempDir(): got %v, want no error", err)
	}

	if err := ioutil.WriteFile(path.Join(tmpdir, "sfmtest.txt"), []byte("test file"), 0777); err != nil {
		t.Fatalf("ioutil.WriteFile(): got %v, want no error", err)
	}

	req, err := http.NewRequest("GET", "/sfmtest.txt", nil)
	if err != nil {
		t.Fatalf("NewRequest(): got %v, want no error", err)
	}

	_, remove, err := martian.TestContext(req, nil, nil)
	if err != nil {
		t.Fatalf("TestContext(): got %v, want no error", err)
	}
	defer remove()

	res := proxyutil.NewResponse(http.StatusOK, nil, req)

	mod := NewModifier(tmpdir)
	if err := mod.ModifyRequest(req); err != nil {
		t.Fatalf("ModifyRequest(): got %v, want no error", err)

	}
	if err := mod.ModifyResponse(res); err != nil {
		t.Fatalf("ModifyResponse(): got %v, want no error", err)
	}

	if got, want := res.Header.Get("Content-Type"), "text/plain; charset=utf-8"; got != want {
		t.Errorf("res.Header.Get('Content-Type'): got %v, want %v", got, want)
	}

	got, err := ioutil.ReadAll(res.Body)
	if err != nil {
		t.Fatalf("ioutil.ReadAll(): got %v, want no error", err)
	}
	res.Body.Close()

	if want := []byte("test file"); !bytes.Equal(got, want) {
		t.Errorf("res.Body: got %q, want %q", got, want)
	}

	if got, want := res.ContentLength, int64(len("test file")); got != want {
		t.Errorf("res.ContentLength: got %v, want %v", got, want)
	}
}

func TestModifierFromJSON(t *testing.T) {
	tmpdir, err := ioutil.TempDir("", "test_static_modifier_on_request_")
	if err != nil {
		t.Fatalf("ioutil.TempDir(): got %v, want no error", err)
	}

	if err := ioutil.WriteFile(path.Join(tmpdir, "sfmtest.txt"), []byte("test file"), 0777); err != nil {
		t.Fatalf("ioutil.WriteFile(): got %v, want no error", err)
	}

	jsonStr := `{
    "static.Modifier": {
      "scope": ["request", "response"],
      "rootPath": "<path>"
    }
  }`

	msg := []byte(strings.Replace(jsonStr, "<path>", tmpdir, 1))

	r, err := parse.FromJSON(msg)
	if err != nil {
		t.Fatalf("parse.FromJSON(): got %v, want no error", err)
	}

	reqmod := r.RequestModifier()
	if reqmod == nil {
		t.Fatal("reqmod: got nil, want not nil")
	}

	resmod := r.ResponseModifier()
	if resmod == nil {
		t.Fatal("resmod: got nil, want not nil")
	}

	req, err := http.NewRequest("GET", "/sfmtest.txt", nil)
	if err != nil {
		t.Fatalf("NewRequest(): got %v, want no error", err)
	}

	_, remove, err := martian.TestContext(req, nil, nil)
	if err != nil {
		t.Fatalf("TestContext(): got %v, want no error", err)
	}
	defer remove()

	res := proxyutil.NewResponse(http.StatusOK, nil, req)

	if err := reqmod.ModifyRequest(req); err != nil {
		t.Fatalf("ModifyRequest(): got %v, want no error", err)
	}

	if err := resmod.ModifyResponse(res); err != nil {
		t.Fatalf("ModifyResponse(): got %v, want no error", err)
	}

	if got, want := res.Header.Get("Content-Type"), "text/plain; charset=utf-8"; got != want {
		t.Errorf("res.Header.Get('Content-Type'): got %v, want %v", got, want)
	}

	got, err := ioutil.ReadAll(res.Body)
	if err != nil {
		t.Fatalf("ioutil.ReadAll(): got %v, want no error", err)
	}
	res.Body.Close()

	if want := []byte("test file"); !bytes.Equal(got, want) {
		t.Errorf("res.Body: got %q, want %q", got, want)
	}

	if got, want := res.ContentLength, int64(len("test file")); got != want {
		t.Errorf("res.ContentLength: got %v, want %v", got, want)
	}
}
<|endoftext|>


<commit_before>package textable

import (
	"testing"

	"github.com/hgfischer/textable/column"
	"github.com/stretchr/testify/assert"
)

var (
	TEST_HEADERS = []string{"Country", "Population", "Educ. Exp. (% GDP)", "HDI", "HDI change (*1000)"}
	TEST_DATA    = [][]interface{}{
		{"Algeria", uint(38700000), 4.3, 0.713, 3},
		{"Argentina", uint(42669500), 6.5, 0.811, 6},
		{"Australia", uint(23536500), 4.5, 0.938, 3},
		{"Brazil", uint(202756000), 5.1, 0.730, 4},
		{"Canada", uint(35427524), 5.5, 0.911, 2},
		{"China", uint(1365220000), 4.3, 0.699, 10},
		{"Greece", uint(10816286), 4.0, 0.860, -6},
		{"India", uint(1245830000), 3.1, 0.554, 7},
		{"Kazakhstan", uint(17244000), 2.8, 0.754, 10},
		{"Portugal", uint(10477800), 5.2, 0.816, -1},
		{"Russia", uint(146000000), 3.9, 0.788, 6},
		{"United States", uint(318275000), 5.5, 0.937, 3},
	}
)

func TestNewTableShouldStoreHeadersAndInitializeColumns(t *testing.T) {
	table := New(TEST_HEADERS...)
	assert.Equal(t, len(TEST_HEADERS), len(table.Headers))
	assert.Equal(t, len(TEST_HEADERS), len(table.Columns))
}

func TestAddRowShouldSetProperTypeForEachColumnAndAppendValues(t *testing.T) {
	tt := New(TEST_HEADERS...)
	for nrow, data := range TEST_DATA {
		tt.AddRow(data...)
		for ncol, col := range data {
			assert.IsType(t, column.NewForTypeOf(col), tt.Columns[ncol])
			assert.Equal(t, nrow+1, tt.Columns[ncol].Len())
		}
	}
}
<commit_msg>Add more tests<commit_after>package textable

import (
	"testing"

	"github.com/hgfischer/textable/column"
	"github.com/stretchr/testify/assert"
)

var (
	TEST_HEADERS = []string{"Country", "Population", "Educ. Exp. (% GDP)", "HDI", "HDI change (*1000)"}
	TEST_DATA    = [][]interface{}{
		{"Algeria", uint(38700000), 4.3, 0.713, 3},
		{"Argentina", uint(42669500), 6.5, 0.811, 6},
		{"Australia", uint(23536500), 4.5, 0.938, 3},
		{"Brazil", uint(202756000), 5.1, 0.730, 4},
		{"Canada", uint(35427524), 5.5, 0.911, 2},
		{"China", uint(1365220000), 4.3, 0.699, 10},
		{"Greece", uint(10816286), 4.0, 0.860, -6},
		{"India", uint(1245830000), 3.1, 0.554, 7},
		{"Kazakhstan", uint(17244000), 2.8, 0.754, 10},
		{"Portugal", uint(10477800), 5.2, 0.816, -1},
		{"Russia", uint(146000000), 3.9, 0.788, 6},
		{"United States", uint(318275000), 5.5, 0.937, 3},
	}
)

func TestNewTableShouldStoreHeadersAndInitializeColumns(t *testing.T) {
	table := New(TEST_HEADERS...)
	assert.Equal(t, len(TEST_HEADERS), len(table.Headers))
	assert.Equal(t, len(TEST_HEADERS), len(table.Columns))
}

func TestAddRowShouldSetProperTypeForEachColumnAndAppendValues(t *testing.T) {
	tt := New(TEST_HEADERS...)
	for nrow, data := range TEST_DATA {
		err := tt.AddRow(data...)
		assert.Nil(t, err)
		for ncol, col := range data {
			assert.IsType(t, column.NewForTypeOf(col), tt.Columns[ncol])
			assert.Equal(t, nrow+1, tt.Columns[ncol].Len())
		}
	}
}

func TestAddRowWithWrongNumberOfColumnsShouldReturnError(t *testing.T) {
	tt := New(TEST_HEADERS...)
	err := tt.AddRow()
	assert.NotNil(t, err)
}

func TestAddRowWithWrongTypeOfColumnShouldReturnError(t *testing.T) {
	tt := New(TEST_HEADERS...)
	err := tt.AddRow(TEST_DATA[0]...)
	assert.Nil(t, err)
	wrongData := []interface{}{
		1,   // StringCol accepts everything
		1.2, // UIntCol also accepts floats, dropping the decimal part
		10,  // FloatCol also accepts integers
		1,   // Same as above
		"-", // This should fail, since this string does not have a number inside
	}
	err = tt.AddRow(wrongData...)
	assert.NotNil(t, err)
}
<|endoftext|>