<commit_before>package logrusmiddleware

import (
	"bytes"
	"fmt"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	"github.com/Sirupsen/logrus"
	h "github.com/bakins/test-helpers"
)

func newRequest(method, url string) *http.Request {
	req, err := http.NewRequest(method, url, nil)
	if err != nil {
		panic(err)
	}
	return req
}

func TestHandler(t *testing.T) {
	var buf bytes.Buffer

	handler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		w.WriteHeader(200)
		fmt.Fprint(w, "Hello World\n")
	})

	logger := logrus.New()
	logger.Level = logrus.InfoLevel
	logger.Formatter = &logrus.JSONFormatter{}
	logger.Out = &buf

	l := Middleware{
		Name:   "example",
		Logger: logger,
	}

	lh := l.Handler(http.HandlerFunc(handler), "homepage")
	http.Handle("/", lh)

	lh.ServeHTTP(httptest.NewRecorder(), newRequest("GET", "/foo"))

	h.Assert(t, buf.Len() > 0, "buffer should not be empty")
	h.Assert(t, strings.Contains(buf.String(), `"component":"homepage"`), "buffer did not match expected result")
}
<commit_msg>Changed logrus import to lower case<commit_after>package logrusmiddleware

import (
	"bytes"
	"fmt"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	"github.com/sirupsen/logrus"
	h "github.com/bakins/test-helpers"
)

func newRequest(method, url string) *http.Request {
	req, err := http.NewRequest(method, url, nil)
	if err != nil {
		panic(err)
	}
	return req
}

func TestHandler(t *testing.T) {
	var buf bytes.Buffer

	handler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		w.WriteHeader(200)
		fmt.Fprint(w, "Hello World\n")
	})

	logger := logrus.New()
	logger.Level = logrus.InfoLevel
	logger.Formatter = &logrus.JSONFormatter{}
	logger.Out = &buf

	l := Middleware{
		Name:   "example",
		Logger: logger,
	}

	lh := l.Handler(http.HandlerFunc(handler), "homepage")
	http.Handle("/", lh)

	lh.ServeHTTP(httptest.NewRecorder(), newRequest("GET", "/foo"))

	h.Assert(t, buf.Len() > 0, "buffer should not be empty")
	h.Assert(t, strings.Contains(buf.String(), `"component":"homepage"`), "buffer did not match expected result")
}
<|endoftext|>


<commit_before>// Copyright 2017 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package servicecontrol

import (
	"context"
	"errors"
	"fmt"
	"io"
	"sync"

	sc "google.golang.org/api/servicecontrol/v1"

	"istio.io/istio/mixer/adapter/servicecontrol/config"
	"istio.io/istio/mixer/adapter/servicecontrol/template/servicecontrolreport"
	"istio.io/istio/mixer/pkg/adapter"
	"istio.io/istio/mixer/pkg/status"
	"istio.io/istio/mixer/template/apikey"
	"istio.io/istio/mixer/template/quota"
	"istio.io/istio/pkg/cache"
)

type (
	serviceControlClient interface {
		Check(googleServiceName string, request *sc.CheckRequest) (*sc.CheckResponse, error)
		Report(googleServiceName string, request *sc.ReportRequest) (*sc.ReportResponse, error)
		AllocateQuota(googleServiceName string, request *sc.AllocateQuotaRequest) (*sc.AllocateQuotaResponse, error)
	}

	checkProcessor interface {
		ProcessCheck(ctx context.Context, instance *apikey.Instance) (adapter.CheckResult, error)
	}

	reportProcessor interface {
		io.Closer
		ProcessReport(ctx context.Context, instances []*servicecontrolreport.Instance) error
	}

	quotaProcessor interface {
		ProcessQuota(ctx context.Context, instances *quota.Instance, args adapter.QuotaArgs) (adapter.QuotaResult, error)
	}

	serviceProcessor struct {
		checkProcessor
		reportProcessor
		quotaProcessor
	}

	handlerContext struct {
		env    adapter.Env
		config *config.Params
		// A map keyed by mesh service name to service config in adapter config
		serviceConfigIndex map[string]*config.GcpServiceSetting
		checkDataShape     map[string]*apikey.Type
		reportDataShape    map[string]*servicecontrolreport.Type
		checkResponseCache cache.ExpiringCache // A LRU cache for check response
		client             serviceControlClient
	}

	handler struct {
		ctx *handlerContext

		// lock protects svcProcMap.
		lock sync.Mutex
		// Istio mesh service name to serviceProcessor map. Each serviceProcessor instance handles a single
		// service.
		svcProcMap map[string]*serviceProcessor
	}
)

func newServiceProcessor(meshServiceName string, ctx *handlerContext) (*serviceProcessor, error) {
	checkProc, err := newCheckProcessor(meshServiceName, ctx)
	if err != nil {
		return nil, err
	}

	reportProc, err := newReportProcessor(meshServiceName, ctx, checkProc)
	if err != nil {
		return nil, err
	}
	quotaProc, err := newQuotaProcessor(meshServiceName, ctx)
	if err != nil {
		return nil, err
	}

	return &serviceProcessor{
		checkProcessor:  checkProc,
		reportProcessor: reportProc,
		quotaProcessor:  quotaProc,
	}, nil
}

// HandleApiKey handles apikey check.
// nolint:golint
// Disable lint warning of HandleApiKey name
func (h *handler) HandleApiKey(ctx context.Context, instance *apikey.Instance) (adapter.CheckResult, error) {
	svcProc, err := h.getServiceProcessor(instance.Api)
	if err != nil {
		return adapter.CheckResult{
			Status: status.WithPermissionDenied(err.Error()),
		}, nil
	}
	return svcProc.ProcessCheck(ctx, instance)
}

// HandleServicecontrolReport handles reporting metrics and logs.
func (h *handler) HandleServicecontrolReport(ctx context.Context, instances []*servicecontrolreport.Instance) error {
	// TODO: this is inefficient as it dispatches each report individually, instead of grouping them by the service
	for _, instance := range instances {
		svcProc, err := h.getServiceProcessor(instance.ApiService)
		if err != nil {
			return err
		}
		if err = svcProc.ProcessReport(ctx, []*servicecontrolreport.Instance{instance}); err != nil {
			return err
		}
	}
	return nil
}

// HandleQuota handles rate limiting quota.
func (h *handler) HandleQuota(ctx context.Context, instance *quota.Instance,
	args adapter.QuotaArgs) (adapter.QuotaResult, error) {

	/*
		svcProc, err := h.getServiceProcessor(instance.Api)
		if err != nil {
			return adapter.QuotaResult{
				// This map to rpc.INTERNAL.
				Status: status.WithError(err),
			}, nil
		}
		return svcProc.ProcessQuota(ctx, instance, args)
	*/
	return adapter.QuotaResult{}, errors.New("not implemented")
}

// Close closes a handler.
// TODO(manlinl): Run svcProc.Close in goroutine after reportProcessor implements buffering.
func (h *handler) Close() error {
	h.lock.Lock()
	defer h.lock.Lock()
	for _, svcProc := range h.svcProcMap {
		// TODO: handle Close errors
		_ = svcProc.Close()
	}
	return nil
}

func (h *handler) getServiceProcessor(serviceFullName string) (*serviceProcessor, error) {
	if _, ok := h.ctx.serviceConfigIndex[serviceFullName]; !ok {
		return nil, fmt.Errorf("unknown service %v", serviceFullName)
	}

	h.lock.Lock()
	defer h.lock.Unlock()
	svcProc, found := h.svcProcMap[serviceFullName]
	if found {
		return svcProc, nil
	}

	svcProc, err := newServiceProcessor(serviceFullName, h.ctx)
	if err != nil {
		return nil, err
	}
	h.svcProcMap[serviceFullName] = svcProc
	return svcProc, nil
}

func newHandler(ctx *handlerContext) (*handler, error) {
	return &handler{
		ctx:        ctx,
		svcProcMap: make(map[string]*serviceProcessor, len(ctx.config.ServiceConfigs)),
	}, nil
}
<commit_msg>servicecontrol adapter: handle Close errors (#8104)<commit_after>// Copyright 2017 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package servicecontrol

import (
	"context"
	"errors"
	"fmt"
	"io"
	"sync"

	multierror "github.com/hashicorp/go-multierror"
	sc "google.golang.org/api/servicecontrol/v1"

	"istio.io/istio/mixer/adapter/servicecontrol/config"
	"istio.io/istio/mixer/adapter/servicecontrol/template/servicecontrolreport"
	"istio.io/istio/mixer/pkg/adapter"
	"istio.io/istio/mixer/pkg/status"
	"istio.io/istio/mixer/template/apikey"
	"istio.io/istio/mixer/template/quota"
	"istio.io/istio/pkg/cache"
)

type (
	serviceControlClient interface {
		Check(googleServiceName string, request *sc.CheckRequest) (*sc.CheckResponse, error)
		Report(googleServiceName string, request *sc.ReportRequest) (*sc.ReportResponse, error)
		AllocateQuota(googleServiceName string, request *sc.AllocateQuotaRequest) (*sc.AllocateQuotaResponse, error)
	}

	checkProcessor interface {
		ProcessCheck(ctx context.Context, instance *apikey.Instance) (adapter.CheckResult, error)
	}

	reportProcessor interface {
		io.Closer
		ProcessReport(ctx context.Context, instances []*servicecontrolreport.Instance) error
	}

	quotaProcessor interface {
		ProcessQuota(ctx context.Context, instances *quota.Instance, args adapter.QuotaArgs) (adapter.QuotaResult, error)
	}

	serviceProcessor struct {
		checkProcessor
		reportProcessor
		quotaProcessor
	}

	handlerContext struct {
		env    adapter.Env
		config *config.Params
		// A map keyed by mesh service name to service config in adapter config
		serviceConfigIndex map[string]*config.GcpServiceSetting
		checkDataShape     map[string]*apikey.Type
		reportDataShape    map[string]*servicecontrolreport.Type
		checkResponseCache cache.ExpiringCache // A LRU cache for check response
		client             serviceControlClient
	}

	handler struct {
		ctx *handlerContext

		// lock protects svcProcMap.
		lock sync.Mutex
		// Istio mesh service name to serviceProcessor map. Each serviceProcessor instance handles a single
		// service.
		svcProcMap map[string]*serviceProcessor
	}
)

func newServiceProcessor(meshServiceName string, ctx *handlerContext) (*serviceProcessor, error) {
	checkProc, err := newCheckProcessor(meshServiceName, ctx)
	if err != nil {
		return nil, err
	}

	reportProc, err := newReportProcessor(meshServiceName, ctx, checkProc)
	if err != nil {
		return nil, err
	}
	quotaProc, err := newQuotaProcessor(meshServiceName, ctx)
	if err != nil {
		return nil, err
	}

	return &serviceProcessor{
		checkProcessor:  checkProc,
		reportProcessor: reportProc,
		quotaProcessor:  quotaProc,
	}, nil
}

// HandleApiKey handles apikey check.
// nolint:golint
// Disable lint warning of HandleApiKey name
func (h *handler) HandleApiKey(ctx context.Context, instance *apikey.Instance) (adapter.CheckResult, error) {
	svcProc, err := h.getServiceProcessor(instance.Api)
	if err != nil {
		return adapter.CheckResult{
			Status: status.WithPermissionDenied(err.Error()),
		}, nil
	}
	return svcProc.ProcessCheck(ctx, instance)
}

// HandleServicecontrolReport handles reporting metrics and logs.
func (h *handler) HandleServicecontrolReport(ctx context.Context, instances []*servicecontrolreport.Instance) error {
	// TODO: this is inefficient as it dispatches each report individually, instead of grouping them by the service
	for _, instance := range instances {
		svcProc, err := h.getServiceProcessor(instance.ApiService)
		if err != nil {
			return err
		}
		if err = svcProc.ProcessReport(ctx, []*servicecontrolreport.Instance{instance}); err != nil {
			return err
		}
	}
	return nil
}

// HandleQuota handles rate limiting quota.
func (h *handler) HandleQuota(ctx context.Context, instance *quota.Instance,
	args adapter.QuotaArgs) (adapter.QuotaResult, error) {

	/*
		svcProc, err := h.getServiceProcessor(instance.Api)
		if err != nil {
			return adapter.QuotaResult{
				// This map to rpc.INTERNAL.
				Status: status.WithError(err),
			}, nil
		}
		return svcProc.ProcessQuota(ctx, instance, args)
	*/
	return adapter.QuotaResult{}, errors.New("not implemented")
}

// Close closes a handler.
// TODO(manlinl): Run svcProc.Close in goroutine after reportProcessor implements buffering.
func (h *handler) Close() error {
	var errors *multierror.Error

	h.lock.Lock()
	defer h.lock.Lock()
	for _, svcProc := range h.svcProcMap {
		if err := svcProc.Close(); err != nil {
			errors = multierror.Append(errors, err)
		}
	}

	return errors.ErrorOrNil()
}

func (h *handler) getServiceProcessor(serviceFullName string) (*serviceProcessor, error) {
	if _, ok := h.ctx.serviceConfigIndex[serviceFullName]; !ok {
		return nil, fmt.Errorf("unknown service %v", serviceFullName)
	}

	h.lock.Lock()
	defer h.lock.Unlock()
	svcProc, found := h.svcProcMap[serviceFullName]
	if found {
		return svcProc, nil
	}

	svcProc, err := newServiceProcessor(serviceFullName, h.ctx)
	if err != nil {
		return nil, err
	}
	h.svcProcMap[serviceFullName] = svcProc
	return svcProc, nil
}

func newHandler(ctx *handlerContext) (*handler, error) {
	return &handler{
		ctx:        ctx,
		svcProcMap: make(map[string]*serviceProcessor, len(ctx.config.ServiceConfigs)),
	}, nil
}
<|endoftext|>


<commit_before>package model

import (
	"context"
	"errors"
	"fmt"
	"testing"
	"time"

	fb "github.com/FlashbackSRS/flashback-model"
	"github.com/flimzy/diff"
	"github.com/flimzy/kivik"
)

type buryTest struct {
	Bury     fb.Interval
	Interval fb.Interval
	New      bool
	Expected fb.Interval
}

func TestBuryInterval(t *testing.T) {
	tests := []buryTest{
		buryTest{
			Bury:     10 * fb.Day,
			Interval: 20 * fb.Day,
			New:      false,
			Expected: 4 * fb.Day,
		},
		buryTest{
			Bury:     10 * fb.Day,
			Interval: 20 * fb.Day,
			New:      true,
			Expected: 7 * fb.Day,
		},
		buryTest{
			Bury:     10 * fb.Day,
			Interval: 1 * fb.Day,
			New:      false,
			Expected: 1 * fb.Day,
		},
	}
	for _, test := range tests {
		result := buryInterval(test.Bury, test.Interval, test.New)
		if result != test.Expected {
			t.Errorf("%s / %s / %t:\n\tExpected: %s\n\t  Actual: %s\n", test.Bury, test.Interval, test.New, test.Expected, result)
		}
	}
}

func TestFetchRelatedCards(t *testing.T) {
	tests := []struct {
		name     string
		db       allDocer
		cardID   string
		expected []*fb.Card
		err      string
	}{
		{
			name:   "db error",
			db:     &mockAllDocer{err: errors.New("db error")},
			cardID: "card-foo.bar.0",
			err:    "db error",
		},
		{
			name:   "iteration error",
			db:     &mockAllDocer{rows: &mockRows{err: errors.New("db error")}},
			cardID: "card-foo.bar.0",
			err:    "db error",
		},
		{
			name: "invalid json",
			db: &mockAllDocer{
				rows: &mockRows{
					rows: []string{
						`{"_id":"card-foo.bar.1", "created":"2017-01-01T01:01:01Z", "modified":12345, "model": "theme-Zm9v/0"}`,
					},
				},
			},
			cardID: "card-foo.bar.0",
			err:    `scan doc: parsing time "12345" as ""2006-01-02T15:04:05Z07:00"": cannot parse "12345" as """`,
		},
		{
			name: "success",
			db: &mockAllDocer{
				rows: &mockRows{
					rows: []string{
						`{"_id":"card-foo.bar.0", "created":"2017-01-01T01:01:01Z", "modified":"2017-01-01T01:01:01Z", "model": "theme-Zm9v/0"}`,
						`{"_id":"card-foo.bar.1", "created":"2017-01-01T01:01:01Z", "modified":"2017-01-01T01:01:01Z", "model": "theme-Zm9v/0"}`,
					},
				},
			},
			cardID: "card-foo.bar.0",
			expected: []*fb.Card{
				{
					ID:       "card-foo.bar.1",
					ModelID:  "theme-Zm9v/0",
					Created:  parseTime(t, "2017-01-01T01:01:01Z"),
					Modified: parseTime(t, "2017-01-01T01:01:01Z"),
				},
			},
		},
	}
	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			result, err := fetchRelatedCards(context.Background(), test.db, test.cardID)
			checkErr(t, test.err, err)
			if err != nil {
				return
			}
			if d := diff.Interface(test.expected, result); d != nil {
				t.Error(d)
			}
		})
	}
}

type buryClient struct {
	kivikClient
	db kivikDB
}

var _ kivikClient = &buryClient{}

func (c *buryClient) DB(_ context.Context, _ string, _ ...kivik.Options) (kivikDB, error) {
	return c.db, nil
}

func TestBuryRelatedCards(t *testing.T) {
	tests := []struct {
		name string
		repo *Repo
		card *fb.Card
		err  string
	}{
		{
			name: "not logged in",
			repo: &Repo{},
			card: &fb.Card{ID: "card-foo.bar.0"},
			err:  "not logged in",
		},
		{
			name: "fetch error",
			repo: &Repo{user: "bob",
				local: &buryClient{
					db: &mockAllDocer{
						err: errors.New("db error"),
					},
				},
			},
			card: &fb.Card{ID: "card-foo.bar.0"},
			err:  "db error",
		},
		{
			name: "no related cards",
			repo: &Repo{user: "bob",
				local: &buryClient{db: &mockAllDocer{
					rows: &mockRows{},
				}}},
			card: &fb.Card{ID: "card-foo.bar.0"},
		},
	}
	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			err := test.repo.BuryRelatedCards(context.Background(), test.card)
			checkErr(t, test.err, err)
		})
	}
}

func TestSetBurials(t *testing.T) {
	tests := []struct {
		name     string
		interval fb.Interval
		cards    []*fb.Card
		expected []*fb.Card
	}{
		{
			name:     "no cards",
			cards:    []*fb.Card{},
			expected: []*fb.Card{},
		},
		{
			name:     "two cards",
			interval: fb.Interval(24 * time.Hour),
			cards: []*fb.Card{
				{}, // new
				{
					ReviewCount: 1,
					Interval:    fb.Interval(24 * time.Hour),
				}, // Minimal burial
				{
					ReviewCount: 1,
					BuriedUntil: fb.Due(parseTime(t, "2018-01-01T00:00:00Z")),
				}, // Should not be re-buried
			},
			expected: []*fb.Card{
				{BuriedUntil: fb.Due(parseTime(t, "2017-01-08T00:00:00Z"))},
				{
					ReviewCount: 1,
					Interval:    fb.Interval(24 * time.Hour),
					BuriedUntil: fb.Due(parseTime(t, "2017-01-02T00:00:00Z")),
				},
			},
		},
	}
	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			result := setBurials(test.interval, test.cards)
			if d := diff.Interface(test.expected, result); d != nil {
				t.Error(d)
			}
			for _, x := range result {
				fmt.Printf("%v\n", x.BuriedUntil)
			}
		})
	}
}
<commit_msg>Remove some debugging code<commit_after>package model

import (
	"context"
	"errors"
	"testing"
	"time"

	fb "github.com/FlashbackSRS/flashback-model"
	"github.com/flimzy/diff"
	"github.com/flimzy/kivik"
)

type buryTest struct {
	Bury     fb.Interval
	Interval fb.Interval
	New      bool
	Expected fb.Interval
}

func TestBuryInterval(t *testing.T) {
	tests := []buryTest{
		buryTest{
			Bury:     10 * fb.Day,
			Interval: 20 * fb.Day,
			New:      false,
			Expected: 4 * fb.Day,
		},
		buryTest{
			Bury:     10 * fb.Day,
			Interval: 20 * fb.Day,
			New:      true,
			Expected: 7 * fb.Day,
		},
		buryTest{
			Bury:     10 * fb.Day,
			Interval: 1 * fb.Day,
			New:      false,
			Expected: 1 * fb.Day,
		},
	}
	for _, test := range tests {
		result := buryInterval(test.Bury, test.Interval, test.New)
		if result != test.Expected {
			t.Errorf("%s / %s / %t:\n\tExpected: %s\n\t  Actual: %s\n", test.Bury, test.Interval, test.New, test.Expected, result)
		}
	}
}

func TestFetchRelatedCards(t *testing.T) {
	tests := []struct {
		name     string
		db       allDocer
		cardID   string
		expected []*fb.Card
		err      string
	}{
		{
			name:   "db error",
			db:     &mockAllDocer{err: errors.New("db error")},
			cardID: "card-foo.bar.0",
			err:    "db error",
		},
		{
			name:   "iteration error",
			db:     &mockAllDocer{rows: &mockRows{err: errors.New("db error")}},
			cardID: "card-foo.bar.0",
			err:    "db error",
		},
		{
			name: "invalid json",
			db: &mockAllDocer{
				rows: &mockRows{
					rows: []string{
						`{"_id":"card-foo.bar.1", "created":"2017-01-01T01:01:01Z", "modified":12345, "model": "theme-Zm9v/0"}`,
					},
				},
			},
			cardID: "card-foo.bar.0",
			err:    `scan doc: parsing time "12345" as ""2006-01-02T15:04:05Z07:00"": cannot parse "12345" as """`,
		},
		{
			name: "success",
			db: &mockAllDocer{
				rows: &mockRows{
					rows: []string{
						`{"_id":"card-foo.bar.0", "created":"2017-01-01T01:01:01Z", "modified":"2017-01-01T01:01:01Z", "model": "theme-Zm9v/0"}`,
						`{"_id":"card-foo.bar.1", "created":"2017-01-01T01:01:01Z", "modified":"2017-01-01T01:01:01Z", "model": "theme-Zm9v/0"}`,
					},
				},
			},
			cardID: "card-foo.bar.0",
			expected: []*fb.Card{
				{
					ID:       "card-foo.bar.1",
					ModelID:  "theme-Zm9v/0",
					Created:  parseTime(t, "2017-01-01T01:01:01Z"),
					Modified: parseTime(t, "2017-01-01T01:01:01Z"),
				},
			},
		},
	}
	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			result, err := fetchRelatedCards(context.Background(), test.db, test.cardID)
			checkErr(t, test.err, err)
			if err != nil {
				return
			}
			if d := diff.Interface(test.expected, result); d != nil {
				t.Error(d)
			}
		})
	}
}

type buryClient struct {
	kivikClient
	db kivikDB
}

var _ kivikClient = &buryClient{}

func (c *buryClient) DB(_ context.Context, _ string, _ ...kivik.Options) (kivikDB, error) {
	return c.db, nil
}

func TestBuryRelatedCards(t *testing.T) {
	tests := []struct {
		name string
		repo *Repo
		card *fb.Card
		err  string
	}{
		{
			name: "not logged in",
			repo: &Repo{},
			card: &fb.Card{ID: "card-foo.bar.0"},
			err:  "not logged in",
		},
		{
			name: "fetch error",
			repo: &Repo{user: "bob",
				local: &buryClient{
					db: &mockAllDocer{
						err: errors.New("db error"),
					},
				},
			},
			card: &fb.Card{ID: "card-foo.bar.0"},
			err:  "db error",
		},
		{
			name: "no related cards",
			repo: &Repo{user: "bob",
				local: &buryClient{db: &mockAllDocer{
					rows: &mockRows{},
				}}},
			card: &fb.Card{ID: "card-foo.bar.0"},
		},
	}
	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			err := test.repo.BuryRelatedCards(context.Background(), test.card)
			checkErr(t, test.err, err)
		})
	}
}

func TestSetBurials(t *testing.T) {
	tests := []struct {
		name     string
		interval fb.Interval
		cards    []*fb.Card
		expected []*fb.Card
	}{
		{
			name:     "no cards",
			cards:    []*fb.Card{},
			expected: []*fb.Card{},
		},
		{
			name:     "two cards",
			interval: fb.Interval(24 * time.Hour),
			cards: []*fb.Card{
				{}, // new
				{
					ReviewCount: 1,
					Interval:    fb.Interval(24 * time.Hour),
				}, // Minimal burial
				{
					ReviewCount: 1,
					BuriedUntil: fb.Due(parseTime(t, "2018-01-01T00:00:00Z")),
				}, // Should not be re-buried
			},
			expected: []*fb.Card{
				{BuriedUntil: fb.Due(parseTime(t, "2017-01-08T00:00:00Z"))},
				{
					ReviewCount: 1,
					Interval:    fb.Interval(24 * time.Hour),
					BuriedUntil: fb.Due(parseTime(t, "2017-01-02T00:00:00Z")),
				},
			},
		},
	}
	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			result := setBurials(test.interval, test.cards)
			if d := diff.Interface(test.expected, result); d != nil {
				t.Error(d)
			}
		})
	}
}
<|endoftext|>


<commit_before>// +build e2e

/*
Copyright 2019 The Knative Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"testing"

	"github.com/knative/eventing/pkg/apis/eventing/v1alpha1"
	"github.com/knative/eventing/test"
	pkgTest "github.com/knative/pkg/test"
	"k8s.io/apimachinery/pkg/util/uuid"
)

/*
TestEventTransformationForTrigger tests the following scenario:

                         5                 4
                   ------------- ----------------------
                   |           | |                    |
             1     v	 2     | v        3           |
EventSource ---> Broker ---> Trigger1 -------> Service(Transformation)
                   |
                   | 6                   7
                   |-------> Trigger2 -------> Service(Logger)

Note: the number denotes the sequence of the event that flows in this test case.
*/
func TestEventTransformationForTrigger(t *testing.T) {
	const (
		brokerName = "e2e-eventtransformation-broker"
		saName     = "eventing-broker-filter"
		// This ClusterRole is installed in Knative Eventing setup, see https://github.com/knative/eventing/tree/master/docs/broker#manual-setup.
		crName = "eventing-broker-filter"

		any          = v1alpha1.TriggerAnyFilter
		eventType1   = "type1"
		eventType2   = "type2"
		eventSource1 = "source1"
		eventSource2 = "source2"
		eventBody    = "e2e-eventtransformation-body"

		triggerName1 = "trigger1"
		triggerName2 = "trigger2"

		transformationPodName = "trans-pod"
		loggerPodName         = "logger-pod"
	)

	clients, ns, provisioner, cleaner := Setup(t, true, t.Logf)
	defer TearDown(clients, ns, cleaner, t.Logf)

	// creates ServiceAccount and ClusterRoleBinding with default cluster-admin role
	err := CreateServiceAccountAndBinding(clients, saName, crName, ns, t.Logf, cleaner)
	if err != nil {
		t.Fatalf("Failed to create the ServiceAccount and ServiceAccountRoleBinding: %v", err)
	}

	// create a new broker
	broker := test.Broker(brokerName, ns, test.ClusterChannelProvisioner(provisioner))
	t.Logf("provisioner name is: %s", broker.Spec.ChannelTemplate.Provisioner.Name)
	err = WithBrokerReady(clients, broker, t.Logf, cleaner)
	if err != nil {
		t.Fatalf("Error waiting for the broker to become ready: %v, %v", err, broker)
	}
	brokerUrl := fmt.Sprintf("http://%s", broker.Status.Address.Hostname)
	t.Logf("The broker is ready with url: %q", brokerUrl)

	// create an event we want to send
	eventToSend := &test.CloudEvent{
		Source:   eventSource1,
		Type:     eventType1,
		Data:     fmt.Sprintf(`{"msg":%q}`, eventBody),
		Encoding: test.CloudEventDefaultEncoding,
	}

	// create the event we want to tranform to
	transformedEventBody := fmt.Sprintf("%s %s", eventBody, string(uuid.NewUUID()))
	eventAfterTransformation := &test.CloudEvent{
		Source:   eventSource2,
		Type:     eventType2,
		Data:     fmt.Sprintf(`{"msg":%q}`, transformedEventBody),
		Encoding: test.CloudEventDefaultEncoding,
	}

	// create the transformation pod and service, and get them ready
	transformationPodSelector := map[string]string{"e2etest": string(uuid.NewUUID())}
	transformationPod := test.EventTransformationPod(transformationPodName, ns, transformationPodSelector, eventAfterTransformation)
	transformationSvc := test.Service(transformationPodName, ns, transformationPodSelector)
	transformationPod, err = CreatePodAndServiceReady(clients, transformationPod, transformationSvc, t.Logf, cleaner)
	if err != nil {
		t.Fatalf("Failed to create transformation pod and service, and get them ready: %v", err)
	}

	trigger1 := test.NewTriggerBuilder(triggerName1, ns).
		EventType(eventType1).
		EventSource(eventSource1).
		Broker(brokerName).
		SubscriberSvc(transformationPodName).
		Build()
	err = CreateTrigger(clients, trigger1, t.Logf, cleaner)
	if err != nil {
		t.Fatalf("Error creating trigger1: %v", err)
	}

	// create logger pod and service, and get them ready
	loggerPodSelector := map[string]string{"e2etest": string(uuid.NewUUID())}
	loggerPod := test.EventLoggerPod(loggerPodName, ns, loggerPodSelector)
	loggerSvc := test.Service(loggerPodName, ns, loggerPodSelector)
	loggerPod, err = CreatePodAndServiceReady(clients, loggerPod, loggerSvc, t.Logf, cleaner)
	if err != nil {
		t.Fatalf("Failed to create logger pod and service, and get them ready: %v", err)
	}

	trigger2 := test.NewTriggerBuilder(triggerName2, ns).
		EventType(eventType2).
		EventSource(eventSource2).
		Broker(brokerName).
		SubscriberSvc(loggerPodName).
		Build()
	err = CreateTrigger(clients, trigger2, t.Logf, cleaner)
	if err != nil {
		t.Fatalf("Error creating trigger2: %v", err)
	}

	// Wait for all of the triggers in the namespace to be ready.
	if err := WaitForAllTriggersReady(clients, ns, t.Logf); err != nil {
		t.Fatalf("Error waiting for triggers to become ready: %v", err)
	}

	// send fake CloudEvent to the broker
	if err := SendFakeEventToBroker(clients, eventToSend, broker, t.Logf, cleaner); err != nil {
		t.Fatalf("Failed to send fake CloudEvent to the broker %q", broker.Name)
	}

	if err := pkgTest.WaitForLogContent(clients.Kube, loggerPodName, loggerPod.Spec.Containers[0].Name, ns, transformedEventBody); err != nil {
		logPodLogsForDebugging(clients, transformationPodName, transformationPod.Spec.Containers[0].Name, ns, t.Logf)
		logPodLogsForDebugging(clients, loggerPodName, loggerPod.Spec.Containers[0].Name, ns, t.Logf)
		logPodLogsForDebugging(clients, eventSource1, "sendevent", ns, t.Logf)
		t.Fatalf("String %q not found in logs of logger pod %q: %v", transformedEventBody, loggerPodName, err)
	}
}
<commit_msg>Fix spelling errors (#1150)<commit_after>// +build e2e

/*
Copyright 2019 The Knative Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"testing"

	"github.com/knative/eventing/pkg/apis/eventing/v1alpha1"
	"github.com/knative/eventing/test"
	pkgTest "github.com/knative/pkg/test"
	"k8s.io/apimachinery/pkg/util/uuid"
)

/*
TestEventTransformationForTrigger tests the following scenario:

                         5                 4
                   ------------- ----------------------
                   |           | |                    |
             1     v	 2     | v        3           |
EventSource ---> Broker ---> Trigger1 -------> Service(Transformation)
                   |
                   | 6                   7
                   |-------> Trigger2 -------> Service(Logger)

Note: the number denotes the sequence of the event that flows in this test case.
*/
func TestEventTransformationForTrigger(t *testing.T) {
	const (
		brokerName = "e2e-eventtransformation-broker"
		saName     = "eventing-broker-filter"
		// This ClusterRole is installed in Knative Eventing setup, see https://github.com/knative/eventing/tree/master/docs/broker#manual-setup.
		crName = "eventing-broker-filter"

		any          = v1alpha1.TriggerAnyFilter
		eventType1   = "type1"
		eventType2   = "type2"
		eventSource1 = "source1"
		eventSource2 = "source2"
		eventBody    = "e2e-eventtransformation-body"

		triggerName1 = "trigger1"
		triggerName2 = "trigger2"

		transformationPodName = "trans-pod"
		loggerPodName         = "logger-pod"
	)

	clients, ns, provisioner, cleaner := Setup(t, true, t.Logf)
	defer TearDown(clients, ns, cleaner, t.Logf)

	// creates ServiceAccount and ClusterRoleBinding with default cluster-admin role
	err := CreateServiceAccountAndBinding(clients, saName, crName, ns, t.Logf, cleaner)
	if err != nil {
		t.Fatalf("Failed to create the ServiceAccount and ServiceAccountRoleBinding: %v", err)
	}

	// create a new broker
	broker := test.Broker(brokerName, ns, test.ClusterChannelProvisioner(provisioner))
	t.Logf("provisioner name is: %s", broker.Spec.ChannelTemplate.Provisioner.Name)
	err = WithBrokerReady(clients, broker, t.Logf, cleaner)
	if err != nil {
		t.Fatalf("Error waiting for the broker to become ready: %v, %v", err, broker)
	}
	brokerUrl := fmt.Sprintf("http://%s", broker.Status.Address.Hostname)
	t.Logf("The broker is ready with url: %q", brokerUrl)

	// create an event we want to send
	eventToSend := &test.CloudEvent{
		Source:   eventSource1,
		Type:     eventType1,
		Data:     fmt.Sprintf(`{"msg":%q}`, eventBody),
		Encoding: test.CloudEventDefaultEncoding,
	}

	// create the event we want to transform to
	transformedEventBody := fmt.Sprintf("%s %s", eventBody, string(uuid.NewUUID()))
	eventAfterTransformation := &test.CloudEvent{
		Source:   eventSource2,
		Type:     eventType2,
		Data:     fmt.Sprintf(`{"msg":%q}`, transformedEventBody),
		Encoding: test.CloudEventDefaultEncoding,
	}

	// create the transformation pod and service, and get them ready
	transformationPodSelector := map[string]string{"e2etest": string(uuid.NewUUID())}
	transformationPod := test.EventTransformationPod(transformationPodName, ns, transformationPodSelector, eventAfterTransformation)
	transformationSvc := test.Service(transformationPodName, ns, transformationPodSelector)
	transformationPod, err = CreatePodAndServiceReady(clients, transformationPod, transformationSvc, t.Logf, cleaner)
	if err != nil {
		t.Fatalf("Failed to create transformation pod and service, and get them ready: %v", err)
	}

	trigger1 := test.NewTriggerBuilder(triggerName1, ns).
		EventType(eventType1).
		EventSource(eventSource1).
		Broker(brokerName).
		SubscriberSvc(transformationPodName).
		Build()
	err = CreateTrigger(clients, trigger1, t.Logf, cleaner)
	if err != nil {
		t.Fatalf("Error creating trigger1: %v", err)
	}

	// create logger pod and service, and get them ready
	loggerPodSelector := map[string]string{"e2etest": string(uuid.NewUUID())}
	loggerPod := test.EventLoggerPod(loggerPodName, ns, loggerPodSelector)
	loggerSvc := test.Service(loggerPodName, ns, loggerPodSelector)
	loggerPod, err = CreatePodAndServiceReady(clients, loggerPod, loggerSvc, t.Logf, cleaner)
	if err != nil {
		t.Fatalf("Failed to create logger pod and service, and get them ready: %v", err)
	}

	trigger2 := test.NewTriggerBuilder(triggerName2, ns).
		EventType(eventType2).
		EventSource(eventSource2).
		Broker(brokerName).
		SubscriberSvc(loggerPodName).
		Build()
	err = CreateTrigger(clients, trigger2, t.Logf, cleaner)
	if err != nil {
		t.Fatalf("Error creating trigger2: %v", err)
	}

	// Wait for all of the triggers in the namespace to be ready.
	if err := WaitForAllTriggersReady(clients, ns, t.Logf); err != nil {
		t.Fatalf("Error waiting for triggers to become ready: %v", err)
	}

	// send fake CloudEvent to the broker
	if err := SendFakeEventToBroker(clients, eventToSend, broker, t.Logf, cleaner); err != nil {
		t.Fatalf("Failed to send fake CloudEvent to the broker %q", broker.Name)
	}

	if err := pkgTest.WaitForLogContent(clients.Kube, loggerPodName, loggerPod.Spec.Containers[0].Name, ns, transformedEventBody); err != nil {
		logPodLogsForDebugging(clients, transformationPodName, transformationPod.Spec.Containers[0].Name, ns, t.Logf)
		logPodLogsForDebugging(clients, loggerPodName, loggerPod.Spec.Containers[0].Name, ns, t.Logf)
		logPodLogsForDebugging(clients, eventSource1, "sendevent", ns, t.Logf)
		t.Fatalf("String %q not found in logs of logger pod %q: %v", transformedEventBody, loggerPodName, err)
	}
}
<|endoftext|>


<commit_before>package models

import (
	"bufio"
	"encoding/csv"
	"os"
	"path/filepath"

	"github.com/nicomo/abacaxi/logger"
)

func createFile(fname string) (*os.File, error) {
	// create dirs if they don't exist
	path := filepath.Join("static", "downloads")
	ErrPath := os.MkdirAll(path, os.ModePerm)
	if ErrPath != nil {
		logger.Error.Println(path)
	}

	// create file
	f, err := os.Create(filepath.Join(path, fname))
	if err != nil {
		logger.Error.Println(err)
		return nil, err
	}
	return f, nil
}

// CreateKbartFile creates csv file with KBART fields from records
func CreateKbartFile(records []Record, fname string) (int64, error) {

	logger.Debug.Printf("in CreateKbartFile, fname : %s", fname)

	f, err := createFile(fname)
	if err != nil {
		return 0, err
	}
	defer f.Close()

	// create a new writer and change default separator
	w := csv.NewWriter(f)
	w.Comma = ';'
	defer w.Flush()

	// write header to csv file
	kbartHeader := []string{
		"publication_title",
		"print_identifier",
		"online_identifier",
		"date_first_issue_online",
		"num_first_vol_online",
		"num_first_issue_online",
		"date_last_issue_online",
		"num_last_vol_online",
		"num_last_issue_online",
		"title_url",
		"first_author",
		"title_id",
		"embargo_info",
		"coverage_depth",
		"coverage_notes",
		"publisher_name",
	}
	if err := w.Write(kbartHeader); err != nil {
		return 0, err
	}

	for _, record := range records {
		if err := w.Write(recordToKbart(record)); err != nil {
			logger.Error.Printf("couldn't write to csv file: %v", err)
			continue
		}
	}

	return getFileSize(f), nil
}

// CreateUnimarcFile creates the file to be exported
func CreateUnimarcFile(records []Record, fname string) (int64, error) {

	f, err := createFile(fname)
	if err != nil {
		return 0, err
	}
	defer f.Close()

	// get a buffered writer and write to file
	w := bufio.NewWriter(f)
	_, ErrWriteHeader := w.WriteString("<?xml version=\"1.0\"?>\n")
	if ErrWriteHeader != nil {
		return 0, ErrWriteHeader
	}

	// write each marc record in turn
	for _, record := range records {
		_, ErrWriteRecord := w.WriteString(record.RecordUnimarc)
		if ErrWriteRecord != nil {
			return 0, ErrWriteRecord
		}
	}

	w.Flush() // flush the buffer

	return getFileSize(f), nil
}

func getFileSize(f *os.File) int64 {
	// get & return the size of the written file
	fi, err := f.Stat()
	if err != nil {
		logger.Error.Printf("couldn't get file info: %v", err)
		return 0
	}
	return fi.Size()
}
<commit_msg>fixes #22<commit_after>package models

import (
	"bufio"
	"encoding/csv"
	"os"
	"path/filepath"

	"github.com/nicomo/abacaxi/logger"
)

func createFile(fname string) (*os.File, error) {
	// create dirs if they don't exist
	path := filepath.Join("static", "downloads")
	ErrPath := os.MkdirAll(path, os.ModePerm)
	if ErrPath != nil {
		logger.Error.Println(path)
	}

	// create file
	f, err := os.Create(filepath.Join(path, fname))
	if err != nil {
		logger.Error.Println(err)
		return nil, err
	}
	return f, nil
}

// CreateKbartFile creates csv file with KBART fields from records
func CreateKbartFile(records []Record, fname string) (int64, error) {

	f, err := createFile(fname)
	if err != nil {
		return 0, err
	}
	defer f.Close()

	// create a new writer and change default separator
	w := csv.NewWriter(f)
	w.Comma = ';'

	// write header to csv file
	kbartHeader := []string{
		"publication_title",
		"print_identifier",
		"online_identifier",
		"date_first_issue_online",
		"num_first_vol_online",
		"num_first_issue_online",
		"date_last_issue_online",
		"num_last_vol_online",
		"num_last_issue_online",
		"title_url",
		"first_author",
		"title_id",
		"embargo_info",
		"coverage_depth",
		"coverage_notes",
		"publisher_name",
	}

	// write the header
	if err := w.Write(kbartHeader); err != nil {
		return 0, err
	}

	// write each record in turn
	for _, record := range records {
		if err := w.Write(recordToKbart(record)); err != nil {
			logger.Error.Printf("couldn't write to csv file: %v", err)
			continue
		}
	}

	w.Flush()
	return getFileSize(f), nil
}

// CreateUnimarcFile creates the file to be exported
func CreateUnimarcFile(records []Record, fname string) (int64, error) {

	f, err := createFile(fname)
	if err != nil {
		return 0, err
	}
	defer f.Close()

	// get a buffered writer and write to file
	w := bufio.NewWriter(f)
	_, ErrWriteHeader := w.WriteString("<?xml version=\"1.0\"?>\n")
	if ErrWriteHeader != nil {
		return 0, ErrWriteHeader
	}

	// write each marc record in turn
	for _, record := range records {
		_, ErrWriteRecord := w.WriteString(record.RecordUnimarc)
		if ErrWriteRecord != nil {
			return 0, ErrWriteRecord
		}
	}

	w.Flush() // flush the buffer

	return getFileSize(f), nil
}

func getFileSize(f *os.File) int64 {
	// get & return the size of the written file
	fi, err := f.Stat()
	if err != nil {
		logger.Error.Printf("couldn't get file info: %v", err)
		return 0
	}
	return fi.Size()
}
<|endoftext|>


<commit_before>// Package output defines an interface to write
// text messages. It provides implementations
// of this interface that use standard output,
// Cothority's logging infrastructure to write
// the messages or simply discards them.
package output

import (
	"fmt"
	"github.com/dedis/cothority/log"
	"os"
)

// Output Interface represents a generic output.
type Output interface {
	// Print prints a message to the output.
	Print(text string)
}

// PrintOutput prints it's messages to the standard output.
type PrintOutput struct{}

// Print implements Output interface.
func (o *PrintOutput) Print(text string) {
	fmt.Println(text)
}

// LogOutput prints it's messages using Cothority's logging infrastructure.
type LogOutput struct {
	Level int
	Info  bool
}

// Print implements Output interface.
func (o *LogOutput) Print(text string) {
	if o.Info {
		log.Info(text)
	} else {
		switch o.Level {
		case 1:
			log.Lvl1(text)
		case 2:
			log.Lvl2(text)
		case 3:
			log.Lvl3(text)
		case 4:
			log.Lvl4(text)
		case 5:
			log.Lvl5(text)
		default:
			log.Print(text)
		}
	}
}

// NullOutput prints discards all messages.
type NullOutput struct{}

// Print implements Output interface.
func (o *NullOutput) Print(text string) {}

type FileOutput struct {
	Filename string
	file *os.File
}

func (o *FileOutput) Print(text string) {
	if o.file == nil {
		f, err := os.Create(o.Filename)
		if err != nil {
			fmt.Print("Unable to open or create file:", err)
			return
		}
		o.file = f
	}

	o.file.WriteString(text + "\n")
}
<commit_msg>Run go fmt<commit_after>// Package output defines an interface to write
// text messages. It provides implementations
// of this interface that use standard output,
// Cothority's logging infrastructure to write
// the messages or simply discards them.
package output

import (
	"fmt"
	"github.com/dedis/cothority/log"
	"os"
)

// Output Interface represents a generic output.
type Output interface {
	// Print prints a message to the output.
	Print(text string)
}

// PrintOutput prints it's messages to the standard output.
type PrintOutput struct{}

// Print implements Output interface.
func (o *PrintOutput) Print(text string) {
	fmt.Println(text)
}

// LogOutput prints it's messages using Cothority's logging infrastructure.
type LogOutput struct {
	Level int
	Info  bool
}

// Print implements Output interface.
func (o *LogOutput) Print(text string) {
	if o.Info {
		log.Info(text)
	} else {
		switch o.Level {
		case 1:
			log.Lvl1(text)
		case 2:
			log.Lvl2(text)
		case 3:
			log.Lvl3(text)
		case 4:
			log.Lvl4(text)
		case 5:
			log.Lvl5(text)
		default:
			log.Print(text)
		}
	}
}

// NullOutput prints discards all messages.
type NullOutput struct{}

// Print implements Output interface.
func (o *NullOutput) Print(text string) {}

type FileOutput struct {
	Filename string
	file     *os.File
}

func (o *FileOutput) Print(text string) {
	if o.file == nil {
		f, err := os.Create(o.Filename)
		if err != nil {
			fmt.Print("Unable to open or create file:", err)
			return
		}
		o.file = f
	}

	o.file.WriteString(text + "\n")
}
<|endoftext|>


<commit_before>// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package git

import (
	"context"
	"fmt"
	"os/exec"
	"runtime"
	"strings"
	"time"

	"code.gitea.io/gitea/modules/process"
	"code.gitea.io/gitea/modules/setting"

	"github.com/hashicorp/go-version"
)

var (
	// Prefix the log prefix
	Prefix = "[git-module] "
	// GitVersionRequired is the minimum Git version required
	GitVersionRequired = "1.7.2"

	// GitExecutable is the command name of git
	// Could be updated to an absolute path while initialization
	GitExecutable = "git"

	// DefaultContext is the default context to run git commands in
	// will be overwritten by Init with HammerContext
	DefaultContext = context.Background()

	gitVersion *version.Version

	// SupportProcReceive version >= 2.29.0
	SupportProcReceive bool
)

// LocalVersion returns current Git version from shell.
func LocalVersion() (*version.Version, error) {
	if err := LoadGitVersion(); err != nil {
		return nil, err
	}
	return gitVersion, nil
}

// LoadGitVersion returns current Git version from shell.
func LoadGitVersion() error {
	// doesn't need RWMutex because its exec by Init()
	if gitVersion != nil {
		return nil
	}

	stdout, _, runErr := NewCommand(context.Background(), "version").RunStdString(nil)
	if runErr != nil {
		return runErr
	}

	fields := strings.Fields(stdout)
	if len(fields) < 3 {
		return fmt.Errorf("not enough output: %s", stdout)
	}

	var versionString string

	// Handle special case on Windows.
	i := strings.Index(fields[2], "windows")
	if i >= 1 {
		versionString = fields[2][:i-1]
	} else {
		versionString = fields[2]
	}

	var err error
	gitVersion, err = version.NewVersion(versionString)
	return err
}

// SetExecutablePath changes the path of git executable and checks the file permission and version.
func SetExecutablePath(path string) error {
	// If path is empty, we use the default value of GitExecutable "git" to search for the location of git.
	if path != "" {
		GitExecutable = path
	}
	absPath, err := exec.LookPath(GitExecutable)
	if err != nil {
		return fmt.Errorf("Git not found: %v", err)
	}
	GitExecutable = absPath

	err = LoadGitVersion()
	if err != nil {
		return fmt.Errorf("Git version missing: %v", err)
	}

	versionRequired, err := version.NewVersion(GitVersionRequired)
	if err != nil {
		return err
	}

	if gitVersion.LessThan(versionRequired) {
		return fmt.Errorf("Git version not supported. Requires version > %v", GitVersionRequired)
	}

	return nil
}

// VersionInfo returns git version information
func VersionInfo() string {
	format := "Git Version: %s"
	args := []interface{}{gitVersion.Original()}
	// Since git wire protocol has been released from git v2.18
	if setting.Git.EnableAutoGitWireProtocol && CheckGitVersionAtLeast("2.18") == nil {
		format += ", Wire Protocol %s Enabled"
		args = append(args, "Version 2") // for focus color
	}

	return fmt.Sprintf(format, args...)
}

// Init initializes git module
func Init(ctx context.Context) error {
	DefaultContext = ctx

	if setting.Git.Timeout.Default > 0 {
		defaultCommandExecutionTimeout = time.Duration(setting.Git.Timeout.Default) * time.Second
	}

	if err := SetExecutablePath(setting.Git.Path); err != nil {
		return err
	}

	// force cleanup args
	globalCommandArgs = []string{}

	if CheckGitVersionAtLeast("2.9") == nil {
		// Explicitly disable credential helper, otherwise Git credentials might leak
		globalCommandArgs = append(globalCommandArgs, "-c", "credential.helper=")
	}

	// Since git wire protocol has been released from git v2.18
	if setting.Git.EnableAutoGitWireProtocol && CheckGitVersionAtLeast("2.18") == nil {
		globalCommandArgs = append(globalCommandArgs, "-c", "protocol.version=2")
	}

	// By default partial clones are disabled, enable them from git v2.22
	if !setting.Git.DisablePartialClone && CheckGitVersionAtLeast("2.22") == nil {
		globalCommandArgs = append(globalCommandArgs, "-c", "uploadpack.allowfilter=true")
	}

	// Save current git version on init to gitVersion otherwise it would require an RWMutex
	if err := LoadGitVersion(); err != nil {
		return err
	}

	// Git requires setting user.name and user.email in order to commit changes - if they're not set just add some defaults
	for configKey, defaultValue := range map[string]string{"user.name": "Gitea", "user.email": "gitea@fake.local"} {
		if err := checkAndSetConfig(configKey, defaultValue, false); err != nil {
			return err
		}
	}

	// Set git some configurations - these must be set to these values for gitea to work correctly
	if err := checkAndSetConfig("core.quotePath", "false", true); err != nil {
		return err
	}

	if CheckGitVersionAtLeast("2.10") == nil {
		if err := checkAndSetConfig("receive.advertisePushOptions", "true", true); err != nil {
			return err
		}
	}

	if CheckGitVersionAtLeast("2.18") == nil {
		if err := checkAndSetConfig("core.commitGraph", "true", true); err != nil {
			return err
		}
		if err := checkAndSetConfig("gc.writeCommitGraph", "true", true); err != nil {
			return err
		}
	}

	if CheckGitVersionAtLeast("2.29") == nil {
		// set support for AGit flow
		if err := checkAndAddConfig("receive.procReceiveRefs", "refs/for"); err != nil {
			return err
		}
		SupportProcReceive = true
	} else {
		if err := checkAndRemoveConfig("receive.procReceiveRefs", "refs/for"); err != nil {
			return err
		}
		SupportProcReceive = false
	}

	if runtime.GOOS == "windows" {
		if err := checkAndSetConfig("core.longpaths", "true", true); err != nil {
			return err
		}
	}
	if setting.Git.DisableCoreProtectNTFS {
		if err := checkAndSetConfig("core.protectntfs", "false", true); err != nil {
			return err
		}
		globalCommandArgs = append(globalCommandArgs, "-c", "core.protectntfs=false")
	}
	return nil
}

// CheckGitVersionAtLeast check git version is at least the constraint version
func CheckGitVersionAtLeast(atLeast string) error {
	if err := LoadGitVersion(); err != nil {
		return err
	}
	atLeastVersion, err := version.NewVersion(atLeast)
	if err != nil {
		return err
	}
	if gitVersion.Compare(atLeastVersion) < 0 {
		return fmt.Errorf("installed git binary version %s is not at least %s", gitVersion.Original(), atLeast)
	}
	return nil
}

func checkAndSetConfig(key, defaultValue string, forceToDefault bool) error {
	stdout, stderr, err := process.GetManager().Exec("git.Init(get setting)", GitExecutable, "config", "--get", key)
	if err != nil {
		perr, ok := err.(*process.Error)
		if !ok {
			return fmt.Errorf("Failed to get git %s(%v) errType %T: %s", key, err, err, stderr)
		}
		eerr, ok := perr.Err.(*exec.ExitError)
		if !ok || eerr.ExitCode() != 1 {
			return fmt.Errorf("Failed to get git %s(%v) errType %T: %s", key, err, err, stderr)
		}
	}

	currValue := strings.TrimSpace(stdout)

	if currValue == defaultValue || (!forceToDefault && len(currValue) > 0) {
		return nil
	}

	if _, stderr, err = process.GetManager().Exec(fmt.Sprintf("git.Init(set %s)", key), "git", "config", "--global", key, defaultValue); err != nil {
		return fmt.Errorf("Failed to set git %s(%s): %s", key, err, stderr)
	}

	return nil
}

func checkAndAddConfig(key, value string) error {
	_, stderr, err := process.GetManager().Exec("git.Init(get setting)", GitExecutable, "config", "--get", key, value)
	if err != nil {
		perr, ok := err.(*process.Error)
		if !ok {
			return fmt.Errorf("Failed to get git %s(%v) errType %T: %s", key, err, err, stderr)
		}
		eerr, ok := perr.Err.(*exec.ExitError)
		if !ok || eerr.ExitCode() != 1 {
			return fmt.Errorf("Failed to get git %s(%v) errType %T: %s", key, err, err, stderr)
		}
		if eerr.ExitCode() == 1 {
			if _, stderr, err = process.GetManager().Exec(fmt.Sprintf("git.Init(set %s)", key), "git", "config", "--global", "--add", key, value); err != nil {
				return fmt.Errorf("Failed to set git %s(%s): %s", key, err, stderr)
			}
			return nil
		}
	}

	return nil
}

func checkAndRemoveConfig(key, value string) error {
	_, stderr, err := process.GetManager().Exec("git.Init(get setting)", GitExecutable, "config", "--get", key, value)
	if err != nil {
		perr, ok := err.(*process.Error)
		if !ok {
			return fmt.Errorf("Failed to get git %s(%v) errType %T: %s", key, err, err, stderr)
		}
		eerr, ok := perr.Err.(*exec.ExitError)
		if !ok || eerr.ExitCode() != 1 {
			return fmt.Errorf("Failed to get git %s(%v) errType %T: %s", key, err, err, stderr)
		}
		if eerr.ExitCode() == 1 {
			return nil
		}
	}

	if _, stderr, err = process.GetManager().Exec(fmt.Sprintf("git.Init(set %s)", key), "git", "config", "--global", "--unset-all", key, value); err != nil {
		return fmt.Errorf("Failed to set git %s(%s): %s", key, err, stderr)
	}

	return nil
}

// Fsck verifies the connectivity and validity of the objects in the database
func Fsck(ctx context.Context, repoPath string, timeout time.Duration, args ...string) error {
	return NewCommand(ctx, "fsck").AddArguments(args...).Run(&RunOpts{Timeout: timeout, Dir: repoPath})
}
<commit_msg>Add uploadpack.allowAnySHA1InWant to allow --filter=blob:none with older git clients (#19430)<commit_after>// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package git

import (
	"context"
	"fmt"
	"os/exec"
	"runtime"
	"strings"
	"time"

	"code.gitea.io/gitea/modules/process"
	"code.gitea.io/gitea/modules/setting"

	"github.com/hashicorp/go-version"
)

var (
	// Prefix the log prefix
	Prefix = "[git-module] "
	// GitVersionRequired is the minimum Git version required
	GitVersionRequired = "1.7.2"

	// GitExecutable is the command name of git
	// Could be updated to an absolute path while initialization
	GitExecutable = "git"

	// DefaultContext is the default context to run git commands in
	// will be overwritten by Init with HammerContext
	DefaultContext = context.Background()

	gitVersion *version.Version

	// SupportProcReceive version >= 2.29.0
	SupportProcReceive bool
)

// LocalVersion returns current Git version from shell.
func LocalVersion() (*version.Version, error) {
	if err := LoadGitVersion(); err != nil {
		return nil, err
	}
	return gitVersion, nil
}

// LoadGitVersion returns current Git version from shell.
func LoadGitVersion() error {
	// doesn't need RWMutex because its exec by Init()
	if gitVersion != nil {
		return nil
	}

	stdout, _, runErr := NewCommand(context.Background(), "version").RunStdString(nil)
	if runErr != nil {
		return runErr
	}

	fields := strings.Fields(stdout)
	if len(fields) < 3 {
		return fmt.Errorf("not enough output: %s", stdout)
	}

	var versionString string

	// Handle special case on Windows.
	i := strings.Index(fields[2], "windows")
	if i >= 1 {
		versionString = fields[2][:i-1]
	} else {
		versionString = fields[2]
	}

	var err error
	gitVersion, err = version.NewVersion(versionString)
	return err
}

// SetExecutablePath changes the path of git executable and checks the file permission and version.
func SetExecutablePath(path string) error {
	// If path is empty, we use the default value of GitExecutable "git" to search for the location of git.
	if path != "" {
		GitExecutable = path
	}
	absPath, err := exec.LookPath(GitExecutable)
	if err != nil {
		return fmt.Errorf("Git not found: %v", err)
	}
	GitExecutable = absPath

	err = LoadGitVersion()
	if err != nil {
		return fmt.Errorf("Git version missing: %v", err)
	}

	versionRequired, err := version.NewVersion(GitVersionRequired)
	if err != nil {
		return err
	}

	if gitVersion.LessThan(versionRequired) {
		return fmt.Errorf("Git version not supported. Requires version > %v", GitVersionRequired)
	}

	return nil
}

// VersionInfo returns git version information
func VersionInfo() string {
	format := "Git Version: %s"
	args := []interface{}{gitVersion.Original()}
	// Since git wire protocol has been released from git v2.18
	if setting.Git.EnableAutoGitWireProtocol && CheckGitVersionAtLeast("2.18") == nil {
		format += ", Wire Protocol %s Enabled"
		args = append(args, "Version 2") // for focus color
	}

	return fmt.Sprintf(format, args...)
}

// Init initializes git module
func Init(ctx context.Context) error {
	DefaultContext = ctx

	if setting.Git.Timeout.Default > 0 {
		defaultCommandExecutionTimeout = time.Duration(setting.Git.Timeout.Default) * time.Second
	}

	if err := SetExecutablePath(setting.Git.Path); err != nil {
		return err
	}

	// force cleanup args
	globalCommandArgs = []string{}

	if CheckGitVersionAtLeast("2.9") == nil {
		// Explicitly disable credential helper, otherwise Git credentials might leak
		globalCommandArgs = append(globalCommandArgs, "-c", "credential.helper=")
	}

	// Since git wire protocol has been released from git v2.18
	if setting.Git.EnableAutoGitWireProtocol && CheckGitVersionAtLeast("2.18") == nil {
		globalCommandArgs = append(globalCommandArgs, "-c", "protocol.version=2")
	}

	// By default partial clones are disabled, enable them from git v2.22
	if !setting.Git.DisablePartialClone && CheckGitVersionAtLeast("2.22") == nil {
		globalCommandArgs = append(globalCommandArgs, "-c", "uploadpack.allowfilter=true", "-c", "uploadpack.allowAnySHA1InWant=true")
	}

	// Save current git version on init to gitVersion otherwise it would require an RWMutex
	if err := LoadGitVersion(); err != nil {
		return err
	}

	// Git requires setting user.name and user.email in order to commit changes - if they're not set just add some defaults
	for configKey, defaultValue := range map[string]string{"user.name": "Gitea", "user.email": "gitea@fake.local"} {
		if err := checkAndSetConfig(configKey, defaultValue, false); err != nil {
			return err
		}
	}

	// Set git some configurations - these must be set to these values for gitea to work correctly
	if err := checkAndSetConfig("core.quotePath", "false", true); err != nil {
		return err
	}

	if CheckGitVersionAtLeast("2.10") == nil {
		if err := checkAndSetConfig("receive.advertisePushOptions", "true", true); err != nil {
			return err
		}
	}

	if CheckGitVersionAtLeast("2.18") == nil {
		if err := checkAndSetConfig("core.commitGraph", "true", true); err != nil {
			return err
		}
		if err := checkAndSetConfig("gc.writeCommitGraph", "true", true); err != nil {
			return err
		}
	}

	if CheckGitVersionAtLeast("2.29") == nil {
		// set support for AGit flow
		if err := checkAndAddConfig("receive.procReceiveRefs", "refs/for"); err != nil {
			return err
		}
		SupportProcReceive = true
	} else {
		if err := checkAndRemoveConfig("receive.procReceiveRefs", "refs/for"); err != nil {
			return err
		}
		SupportProcReceive = false
	}

	if runtime.GOOS == "windows" {
		if err := checkAndSetConfig("core.longpaths", "true", true); err != nil {
			return err
		}
	}
	if setting.Git.DisableCoreProtectNTFS {
		if err := checkAndSetConfig("core.protectntfs", "false", true); err != nil {
			return err
		}
		globalCommandArgs = append(globalCommandArgs, "-c", "core.protectntfs=false")
	}
	return nil
}

// CheckGitVersionAtLeast check git version is at least the constraint version
func CheckGitVersionAtLeast(atLeast string) error {
	if err := LoadGitVersion(); err != nil {
		return err
	}
	atLeastVersion, err := version.NewVersion(atLeast)
	if err != nil {
		return err
	}
	if gitVersion.Compare(atLeastVersion) < 0 {
		return fmt.Errorf("installed git binary version %s is not at least %s", gitVersion.Original(), atLeast)
	}
	return nil
}

func checkAndSetConfig(key, defaultValue string, forceToDefault bool) error {
	stdout, stderr, err := process.GetManager().Exec("git.Init(get setting)", GitExecutable, "config", "--get", key)
	if err != nil {
		perr, ok := err.(*process.Error)
		if !ok {
			return fmt.Errorf("Failed to get git %s(%v) errType %T: %s", key, err, err, stderr)
		}
		eerr, ok := perr.Err.(*exec.ExitError)
		if !ok || eerr.ExitCode() != 1 {
			return fmt.Errorf("Failed to get git %s(%v) errType %T: %s", key, err, err, stderr)
		}
	}

	currValue := strings.TrimSpace(stdout)

	if currValue == defaultValue || (!forceToDefault && len(currValue) > 0) {
		return nil
	}

	if _, stderr, err = process.GetManager().Exec(fmt.Sprintf("git.Init(set %s)", key), "git", "config", "--global", key, defaultValue); err != nil {
		return fmt.Errorf("Failed to set git %s(%s): %s", key, err, stderr)
	}

	return nil
}

func checkAndAddConfig(key, value string) error {
	_, stderr, err := process.GetManager().Exec("git.Init(get setting)", GitExecutable, "config", "--get", key, value)
	if err != nil {
		perr, ok := err.(*process.Error)
		if !ok {
			return fmt.Errorf("Failed to get git %s(%v) errType %T: %s", key, err, err, stderr)
		}
		eerr, ok := perr.Err.(*exec.ExitError)
		if !ok || eerr.ExitCode() != 1 {
			return fmt.Errorf("Failed to get git %s(%v) errType %T: %s", key, err, err, stderr)
		}
		if eerr.ExitCode() == 1 {
			if _, stderr, err = process.GetManager().Exec(fmt.Sprintf("git.Init(set %s)", key), "git", "config", "--global", "--add", key, value); err != nil {
				return fmt.Errorf("Failed to set git %s(%s): %s", key, err, stderr)
			}
			return nil
		}
	}

	return nil
}

func checkAndRemoveConfig(key, value string) error {
	_, stderr, err := process.GetManager().Exec("git.Init(get setting)", GitExecutable, "config", "--get", key, value)
	if err != nil {
		perr, ok := err.(*process.Error)
		if !ok {
			return fmt.Errorf("Failed to get git %s(%v) errType %T: %s", key, err, err, stderr)
		}
		eerr, ok := perr.Err.(*exec.ExitError)
		if !ok || eerr.ExitCode() != 1 {
			return fmt.Errorf("Failed to get git %s(%v) errType %T: %s", key, err, err, stderr)
		}
		if eerr.ExitCode() == 1 {
			return nil
		}
	}

	if _, stderr, err = process.GetManager().Exec(fmt.Sprintf("git.Init(set %s)", key), "git", "config", "--global", "--unset-all", key, value); err != nil {
		return fmt.Errorf("Failed to set git %s(%s): %s", key, err, stderr)
	}

	return nil
}

// Fsck verifies the connectivity and validity of the objects in the database
func Fsck(ctx context.Context, repoPath string, timeout time.Duration, args ...string) error {
	return NewCommand(ctx, "fsck").AddArguments(args...).Run(&RunOpts{Timeout: timeout, Dir: repoPath})
}
<|endoftext|>


<commit_before>// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see <http://www.gnu.org/licenses/>.

package url

import (
	"errors"
	"fmt"
	"github.com/nmeum/marvin/irc"
	"github.com/nmeum/marvin/modules"
	"golang.org/x/net/html"
	"io"
	"mime"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
)

var extractError = errors.New("couldn't extract title")

type Module struct {
	regex    *regexp.Regexp
	RegexStr string   `json:"regex"`
	Exclude  []string `json:"exclude"`
}

func Init(moduleSet *modules.ModuleSet) {
	moduleSet.Register(new(Module))
}

func (m *Module) Name() string {
	return "url"
}

func (m *Module) Help() string {
	return "Displays HTML titles for HTTP links."
}

func (m *Module) Defaults() {
	m.RegexStr = `(http|https)\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(:[a-zA-Z0-9]*)?/?([a-zA-Z0-9\-\._\?\,\'/\\\+&%\$#\=~])*`
}

func (m *Module) Load(client *irc.Client) error {
	regex, err := regexp.Compile(m.RegexStr)
	if err != nil {
		return err
	}

	m.regex = regex
	client.CmdHook("privmsg", m.urlCmd)

	return nil
}

func (m *Module) urlCmd(client *irc.Client, msg irc.Message) error {
	link := m.regex.FindString(msg.Data)
	if len(link) <= 0 {
		return nil
	}

	purl, err := url.Parse(link)
	if err != nil || m.isExcluded(purl.Host) {
		return nil
	}

	resp, err := http.Get(purl.String())
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	info := m.infoString(resp)
	if len(info) <= 0 {
		return nil
	}

	return client.Write("NOTICE %s :%s", msg.Receiver, info)
}

func (m *Module) infoString(resp *http.Response) string {
	var mtype string
	var infos []string

	ctype := resp.Header.Get("Content-Type")
	if len(ctype) > 0 {
		m, _, err := mime.ParseMediaType(ctype)
		if err == nil {
			mtype = m
			infos = append(infos, fmt.Sprintf("Type: %s", mtype))
		}
	}

	csize := resp.Header.Get("Content-Length")
	if len(csize) > 0 {
		size, err := strconv.Atoi(csize)
		if err == nil {
			infos = append(infos, fmt.Sprintf("Size: %s", m.humanize(size)))
		} else {
			infos = append(infos, fmt.Sprintf("Size: %s B", csize))
		}
	}

	if mtype == "text/html" {
		title, err := m.extractTitle(resp.Body)
		if err == nil {
			infos = append(infos, fmt.Sprintf("Title: %s", title))
		}
	}

	info := strings.Join(infos, " | ")
	if len(info) > 0 {
		info = fmt.Sprintf("%s -- %s", strings.ToUpper(m.Name()), info)
	}

	return info
}

func (m *Module) extractTitle(reader io.Reader) (title string, err error) {
	doc, err := html.Parse(reader)
	if err != nil {
		return
	}

	var parseFunc func(n *html.Node)
	parseFunc = func(n *html.Node) {
		if n.Type == html.ElementNode && n.Data == "title" {
			child := n.FirstChild
			if child != nil {
				title = html.UnescapeString(child.Data)
			} else {
				return
			}
		}

		for c := n.FirstChild; c != nil; c = c.NextSibling {
			parseFunc(c)
		}
	}

	parseFunc(doc)
	title = m.sanitize(title)

	if len(title) <= 0 {
		return "", extractError
	}

	return
}

func (m *Module) sanitize(title string) string {
	normalized := strings.Replace(title, "\n", " ", -1)
	for strings.Contains(normalized, "  ") {
		normalized = strings.Replace(normalized, "  ", " ", -1)
	}

	return strings.TrimSpace(normalized)
}

func (m *Module) isExcluded(host string) bool {
	for _, h := range m.Exclude {
		if host == h {
			return true
		}
	}

	return false
}

func (m *Module) humanize(c int) string {
	switch {
	case c > (1 << 40):
		return fmt.Sprintf("%v TiB", c/(1<<40))
	case c > (1 << 30):
		return fmt.Sprintf("%v GiB", c/(1<<30))
	case c > (1 << 20):
		return fmt.Sprintf("%v MiB", c/(1<<20))
	case c > (1 << 10):
		return fmt.Sprintf("%v KiB", c/(1<<10))
	default:
		return fmt.Sprintf("%v B", c)
	}
}
<commit_msg>url: Content-Length needs to be an int<commit_after>// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see <http://www.gnu.org/licenses/>.

package url

import (
	"errors"
	"fmt"
	"github.com/nmeum/marvin/irc"
	"github.com/nmeum/marvin/modules"
	"golang.org/x/net/html"
	"io"
	"mime"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
)

var extractError = errors.New("couldn't extract title")

type Module struct {
	regex    *regexp.Regexp
	RegexStr string   `json:"regex"`
	Exclude  []string `json:"exclude"`
}

func Init(moduleSet *modules.ModuleSet) {
	moduleSet.Register(new(Module))
}

func (m *Module) Name() string {
	return "url"
}

func (m *Module) Help() string {
	return "Displays HTML titles for HTTP links."
}

func (m *Module) Defaults() {
	m.RegexStr = `(http|https)\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(:[a-zA-Z0-9]*)?/?([a-zA-Z0-9\-\._\?\,\'/\\\+&%\$#\=~])*`
}

func (m *Module) Load(client *irc.Client) error {
	regex, err := regexp.Compile(m.RegexStr)
	if err != nil {
		return err
	}

	m.regex = regex
	client.CmdHook("privmsg", m.urlCmd)

	return nil
}

func (m *Module) urlCmd(client *irc.Client, msg irc.Message) error {
	link := m.regex.FindString(msg.Data)
	if len(link) <= 0 {
		return nil
	}

	purl, err := url.Parse(link)
	if err != nil || m.isExcluded(purl.Host) {
		return nil
	}

	resp, err := http.Get(purl.String())
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	info := m.infoString(resp)
	if len(info) <= 0 {
		return nil
	}

	return client.Write("NOTICE %s :%s", msg.Receiver, info)
}

func (m *Module) infoString(resp *http.Response) string {
	var mtype string
	var infos []string

	ctype := resp.Header.Get("Content-Type")
	if len(ctype) > 0 {
		m, _, err := mime.ParseMediaType(ctype)
		if err == nil {
			mtype = m
			infos = append(infos, fmt.Sprintf("Type: %s", mtype))
		}
	}

	csize := resp.Header.Get("Content-Length")
	if len(csize) > 0 {
		size, err := strconv.Atoi(csize)
		if err == nil {
			infos = append(infos, fmt.Sprintf("Size: %s", m.humanize(size)))
		}
	}

	if mtype == "text/html" {
		title, err := m.extractTitle(resp.Body)
		if err == nil {
			infos = append(infos, fmt.Sprintf("Title: %s", title))
		}
	}

	info := strings.Join(infos, " | ")
	if len(info) > 0 {
		info = fmt.Sprintf("%s -- %s", strings.ToUpper(m.Name()), info)
	}

	return info
}

func (m *Module) extractTitle(reader io.Reader) (title string, err error) {
	doc, err := html.Parse(reader)
	if err != nil {
		return
	}

	var parseFunc func(n *html.Node)
	parseFunc = func(n *html.Node) {
		if n.Type == html.ElementNode && n.Data == "title" {
			child := n.FirstChild
			if child != nil {
				title = html.UnescapeString(child.Data)
			} else {
				return
			}
		}

		for c := n.FirstChild; c != nil; c = c.NextSibling {
			parseFunc(c)
		}
	}

	parseFunc(doc)
	title = m.sanitize(title)

	if len(title) <= 0 {
		return "", extractError
	}

	return
}

func (m *Module) sanitize(title string) string {
	normalized := strings.Replace(title, "\n", " ", -1)
	for strings.Contains(normalized, "  ") {
		normalized = strings.Replace(normalized, "  ", " ", -1)
	}

	return strings.TrimSpace(normalized)
}

func (m *Module) isExcluded(host string) bool {
	for _, h := range m.Exclude {
		if host == h {
			return true
		}
	}

	return false
}

func (m *Module) humanize(c int) string {
	switch {
	case c > (1 << 40):
		return fmt.Sprintf("%v TiB", c/(1<<40))
	case c > (1 << 30):
		return fmt.Sprintf("%v GiB", c/(1<<30))
	case c > (1 << 20):
		return fmt.Sprintf("%v MiB", c/(1<<20))
	case c > (1 << 10):
		return fmt.Sprintf("%v KiB", c/(1<<10))
	default:
		return fmt.Sprintf("%v B", c)
	}
}
<|endoftext|>


<commit_before>package contractor

// TODO: We are in the middle of migrating the contractor to a new concurrency
// model. The contractor should never call out to another package while under a
// lock (except for the proto package). This is because the renter is going to
// start calling contractor methods while holding the renter lock, so we need to
// be absolutely confident that no contractor thread will attempt to grab a
// renter lock.

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sync"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/modules/renter/proto"
	"github.com/NebulousLabs/Sia/persist"
	siasync "github.com/NebulousLabs/Sia/sync"
	"github.com/NebulousLabs/Sia/types"
)

var (
	errNilCS     = errors.New("cannot create contractor with nil consensus set")
	errNilTpool  = errors.New("cannot create contractor with nil transaction pool")
	errNilWallet = errors.New("cannot create contractor with nil wallet")

	// COMPATv1.0.4-lts
	// metricsContractID identifies a special contract that contains aggregate
	// financial metrics from older contractors
	metricsContractID = types.FileContractID{'m', 'e', 't', 'r', 'i', 'c', 's'}
)

// A cachedRevision contains changes that would be applied to a RenterContract
// if a contract revision succeeded. The contractor must cache these changes
// as a safeguard against desynchronizing with the host.
// TODO: save a diff of the Merkle roots instead of all of them.
type cachedRevision struct {
	Revision    types.FileContractRevision `json:"revision"`
	MerkleRoots modules.MerkleRootSet      `json:"merkleroots"`
}

// contractUtility contains metrics internal to the contractor that reflect
// the utility of a given contract.
type contractUtility struct {
	GoodForUpload bool
	GoodForRenew  bool
}

// A Contractor negotiates, revises, renews, and provides access to file
// contracts.
type Contractor struct {
	// dependencies
	cs      consensusSet
	hdb     hostDB
	log     *persist.Logger
	persist persister
	mu      sync.RWMutex
	tg      siasync.ThreadGroup
	tpool   transactionPool
	wallet  wallet

	// Only one thread should be performing contract maintenance at a time.
	interruptMaintenance chan struct{}
	maintenanceLock      siasync.TryMutex

	allowance     modules.Allowance
	blockHeight   types.BlockHeight
	currentPeriod types.BlockHeight
	lastChange    modules.ConsensusChangeID

	downloaders map[types.FileContractID]*hostDownloader
	editors     map[types.FileContractID]*hostEditor
	renewing    map[types.FileContractID]bool // prevent revising during renewal
	revising    map[types.FileContractID]bool // prevent overlapping revisions

	contracts         *proto.ContractSet
	contractUtilities map[types.FileContractID]contractUtility
	oldContracts      map[types.FileContractID]modules.RenterContract
	renewedIDs        map[types.FileContractID]types.FileContractID
}

// resolveID returns the ID of the most recent renewal of id.
func (c *Contractor) resolveID(id types.FileContractID) types.FileContractID {
	newID, exists := c.renewedIDs[id]
	for exists {
		id = newID
		newID, exists = c.renewedIDs[id]
	}
	return id
}

// Allowance returns the current allowance.
func (c *Contractor) Allowance() modules.Allowance {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return c.allowance
}

// PeriodSpending returns the amount spent on contracts during the current
// billing period.
func (c *Contractor) PeriodSpending() modules.ContractorSpending {
	c.mu.RLock()
	defer c.mu.RUnlock()

	var spending modules.ContractorSpending
	for _, contract := range c.contracts.ViewAll() {
		spending.ContractSpending = spending.ContractSpending.Add(contract.TotalCost)
		spending.DownloadSpending = spending.DownloadSpending.Add(contract.DownloadSpending)
		spending.UploadSpending = spending.UploadSpending.Add(contract.UploadSpending)
		spending.StorageSpending = spending.StorageSpending.Add(contract.StorageSpending)
		// TODO: fix PreviousContracts
		// for _, pre := range contract.PreviousContracts {
		// 	spending.ContractSpending = spending.ContractSpending.Add(pre.TotalCost)
		// 	spending.DownloadSpending = spending.DownloadSpending.Add(pre.DownloadSpending)
		// 	spending.UploadSpending = spending.UploadSpending.Add(pre.UploadSpending)
		// 	spending.StorageSpending = spending.StorageSpending.Add(pre.StorageSpending)
		// }
	}
	allSpending := spending.ContractSpending.Add(spending.DownloadSpending).Add(spending.UploadSpending).Add(spending.StorageSpending)
	spending.Unspent = c.allowance.Funds.Sub(allSpending)
	return spending
}

// ContractByID returns the contract with the id specified, if it exists.
func (c *Contractor) ContractByID(id types.FileContractID) (modules.RenterContract, bool) {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return c.contracts.View(id)
}

// Contracts returns the contracts formed by the contractor in the current
// allowance period. Only contracts formed with currently online hosts are
// returned.
func (c *Contractor) Contracts() []modules.RenterContract {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return c.contracts.ViewAll()
}

// CurrentPeriod returns the height at which the current allowance period
// began.
func (c *Contractor) CurrentPeriod() types.BlockHeight {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return c.currentPeriod
}

// GoodForUpload returns whether the given contract should be uploaded to.
func (c *Contractor) GoodForUpload(id types.FileContractID) bool {
	c.mu.RLock()
	good := c.contractUtilities[c.resolveID(id)].GoodForUpload
	c.mu.RUnlock()
	return good
}

// ResolveID returns the ID of the most recent renewal of id.
func (c *Contractor) ResolveID(id types.FileContractID) types.FileContractID {
	c.mu.RLock()
	newID := c.resolveID(id)
	c.mu.RUnlock()
	return newID
}

// Close closes the Contractor.
func (c *Contractor) Close() error {
	return c.tg.Stop()
}

// New returns a new Contractor.
func New(cs consensusSet, wallet walletShim, tpool transactionPool, hdb hostDB, persistDir string) (*Contractor, error) {
	// Check for nil inputs.
	if cs == nil {
		return nil, errNilCS
	}
	if wallet == nil {
		return nil, errNilWallet
	}
	if tpool == nil {
		return nil, errNilTpool
	}

	// Create the persist directory if it does not yet exist.
	err := os.MkdirAll(persistDir, 0700)
	if err != nil {
		return nil, err
	}
	// Create the contract set.
	// TODO: put these in subdirectory?
	contractSet, err := proto.NewContractSet(persistDir)
	if err != nil {
		return nil, err
	}
	// Create the logger.
	logger, err := persist.NewFileLogger(filepath.Join(persistDir, "contractor.log"))
	if err != nil {
		return nil, err
	}

	// Create Contractor using production dependencies.
	return newContractor(cs, &walletBridge{w: wallet}, tpool, hdb, contractSet, newPersist(persistDir), logger)
}

// newContractor creates a Contractor using the provided dependencies.
func newContractor(cs consensusSet, w wallet, tp transactionPool, hdb hostDB, contractSet *proto.ContractSet, p persister, l *persist.Logger) (*Contractor, error) {
	// Create the Contractor object.
	c := &Contractor{
		cs:      cs,
		hdb:     hdb,
		log:     l,
		persist: p,
		tpool:   tp,
		wallet:  w,

		interruptMaintenance: make(chan struct{}),

		contracts:         contractSet,
		downloaders:       make(map[types.FileContractID]*hostDownloader),
		editors:           make(map[types.FileContractID]*hostEditor),
		contractUtilities: make(map[types.FileContractID]contractUtility),
		oldContracts:      make(map[types.FileContractID]modules.RenterContract),
		renewedIDs:        make(map[types.FileContractID]types.FileContractID),
		renewing:          make(map[types.FileContractID]bool),
		revising:          make(map[types.FileContractID]bool),
	}

	// Close the logger (provided as a dependency) upon shutdown.
	c.tg.AfterStop(func() {
		if err := c.log.Close(); err != nil {
			fmt.Println("Failed to close the contractor logger:", err)
		}
	})

	// Load the prior persistence structures.
	err := c.load()
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	// Subscribe to the consensus set.
	err = cs.ConsensusSetSubscribe(c, c.lastChange, c.tg.StopChan())
	if err == modules.ErrInvalidConsensusChangeID {
		// Reset the contractor consensus variables and try rescanning.
		c.blockHeight = 0
		c.lastChange = modules.ConsensusChangeBeginning
		err = cs.ConsensusSetSubscribe(c, c.lastChange, c.tg.StopChan())
	}
	if err != nil {
		return nil, errors.New("contractor subscription failed: " + err.Error())
	}
	// Unsubscribe from the consensus set upon shutdown.
	c.tg.OnStop(func() {
		cs.Unsubscribe(c)
	})

	// We may have upgraded persist or resubscribed. Save now so that we don't
	// lose our work.
	c.mu.Lock()
	err = c.save()
	c.mu.Unlock()
	if err != nil {
		return nil, err
	}

	return c, nil
}
<commit_msg>close contractset on contractor close<commit_after>package contractor

// TODO: We are in the middle of migrating the contractor to a new concurrency
// model. The contractor should never call out to another package while under a
// lock (except for the proto package). This is because the renter is going to
// start calling contractor methods while holding the renter lock, so we need to
// be absolutely confident that no contractor thread will attempt to grab a
// renter lock.

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sync"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/modules/renter/proto"
	"github.com/NebulousLabs/Sia/persist"
	siasync "github.com/NebulousLabs/Sia/sync"
	"github.com/NebulousLabs/Sia/types"
)

var (
	errNilCS     = errors.New("cannot create contractor with nil consensus set")
	errNilTpool  = errors.New("cannot create contractor with nil transaction pool")
	errNilWallet = errors.New("cannot create contractor with nil wallet")

	// COMPATv1.0.4-lts
	// metricsContractID identifies a special contract that contains aggregate
	// financial metrics from older contractors
	metricsContractID = types.FileContractID{'m', 'e', 't', 'r', 'i', 'c', 's'}
)

// A cachedRevision contains changes that would be applied to a RenterContract
// if a contract revision succeeded. The contractor must cache these changes
// as a safeguard against desynchronizing with the host.
// TODO: save a diff of the Merkle roots instead of all of them.
type cachedRevision struct {
	Revision    types.FileContractRevision `json:"revision"`
	MerkleRoots modules.MerkleRootSet      `json:"merkleroots"`
}

// contractUtility contains metrics internal to the contractor that reflect
// the utility of a given contract.
type contractUtility struct {
	GoodForUpload bool
	GoodForRenew  bool
}

// A Contractor negotiates, revises, renews, and provides access to file
// contracts.
type Contractor struct {
	// dependencies
	cs      consensusSet
	hdb     hostDB
	log     *persist.Logger
	persist persister
	mu      sync.RWMutex
	tg      siasync.ThreadGroup
	tpool   transactionPool
	wallet  wallet

	// Only one thread should be performing contract maintenance at a time.
	interruptMaintenance chan struct{}
	maintenanceLock      siasync.TryMutex

	allowance     modules.Allowance
	blockHeight   types.BlockHeight
	currentPeriod types.BlockHeight
	lastChange    modules.ConsensusChangeID

	downloaders map[types.FileContractID]*hostDownloader
	editors     map[types.FileContractID]*hostEditor
	renewing    map[types.FileContractID]bool // prevent revising during renewal
	revising    map[types.FileContractID]bool // prevent overlapping revisions

	contracts         *proto.ContractSet
	contractUtilities map[types.FileContractID]contractUtility
	oldContracts      map[types.FileContractID]modules.RenterContract
	renewedIDs        map[types.FileContractID]types.FileContractID
}

// resolveID returns the ID of the most recent renewal of id.
func (c *Contractor) resolveID(id types.FileContractID) types.FileContractID {
	newID, exists := c.renewedIDs[id]
	for exists {
		id = newID
		newID, exists = c.renewedIDs[id]
	}
	return id
}

// Allowance returns the current allowance.
func (c *Contractor) Allowance() modules.Allowance {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return c.allowance
}

// PeriodSpending returns the amount spent on contracts during the current
// billing period.
func (c *Contractor) PeriodSpending() modules.ContractorSpending {
	c.mu.RLock()
	defer c.mu.RUnlock()

	var spending modules.ContractorSpending
	for _, contract := range c.contracts.ViewAll() {
		spending.ContractSpending = spending.ContractSpending.Add(contract.TotalCost)
		spending.DownloadSpending = spending.DownloadSpending.Add(contract.DownloadSpending)
		spending.UploadSpending = spending.UploadSpending.Add(contract.UploadSpending)
		spending.StorageSpending = spending.StorageSpending.Add(contract.StorageSpending)
		// TODO: fix PreviousContracts
		// for _, pre := range contract.PreviousContracts {
		// 	spending.ContractSpending = spending.ContractSpending.Add(pre.TotalCost)
		// 	spending.DownloadSpending = spending.DownloadSpending.Add(pre.DownloadSpending)
		// 	spending.UploadSpending = spending.UploadSpending.Add(pre.UploadSpending)
		// 	spending.StorageSpending = spending.StorageSpending.Add(pre.StorageSpending)
		// }
	}
	allSpending := spending.ContractSpending.Add(spending.DownloadSpending).Add(spending.UploadSpending).Add(spending.StorageSpending)
	spending.Unspent = c.allowance.Funds.Sub(allSpending)
	return spending
}

// ContractByID returns the contract with the id specified, if it exists.
func (c *Contractor) ContractByID(id types.FileContractID) (modules.RenterContract, bool) {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return c.contracts.View(id)
}

// Contracts returns the contracts formed by the contractor in the current
// allowance period. Only contracts formed with currently online hosts are
// returned.
func (c *Contractor) Contracts() []modules.RenterContract {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return c.contracts.ViewAll()
}

// CurrentPeriod returns the height at which the current allowance period
// began.
func (c *Contractor) CurrentPeriod() types.BlockHeight {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return c.currentPeriod
}

// GoodForUpload returns whether the given contract should be uploaded to.
func (c *Contractor) GoodForUpload(id types.FileContractID) bool {
	c.mu.RLock()
	good := c.contractUtilities[c.resolveID(id)].GoodForUpload
	c.mu.RUnlock()
	return good
}

// ResolveID returns the ID of the most recent renewal of id.
func (c *Contractor) ResolveID(id types.FileContractID) types.FileContractID {
	c.mu.RLock()
	newID := c.resolveID(id)
	c.mu.RUnlock()
	return newID
}

// Close closes the Contractor.
func (c *Contractor) Close() error {
	return c.tg.Stop()
}

// New returns a new Contractor.
func New(cs consensusSet, wallet walletShim, tpool transactionPool, hdb hostDB, persistDir string) (*Contractor, error) {
	// Check for nil inputs.
	if cs == nil {
		return nil, errNilCS
	}
	if wallet == nil {
		return nil, errNilWallet
	}
	if tpool == nil {
		return nil, errNilTpool
	}

	// Create the persist directory if it does not yet exist.
	err := os.MkdirAll(persistDir, 0700)
	if err != nil {
		return nil, err
	}
	// Create the contract set.
	// TODO: put these in subdirectory?
	contractSet, err := proto.NewContractSet(persistDir)
	if err != nil {
		return nil, err
	}
	// Create the logger.
	logger, err := persist.NewFileLogger(filepath.Join(persistDir, "contractor.log"))
	if err != nil {
		return nil, err
	}

	// Create Contractor using production dependencies.
	return newContractor(cs, &walletBridge{w: wallet}, tpool, hdb, contractSet, newPersist(persistDir), logger)
}

// newContractor creates a Contractor using the provided dependencies.
func newContractor(cs consensusSet, w wallet, tp transactionPool, hdb hostDB, contractSet *proto.ContractSet, p persister, l *persist.Logger) (*Contractor, error) {
	// Create the Contractor object.
	c := &Contractor{
		cs:      cs,
		hdb:     hdb,
		log:     l,
		persist: p,
		tpool:   tp,
		wallet:  w,

		interruptMaintenance: make(chan struct{}),

		contracts:         contractSet,
		downloaders:       make(map[types.FileContractID]*hostDownloader),
		editors:           make(map[types.FileContractID]*hostEditor),
		contractUtilities: make(map[types.FileContractID]contractUtility),
		oldContracts:      make(map[types.FileContractID]modules.RenterContract),
		renewedIDs:        make(map[types.FileContractID]types.FileContractID),
		renewing:          make(map[types.FileContractID]bool),
		revising:          make(map[types.FileContractID]bool),
	}

	// Close the contract set and logger upon shutdown.
	c.tg.AfterStop(func() {
		if err := c.contracts.Close(); err != nil {
			c.log.Println("Failed to close contract set:", err)
		}
		if err := c.log.Close(); err != nil {
			fmt.Println("Failed to close the contractor logger:", err)
		}
	})

	// Load the prior persistence structures.
	err := c.load()
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	// Subscribe to the consensus set.
	err = cs.ConsensusSetSubscribe(c, c.lastChange, c.tg.StopChan())
	if err == modules.ErrInvalidConsensusChangeID {
		// Reset the contractor consensus variables and try rescanning.
		c.blockHeight = 0
		c.lastChange = modules.ConsensusChangeBeginning
		err = cs.ConsensusSetSubscribe(c, c.lastChange, c.tg.StopChan())
	}
	if err != nil {
		return nil, errors.New("contractor subscription failed: " + err.Error())
	}
	// Unsubscribe from the consensus set upon shutdown.
	c.tg.OnStop(func() {
		cs.Unsubscribe(c)
	})

	// We may have upgraded persist or resubscribed. Save now so that we don't
	// lose our work.
	c.mu.Lock()
	err = c.save()
	c.mu.Unlock()
	if err != nil {
		return nil, err
	}

	return c, nil
}
<|endoftext|>